From 1949adfff9403245c31da97d26a83dd5405f3cc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20De=20Revi=C3=A8re?= Date: Thu, 16 Apr 2026 00:08:56 +0200 Subject: [PATCH] Add `gpt-5.4` models and request options * Add `lmterminal/model_registry.py` for aliases, pricing, and tokenizer fallbacks. * Add GPT-5.3, GPT-5.4, GPT-5 Pro, `o3-pro`, and newer Codex model names. * Add `--reasoning-effort` and `-o/--option key=value` for Chat Completions. * Apply `gpt-5.4` short vs long prompt pricing in `--tokens`. * Update README, template comments, changelog fragment, and tests. Summary: This keeps model support and prompt pricing in one registry and exposes request controls without hardcoding each new Chat Completions field. Co-authored-by: AI --- README.md | 63 +++- changelog.d/20260415_235800_ai.md | 5 + lmterminal/cli.py | 172 ++++++----- lmterminal/gpt_integration.py | 109 +++---- lmterminal/lib.py | 47 +-- lmterminal/model_registry.py | 466 ++++++++++++++++++++++++++++++ lmterminal/templates.py | 23 +- tests/cli_test.py | 160 ++++++++++ tests/gpt_integration_test.py | 92 ++++++ 9 files changed, 953 insertions(+), 184 deletions(-) create mode 100644 changelog.d/20260415_235800_ai.md create mode 100644 lmterminal/model_registry.py diff --git a/README.md b/README.md index ca1fab1..baa2a9b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # LMterminal (`lmt`) -LMterminal (`lmt`) is a CLI tool that enables you to interact directly with OpenAI's ChatGPT models from the comfort of your terminal. +LMterminal (`lmt`) is a CLI tool that enables you to interact directly with OpenAI models from the comfort of your terminal. ![demo](https://github.com/sderev/lmt/assets/24412384/5cb2c7b2-7edd-4b24-919d-581e5cd7c5b5) @@ -20,6 +20,7 @@ LMterminal (`lmt`) is a CLI tool that enables you to interact directly with Open 1. [Template Utilization](#template-utilization) 1. [Emoji Integration](#emoji-integration) 1. [Prompt Cost Estimation](#prompt-cost-estimation) + 1. [Reasoning Effort and Request Options](#reasoning-effort-and-request-options) 1. [Reading from `stdin`](#reading-from-stdin) 1. [Append an Additional Prompt to Piped `stdin`](#append-an-additional-prompt-to-piped-stdin) 1. [Output Redirection](#output-redirection) @@ -31,7 +32,7 @@ LMterminal (`lmt`) is a CLI tool that enables you to interact directly with Open ## Features -* **Access All ChatGPT Models**: `lmt` supports all available ChatGPT models (gpt-3.5-turbo, gpt-3.5-turbo-16k, gpt-4, gpt-4-32k), giving you the power to choose the most suitable one for your task. +* **Access Current OpenAI Models**: `lmt` supports current ChatGPT, GPT-5, Codex, and o-series models. Use `lmt models` to see the full list supported by your installed version. * **Custom Templates**: Design your own toolbox of templates to streamline your workflow. * **Read File**: Incorporate file content into your prompts seamlessly. * **Output to a File**: Redirect standard output (`stdout`) to a file or another program as needed. @@ -113,20 +114,18 @@ Switching between different models is a breeze with `lmt`. Use the `-m` flag fol lmt "Explain what is a large language model" -m 4 ``` -Below is a table outlining available model aliases for your convenience: +Run `lmt models` to print the full list of supported model names and aliases. + +A few examples: | Alias | Corresponding Model | | --- | --- | -| chatgpt | gpt-3.5-turbo | -| chatgpt-16k | gpt-3.5-turbo-16k | -| 3.5 | gpt-3.5-turbo | -| 3.5-16k | gpt-3.5-turbo-16k | -| 4 | gpt-4 | -| gpt4 | gpt-4 | -| 4-32k | gpt-4-32k | -| gpt4-32k | gpt-4-32k | +| `chatgpt` | `gpt-3.5-turbo` | +| `4o` | `gpt-4o` | +| `5` | `gpt-5` | +| `5.4` | `gpt-5.4` | -For instance, if you want to use the `gpt-4` model, simply include `-m 4` in your command. +For instance, if you want to use `gpt-5.4`, simply include `-m 5.4` in your command. ### Template Utilization @@ -158,6 +157,46 @@ To infuse a touch of emotion into your requests, append the `--emoji` flag optio For an estimation of your prompt's cost before sending, utilize the `--tokens` flag option. +`lmt` reports the prompt token count and the prompt-side input cost for the selected model. +For the `gpt-5.4` family, `lmt` applies short-context pricing below `272000` prompt tokens and long-context pricing at `272000` tokens and above when the model has a separate long-context rate. + +### Reasoning Effort and Request Options + +For models that support reasoning control, use `--reasoning-effort`: + +```bash +lmt "Explain this diff" -m gpt-5.4 --reasoning-effort high +``` + +Accepted values are: + +* `none` +* `minimal` +* `low` +* `medium` +* `high` +* `xhigh` + +You can also pass additional Chat Completions request options with repeatable `-o/--option key=value` pairs: + +```bash +lmt "Summarize this file" -m gpt-5.4 \ + --reasoning-effort medium \ + -o verbosity=low \ + -o top_p=0.9 \ + -o metadata.topic="cli" +``` + +Nested request objects can be passed with dotted keys or JSON values: + +```bash +lmt "Search the web for recent releases" \ + -o web_search_options.search_context_size=high \ + -o response_format='{"type":"json_object"}' +``` + +If a request option already has a dedicated CLI flag such as `--model`, `--temperature`, `--no-stream`, or `--reasoning-effort`, use that dedicated flag instead of `-o`. + ### Reading from `stdin` `lmt` facilitates reading inputs directly from `stdin`, allowing you to pipe in the content of a file as a prompt. This feature can be particularly useful when dealing with longer or more complex prompts, or when you want to streamline your workflow by incorporating `lmt` into a larger pipeline of commands. diff --git a/changelog.d/20260415_235800_ai.md b/changelog.d/20260415_235800_ai.md new file mode 100644 index 0000000..b1b2a10 --- /dev/null +++ b/changelog.d/20260415_235800_ai.md @@ -0,0 +1,5 @@ +Changed +------- +* Add GPT-5.3, GPT-5.4, GPT-5 Pro, `o3-pro`, and newer Codex model names to the CLI model registry. +* Use shared model metadata for alias resolution and prompt-cost estimation, including short vs long pricing for the `gpt-5.4` family. +* Add `--reasoning-effort` and repeatable `-o/--option key=value` request passthrough for Chat Completions. diff --git a/lmterminal/cli.py b/lmterminal/cli.py index b6c9aaf..a8de20b 100644 --- a/lmterminal/cli.py +++ b/lmterminal/cli.py @@ -1,4 +1,5 @@ import filecmp +import json import shutil import sys @@ -6,89 +7,29 @@ from click_default_group import DefaultGroup from .lib import DEFAULT_MODEL, edit_key, prepare_and_generate_response, set_key +from .model_registry import REASONING_EFFORTS, get_valid_models, resolve_model_name from .templates import TEMPLATES_DIR, get_default_template_file_path -VALID_MODELS = { - "gpt-3.5-turbo": ( - "chatgpt", - "3.5", - ), - "gpt-3.5-turbo-instruct": None, - "gpt-4": ( - "4", - "gpt4", - ), - "gpt-4-turbo": ( - "4t", - "4-turbo", - "gpt4-turbo", - ), - "gpt-4-32k": ( - "4-32k", - "gpt4-32k", - ), - "gpt-4o": ("4o",), - "gpt-4o-2024-05-13": None, - "gpt-4o-2024-08-06": None, - "gpt-4o-2024-11-20": None, - "gpt-4o-mini": ( - "4o-mini", - "4omini", - "4om", - ), - "gpt-4o-mini-2024-07-18": None, - "chatgpt-4o-latest": None, - "o1": None, - "o1-2024-12-17": None, - "o1-preview": None, - "o1-preview-2024-09-12": None, - "o1-mini": None, - "o1-mini-2024-09-12": None, - "o1-pro": None, - "o1-pro-2025-03-19": None, - "gpt-4.1": ("4.1",), - "gpt-4.1-2025-04-14": None, - "gpt-4.1-mini": ("4.1-mini",), - "gpt-4.1-mini-2025-04-14": None, - "gpt-4.1-nano": ("4.1-nano",), - "gpt-4.1-nano-2025-04-14": None, - "gpt-4.5-preview": None, - "o3": None, - "o3-2025-04-16": None, - "o3-mini": None, - "o3-mini-2025-01-31": None, - "o4-mini": None, - "o4-mini-2025-04-16": None, - "codex-mini-latest": None, - "gpt-4o-search-preview": None, - "gpt-4o-search-preview-2025-03-11": None, - "gpt-4o-mini-search-preview": None, - "gpt-4o-mini-search-preview-2025-03-11": None, - "gpt-5": ( - "5", - "gpt5", - ), - "gpt-5-mini": ("5-mini",), - "gpt-5-nano": ("5-nano",), - "gpt-5-chat-latest": None, - "gpt-5.1": ("5.1",), - "gpt-5.2": ("5.2",), +RESERVED_REQUEST_OPTION_KEYS = { + "messages": None, + "model": "--model", + "n": None, + "reasoning_effort": "--reasoning-effort", + "stream": "--no-stream", + "temperature": "--temperature", } +VALID_MODELS = get_valid_models() + # The first two parameters are required by Click for a callback. def validate_model_name(ctx, param, value): """ Validates the model name parameter. """ - # This is the value that the user entered for the model name. - model_name = value.lower() - - for model, aliases in VALID_MODELS.items(): - if model_name == model: - return model - if aliases is not None and model_name in aliases: - return model + canonical_model_name = resolve_model_name(value) + if canonical_model_name is not None: + return canonical_model_name error_message = ( f"{click.style('Invalid model name.', fg='red')}\n" @@ -109,11 +50,59 @@ def validate_temperature(ctx, param, value): raise click.BadParameter("Temperature must be between 0 and 2.") +def parse_request_option_value(raw_value): + """Parses a request option value from the CLI.""" + try: + return json.loads(raw_value) + except json.JSONDecodeError: + return raw_value + + +def add_request_option(options, key, value): + """Adds a parsed request option to a nested mapping.""" + key_parts = key.split(".") + if any(not key_part for key_part in key_parts): + raise click.BadParameter("Option keys cannot contain empty path segments.") + + current_level = options + for key_part in key_parts[:-1]: + if key_part not in current_level: + current_level[key_part] = {} + existing_value = current_level[key_part] + if not isinstance(existing_value, dict): + raise click.BadParameter( + f"Option `{key}` conflicts with an existing non-object option." + ) + current_level = existing_value + + leaf_key = key_parts[-1] + if leaf_key in current_level: + raise click.BadParameter(f"Option `{key}` was provided more than once.") + current_level[leaf_key] = value + + +def parse_request_options(ctx, param, values): + """Parses repeatable `key=value` request options from the CLI.""" + options = {} + + for raw_option in values: + if "=" not in raw_option: + raise click.BadParameter("Options must use the `key=value` form.") + + key, raw_value = raw_option.split("=", 1) + if not key: + raise click.BadParameter("Option keys cannot be empty.") + + add_request_option(options, key, parse_request_option_value(raw_value)) + + return options + + @click.group(cls=DefaultGroup, default="prompt", default_if_no_args=True) @click.version_option(package_name="lmterminal") def lmt(): """ - Talk to ChatGPT. + Talk to OpenAI models. Documentation: https://github.com/sderev/lmterminal """ @@ -152,6 +141,19 @@ def lmt(): help="The temperature to use for the requests.", show_default=True, ) +@click.option( + "--reasoning-effort", + type=click.Choice(REASONING_EFFORTS), + help="Set the reasoning effort for supported models.", +) +@click.option( + "-o", + "--option", + "request_options", + multiple=True, + callback=parse_request_options, + help="Pass additional Chat Completions request options as `key=value`.", +) @click.option( "--tokens", is_flag=True, @@ -191,6 +193,8 @@ def prompt( system, emoji, temperature, + reasoning_effort, + request_options, tokens, no_stream, raw, @@ -199,7 +203,7 @@ def prompt( debug, ): """ - Talk to ChatGPT. + Talk to OpenAI models. Example: lmt prompt "Say hello" --emoji """ @@ -244,6 +248,22 @@ def prompt( ), ) + conflicting_request_keys = RESERVED_REQUEST_OPTION_KEYS.keys() & request_options.keys() + if conflicting_request_keys: + conflicting_request_key = sorted(conflicting_request_keys)[0] + dedicated_option = RESERVED_REQUEST_OPTION_KEYS[conflicting_request_key] + if dedicated_option is None: + detail = f"`-o/--option` cannot set `{conflicting_request_key}`." + else: + detail = ( + f"`-o/--option` cannot override `{conflicting_request_key}`." + f" Use `{dedicated_option}` instead." + ) + raise click.BadOptionUsage( + option_name="option", + message=click.style(detail, fg="red"), + ) + # If *not* in an interactive shell or redirecting to a file, # enable the `--raw` option, viz. disabling `Rich` formatting if not sys.stdout.isatty(): @@ -264,6 +284,8 @@ def prompt( emoji, prompt_input, temperature, + reasoning_effort, + request_options, tokens, no_stream, raw, diff --git a/lmterminal/gpt_integration.py b/lmterminal/gpt_integration.py index f299abd..bc94b25 100644 --- a/lmterminal/gpt_integration.py +++ b/lmterminal/gpt_integration.py @@ -1,9 +1,12 @@ import sys import time +from dataclasses import dataclass import openai import tiktoken +from .model_registry import get_input_price_per_million, get_price_band, get_tokenizer_model + _client = None @@ -45,6 +48,8 @@ def chatgpt_request( temperature=1, stop=None, stream=False, + reasoning_effort=None, + request_options=None, update_markdown_stream=None, ): """ @@ -60,14 +65,22 @@ def chatgpt_request( client = _get_client(api_key) + request_kwargs = { + "messages": prompt, + "model": model, + "n": n, + "temperature": temperature, + "stream": stream, + } + if stop is not None: + request_kwargs["stop"] = stop + if reasoning_effort is not None: + request_kwargs["reasoning_effort"] = reasoning_effort + if request_options: + request_kwargs.update(request_options) + # Make the API request - response = client.chat.completions.create( - messages=prompt, - model=model, - n=n, - temperature=temperature, - stream=stream, - ) + response = client.chat.completions.create(**request_kwargs) if stream: # Create variables to collect the stream of chunks @@ -108,6 +121,7 @@ def chatgpt_request( def num_tokens_from_string(string, model=DEFAULT_MODEL): """Returns the number of tokens in a text string.""" + model = get_tokenizer_model(model) try: encoding = tiktoken.encoding_for_model(model) except KeyError: @@ -118,6 +132,7 @@ def num_tokens_from_string(string, model=DEFAULT_MODEL): def num_tokens_from_messages(messages, model=DEFAULT_MODEL): """Returns the number of tokens used by a list of messages.""" + model = get_tokenizer_model(model) try: encoding = tiktoken.encoding_for_model(model) except KeyError: @@ -143,67 +158,31 @@ def estimated_cost(num_tokens, price_per_1M_tokens): return f"{num_tokens / 10**6 * price_per_1M_tokens:.6f}" -def estimate_prompt_cost(message, model): - """Returns the estimated cost of a prompt.""" +@dataclass(frozen=True) +class PromptCostEstimate: + num_tokens: int + price_per_1m_tokens: float + cost: str + pricing_context: str | None + + +def estimate_prompt_cost_details(message, model): + """Returns prompt token and pricing details for a model.""" num_tokens = num_tokens_from_messages(message, model) + price_per_1m_tokens = get_input_price_per_million(model, num_tokens) + _, pricing_context = get_price_band(model, num_tokens) + + return PromptCostEstimate( + num_tokens=num_tokens, + price_per_1m_tokens=price_per_1m_tokens, + cost=estimated_cost(num_tokens, price_per_1m_tokens), + pricing_context=pricing_context, + ) - # Prices in USD per 1M input tokens - prices = { - "gpt-3.5-turbo": 0.50, - "gpt-3.5-turbo-0125": 0.50, - "gpt-3.5-turbo-1106": 0.50, - "gpt-3.5-turbo-instruct": 1.50, - "gpt-4": 30, - "gpt-4-turbo-preview": 10, - "gpt-4-turbo": 10, - "gpt-4-turbo-2024-04-09": 0.01, - "gpt-4-0613": 0.03, - "gpt-4-1106-preview": 10, - "gpt-4-0125-preview": 10, - "gpt-4-32k": 60, - "gpt-4-32k-0613": 60, - "gpt-4o": 2.50, - "gpt-4o-2024-05-13": 5, - "gpt-4o-2024-08-06": 2.50, - "gpt-4o-2024-11-20": 2.50, - "gpt-4o-mini": 0.15, - "gpt-4o-mini-2024-07-18": 0.15, - "chatgpt-4o-latest": 5, - "o1": 15, - "o1-2024-12-17": 15, - "o1-preview": 15, - "o1-preview-2024-09-12": 15, - "o1-mini": 1.10, - "o1-mini-2024-09-12": 1.10, - "o1-pro": 150, - "o1-pro-2025-03-19": 150, - "gpt-4.1": 2, - "gpt-4.1-2025-04-14": 2, - "gpt-4.1-mini": 0.40, - "gpt-4.1-mini-2025-04-14": 0.40, - "gpt-4.1-nano": 0.1, - "gpt-4.1-nano-2025-04-14": 0.10, - "gpt-4.5-preview": 75, - "o3": 2, - "o3-2025-04-16": 2, - "o3-mini": 1.10, - "o3-mini-2025-01-31": 1.10, - "o4-mini": 1.10, - "o4-mini-2025-04-16": 1.10, - "codex-mini-latest": 1.50, - "gpt-4o-search-preview": 2.50, - "gpt-4o-search-preview-2025-03-11": 2.50, - "gpt-4o-mini-search-preview": 0.15, - "gpt-4o-mini-search-preview-2025-03-11": 0.15, - "gpt-5": 1.25, - "gpt-5-mini": 0.25, - "gpt-5-nano": 0.05, - "gpt-5-chat-latest": 1.25, - "gpt-5.1": 1.25, - "gpt-5.2": 1.75, - } - return estimated_cost(num_tokens, prices[model]) +def estimate_prompt_cost(message, model): + """Returns the estimated cost of a prompt.""" + return estimate_prompt_cost_details(message, model).cost def handle_rate_limit_error(): diff --git a/lmterminal/lib.py b/lmterminal/lib.py index e3d8947..fc9d738 100644 --- a/lmterminal/lib.py +++ b/lmterminal/lib.py @@ -26,6 +26,8 @@ def prepare_and_generate_response( emoji: bool, prompt_input: str, temperature: float, + reasoning_effort: str | None, + request_options: dict | None, tokens: bool, no_stream: bool, raw: bool, @@ -62,7 +64,7 @@ def prepare_and_generate_response( ] if debug: - display_debug_information(prompt, model, temperature) + display_debug_information(prompt, model, temperature, reasoning_effort, request_options) if tokens: display_tokens_count_and_cost(prompt, model) @@ -75,6 +77,8 @@ def prepare_and_generate_response( raw, stream, temperature, + reasoning_effort, + request_options, ) return content, response_time, response @@ -159,6 +163,8 @@ def generate_response( raw: bool = False, stream: bool = True, temperature: float = 1, + reasoning_effort: str | None = None, + request_options: dict | None = None, ): """ Generates a response from a ChatGPT. @@ -200,6 +206,8 @@ def update_markdown_stream(chunk: str) -> None: model=model, stream=stream, temperature=temperature, + reasoning_effort=reasoning_effort, + request_options=request_options, update_markdown_stream=update_markdown_stream, ) @@ -235,7 +243,9 @@ def update_markdown_stream(chunk: str) -> None: return content, response_time, response -def display_debug_information(prompt, model, temperature): +def display_debug_information( + prompt, model, temperature, reasoning_effort=None, request_options=None +): """ Displays debug information. """ @@ -258,6 +268,14 @@ def display_debug_information(prompt, model, temperature): click.echo(f"{temperature=}", err=True) click.echo(err=True) + click.secho("Reasoning effort:", fg="red", err=True) + click.echo(f"{reasoning_effort=}", err=True) + click.echo(err=True) + + click.secho("Request options:", fg="red", err=True) + click.echo(f"{request_options=}", err=True) + click.echo(err=True) + click.secho("End of debug information.", fg="yellow", err=True) click.echo("---\n", err=True) @@ -266,28 +284,21 @@ def display_tokens_count_and_cost(prompt, model): """ Displays the number of tokens in the prompt and the cost of the prompt. """ - # If the model is of the `o1` variant, it will ignore the system message. - # This is a temporary solution until the `o1` models support system messages. - if "o1" in model: - full_prompt = prompt[0]["content"] - else: - full_prompt = prompt[0]["content"] + prompt[1]["content"] - - # The model name `chatgpt-4o-latest` is not found in the `tiktoken` OpenAI API (as of 2024-08-14). - # Since it's a `gpt-4o` variant anyway, we can just use `gpt-4o`. - if model == "chatgpt-4o-latest": - model = "gpt-4o" - - number_of_tokens = openai_utils.num_tokens_from_string(full_prompt, model) - cost = openai_utils.estimate_prompt_cost(prompt, model) + prompt_cost_estimate = openai_utils.estimate_prompt_cost_details(prompt, model) click.echo( - f"Number of tokens in the prompt: {click.style(str(number_of_tokens), fg='yellow')}." + "Number of tokens in the prompt:" + f" {click.style(str(prompt_cost_estimate.num_tokens), fg='yellow')}." ) click.echo( f"Cost of the prompt for the {click.style(model, fg='blue')} model is:" - f" {click.style(f'${cost}', fg='yellow')}." + f" {click.style(f'${prompt_cost_estimate.cost}', fg='yellow')}." ) + if prompt_cost_estimate.pricing_context: + click.echo( + "Pricing tier used for this estimate:" + f" {click.style(prompt_cost_estimate.pricing_context, fg='yellow')} context." + ) click.echo( "Please note that this cost applies only to the prompt, not the subsequent response." ) diff --git a/lmterminal/model_registry.py b/lmterminal/model_registry.py new file mode 100644 index 0000000..1a2d9c9 --- /dev/null +++ b/lmterminal/model_registry.py @@ -0,0 +1,466 @@ +from __future__ import annotations + +from dataclasses import dataclass + +SHORT_CONTEXT_TOKEN_THRESHOLD = 272_000 +REASONING_EFFORTS = ( + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", +) + + +@dataclass(frozen=True) +class PriceBand: + input: float | None = None + cached_input: float | None = None + output: float | None = None + + +@dataclass(frozen=True) +class ModelSpec: + aliases: tuple[str, ...] = () + short_context: PriceBand = PriceBand() + long_context: PriceBand | None = None + tokenizer_model: str | None = None + show_in_cli: bool = True + + +def _spec( + *, + aliases: tuple[str, ...] = (), + short_input: float | None = None, + short_cached_input: float | None = None, + short_output: float | None = None, + long_input: float | None = None, + long_cached_input: float | None = None, + long_output: float | None = None, + tokenizer_model: str | None = None, + show_in_cli: bool = True, +) -> ModelSpec: + long_context = None + if any(value is not None for value in (long_input, long_cached_input, long_output)): + long_context = PriceBand(long_input, long_cached_input, long_output) + + return ModelSpec( + aliases=aliases, + short_context=PriceBand(short_input, short_cached_input, short_output), + long_context=long_context, + tokenizer_model=tokenizer_model, + show_in_cli=show_in_cli, + ) + + +MODEL_REGISTRY = { + "gpt-3.5-turbo": _spec( + aliases=("chatgpt", "3.5"), + short_input=0.50, + ), + "gpt-3.5-turbo-0125": _spec( + short_input=0.50, + tokenizer_model="gpt-3.5-turbo", + show_in_cli=False, + ), + "gpt-3.5-turbo-1106": _spec( + short_input=0.50, + tokenizer_model="gpt-3.5-turbo", + show_in_cli=False, + ), + "gpt-3.5-turbo-instruct": _spec(short_input=1.50), + "gpt-4": _spec( + aliases=("4", "gpt4"), + short_input=30, + ), + "gpt-4-turbo": _spec( + aliases=("4t", "4-turbo", "gpt4-turbo"), + short_input=10, + ), + "gpt-4-turbo-preview": _spec( + short_input=10, + tokenizer_model="gpt-4-turbo", + show_in_cli=False, + ), + "gpt-4-turbo-2024-04-09": _spec( + short_input=0.01, + tokenizer_model="gpt-4-turbo", + show_in_cli=False, + ), + "gpt-4-0613": _spec( + short_input=0.03, + tokenizer_model="gpt-4", + show_in_cli=False, + ), + "gpt-4-32k": _spec( + aliases=("4-32k", "gpt4-32k"), + short_input=60, + ), + "gpt-4-1106-preview": _spec( + short_input=10, + tokenizer_model="gpt-4-turbo", + show_in_cli=False, + ), + "gpt-4-0125-preview": _spec( + short_input=10, + tokenizer_model="gpt-4-turbo", + show_in_cli=False, + ), + "gpt-4-32k-0613": _spec( + short_input=60, + tokenizer_model="gpt-4-32k", + show_in_cli=False, + ), + "gpt-4o": _spec( + aliases=("4o",), + short_input=2.50, + short_cached_input=1.25, + short_output=10.00, + ), + "gpt-4o-2024-05-13": _spec( + short_input=5.00, + short_output=15.00, + tokenizer_model="gpt-4o", + ), + "gpt-4o-2024-08-06": _spec( + short_input=2.50, + short_cached_input=1.25, + short_output=10.00, + tokenizer_model="gpt-4o", + ), + "gpt-4o-2024-11-20": _spec( + short_input=2.50, + short_cached_input=1.25, + short_output=10.00, + tokenizer_model="gpt-4o", + ), + "gpt-4o-mini": _spec( + aliases=("4o-mini", "4omini", "4om"), + short_input=0.15, + short_cached_input=0.075, + short_output=0.60, + ), + "gpt-4o-mini-2024-07-18": _spec( + short_input=0.15, + short_cached_input=0.075, + short_output=0.60, + tokenizer_model="gpt-4o-mini", + ), + "chatgpt-4o-latest": _spec( + short_input=5.00, + short_output=15.00, + tokenizer_model="gpt-4o", + ), + "o1": _spec( + short_input=15.00, + short_cached_input=7.50, + short_output=60.00, + ), + "o1-2024-12-17": _spec( + short_input=15.00, + short_cached_input=7.50, + short_output=60.00, + tokenizer_model="o1", + ), + "o1-preview": _spec( + short_input=15.00, + tokenizer_model="o1", + ), + "o1-preview-2024-09-12": _spec( + short_input=15.00, + tokenizer_model="o1", + ), + "o1-mini": _spec( + short_input=1.10, + short_cached_input=0.55, + short_output=4.40, + ), + "o1-mini-2024-09-12": _spec( + short_input=1.10, + short_cached_input=0.55, + short_output=4.40, + tokenizer_model="o1-mini", + ), + "o1-pro": _spec( + short_input=150.00, + short_output=600.00, + ), + "o1-pro-2025-03-19": _spec( + short_input=150.00, + short_output=600.00, + tokenizer_model="o1-pro", + ), + "gpt-4.1": _spec( + aliases=("4.1",), + short_input=2.00, + short_cached_input=0.50, + short_output=8.00, + ), + "gpt-4.1-2025-04-14": _spec( + short_input=2.00, + short_cached_input=0.50, + short_output=8.00, + tokenizer_model="gpt-4.1", + ), + "gpt-4.1-mini": _spec( + aliases=("4.1-mini",), + short_input=0.40, + short_cached_input=0.10, + short_output=1.60, + ), + "gpt-4.1-mini-2025-04-14": _spec( + short_input=0.40, + short_cached_input=0.10, + short_output=1.60, + tokenizer_model="gpt-4.1-mini", + ), + "gpt-4.1-nano": _spec( + aliases=("4.1-nano",), + short_input=0.10, + short_cached_input=0.025, + short_output=0.40, + ), + "gpt-4.1-nano-2025-04-14": _spec( + short_input=0.10, + short_cached_input=0.025, + short_output=0.40, + tokenizer_model="gpt-4.1-nano", + ), + "gpt-4.5-preview": _spec(short_input=75), + "o3": _spec( + short_input=2.00, + short_cached_input=0.50, + short_output=8.00, + ), + "o3-2025-04-16": _spec( + short_input=2.00, + short_cached_input=0.50, + short_output=8.00, + tokenizer_model="o3", + ), + "o3-mini": _spec( + short_input=1.10, + short_cached_input=0.55, + short_output=4.40, + ), + "o3-mini-2025-01-31": _spec( + short_input=1.10, + short_cached_input=0.55, + short_output=4.40, + tokenizer_model="o3-mini", + ), + "o3-pro": _spec( + short_input=20.00, + short_output=80.00, + ), + "o4-mini": _spec( + short_input=1.10, + short_cached_input=0.275, + short_output=4.40, + ), + "o4-mini-2025-04-16": _spec( + short_input=1.10, + short_cached_input=0.275, + short_output=4.40, + tokenizer_model="o4-mini", + ), + "codex-mini-latest": _spec( + short_input=1.50, + short_cached_input=0.375, + short_output=6.00, + ), + "gpt-4o-search-preview": _spec(short_input=2.50), + "gpt-4o-search-preview-2025-03-11": _spec( + short_input=2.50, + tokenizer_model="gpt-4o", + ), + "gpt-4o-mini-search-preview": _spec(short_input=0.15), + "gpt-4o-mini-search-preview-2025-03-11": _spec( + short_input=0.15, + tokenizer_model="gpt-4o-mini", + ), + "gpt-5": _spec( + aliases=("5", "gpt5"), + short_input=1.25, + short_cached_input=0.125, + short_output=10.00, + ), + "gpt-5-mini": _spec( + aliases=("5-mini",), + short_input=0.25, + short_cached_input=0.025, + short_output=2.00, + ), + "gpt-5-nano": _spec( + aliases=("5-nano",), + short_input=0.05, + short_cached_input=0.005, + short_output=0.40, + ), + "gpt-5-chat-latest": _spec( + short_input=1.25, + short_cached_input=0.125, + short_output=10.00, + tokenizer_model="gpt-5", + ), + "gpt-5-codex": _spec( + short_input=1.25, + short_cached_input=0.125, + short_output=10.00, + tokenizer_model="gpt-5", + ), + "gpt-5-pro": _spec( + aliases=("5-pro",), + short_input=15.00, + short_output=120.00, + tokenizer_model="gpt-5", + ), + "gpt-5.1": _spec( + aliases=("5.1",), + short_input=1.25, + short_cached_input=0.125, + short_output=10.00, + tokenizer_model="gpt-5", + ), + "gpt-5.1-chat-latest": _spec( + short_input=1.25, + short_cached_input=0.125, + short_output=10.00, + tokenizer_model="gpt-5", + ), + "gpt-5.1-codex": _spec( + short_input=1.25, + short_cached_input=0.125, + short_output=10.00, + tokenizer_model="gpt-5", + ), + "gpt-5.1-codex-max": _spec( + short_input=1.25, + short_cached_input=0.125, + short_output=10.00, + tokenizer_model="gpt-5", + ), + "gpt-5.1-codex-mini": _spec( + short_input=0.25, + short_cached_input=0.025, + short_output=2.00, + tokenizer_model="gpt-5", + ), + "gpt-5.2": _spec( + aliases=("5.2",), + short_input=1.75, + short_cached_input=0.175, + short_output=14.00, + tokenizer_model="gpt-5", + ), + "gpt-5.2-chat-latest": _spec( + short_input=1.75, + short_cached_input=0.175, + short_output=14.00, + tokenizer_model="gpt-5", + ), + "gpt-5.2-codex": _spec( + short_input=1.75, + short_cached_input=0.175, + short_output=14.00, + tokenizer_model="gpt-5", + ), + "gpt-5.2-pro": _spec( + aliases=("5.2-pro",), + short_input=21.00, + short_output=168.00, + tokenizer_model="gpt-5", + ), + "gpt-5.3-chat-latest": _spec( + short_input=1.75, + short_cached_input=0.175, + short_output=14.00, + tokenizer_model="gpt-5", + ), + "gpt-5.3-codex": _spec( + short_input=1.75, + short_cached_input=0.175, + short_output=14.00, + tokenizer_model="gpt-5", + ), + "gpt-5.4": _spec( + aliases=("5.4",), + short_input=2.50, + short_cached_input=0.25, + short_output=15.00, + long_input=5.00, + long_cached_input=0.50, + long_output=22.50, + tokenizer_model="gpt-5", + ), + "gpt-5.4-mini": _spec( + aliases=("5.4-mini",), + short_input=0.75, + short_cached_input=0.075, + short_output=4.50, + tokenizer_model="gpt-5", + ), + "gpt-5.4-nano": _spec( + aliases=("5.4-nano",), + short_input=0.20, + short_cached_input=0.02, + short_output=1.25, + tokenizer_model="gpt-5", + ), + "gpt-5.4-pro": _spec( + aliases=("5.4-pro",), + short_input=30.00, + short_output=180.00, + long_input=60.00, + long_output=270.00, + tokenizer_model="gpt-5", + ), +} + + +def get_valid_models() -> dict[str, tuple[str, ...] | None]: + return { + model_name: spec.aliases or None + for model_name, spec in MODEL_REGISTRY.items() + if spec.show_in_cli + } + + +def resolve_model_name(model_name: str) -> str | None: + normalized_name = model_name.lower() + + for canonical_model_name, spec in MODEL_REGISTRY.items(): + if not spec.show_in_cli: + continue + if normalized_name == canonical_model_name: + return canonical_model_name + if normalized_name in spec.aliases: + return canonical_model_name + + return None + + +def get_model_spec(model_name: str) -> ModelSpec: + return MODEL_REGISTRY[model_name] + + +def get_tokenizer_model(model_name: str) -> str: + spec = MODEL_REGISTRY.get(model_name) + if spec is None: + return model_name + return spec.tokenizer_model or model_name + + +def get_price_band(model_name: str, prompt_tokens: int) -> tuple[PriceBand, str | None]: + spec = get_model_spec(model_name) + if spec.long_context and prompt_tokens >= SHORT_CONTEXT_TOKEN_THRESHOLD: + return spec.long_context, "long" + return spec.short_context, "short" if spec.long_context else None + + +def get_input_price_per_million(model_name: str, prompt_tokens: int) -> float: + price_band, _ = get_price_band(model_name, prompt_tokens) + if price_band.input is None: + raise KeyError(f"No input price configured for model: {model_name}") + return price_band.input diff --git a/lmterminal/templates.py b/lmterminal/templates.py index 61a5a3d..21a2278 100644 --- a/lmterminal/templates.py +++ b/lmterminal/templates.py @@ -96,24 +96,19 @@ def get_default_template_file_path() -> Path: -# You can change the model according to the list below. -# The default model is "gpt-3.5-turbo". +# You can change the model according to the list printed by `lmt models`. +# The default model is "gpt-5-nano". # More advanced models might provide more accurate or detailed responses, -# but they also have a higher "cost per 1K tokens". Tokens refer to chunks of text that AI models read. More tokens usually mean more cost. -model: "gpt-3.5-turbo" +# but they also have a higher prompt and response cost. +model: "gpt-5-nano" -# Valid Models +# Model examples # # "chatgpt" or "gpt-3.5-turbo" -# "chatgpt-16k" or "gpt-3.5-turbo-16k" +# "4o" or "gpt-4o" +# "5" or "gpt-5" +# "5.4" or "gpt-5.4" # -# "3.5" or "gpt-3.5-turbo" -# "3.5-16k" or "gpt-3.5-turbo-16k" -# -# "4" or "gpt-4" -# "gpt4" or "gpt-4" -# -# "4-32k" or "gpt-4-32k" -# "gpt4-32k" or "gpt-4-32k" +# Run `lmt models` to see the full list available in your installed version. """ diff --git a/tests/cli_test.py b/tests/cli_test.py index 959c0bd..1e0e47c 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -61,6 +61,40 @@ ("gpt-4o-search-preview-2025-03-11", "gpt-4o-search-preview-2025-03-11"), ("gpt-4o-mini-search-preview", "gpt-4o-mini-search-preview"), ("gpt-4o-mini-search-preview-2025-03-11", "gpt-4o-mini-search-preview-2025-03-11"), + ("gpt-5", "gpt-5"), + ("5", "gpt-5"), + ("gpt5", "gpt-5"), + ("gpt-5-mini", "gpt-5-mini"), + ("5-mini", "gpt-5-mini"), + ("gpt-5-nano", "gpt-5-nano"), + ("5-nano", "gpt-5-nano"), + ("gpt-5-chat-latest", "gpt-5-chat-latest"), + ("gpt-5-codex", "gpt-5-codex"), + ("gpt-5-pro", "gpt-5-pro"), + ("5-pro", "gpt-5-pro"), + ("gpt-5.1", "gpt-5.1"), + ("5.1", "gpt-5.1"), + ("gpt-5.1-chat-latest", "gpt-5.1-chat-latest"), + ("gpt-5.1-codex", "gpt-5.1-codex"), + ("gpt-5.1-codex-max", "gpt-5.1-codex-max"), + ("gpt-5.1-codex-mini", "gpt-5.1-codex-mini"), + ("gpt-5.2", "gpt-5.2"), + ("5.2", "gpt-5.2"), + ("gpt-5.2-chat-latest", "gpt-5.2-chat-latest"), + ("gpt-5.2-codex", "gpt-5.2-codex"), + ("gpt-5.2-pro", "gpt-5.2-pro"), + ("5.2-pro", "gpt-5.2-pro"), + ("gpt-5.3-chat-latest", "gpt-5.3-chat-latest"), + ("gpt-5.3-codex", "gpt-5.3-codex"), + ("gpt-5.4", "gpt-5.4"), + ("5.4", "gpt-5.4"), + ("gpt-5.4-mini", "gpt-5.4-mini"), + ("5.4-mini", "gpt-5.4-mini"), + ("gpt-5.4-nano", "gpt-5.4-nano"), + ("5.4-nano", "gpt-5.4-nano"), + ("gpt-5.4-pro", "gpt-5.4-pro"), + ("5.4-pro", "gpt-5.4-pro"), + ("o3-pro", "o3-pro"), ], ) def test_validate_model_name(alias, canonical): @@ -92,6 +126,132 @@ def test_validate_temperature_invalid(value): cli.validate_temperature(None, None, value) +def test_reasoning_effort_rejects_unknown_value(): + runner = CliRunner() + result = runner.invoke(cli.lmt, ["hello", "--reasoning-effort", "extreme"]) + + assert result.exit_code != 0 + assert "Invalid value for '--reasoning-effort'" in result.output + + +def test_parse_request_options_supports_json_scalars_and_nested_keys(): + parsed_options = cli.parse_request_options( + None, + None, + ( + "top_p=0.9", + "store=true", + 'metadata.topic="cli"', + 'response_format={"type":"json_object"}', + ), + ) + + assert parsed_options == { + "top_p": 0.9, + "store": True, + "metadata": {"topic": "cli"}, + "response_format": {"type": "json_object"}, + } + + +def test_parse_request_options_leaves_plain_strings_unchanged(): + parsed_options = cli.parse_request_options(None, None, ("service_tier=priority",)) + + assert parsed_options == {"service_tier": "priority"} + + +@pytest.mark.parametrize( + "raw_option", + [ + "missing-separator", + "=value", + 'metadata..topic="cli"', + ], +) +def test_parse_request_options_rejects_invalid_input(raw_option): + with pytest.raises(BadParameter): + cli.parse_request_options(None, None, (raw_option,)) + + +def test_parse_request_options_rejects_duplicate_keys(): + with pytest.raises(BadParameter): + cli.parse_request_options(None, None, ("top_p=0.9", "top_p=0.8")) + + +def test_parse_request_options_rejects_nested_override_after_null(): + with pytest.raises(BadParameter): + cli.parse_request_options(None, None, ("metadata=null", 'metadata.topic="cli"')) + + +def test_prompt_forwards_reasoning_effort_and_request_options(monkeypatch): + captured_call = {} + + def fake_prepare_and_generate_response(*args): + captured_call.update( + { + "system": args[0], + "template": args[1], + "model": args[2], + "emoji": args[3], + "prompt_input": args[4], + "temperature": args[5], + "reasoning_effort": args[6], + "request_options": args[7], + "tokens": args[8], + "no_stream": args[9], + "raw": args[10], + "debug": args[11], + } + ) + + monkeypatch.setattr(cli, "prepare_and_generate_response", fake_prepare_and_generate_response) + monkeypatch.setattr(cli.sys.stdin, "isatty", lambda: True) + + runner = CliRunner() + result = runner.invoke( + cli.lmt, + [ + "hello", + "--reasoning-effort", + "high", + "-o", + "top_p=0.9", + "-o", + 'metadata.topic="cli"', + ], + ) + + assert result.exit_code == 0 + assert captured_call["model"] == cli.DEFAULT_MODEL + assert captured_call["prompt_input"].endswith("hello") + assert captured_call["reasoning_effort"] == "high" + assert captured_call["request_options"] == { + "top_p": 0.9, + "metadata": {"topic": "cli"}, + } + + +@pytest.mark.parametrize( + "reserved_key, dedicated_flag", + [ + ("model=gpt-5.4", "--model"), + ("n=2", None), + ("temperature=0.9", "--temperature"), + ("stream=true", "--no-stream"), + ("reasoning_effort=high", "--reasoning-effort"), + ], +) +def test_prompt_rejects_reserved_request_options(reserved_key, dedicated_flag): + runner = CliRunner() + result = runner.invoke(cli.lmt, ["hello", "-o", reserved_key]) + + assert result.exit_code != 0 + if dedicated_flag is None: + assert f"cannot set `{reserved_key.split('=', 1)[0]}`" in result.output + else: + assert dedicated_flag in result.output + + @pytest.mark.integration @pytest.mark.parametrize("model", list(cli.VALID_MODELS), ids=str) def test_live_call(model, request): diff --git a/tests/gpt_integration_test.py b/tests/gpt_integration_test.py index 6cda9cc..54cab89 100644 --- a/tests/gpt_integration_test.py +++ b/tests/gpt_integration_test.py @@ -101,3 +101,95 @@ def fake_print(*args, **kwargs): (("Hel",), {"end": "", "flush": True}), (("lo",), {"end": "", "flush": True}), ] + + +def test_estimate_prompt_cost_details_uses_short_context_pricing(monkeypatch): + monkeypatch.setattr( + gpt_integration, "num_tokens_from_messages", lambda *_args, **_kwargs: 271_999 + ) + + estimate = gpt_integration.estimate_prompt_cost_details( + [{"role": "user", "content": "hello"}], + "gpt-5.4", + ) + + assert estimate.num_tokens == 271_999 + assert estimate.price_per_1m_tokens == 2.50 + assert estimate.pricing_context == "short" + assert estimate.cost == "0.679998" + + +def test_estimate_prompt_cost_details_uses_long_context_pricing(monkeypatch): + monkeypatch.setattr( + gpt_integration, "num_tokens_from_messages", lambda *_args, **_kwargs: 272_000 + ) + + estimate = gpt_integration.estimate_prompt_cost_details( + [{"role": "user", "content": "hello"}], + "gpt-5.4", + ) + + assert estimate.num_tokens == 272_000 + assert estimate.price_per_1m_tokens == 5.00 + assert estimate.pricing_context == "long" + assert estimate.cost == "1.360000" + + +def test_estimate_prompt_cost_details_keeps_short_pricing_when_long_band_missing(monkeypatch): + monkeypatch.setattr( + gpt_integration, "num_tokens_from_messages", lambda *_args, **_kwargs: 272_000 + ) + + estimate = gpt_integration.estimate_prompt_cost_details( + [{"role": "user", "content": "hello"}], + "gpt-5.4-mini", + ) + + assert estimate.num_tokens == 272_000 + assert estimate.price_per_1m_tokens == 0.75 + assert estimate.pricing_context is None + assert estimate.cost == "0.204000" + + +def test_estimate_prompt_cost_preserves_legacy_price_only_models(monkeypatch): + monkeypatch.setattr( + gpt_integration, "num_tokens_from_messages", lambda *_args, **_kwargs: 1_000_000 + ) + + cost = gpt_integration.estimate_prompt_cost( + [{"role": "user", "content": "hello"}], + "gpt-4-turbo-preview", + ) + + assert cost == "10.000000" + + +def test_chatgpt_request_forwards_reasoning_effort_and_request_options(monkeypatch): + response = SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content="pong"))]) + client, completions = _build_client(response) + monkeypatch.setattr(gpt_integration, "_get_client", lambda _api_key: client) + + prompt = [{"role": "user", "content": "ping"}] + gpt_integration.chatgpt_request( + api_key="test-key", + prompt=prompt, + model="gpt-5.4", + n=1, + temperature=0.3, + stream=False, + reasoning_effort="high", + request_options={"top_p": 0.9, "metadata": {"topic": "test"}}, + ) + + assert completions.calls == [ + { + "messages": prompt, + "model": "gpt-5.4", + "n": 1, + "temperature": 0.3, + "stream": False, + "reasoning_effort": "high", + "top_p": 0.9, + "metadata": {"topic": "test"}, + } + ]