| """Regression test for the `/model` picker confirmation display. |
| |
| Bug (April 2026): after choosing a model from the interactive `/model` picker, |
| ``HermesCLI._apply_model_switch_result()`` printed ``ModelInfo.context_window`` |
| straight from models.dev, which always reports the vendor-wide value (e.g. |
| gpt-5.5 = 1,050,000 on ``openai``). That ignored provider-specific caps — in |
| particular, ChatGPT Codex OAuth enforces 272K on the same slug. The sibling |
| ``_handle_model_switch()`` (typed ``/model <name>``) was already fixed to use |
| ``resolve_display_context_length()``; the picker path was missed, causing |
| "sometimes 1M, sometimes 272K" for the same model across sibling UI paths. |
| |
| Fix: both display paths now go through ``resolve_display_context_length()``. |
| """ |
| from __future__ import annotations |
|
|
| from unittest.mock import patch |
|
|
| from hermes_cli.model_switch import ModelSwitchResult |
|
|
|
|
| class _FakeModelInfo: |
| context_window = 1_050_000 |
| max_output = 0 |
|
|
| def has_cost_data(self): |
| return False |
|
|
| def format_capabilities(self): |
| return "" |
|
|
|
|
| class _StubCLI: |
| """Minimum attrs ``_apply_model_switch_result`` reads on ``self``.""" |
| def _stage_and_swap_model(self, result, old_model): |
| |
| import cli as _cli_mod |
| return _cli_mod.HermesCLI._stage_and_swap_model(self, result, old_model) |
|
|
| agent = None |
| model = "" |
| provider = "" |
| requested_provider = "" |
| api_key = "" |
| _explicit_api_key = "" |
| base_url = "" |
| _explicit_base_url = "" |
| api_mode = "" |
| _pending_model_switch_note = "" |
|
|
|
|
| def _run_display(monkeypatch, result): |
| import cli as cli_mod |
|
|
| captured: list[str] = [] |
| monkeypatch.setattr(cli_mod, "_cprint", lambda s, *a, **k: captured.append(str(s))) |
| |
| monkeypatch.setattr(cli_mod, "save_config_value", lambda *a, **k: None) |
| cli_mod.HermesCLI._apply_model_switch_result(_StubCLI(), result, False) |
| return captured |
|
|
|
|
| def test_picker_path_uses_provider_aware_context_on_codex(monkeypatch): |
| """``_apply_model_switch_result`` must prefer the provider-aware resolver |
| (272K on Codex) over the raw models.dev value (1.05M for gpt-5.5). |
| """ |
| result = ModelSwitchResult( |
| success=True, |
| new_model="gpt-5.5", |
| target_provider="openai-codex", |
| provider_changed=True, |
| api_key="", |
| base_url="https://chatgpt.com/backend-api/codex", |
| api_mode="codex_responses", |
| warning_message="", |
| provider_label="ChatGPT Codex", |
| resolved_via_alias=False, |
| capabilities=None, |
| model_info=_FakeModelInfo(), |
| is_global=False, |
| ) |
| with patch( |
| "agent.model_metadata.get_model_context_length", |
| return_value=272_000, |
| ): |
| lines = _run_display(monkeypatch, result) |
|
|
| ctx_line = next((l for l in lines if "Context:" in l), "") |
| assert "272,000" in ctx_line, ( |
| f"picker-path display must show Codex's 272K cap, got: {ctx_line!r}" |
| ) |
| assert "1,050,000" not in ctx_line, ( |
| f"picker-path display leaked models.dev's 1.05M for Codex: {ctx_line!r}" |
| ) |
|
|
|
|
| def test_picker_path_falls_back_to_model_info_when_resolver_empty(monkeypatch): |
| """If ``get_model_context_length`` returns nothing (rare — truly unknown |
| endpoint), the display still surfaces ``ModelInfo.context_window`` so the |
| user sees *something* rather than a silent blank. |
| """ |
| result = ModelSwitchResult( |
| success=True, |
| new_model="some-model", |
| target_provider="some-provider", |
| provider_changed=True, |
| api_key="", |
| base_url="", |
| api_mode="chat_completions", |
| warning_message="", |
| provider_label="Some Provider", |
| resolved_via_alias=False, |
| capabilities=None, |
| model_info=_FakeModelInfo(), |
| is_global=False, |
| ) |
| with patch( |
| "agent.model_metadata.get_model_context_length", |
| return_value=None, |
| ): |
| lines = _run_display(monkeypatch, result) |
|
|
| ctx_line = next((l for l in lines if "Context:" in l), "") |
| assert "1,050,000" in ctx_line, ( |
| f"resolver-empty path should fall back to ModelInfo, got: {ctx_line!r}" |
| ) |
|
|
|
|
| def test_global_switch_clears_context_pin_owned_by_previous_route(monkeypatch): |
| import cli as cli_mod |
|
|
| writes = [] |
| monkeypatch.setattr(cli_mod, "_cprint", lambda *_a, **_k: None) |
| monkeypatch.setattr( |
| "utils.atomic_roundtrip_yaml_update", |
| lambda path, key, value: writes.append((key, value)), |
| ) |
| cli = _StubCLI() |
| cli.model = "shared-model" |
| cli.provider = "custom" |
| |
| cli.base_url = "https://small.example/v1" |
| result = ModelSwitchResult( |
| success=True, |
| new_model="shared-model", |
| target_provider="custom", |
| provider_changed=False, |
| api_key="", |
| base_url="https://small.example/v1", |
| api_mode="chat_completions", |
| warning_message="", |
| provider_label="Custom", |
| resolved_via_alias=False, |
| capabilities=None, |
| model_info=_FakeModelInfo(), |
| is_global=True, |
| ) |
|
|
| configured = { |
| "model": { |
| "default": "shared-model", |
| "provider": "custom", |
| "base_url": "https://large.example/v1", |
| "context_length": 1_048_576, |
| } |
| } |
| with ( |
| patch( |
| "agent.model_metadata.get_model_context_length", |
| return_value=256_000, |
| ), |
| patch("hermes_cli.config.read_user_config_raw", return_value=configured), |
| ): |
| cli_mod.HermesCLI._apply_model_switch_result(cli, result, True) |
|
|
| assert ("model.context_length", None) in writes |
|
|
