Spaces:
Running
Running
Download e2e/test_admin_models.py from dherain24/claude-proxy: direct link, hf CLI and curl.
- Browser
- Download file 28.2 kB
-
https://huggingface.co/spaces/dherain24/claude-proxy/resolve/main/e2e/test_admin_models.py
- Command line
-
hf download hf://spaces/dherain24/claude-proxy/e2e/test_admin_models.py
-
curl -L -o test_admin_models.py https://huggingface.co/spaces/dherain24/claude-proxy/resolve/main/e2e/test_admin_models.py
28.2 kB
| """Rendered model-fallback controls for the local Admin UI.""" | |
| import json | |
| from typing import cast | |
| import pytest | |
| from playwright.sync_api import Locator, Page, expect | |
| from e2e.provider_support import close_provider, open_provider | |
| def _open_models(page: Page, admin_base_url: str) -> None: | |
| page.emulate_media(reduced_motion="reduce") | |
| page.goto(f"{admin_base_url}/admin") | |
| expect(page.locator("#messageArea")).to_have_text("") | |
| page.get_by_role("button", name="Model Config", exact=True).click() | |
| def _refresh_openrouter_models(page: Page) -> None: | |
| page.get_by_role("button", name="Providers", exact=True).click() | |
| card = page.locator('[data-provider="open_router"]') | |
| dialog = open_provider(page, "open_router") | |
| dialog.get_by_role("button", name="Refresh models", exact=True).click() | |
| expect(card.locator(".provider-check-result")).to_have_text("3 models available") | |
| close_provider(page) | |
| page.get_by_role("button", name="Model Config", exact=True).click() | |
| def _fallbacks(editor: Locator) -> list[dict[str, str | int]]: | |
| value = editor.locator("#field-MODEL_FALLBACKS").input_value() | |
| return json.loads(value) if value else [] | |
| def test_model_routing_pairs_five_models_with_effort_and_defaults( | |
| page: Page, admin_base_url: str | |
| ) -> None: | |
| page.set_viewport_size({"width": 1280, "height": 900}) | |
| _open_models(page, admin_base_url) | |
| expect(page.locator("#section-reasoning")).to_have_count(0) | |
| routes = ("Default", "Fable", "Opus", "Sonnet", "Haiku") | |
| previous_bottom = 0.0 | |
| for route in routes: | |
| group = page.get_by_role("group", name=f"{route} model and effort", exact=True) | |
| model_key = "MODEL" if route == "Default" else f"MODEL_{route.upper()}" | |
| effort_key = ( | |
| "REASONING_POLICY" if route == "Default" else f"REASONING_{route.upper()}" | |
| ) | |
| model = group.locator(f"#field-{model_key}") | |
| effort = group.locator(f"#field-{effort_key}") | |
| expect(model).to_be_visible() | |
| expect(effort).to_have_value("client" if route == "Default" else "inherit") | |
| if route != "Default": | |
| expect(model).to_have_value("None") | |
| expect(effort).to_be_enabled() | |
| model_box, effort_box = model.bounding_box(), effort.bounding_box() | |
| assert model_box is not None and effort_box is not None | |
| assert abs(model_box["width"] - effort_box["width"]) <= 1 | |
| assert abs(model_box["height"] - effort_box["height"]) <= 1 | |
| assert abs(model_box["y"] - effort_box["y"]) <= 1 | |
| assert model_box["x"] < effort_box["x"] | |
| assert model_box["y"] > previous_bottom | |
| previous_bottom = model_box["y"] + model_box["height"] | |
| expect(page.locator("#field-REASONING_POLICY option[value=inherit]")).to_have_count( | |
| 0 | |
| ) | |
| fallback_box = page.locator('.field[data-key="MODEL_FALLBACKS"]').bounding_box() | |
| assert fallback_box is not None and fallback_box["y"] > previous_bottom | |
| def test_model_routing_saves_independent_model_and_effort_choices( | |
| page: Page, admin_base_url: str | |
| ) -> None: | |
| _open_models(page, admin_base_url) | |
| page.locator("#field-REASONING_POLICY").select_option("medium") | |
| opus = page.get_by_role("group", name="Opus model and effort", exact=True) | |
| expect(opus.locator("#field-MODEL_OPUS")).to_have_value("None") | |
| opus.locator("#field-REASONING_OPUS").select_option("high") | |
| sonnet = page.get_by_role("group", name="Sonnet model and effort", exact=True) | |
| sonnet.locator("#field-MODEL_SONNET").fill("open_router/vendor/model-b") | |
| sonnet.locator("#field-REASONING_SONNET").select_option("low") | |
| fallback = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| fallback.locator("#field-MODEL_FALLBACKS-add").fill("open_router/vendor/model-a") | |
| fallback.get_by_label("Fallback reasoning effort", exact=True).select_option("off") | |
| fallback.get_by_role("button", name="Add fallback", exact=True).click() | |
| with page.expect_response("**/admin/api/config/apply") as saved: | |
| page.locator("#applyButton").click() | |
| assert saved.value.json()["applied"] is True | |
| payload = saved.value.request.post_data_json | |
| assert isinstance(payload, dict) | |
| values = payload["values"] | |
| assert values["REASONING_POLICY"] == "medium" | |
| assert values["REASONING_OPUS"] == "high" | |
| assert "MODEL_OPUS" not in values | |
| assert values["MODEL_SONNET"] == "open_router/vendor/model-b" | |
| assert values["REASONING_SONNET"] == "low" | |
| assert json.loads(values["MODEL_FALLBACKS"]) == [ | |
| {"model": "open_router/vendor/model-a", "reasoning": "off", "api_key_number": 1} | |
| ] | |
| expect(page.locator("#dirtyState")).to_have_text("No changes") | |
| page.reload() | |
| expect(page.locator("#messageArea")).to_have_text("") | |
| expect(page.locator("#field-REASONING_POLICY")).to_have_value("medium") | |
| expect(opus.locator("#field-MODEL_OPUS")).to_have_value("None") | |
| expect(opus.locator("#field-REASONING_OPUS")).to_have_value("high") | |
| expect(sonnet.locator("#field-MODEL_SONNET")).to_have_value( | |
| "open_router/vendor/model-b" | |
| ) | |
| expect(sonnet.locator("#field-REASONING_SONNET")).to_have_value("low") | |
| assert _fallbacks(fallback) == [ | |
| {"model": "open_router/vendor/model-a", "reasoning": "off", "api_key_number": 1} | |
| ] | |
| expect(page.locator("#dirtyState")).to_have_text("No changes") | |
| def test_process_owned_route_controls_lock_independently( | |
| page: Page, admin_base_url: str | |
| ) -> None: | |
| _open_models(page, admin_base_url) | |
| expect(page.locator("#field-REASONING_POLICY")).to_have_value("client") | |
| expect(page.locator("#field-REASONING_POLICY")).to_be_disabled() | |
| opus = page.get_by_role("group", name="Opus model and effort", exact=True) | |
| expect(opus.locator("#field-MODEL_OPUS")).to_be_disabled() | |
| expect(opus.locator("#field-REASONING_OPUS")).to_be_enabled() | |
| sonnet = page.get_by_role("group", name="Sonnet model and effort", exact=True) | |
| expect(sonnet.locator("#field-MODEL_SONNET")).to_be_enabled() | |
| expect(sonnet.locator("#field-REASONING_SONNET")).to_be_disabled() | |
| def test_fallback_editor_adds_filters_reorders_and_removes_models( | |
| page: Page, | |
| admin_base_url: str, | |
| ) -> None: | |
| page.set_viewport_size({"width": 1280, "height": 720}) | |
| _open_models(page, admin_base_url) | |
| _refresh_openrouter_models(page) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| add_input = editor.locator("#field-MODEL_FALLBACKS-add") | |
| add_effort = editor.get_by_label("Fallback reasoning effort", exact=True) | |
| stored_value = editor.locator("#field-MODEL_FALLBACKS") | |
| expect(editor.locator(".model-list-row")).to_have_count(0) | |
| add_input.fill("model-b") | |
| expect( | |
| editor.get_by_role("option", name="open_router/vendor/model-b", exact=True) | |
| ).to_be_visible() | |
| add_input.press("ArrowDown") | |
| add_input.press("Enter") | |
| expect(add_input).to_have_value("open_router/vendor/model-b") | |
| add_effort.select_option("low") | |
| editor.get_by_role("button", name="Add fallback", exact=True).click() | |
| expect(add_effort).to_have_value("inherit") | |
| expect(add_input).to_have_value("") | |
| add_input.fill("groq/custom-model") | |
| editor.get_by_role("button", name="Add fallback", exact=True).click() | |
| assert _fallbacks(editor) == [ | |
| { | |
| "model": "open_router/vendor/model-b", | |
| "reasoning": "low", | |
| "api_key_number": 1, | |
| }, | |
| {"model": "groq/custom-model", "reasoning": "inherit", "api_key_number": 1}, | |
| ] | |
| expect(page.locator("#dirtyState")).to_have_text("1 unsaved change") | |
| expect(page.locator("#applyButton")).to_be_enabled() | |
| rows = editor.locator(".model-list-row") | |
| expect(rows.locator('input[role="combobox"]')).to_have_count(2) | |
| last_model_box = editor.get_by_role( | |
| "combobox", name="Fallback model 2", exact=True | |
| ).bounding_box() | |
| add_model_box = add_input.bounding_box() | |
| assert last_model_box is not None and add_model_box is not None | |
| assert add_model_box["y"] > last_model_box["y"] | |
| editor.get_by_role("button", name="Move groq/custom-model", exact=True).drag_to( | |
| rows.nth(0), target_position={"x": 10, "y": 5} | |
| ) | |
| assert _fallbacks(editor) == [ | |
| {"model": "groq/custom-model", "reasoning": "inherit", "api_key_number": 1}, | |
| { | |
| "model": "open_router/vendor/model-b", | |
| "reasoning": "low", | |
| "api_key_number": 1, | |
| }, | |
| ] | |
| editor.get_by_role("button", name="Remove groq/custom-model", exact=True).click() | |
| assert _fallbacks(editor) == [ | |
| {"model": "open_router/vendor/model-b", "reasoning": "low", "api_key_number": 1} | |
| ] | |
| editor.get_by_role( | |
| "button", name="Remove open_router/vendor/model-b", exact=True | |
| ).click() | |
| expect(stored_value).to_have_value("") | |
| expect(editor.locator(".model-list-row")).to_have_count(0) | |
| expect(page.locator("#dirtyState")).to_have_text("No changes") | |
| expect(page.locator("#applyButton")).to_be_disabled() | |
| def test_fallback_editor_rejects_blank_and_duplicate_rows( | |
| page: Page, | |
| admin_base_url: str, | |
| ) -> None: | |
| page.set_viewport_size({"width": 1280, "height": 720}) | |
| _open_models(page, admin_base_url) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| add_input = editor.locator("#field-MODEL_FALLBACKS-add") | |
| add_button = editor.get_by_role("button", name="Add fallback", exact=True) | |
| add_button.click() | |
| expect(page.locator("#messageArea")).to_have_text( | |
| "Enter a full provider/model fallback." | |
| ) | |
| add_input.fill("groq/custom-model") | |
| editor.get_by_label("Fallback reasoning effort", exact=True).select_option("high") | |
| add_button.click() | |
| add_input.fill("groq/custom-model") | |
| add_button.click() | |
| expect(page.locator("#messageArea")).to_have_text( | |
| "That model and API key are already in the fallback list." | |
| ) | |
| expect(editor.locator(".model-list-row")).to_have_count(1) | |
| def test_process_owned_fallback_list_locks_every_mutation( | |
| page: Page, | |
| admin_base_url: str, | |
| ) -> None: | |
| page.set_viewport_size({"width": 1280, "height": 720}) | |
| _open_models(page, admin_base_url) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| expect(editor.locator("#field-MODEL_FALLBACKS-add")).to_be_disabled() | |
| expect( | |
| editor.get_by_label("Fallback reasoning effort", exact=True) | |
| ).to_be_disabled() | |
| expect( | |
| editor.get_by_role("button", name="Add fallback", exact=True) | |
| ).to_be_disabled() | |
| expect(editor.locator(".model-list-row .model-list-action")).to_have_count(4) | |
| for button in editor.locator(".model-list-action").all(): | |
| expect(button).to_be_disabled() | |
| for effort in editor.locator(".model-list-row select").all(): | |
| expect(effort).to_be_disabled() | |
| for model in editor.locator(".model-list-row input").all(): | |
| expect(model).to_be_disabled() | |
| for handle in editor.locator(".model-list-drag").all(): | |
| expect(handle).to_have_attribute("draggable", "false") | |
| def test_fallback_editor_pairs_match_existing_controls_and_save_effort( | |
| page: Page, admin_base_url: str | |
| ) -> None: | |
| page.set_viewport_size({"width": 1280, "height": 900}) | |
| _open_models(page, admin_base_url) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| model = editor.locator("#field-MODEL_FALLBACKS-add") | |
| effort = editor.get_by_label("Fallback reasoning effort", exact=True) | |
| expect(effort.locator("option")).to_have_text( | |
| ["Inherit", "Off", "From client", "Low", "Medium", "High", "X-High", "Max"] | |
| ) | |
| bounds = [ | |
| control.bounding_box() | |
| for control in ( | |
| model, | |
| effort, | |
| editor.get_by_role("button", name="Add fallback", exact=True), | |
| ) | |
| ] | |
| assert all(box is not None for box in bounds) | |
| model_box, effort_box, add_box = bounds | |
| assert model_box is not None and effort_box is not None and add_box is not None | |
| assert abs(model_box["width"] - effort_box["width"]) <= 1 | |
| assert abs(model_box["height"] - effort_box["height"]) <= 1 | |
| assert model_box["x"] < effort_box["x"] < add_box["x"] | |
| assert abs(model_box["y"] - effort_box["y"]) <= 1 | |
| style = "el => { const s = getComputedStyle(el); return [s.fontSize, s.borderRadius, s.borderWidth, s.backgroundSize]; }" | |
| assert effort.evaluate(style) == page.locator("#field-REASONING_POLICY").evaluate( | |
| style | |
| ) | |
| # A fallback can match Default today, even with a different effort. | |
| model.fill("open_router/e2e-default") | |
| effort.select_option("off") | |
| editor.get_by_role("button", name="Add fallback", exact=True).click() | |
| row_model = editor.get_by_role("combobox", name="Fallback model 1", exact=True) | |
| row_model.fill("model-b") | |
| editor.get_by_role("option", name="open_router/vendor/model-b", exact=True).click() | |
| expect(row_model).to_have_value("open_router/vendor/model-b") | |
| row_effort = editor.get_by_label( | |
| "Reasoning effort for open_router/vendor/model-b", exact=True | |
| ) | |
| expect(row_effort).to_have_value("off") | |
| row_model_box, row_effort_box = row_model.bounding_box(), row_effort.bounding_box() | |
| assert row_model_box is not None and row_effort_box is not None | |
| assert abs(row_model_box["width"] - model_box["width"]) <= 1 | |
| assert abs(row_effort_box["width"] - effort_box["width"]) <= 1 | |
| assert row_model.evaluate(style) == model.evaluate(style) | |
| row_effort.select_option("max") | |
| with page.expect_response("**/admin/api/config/apply") as saved: | |
| page.locator("#applyButton").click() | |
| assert saved.value.json()["applied"] is True | |
| expect(page.locator("#dirtyState")).to_have_text("No changes") | |
| page.reload() | |
| page.get_by_role("button", name="Model Config", exact=True).click() | |
| expect( | |
| editor.get_by_label( | |
| "Reasoning effort for open_router/vendor/model-b", exact=True | |
| ) | |
| ).to_have_value("max") | |
| assert _fallbacks(editor) == [ | |
| {"model": "open_router/vendor/model-b", "reasoning": "max", "api_key_number": 1} | |
| ] | |
| expect(page.locator("#dirtyState")).to_have_text("No changes") | |
| def test_fallback_drag_and_keyboard_reordering_save_model_effort_pairs( | |
| page: Page, admin_base_url: str | |
| ) -> None: | |
| page.set_viewport_size({"width": 1280, "height": 1100}) | |
| _open_models(page, admin_base_url) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| entries = [ | |
| {"model": "groq/model-a", "reasoning": "low", "api_key_number": 1}, | |
| {"model": "groq/model-b", "reasoning": "client", "api_key_number": 1}, | |
| {"model": "groq/model-c", "reasoning": "max", "api_key_number": 1}, | |
| {"model": "groq/model-d", "reasoning": "high", "api_key_number": 1}, | |
| ] | |
| for entry in entries: | |
| editor.locator("#field-MODEL_FALLBACKS-add").fill(cast(str, entry["model"])) | |
| editor.get_by_label("Fallback reasoning effort", exact=True).select_option( | |
| cast(str, entry["reasoning"]) | |
| ) | |
| editor.get_by_role("button", name="Add fallback", exact=True).click() | |
| rows = editor.locator(".model-list-row") | |
| target_box = rows.nth(3).bounding_box() | |
| assert target_box is not None | |
| editor.get_by_role("button", name="Move groq/model-a", exact=True).drag_to( | |
| rows.nth(3), target_position={"x": 10, "y": target_box["height"] - 5} | |
| ) | |
| assert _fallbacks(editor) == [entries[1], entries[2], entries[3], entries[0]] | |
| handle = editor.get_by_role("button", name="Move groq/model-a", exact=True) | |
| handle.focus() | |
| handle.press("ArrowUp") | |
| expect(handle).to_be_focused() | |
| assert _fallbacks(editor) == [entries[1], entries[2], entries[0], entries[3]] | |
| handle.press("ArrowDown") | |
| handle.press("ArrowDown") # The last row stays in place. | |
| assert _fallbacks(editor) == [entries[1], entries[2], entries[3], entries[0]] | |
| expect(page.locator("#field-MODEL_SONNET")).to_have_value("None") | |
| with page.expect_response("**/admin/api/config/apply") as saved: | |
| page.locator("#applyButton").click() | |
| assert saved.value.json()["applied"] is True | |
| page.reload() | |
| expect(page.locator("#messageArea")).to_have_text("") | |
| assert _fallbacks(editor) == [entries[1], entries[2], entries[3], entries[0]] | |
| expect( | |
| editor.get_by_role("combobox", name="Fallback model 1", exact=True) | |
| ).to_have_value("groq/model-b") | |
| def test_fallback_plus_and_trash_have_matching_shape_and_centered_icons( | |
| page: Page, admin_base_url: str | |
| ) -> None: | |
| _open_models(page, admin_base_url) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| editor.locator("#field-MODEL_FALLBACKS-add").fill("groq/model-a") | |
| plus = editor.get_by_role("button", name="Add fallback", exact=True) | |
| plus.click() | |
| trash = editor.get_by_role("button", name="Remove groq/model-a", exact=True) | |
| shape = "el => { const s = getComputedStyle(el); return [s.width, s.height, s.borderRadius]; }" | |
| assert plus.evaluate(shape) == trash.evaluate(shape) | |
| for button in (plus, trash): | |
| icon = button.locator("svg") | |
| expect(icon).to_be_visible() | |
| button_box, icon_box = button.bounding_box(), icon.bounding_box() | |
| assert button_box is not None and icon_box is not None | |
| for axis, size in (("x", "width"), ("y", "height")): | |
| button_center = button_box[axis] + button_box[size] / 2 | |
| icon_center = icon_box[axis] + icon_box[size] / 2 | |
| assert abs(button_center - icon_center) <= 0.5 | |
| def test_fallback_drag_indicators_only_offer_unique_positions_that_change_order( | |
| page: Page, admin_base_url: str | |
| ) -> None: | |
| page.set_viewport_size({"width": 1280, "height": 1100}) | |
| _open_models(page, admin_base_url) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| for suffix in ("a", "b", "c", "d"): | |
| editor.locator("#field-MODEL_FALLBACKS-add").fill(f"groq/model-{suffix}") | |
| editor.get_by_role("button", name="Add fallback", exact=True).click() | |
| editor.scroll_into_view_if_needed() | |
| original = _fallbacks(editor) | |
| rows = editor.locator(".model-list-row") | |
| handle = editor.get_by_role("button", name="Move groq/model-b", exact=True) | |
| handle_box = handle.bounding_box() | |
| assert handle_box is not None | |
| x = handle_box["x"] + handle_box["width"] / 2 | |
| y = handle_box["y"] + handle_box["height"] / 2 | |
| page.mouse.move(x, y) | |
| page.mouse.down() | |
| page.mouse.move(x + 8, y + 8, steps=5) | |
| expect(editor.locator(".model-list-row.dragging")).to_have_count(1) | |
| indicators = editor.locator(".model-list-row[data-drop-position]") | |
| try: | |
| # Both halves of the source and the two neighboring insertion points | |
| # retain the current order, so they should show no placement line. | |
| for index, after in ((1, False), (1, True), (0, True), (2, False)): | |
| bounds = rows.nth(index).bounding_box() | |
| assert bounds is not None | |
| page.mouse.move( | |
| bounds["x"] + 10, | |
| bounds["y"] + (bounds["height"] - 2 if after else 2), | |
| steps=5, | |
| ) | |
| expect(indicators).to_have_count(0) | |
| assert _fallbacks(editor) == original | |
| for index, position in ((3, "after"), (0, "before")): | |
| bounds = rows.nth(index).bounding_box() | |
| assert bounds is not None | |
| page.mouse.move( | |
| bounds["x"] + 10, | |
| bounds["y"] + (bounds["height"] - 2 if position == "after" else 2), | |
| steps=5, | |
| ) | |
| expect(indicators).to_have_count(1) | |
| expect(rows.nth(index)).to_have_attribute("data-drop-position", position) | |
| line_y = None | |
| for index, after in ((2, True), (3, False)): | |
| bounds = rows.nth(index).bounding_box() | |
| assert bounds is not None | |
| page.mouse.move( | |
| bounds["x"] + 10, | |
| bounds["y"] + (bounds["height"] - 2 if after else 2), | |
| steps=5, | |
| ) | |
| expect(indicators).to_have_count(1) | |
| current_y = indicators.evaluate( | |
| "el => { const r = el.getBoundingClientRect(); return el.dataset.dropPosition === 'before' ? r.top : r.bottom; }" | |
| ) | |
| if line_y is None: | |
| line_y = current_y | |
| else: | |
| assert current_y == line_y | |
| upper, lower = rows.nth(2).bounding_box(), rows.nth(3).bounding_box() | |
| assert upper is not None and lower is not None | |
| page.mouse.move( | |
| upper["x"] + 10, | |
| (upper["y"] + upper["height"] + lower["y"]) / 2, | |
| steps=5, | |
| ) | |
| expect(indicators).to_have_count(1) | |
| assert ( | |
| indicators.evaluate( | |
| "el => { const r = el.getBoundingClientRect(); return el.dataset.dropPosition === 'before' ? r.top : r.bottom; }" | |
| ) | |
| == line_y | |
| ) | |
| finally: | |
| page.mouse.up() | |
| assert _fallbacks(editor) == [original[0], original[2], original[1], original[3]] | |
| expect(indicators).to_have_count(0) | |
| def test_fallback_model_edits_validate_without_losing_saved_configuration( | |
| page: Page, admin_base_url: str, invalid_model: str | |
| ) -> None: | |
| _open_models(page, admin_base_url) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| for model in ("groq/model-a", "groq/model-b"): | |
| editor.locator("#field-MODEL_FALLBACKS-add").fill(model) | |
| editor.get_by_role("button", name="Add fallback", exact=True).click() | |
| with page.expect_response("**/admin/api/config/apply") as saved: | |
| page.locator("#applyButton").click() | |
| assert saved.value.json()["applied"] is True | |
| expect(page.locator("#dirtyState")).to_have_text("No changes") | |
| original = _fallbacks(editor) | |
| model = editor.get_by_role("combobox", name="Fallback model 1", exact=True) | |
| model.fill(invalid_model) | |
| expect(page.locator("#dirtyState")).to_have_text("1 unsaved change") | |
| model.press("Tab") | |
| with page.expect_response("**/admin/api/config/apply") as rejected: | |
| page.locator("#applyButton").click() | |
| assert rejected.value.json()["applied"] is False | |
| expect(model).to_have_value(invalid_model) | |
| fields = page.request.get(f"{admin_base_url}/admin/api/config").json()["fields"] | |
| persisted = next(field for field in fields if field["key"] == "MODEL_FALLBACKS") | |
| assert json.loads(persisted["value"]) == original | |
| def test_editable_fallbacks_keep_focus_and_live_model_suggestions( | |
| page: Page, admin_base_url: str | |
| ) -> None: | |
| _open_models(page, admin_base_url) | |
| page.wait_for_function("!state.startupRequest && !state.startupTimer") | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| editor.locator("#field-MODEL_FALLBACKS-add").fill("groq/custom") | |
| editor.get_by_role("button", name="Add fallback", exact=True).click() | |
| model = editor.get_by_role("combobox", name="Fallback model 1", exact=True) | |
| model.fill("open_router/new-model") | |
| expect(model).to_be_focused() | |
| assert _fallbacks(editor) == [ | |
| {"model": "open_router/new-model", "reasoning": "inherit", "api_key_number": 1} | |
| ] | |
| for effort in ("low", "high", "client"): | |
| editor.get_by_label( | |
| "Reasoning effort for open_router/new-model", exact=True | |
| ).select_option(effort) | |
| model.fill("new-model") | |
| page.route( | |
| "**/admin/api/models", | |
| lambda route: route.fulfill( | |
| json={"models": ["open_router/new-model"], "failed_providers": []} | |
| ), | |
| ) | |
| # Refresh the catalog without closing the currently edited model dropdown. | |
| with page.expect_response("**/admin/api/models"): | |
| page.evaluate("void hydrateModelOptions()") | |
| expect( | |
| editor.get_by_role("option", name="open_router/new-model", exact=True) | |
| ).to_be_visible() | |
| expect(model).to_have_value("new-model") | |
| editor.get_by_role("option", name="open_router/new-model", exact=True).click() | |
| assert _fallbacks(editor) == [ | |
| {"model": "open_router/new-model", "reasoning": "client", "api_key_number": 1} | |
| ] | |
| registered = page.evaluate("state.modelComboboxes.size") | |
| for _ in range(3): | |
| editor.locator("#field-MODEL_FALLBACKS-add").fill("groq/discarded") | |
| editor.get_by_role("button", name="Add fallback", exact=True).click() | |
| editor.get_by_role("button", name="Remove groq/discarded", exact=True).click() | |
| assert page.evaluate("state.modelComboboxes.size") == registered | |
| def test_fallback_editor_remains_usable_at_narrow_viewport( | |
| page: Page, | |
| admin_base_url: str, | |
| ) -> None: | |
| page.set_viewport_size({"width": 390, "height": 844}) | |
| _open_models(page, admin_base_url) | |
| editor = page.locator('.field[data-key="MODEL_FALLBACKS"]') | |
| editor.scroll_into_view_if_needed() | |
| expect(editor.locator("#field-MODEL_FALLBACKS-add")).to_be_visible() | |
| expect( | |
| editor.get_by_role("button", name="Add fallback", exact=True) | |
| ).to_be_visible() | |
| assert page.evaluate( | |
| "document.documentElement.scrollWidth <= document.documentElement.clientWidth" | |
| ) | |
| def test_model_suggestions_keep_server_order_after_provider_check_and_late_response( | |
| page: Page, | |
| admin_base_url: str, | |
| ) -> None: | |
| models = [ | |
| "alpha/Apple", | |
| "alpha/apple", | |
| "alpha/ss", | |
| "alpha/ß", | |
| "alpha/z", | |
| "alpha-2/model", | |
| ] | |
| page.route( | |
| "**/admin/api/models", | |
| lambda route: route.fulfill(json={"models": models, "failed_providers": []}), | |
| ) | |
| _open_models(page, admin_base_url) | |
| page.wait_for_function("!state.startupRequest && !state.startupTimer") | |
| field = page.locator('.field[data-key="MODEL_SONNET"]') | |
| field.locator("input").fill("") | |
| expect(field.get_by_role("option")).to_have_text(["None", *models]) | |
| field.locator("input").fill("open_router/custom-unsaved") | |
| field.locator("input").press("Tab") | |
| models.append("open_router/new-model") | |
| page.route( | |
| "**/admin/api/providers/open_router/test", | |
| lambda route: route.fulfill(json={"ok": True, "models": ["new-model"]}), | |
| ) | |
| page.get_by_role("button", name="Providers", exact=True).click() | |
| dialog = open_provider(page, "open_router") | |
| with page.expect_request("**/admin/api/models"): | |
| dialog.get_by_role("button", name="Refresh models", exact=True).click() | |
| close_provider(page) | |
| page.get_by_role("button", name="Model Config", exact=True).click() | |
| expect(field.locator("input")).to_have_value("open_router/custom-unsaved") | |
| field.locator("input").fill("") | |
| expect(field.get_by_role("option")).to_have_text(["None", *models]) | |
| field.locator("input").press("Escape") | |
| optional = page.locator('.field[data-key="MODEL_OPUS"]') | |
| optional.locator("input").fill("") | |
| expect(optional.get_by_role("option")).to_have_text(["None", *models]) | |
| page.wait_for_function("!state.startupRequest && !state.startupTimer") | |
| page.unroute("**/admin/api/models") | |
| pending = [] | |
| def hold_catalog(route): | |
| pending.append(route) | |
| page.evaluate("count => { window.heldCatalogLoads = count; }", len(pending)) | |
| page.route("**/admin/api/models", hold_catalog) | |
| for _ in range(2): | |
| with page.expect_request("**/admin/api/models"): | |
| page.evaluate("void hydrateModelOptions()") | |
| page.wait_for_function("window.heldCatalogLoads === 2") | |
| with page.expect_response("**/admin/api/models"): | |
| pending[1].fulfill(json={"models": models, "failed_providers": []}) | |
| with page.expect_response("**/admin/api/models") as stale: | |
| pending[0].fulfill(json={"models": ["old/stale"], "failed_providers": []}) | |
| stale.value.finished() | |
| expect(optional.get_by_role("option")).to_have_text(["None", *models]) | |