diff --git a/AGENTS.md b/AGENTS.md index 03f3bd9d98fee110961befde5d4ac148421f4b28..9943ca61cd50bd5e13cf850a91bc75504d4d617d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ Notes: - Vite proxies `/api` and `/auth` to `http://localhost:7860`. - If `127.0.0.1:7860` is already owned by another local process, binding the backend to `::1` lets the Vite proxy resolve `localhost` cleanly. - Prefer `npm ci` over `npm install` for setup, since `npm install` may rewrite `frontend/package-lock.json` metadata depending on npm version. -- Production defaults to the Bedrock Claude model. For local development with a personal Anthropic key, set `ANTHROPIC_API_KEY` and `ML_INTERN_CLAUDE_MODEL_ID=anthropic/claude-opus-4-6` before starting the backend. Other models are selected through the app's model switcher. +- Production defaults to Claude Opus 4.8 through the HF Router (`anthropic/claude-opus-4.8:fal-ai`). Non-local LLM calls use `https://router.huggingface.co/v1` with Hugging Face tokens. Subsidized premium calls use the Space/operator `INFERENCE_TOKEN` plus `X-HF-Bill-To` from `HF_BILL_TO` (default `smolagents`); after the daily allowance, premium calls bill the user's own HF token. For local development, set `HF_TOKEN` and optionally `ML_INTERN_DEFAULT_MODEL_ID`. ## Development Checks diff --git a/agent/__init__.py b/agent/__init__.py index 2e301c8d7b97df90efb932a3685a5c401326232e..0e4cf34cd240ebdb4bcefb7752cb8074a9f5244c 100644 --- a/agent/__init__.py +++ b/agent/__init__.py @@ -8,10 +8,8 @@ import litellm # backend entries share the same config. # drop_params: quietly drop unsupported params rather than raising # suppress_debug_info: hide the noisy "Give Feedback" banner on errors -# modify_params: let LiteLLM patch Anthropic's tool-call requirements -# (synthesize a dummy tool spec when we call completion on a history -# that contains tool_calls but aren't passing `tools=` — happens -# during summarization / session seeding). +# modify_params: let LiteLLM patch provider-specific schema requirements +# for router-compatible request bodies when possible. litellm.drop_params = True litellm.suppress_debug_info = True litellm.modify_params = True diff --git a/agent/config.py b/agent/config.py index 5ad8bd8aa4b2cea4df6aeaed426bc79dd911fe22..c6784db92d0100f033cf497806e7803fcbac8109 100644 --- a/agent/config.py +++ b/agent/config.py @@ -51,12 +51,11 @@ class Config(BaseModel): # Reasoning effort *preference* — the ceiling the user wants. The probe # on `/model` walks a cascade down from here (``max`` → ``xhigh`` → ``high`` # → …) and caches per-model what the provider actually accepted in - # ``Session.model_effective_effort``. Default ``max`` because we'd rather - # burn tokens thinking than ship a wrong ML recipe; the cascade lands on - # whichever level the model supports (``high`` for GPT-5 / HF router, - # ``xhigh`` or ``max`` for Anthropic 4.6 / 4.7). ``None`` = thinking off. + # ``Session.model_effective_effort``. Default ``high`` because HF Router + # accepts low/medium/high generically and provider-specific higher levels + # should be discovered through explicit probes. ``None`` = thinking off. # Valid values: None | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" - reasoning_effort: str | None = "max" + reasoning_effort: str | None = "high" messaging: MessagingConfig = MessagingConfig() diff --git a/agent/context_manager/manager.py b/agent/context_manager/manager.py index 85e96af0f6f3fa6d0426acddcd308281d502558b..42cd6b5c570299e0ab0f659675f655183adbef56 100644 --- a/agent/context_manager/manager.py +++ b/agent/context_manager/manager.py @@ -13,8 +13,6 @@ import yaml from jinja2 import Template from litellm import Message, acompletion -from agent.core.prompt_caching import with_prompt_caching - logger = logging.getLogger(__name__) _HF_WHOAMI_URL = "https://huggingface.co/api/whoami-v2" @@ -91,8 +89,8 @@ class CompactionFailedError(Exception): Typically means an individual preserved message (system, first user, or untouched tail) exceeds what truncation can fix in one pass. The caller - must terminate the session — retrying produces an infinite loop that - burns Bedrock budget for free (~$3 per re-attempt on Opus). + must terminate the session; retrying produces an infinite loop that burns + premium inference budget. """ @@ -133,18 +131,19 @@ async def summarize_messages( ``session`` is optional; when provided, the call is recorded via ``telemetry.record_llm_call`` so its cost lands in the session's ``total_cost_usd``. Without it, the call still happens but is - invisible in telemetry — which used to be the case for every - compaction call until 2026-04-29 (~30-50% of Bedrock spend was - attributed to this single source of dark cost). + invisible in telemetry, which used to hide a significant share of premium + inference spend. Returns ``(summary_text, completion_tokens)``. """ from agent.core.llm_params import _resolve_llm_params prompt_messages = list(messages) + [Message(role="user", content=prompt)] - llm_params = _resolve_llm_params(model_name, hf_token, reasoning_effort="high") - prompt_messages, tool_specs = with_prompt_caching( - prompt_messages, tool_specs, llm_params.get("model") + llm_params = _resolve_llm_params( + model_name, + hf_token, + reasoning_effort="high", + bill_to_user=getattr(session, "premium_user_billed", False), ) _t0 = time.monotonic() response = await acompletion( @@ -184,9 +183,13 @@ class ContextManager: hf_token: str | None = None, local_mode: bool = False, ): + self.prompt_file_suffix = prompt_file_suffix + self.tool_specs = tool_specs or [] + self.hf_token = hf_token + self.local_mode = local_mode self.system_prompt = self._load_system_prompt( - tool_specs or [], - prompt_file_suffix="system_prompt_v3.yaml", + self.tool_specs, + prompt_file_suffix=self.prompt_file_suffix, hf_token=hf_token, local_mode=local_mode, ) @@ -203,6 +206,30 @@ class ContextManager: self.items: list[Message] = [Message(role="system", content=self.system_prompt)] self.on_message_added = None + def refresh_system_prompt( + self, + *, + tool_specs: list[dict[str, Any]] | None = None, + hf_token: str | None = None, + local_mode: bool | None = None, + ) -> Message: + """Re-render the system prompt and return it as a system message.""" + if tool_specs is not None: + self.tool_specs = tool_specs + if hf_token is not None: + self.hf_token = hf_token + if local_mode is not None: + self.local_mode = local_mode + self.system_prompt = self._load_system_prompt( + self.tool_specs, + prompt_file_suffix=getattr( + self, "prompt_file_suffix", "system_prompt_v3.yaml" + ), + hf_token=getattr(self, "hf_token", None), + local_mode=getattr(self, "local_mode", False), + ) + return Message(role="system", content=self.system_prompt) + def _load_system_prompt( self, tool_specs: list[dict[str, Any]], @@ -441,9 +468,8 @@ class ContextManager: ) # Preserve all known assistant-side fields (tool_calls, thinking_blocks, # reasoning_content, provider_specific_fields) even when content is - # replaced. Anthropic extended-thinking models reject the next request - # with "Invalid signature in thinking block" if thinking_blocks is - # dropped from a prior assistant message. + # replaced. Historical traces may still contain provider reasoning + # metadata, and truncation should not silently discard it. kept = { k: getattr(msg, k, None) for k in ( @@ -493,7 +519,7 @@ class ContextManager: a giant tool output stuck in the untouched tail) is too large for truncation to fix. The caller must terminate the session — retrying is what caused the 2026-05-03 infinite-compaction-loop pattern that - burned Bedrock budget invisibly. + burned premium inference budget invisibly. """ if not self.needs_compaction: return @@ -521,7 +547,7 @@ class ContextManager: # otherwise recent_messages overlaps with the messages we put in # head". The walk-back's `idx > 1` guard is necessary (no system in # recent) but insufficient (first_user is also in head and would be - # duplicated). Anthropic API rejects two consecutive user messages + # duplicated). Chat providers can reject two consecutive user messages # with a 400 — bot review on PR #213 caught this on the second clamp # iteration. if idx <= first_user_idx: @@ -583,7 +609,7 @@ class ContextManager: # Hard verify: if compaction didn't bring us below the threshold even # after truncating oversized preserved messages, retrying just burns - # Bedrock budget on the same useless compaction call. Raise so the + # premium inference budget on the same useless compaction call. Raise so the # caller can terminate the session cleanly. Pre-2026-05-04, the # caller looped indefinitely (~$3/Opus retry) until the pod was # killed — invisible to the dataset because the session never diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index e6742efb0605bac6f1b72e811ac44c73fff4137f..8a11de338ea5bc5e4ad5d32f8c8d30982c9d3dc0 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -14,7 +14,6 @@ from litellm import ( ChatCompletionMessageToolCall, Message, acompletion, - stream_chunk_builder, ) from litellm.exceptions import ContextWindowExceededError @@ -28,7 +27,6 @@ from agent.messaging.gateway import NotificationGateway from agent.core import telemetry from agent.core.doom_loop import check_for_doom_loop from agent.core.llm_params import _resolve_llm_params -from agent.core.prompt_caching import with_prompt_caching from agent.core.session import DEFAULT_SESSION_LOG_DIR, Event, OpType, Session from agent.core.tools import ToolRouter from agent.tools.jobs_tool import CPU_FLAVORS @@ -405,7 +403,7 @@ async def _record_manual_approved_spend_if_needed( # -- LLM retry constants -------------------------------------------------- _MAX_LLM_RETRIES = 3 _LLM_RETRY_DELAYS = [5, 15, 30] # seconds between retries -_LLM_RATE_LIMIT_RETRY_DELAYS = [30, 60] # exceed Bedrock's ~60s TPM bucket window +_LLM_RATE_LIMIT_RETRY_DELAYS = [30, 60] def _is_rate_limit_error(error: Exception) -> bool: @@ -541,6 +539,7 @@ async def _heal_effort_and_rebuild_params( model, session.hf_token, reasoning_effort=session.effective_effort_for(model), + bill_to_user=getattr(session, "premium_user_billed", False), ) @@ -554,11 +553,8 @@ def _friendly_error_message(error: Exception) -> str | None: or "invalid x-api-key" in err_str ): return ( - "Authentication failed — your API key is missing or invalid.\n\n" - "To fix this, set the API key for your model provider:\n" - " • Anthropic: export ANTHROPIC_API_KEY=sk-...\n" - " • OpenAI: export OPENAI_API_KEY=sk-...\n" - " • HF Router: export HF_TOKEN=hf_...\n\n" + "Authentication failed - your Hugging Face token is missing or invalid.\n\n" + "To fix this, set HF_TOKEN=hf_... or run `hf auth login`.\n\n" "You can also add it to a .env file in the project root.\n" "To switch models, use the /model command." ) @@ -594,9 +590,8 @@ async def _compact_and_notify(session: Session) -> None: Catches ``CompactionFailedError`` and ends the session cleanly instead of letting the caller retry. Pre-2026-05-04 the caller looped on - ContextWindowExceededError → compact → re-trigger, burning Bedrock - budget at ~$3/Opus retry while the session never reached the upload - path (so the cost was invisible in the dataset). + ContextWindowExceededError → compact → re-trigger, burning premium + inference budget while the session never reached the upload path. """ from agent.context_manager.manager import CompactionFailedError @@ -698,38 +693,10 @@ class LLMResult: token_count: int finish_reason: str | None usage: dict = field(default_factory=dict) - thinking_blocks: list[dict[str, Any]] | None = None - reasoning_content: str | None = None - - -def _extract_thinking_state( - message: Any, -) -> tuple[list[dict[str, Any]] | None, str | None]: - """Return provider reasoning fields that must be replayed after tool calls.""" - provider_fields = getattr(message, "provider_specific_fields", None) - if not isinstance(provider_fields, dict): - provider_fields = {} - - thinking_blocks = ( - getattr(message, "thinking_blocks", None) - or provider_fields.get("thinking_blocks") - or None - ) - reasoning_content = ( - getattr(message, "reasoning_content", None) - or provider_fields.get("reasoning_content") - or None - ) - return thinking_blocks, reasoning_content - - -def _should_replay_thinking_state(model_name: str | None) -> bool: - """Only Anthropic's native adapter accepts replayed thinking metadata.""" - return bool(model_name and model_name.startswith("anthropic/")) def _is_invalid_thinking_signature_error(exc: Exception) -> bool: - """Return True when Anthropic rejected replayed extended-thinking state.""" + """Return True when a provider rejected replayed thinking metadata.""" text = str(exc) return ( "Invalid `signature` in `thinking` block" in text @@ -818,7 +785,7 @@ async def _maybe_heal_invalid_thinking_signature( data={ "tool": "system", "log": ( - "Anthropic rejected stale thinking signatures; retrying " + "The inference provider rejected stale thinking signatures; retrying " "without replayed thinking metadata." ), }, @@ -830,21 +797,15 @@ async def _maybe_heal_invalid_thinking_signature( def _assistant_message_from_result( llm_result: LLMResult, *, - model_name: str | None, tool_calls: list[ToolCall] | None = None, ) -> Message: - """Build an assistant history message without dropping reasoning state.""" + """Build an assistant history message for HF Router-compatible replay.""" kwargs: dict[str, Any] = { "role": "assistant", "content": llm_result.content, } if tool_calls is not None: kwargs["tool_calls"] = tool_calls - if _should_replay_thinking_state(model_name): - if llm_result.thinking_blocks: - kwargs["thinking_blocks"] = llm_result.thinking_blocks - if llm_result.reasoning_content: - kwargs["reasoning_content"] = llm_result.reasoning_content return Message(**kwargs) @@ -855,7 +816,6 @@ async def _call_llm_streaming( response = None _healed_effort = False # one-shot safety net per call _healed_thinking_signature = False - messages, tools = with_prompt_caching(messages, tools, llm_params.get("model")) t_start = time.monotonic() for _llm_attempt in range(_MAX_LLM_RETRIES): try: @@ -924,11 +884,8 @@ async def _call_llm_streaming( token_count = 0 finish_reason = None final_usage_chunk = None - chunks = [] - should_replay_thinking = _should_replay_thinking_state(llm_params.get("model")) async for chunk in response: - chunks.append(chunk) if session.is_cancelled: tool_calls_acc.clear() break @@ -982,27 +939,12 @@ async def _call_llm_streaming( latency_ms=int((time.monotonic() - t_start) * 1000), finish_reason=finish_reason, ) - thinking_blocks = None - reasoning_content = None - if chunks and should_replay_thinking: - try: - rebuilt = stream_chunk_builder(chunks, messages=messages) - if rebuilt and getattr(rebuilt, "choices", None): - rebuilt_msg = rebuilt.choices[0].message - thinking_blocks, reasoning_content = _extract_thinking_state( - rebuilt_msg - ) - except Exception: - logger.debug("Failed to rebuild streaming thinking state", exc_info=True) - return LLMResult( content=full_content or None, tool_calls_acc=tool_calls_acc, token_count=token_count, finish_reason=finish_reason, usage=usage, - thinking_blocks=thinking_blocks, - reasoning_content=reasoning_content, ) @@ -1013,7 +955,6 @@ async def _call_llm_non_streaming( response = None _healed_effort = False _healed_thinking_signature = False - messages, tools = with_prompt_caching(messages, tools, llm_params.get("model")) t_start = time.monotonic() for _llm_attempt in range(_MAX_LLM_RETRIES): try: @@ -1081,7 +1022,6 @@ async def _call_llm_non_streaming( content = message.content or None finish_reason = choice.finish_reason token_count = response.usage.total_tokens if response.usage else 0 - thinking_blocks, reasoning_content = _extract_thinking_state(message) # Build tool_calls_acc in the same format as streaming tool_calls_acc: dict[int, dict] = {} @@ -1116,8 +1056,6 @@ async def _call_llm_non_streaming( token_count=token_count, finish_reason=finish_reason, usage=usage, - thinking_blocks=thinking_blocks, - reasoning_content=reasoning_content, ) @@ -1256,6 +1194,7 @@ class Handlers: reasoning_effort=session.effective_effort_for( session.config.model_name ), + bill_to_user=getattr(session, "premium_user_billed", False), ) if session.stream: llm_result = await _call_llm_streaming( @@ -1296,10 +1235,7 @@ class Handlers: " • For other tools: reduce the size of your arguments or use bash." ) if content: - assistant_msg = _assistant_message_from_result( - llm_result, - model_name=llm_params.get("model"), - ) + assistant_msg = _assistant_message_from_result(llm_result) session.context_manager.add_message(assistant_msg, token_count) session.context_manager.add_message( Message(role="user", content=f"[SYSTEM: {truncation_hint}]") @@ -1356,10 +1292,7 @@ class Handlers: _NO_TOOL_INCOMPLETE_PLAN_RETRY_LIMIT, ) if content: - assistant_msg = _assistant_message_from_result( - llm_result, - model_name=llm_params.get("model"), - ) + assistant_msg = _assistant_message_from_result(llm_result) session.context_manager.add_message( assistant_msg, token_count ) @@ -1403,10 +1336,7 @@ class Handlers: (content or "")[:500], ) if content: - assistant_msg = _assistant_message_from_result( - llm_result, - model_name=llm_params.get("model"), - ) + assistant_msg = _assistant_message_from_result(llm_result) session.context_manager.add_message(assistant_msg, token_count) final_response = content break @@ -1433,7 +1363,6 @@ class Handlers: # Add assistant message with all tool calls to context assistant_msg = _assistant_message_from_result( llm_result, - model_name=llm_params.get("model"), tool_calls=tool_calls, ) session.context_manager.add_message(assistant_msg, token_count) @@ -1756,6 +1685,19 @@ class Handlers: logger.warning("Undo: no user message found to remove") await session.send_event(Event(event_type="undo_complete")) + @staticmethod + async def new_conversation(session: Session, *, clear_screen: bool = False) -> None: + """Start a fresh conversation inside the active runtime.""" + try: + result = session.start_new_conversation() + except Exception as e: + await session.send_event( + Event(event_type="error", data={"error": f"New chat failed: {e}"}) + ) + return + result["clear_screen"] = clear_screen + await session.send_event(Event(event_type="new_complete", data=result)) + @staticmethod async def resume(session: Session, path: str) -> None: """Reload context from a saved session log into the active session.""" @@ -2058,6 +2000,11 @@ async def process_submission(session: Session, submission) -> bool: await Handlers.undo(session) return True + if op.op_type == OpType.NEW: + clear_screen = bool((op.data or {}).get("clear_screen")) + await Handlers.new_conversation(session, clear_screen=clear_screen) + return True + if op.op_type == OpType.RESUME: path = op.data.get("path") if op.data else None if path: diff --git a/agent/core/effort_probe.py b/agent/core/effort_probe.py index dbad4c3da95e939ec9d2dae5c6c7408bcc6ea156..11f6580ae7be527dcd440a9a17f0cffa7e7d6b75 100644 --- a/agent/core/effort_probe.py +++ b/agent/core/effort_probe.py @@ -34,10 +34,9 @@ logger = logging.getLogger(__name__) # Cascade: for each user-stated preference, the ordered list of levels to -# try. First success wins. ``max`` is Anthropic-only; ``xhigh`` is also -# supported on current OpenAI GPT-5 models. Providers that don't accept a -# requested level raise ``UnsupportedEffortError`` synchronously (no wasted -# network round-trip) and we advance to the next level. +# try. First success wins. HF Router accepts low/medium/high generically; +# higher preferences are kept in the cascade for future/provider-specific +# support and are skipped synchronously when unsupported. _EFFORT_CASCADE: dict[str, list[str]] = { "max": ["max", "xhigh", "high", "medium", "low"], "xhigh": ["xhigh", "high", "medium", "low"], @@ -79,9 +78,7 @@ class ProbeOutcome: def _is_thinking_unsupported(e: Exception) -> bool: """Model rejected any thinking config. - Matches Anthropic's 'thinking.type.enabled is not supported for this - model' as well as the adaptive variant. Substring-match because the - exact wording shifts across API versions. + Substring-match because exact wording shifts across models and providers. """ s = str(e).lower() return "thinking" in s and "not supported" in s @@ -90,16 +87,12 @@ def _is_thinking_unsupported(e: Exception) -> bool: def _is_invalid_effort(e: Exception) -> bool: """The requested effort level isn't accepted for this model. - Covers both API responses (Anthropic/OpenAI 400 with "invalid", "must - be one of", etc.) and LiteLLM's local validation that fires *before* - the request (e.g. "effort='max' is only supported by Claude Opus 4.6" - — LiteLLM knows max is Opus-4.6-only and raises synchronously). The - cascade walks down on either. + Covers API responses with "invalid", "must be one of", etc. and local + validation that fires *before* the request. The cascade walks down on + either. Explicitly returns False when the message is really about thinking - itself (e.g. Anthropic's 4.7 error mentions ``output_config.effort`` - in its fix hint, but the actual failure is ``thinking.type.enabled`` - being unsupported). That case is caught by ``_is_thinking_unsupported``. + itself. That case is caught by ``_is_thinking_unsupported``. """ if _is_thinking_unsupported(e): return False @@ -195,6 +188,7 @@ async def probe_effort( hf_token, reasoning_effort=effort, strict=True, + bill_to_user=getattr(session, "premium_user_billed", False), ) except UnsupportedEffortError: # Provider can't even accept this effort name (e.g. "max" on diff --git a/agent/core/llm_params.py b/agent/core/llm_params.py index f95695fb88ff2d6664f3a5be357c97f8b83131d8..89018227f8002bd0524ffb661aac3b7946e7dda6 100644 --- a/agent/core/llm_params.py +++ b/agent/core/llm_params.py @@ -7,7 +7,11 @@ creating circular imports. import os -from agent.core.hf_tokens import get_hf_bill_to, resolve_hf_router_token +from agent.core.hf_tokens import ( + get_hf_bill_to, + resolve_hf_router_token, + resolve_hf_token, +) from agent.core.local_models import ( LOCAL_MODEL_API_KEY_DEFAULT, LOCAL_MODEL_API_KEY_ENV, @@ -16,6 +20,11 @@ from agent.core.local_models import ( local_model_name, local_model_provider, ) +from agent.core.model_ids import ( + HF_ROUTER_BASE_URL, + is_premium_model_id, + strip_huggingface_model_prefix, +) def _resolve_hf_router_token(session_hf_token: str | None = None) -> str | None: @@ -23,81 +32,18 @@ def _resolve_hf_router_token(session_hf_token: str | None = None) -> str | None: return resolve_hf_router_token(session_hf_token) -def _patch_litellm_effort_validation() -> None: - """Neuter LiteLLM 1.83's hardcoded effort-level validation. - - Context: at ``litellm/llms/anthropic/chat/transformation.py:~1443`` the - Anthropic adapter validates ``output_config.effort ∈ {high, medium, - low, max}`` and gates ``max`` behind an ``_is_opus_4_6_model`` check - that only matches the substring ``opus-4-6`` / ``opus_4_6``. Result: - - * ``xhigh`` — valid on Anthropic's real API for Claude 4.7 — is - rejected pre-flight with "Invalid effort value: xhigh". - * ``max`` on Opus 4.7 is rejected with "effort='max' is only supported - by Claude Opus 4.6", even though Opus 4.7 accepts it in practice. - - We don't want to maintain a parallel model table, so we let the - Anthropic API itself be the validator: widen ``_is_opus_4_6_model`` - to also match ``opus-4-7``+ families, and drop the valid-effort-set - check entirely. If Anthropic rejects an effort level, we see a 400 - and the cascade walks down — exactly the behavior we want for any - future model family. - - Removable once litellm ships 1.83.8-stable (which merges PR #25867, - "Litellm day 0 opus 4.7 support") — see commit 0868a82 on their main - branch. Until then, this one-time patch is the escape hatch. - """ - try: - from litellm.llms.anthropic.chat import transformation as _t - except Exception: - return - - cfg = getattr(_t, "AnthropicConfig", None) - if cfg is None: - return - - original = getattr(cfg, "_is_opus_4_6_model", None) - if original is None or getattr(original, "_hf_agent_patched", False): - return - - def _widened(model: str) -> bool: - m = model.lower() - # Original 4.6 match plus any future Opus >= 4.6. We only need this - # to return True for families where "max" / "xhigh" are acceptable - # at the API; the cascade handles the case when they're not. - return any( - v in m - for v in ( - "opus-4-6", - "opus_4_6", - "opus-4.6", - "opus_4.6", - "opus-4-7", - "opus_4_7", - "opus-4.7", - "opus_4.7", - ) - ) - - _widened._hf_agent_patched = True # type: ignore[attr-defined] - cfg._is_opus_4_6_model = staticmethod(_widened) - - -_patch_litellm_effort_validation() - - # Effort levels accepted on the wire. -# Anthropic (4.6+): low | medium | high | xhigh | max (output_config.effort) -# OpenAI direct: minimal | low | medium | high | xhigh (reasoning_effort top-level) -# HF router: low | medium | high (extra_body.reasoning_effort) -# -# We validate *shape* here and let the probe cascade walk down on rejection; -# we deliberately do NOT maintain a per-model capability table. -_ANTHROPIC_EFFORTS = {"low", "medium", "high", "xhigh", "max"} -_OPENAI_EFFORTS = {"minimal", "low", "medium", "high", "xhigh"} +# HF Router exposes reasoning controls through the OpenAI-compatible +# ``extra_body`` field. The probe cascade walks down when a provider rejects +# an accepted-looking value, so this stays intentionally small and generic. _HF_EFFORTS = {"low", "medium", "high"} +def _hf_router_effort_level(reasoning_effort: str) -> str: + level = "low" if reasoning_effort == "minimal" else reasoning_effort + return level + + class UnsupportedEffortError(ValueError): """The requested effort isn't valid for this provider's API surface. @@ -151,40 +97,23 @@ def _resolve_llm_params( session_hf_token: str | None = None, reasoning_effort: str | None = None, strict: bool = False, + bill_to_user: bool = False, ) -> dict: """ Build LiteLLM kwargs for a given model id. - • ``anthropic/`` — native thinking config. We bypass LiteLLM's - ``reasoning_effort`` → ``thinking`` mapping (which lags new Claude - releases like 4.7 and sends the wrong API shape). Instead we pass - both ``thinking={"type": "adaptive"}`` and ``output_config= - {"effort": }`` as top-level kwargs — LiteLLM's Anthropic - adapter forwards unknown top-level kwargs into the request body - verbatim (confirmed by live probe; ``extra_body`` does NOT work - here because Anthropic's API rejects it as "Extra inputs are not - permitted"). This is the stable API for 4.6 and 4.7. Older - extended-thinking models that only accept ``thinking.type.enabled`` - will reject this; the probe's cascade catches that and falls back - to no thinking. - - • ``openai/`` — ``reasoning_effort`` forwarded as a top-level - kwarg (GPT-5 / o-series). LiteLLM uses the user's ``OPENAI_API_KEY``. - • ``ollama/``, ``vllm/``, ``lm_studio/``, and ``llamacpp/`` — local OpenAI-compatible endpoints. The id prefix selects a configurable localhost base URL, and the model suffix is sent to LiteLLM as ``openai/``. These endpoints don't receive ``reasoning_effort``. - • Anything else is treated as a HuggingFace router id. We hit the - auto-routing OpenAI-compatible endpoint at - ``https://router.huggingface.co/v1``. The id can be bare or carry an - HF routing suffix (``:fastest`` / ``:cheapest`` / ``:``). - A leading ``huggingface/`` is stripped. ``reasoning_effort`` is - forwarded via ``extra_body`` (LiteLLM's OpenAI adapter refuses it as - a top-level kwarg for non-OpenAI models). "minimal" normalizes to - "low". + • Anything else is treated as an HF Router id. We hit the auto-routing + OpenAI-compatible endpoint at ``https://router.huggingface.co/v1``. + The id can be bare or carry an HF routing suffix (``:fastest`` / + ``:cheapest`` / ``:``). A leading ``huggingface/`` is + stripped. ``reasoning_effort`` is forwarded via ``extra_body``. + "minimal" normalizes to "low". ``strict=True`` raises ``UnsupportedEffortError`` when the requested effort isn't in the provider's accepted set, instead of silently @@ -193,77 +122,45 @@ def _resolve_llm_params( runtime callers leave ``strict=False``, so a stale cached effort can't crash a turn — it just doesn't get sent. - Token precedence (first non-empty wins): + Token precedence for HF-router calls (first non-empty wins): 1. INFERENCE_TOKEN env — shared key on the hosted Space (inference is free for users, billed to the Space owner via ``X-HF-Bill-To``). 2. session.hf_token — the user's own token (CLI / OAuth / cache file). 3. huggingface_hub cache — ``HF_TOKEN`` / ``HUGGING_FACE_HUB_TOKEN`` / local ``hf auth login`` cache. - """ - if model_name.startswith("anthropic/"): - params: dict = {"model": model_name} - if reasoning_effort: - level = reasoning_effort - if level == "minimal": - level = "low" - if level not in _ANTHROPIC_EFFORTS: - if strict: - raise UnsupportedEffortError( - f"Anthropic doesn't accept effort={level!r}" - ) - else: - # Adaptive thinking + output_config.effort is the stable - # Anthropic API for Claude 4.6 / 4.7. Both kwargs are - # passed top-level: LiteLLM forwards unknown params into - # the request body for Anthropic, so ``output_config`` - # reaches the API. ``extra_body`` does NOT work here — - # Anthropic rejects it as "Extra inputs are not - # permitted". - params["thinking"] = {"type": "adaptive"} - params["output_config"] = {"effort": level} - return params - - if model_name.startswith("bedrock/"): - # LiteLLM routes ``bedrock/...`` through the Converse adapter, which - # picks up AWS credentials from the standard env vars - # (``AWS_ACCESS_KEY_ID`` / ``AWS_SECRET_ACCESS_KEY`` / ``AWS_REGION``). - # The Anthropic thinking/effort shape is not forwarded through Converse - # the same way, so we leave it off for now. - return {"model": model_name} - if model_name.startswith("openai/"): - params = {"model": model_name} - if reasoning_effort: - if reasoning_effort not in _OPENAI_EFFORTS: - if strict: - raise UnsupportedEffortError( - f"OpenAI doesn't accept effort={reasoning_effort!r}" - ) - else: - params["reasoning_effort"] = reasoning_effort - return params + Pass ``bill_to_user=True`` only after the daily subsidized allowance is + spent. Premium router ids then use the caller's own token, skip + ``INFERENCE_TOKEN``, and omit ``X-HF-Bill-To``. + """ + normalized_model = strip_huggingface_model_prefix(model_name) or model_name - if is_reserved_local_model_id(model_name): - raise ValueError(f"Unsupported local model id: {model_name}") + if is_reserved_local_model_id(normalized_model): + raise ValueError(f"Unsupported local model id: {normalized_model}") - if local_model_provider(model_name) is not None: - return _resolve_local_model_params(model_name, reasoning_effort, strict) + if local_model_provider(normalized_model) is not None: + return _resolve_local_model_params(normalized_model, reasoning_effort, strict) - hf_model = model_name.removeprefix("huggingface/") - api_key = _resolve_hf_router_token(session_hf_token) + hf_model = normalized_model + bill_user = bill_to_user and is_premium_model_id(hf_model) + api_key = ( + resolve_hf_token(session_hf_token, include_cached=False) + if bill_user + else _resolve_hf_router_token(session_hf_token) + ) params = { "model": f"openai/{hf_model}", - "api_base": "https://router.huggingface.co/v1", + "api_base": HF_ROUTER_BASE_URL, "api_key": api_key, } - if bill_to := get_hf_bill_to(): + if not bill_user and (bill_to := get_hf_bill_to()): params["extra_headers"] = {"X-HF-Bill-To": bill_to} if reasoning_effort: - hf_level = "low" if reasoning_effort == "minimal" else reasoning_effort + hf_level = _hf_router_effort_level(reasoning_effort) if hf_level not in _HF_EFFORTS: if strict: raise UnsupportedEffortError( - f"HF router doesn't accept effort={hf_level!r}" + f"HF Router doesn't accept effort={hf_level!r}" ) else: params["extra_body"] = {"reasoning_effort": hf_level} diff --git a/agent/core/model_ids.py b/agent/core/model_ids.py new file mode 100644 index 0000000000000000000000000000000000000000..3e3d10f9160d80c9cf4e7a6a9e36aae7ba873a9c --- /dev/null +++ b/agent/core/model_ids.py @@ -0,0 +1,41 @@ +"""Canonical model ids for HF Router inference.""" + +HF_ROUTER_BASE_URL = "https://router.huggingface.co/v1" + +CLAUDE_OPUS_48_MODEL_ID = "anthropic/claude-opus-4.8:fal-ai" +GPT_55_MODEL_ID = "openai/gpt-5.5:fal-ai" +KIMI_K26_MODEL_ID = "moonshotai/Kimi-K2.6" +MINIMAX_M27_MODEL_ID = "MiniMaxAI/MiniMax-M2.7" +GLM_51_MODEL_ID = "zai-org/GLM-5.1" +DEEPSEEK_V4_PRO_MODEL_ID = "deepseek-ai/DeepSeek-V4-Pro:deepinfra" + +DEFAULT_MODEL_ID = CLAUDE_OPUS_48_MODEL_ID + +PREMIUM_MODEL_IDS = { + CLAUDE_OPUS_48_MODEL_ID, + GPT_55_MODEL_ID, +} + +KNOWN_ROUTER_MODEL_IDS = PREMIUM_MODEL_IDS | { + KIMI_K26_MODEL_ID, + MINIMAX_M27_MODEL_ID, + GLM_51_MODEL_ID, + DEEPSEEK_V4_PRO_MODEL_ID, +} + + +def strip_huggingface_model_prefix(model_id: str | None) -> str | None: + """Return model ids without LiteLLM's optional ``huggingface/`` prefix.""" + if not model_id: + return model_id + return model_id.removeprefix("huggingface/") + + +def is_premium_model_id(model_id: str | None) -> bool: + normalized = strip_huggingface_model_prefix(model_id) + return bool(normalized and normalized in PREMIUM_MODEL_IDS) + + +def is_known_router_model_id(model_id: str | None) -> bool: + normalized = strip_huggingface_model_prefix(model_id) + return bool(normalized and normalized in KNOWN_ROUTER_MODEL_IDS) diff --git a/agent/core/model_switcher.py b/agent/core/model_switcher.py index 34eaccdd1f127253bec68b4ccdd1159c7a3c4a0a..fcab7d57545c804ba146144ea485527f99458c24 100644 --- a/agent/core/model_switcher.py +++ b/agent/core/model_switcher.py @@ -26,31 +26,29 @@ from agent.core.local_models import ( is_local_model_id, is_reserved_local_model_id, ) +from agent.core.model_ids import ( + CLAUDE_OPUS_48_MODEL_ID, + GPT_55_MODEL_ID, + KIMI_K26_MODEL_ID, + strip_huggingface_model_prefix, +) # Suggested models shown by `/model` (not a gate). Users can paste any HF -# model id (e.g. "MiniMaxAI/MiniMax-M2.7") or an `anthropic/` / `openai/` -# prefix for direct API access. For HF ids, append ":fastest" / -# ":cheapest" / ":preferred" / ":" to override the default -# routing policy (auto = fastest with failover). +# Router model id (e.g. "MiniMaxAI/MiniMax-M2.7"). Append ":fastest", +# ":cheapest", ":preferred", or ":" to override the default routing +# policy (auto = fastest with failover). SUGGESTED_MODELS = [ - {"id": "openai/gpt-5.5", "label": "GPT-5.5"}, - {"id": "openai/gpt-5.4", "label": "GPT-5.4"}, - {"id": "anthropic/claude-opus-4-7", "label": "Claude Opus 4.7"}, - {"id": "anthropic/claude-opus-4-6", "label": "Claude Opus 4.6"}, - { - "id": "bedrock/us.anthropic.claude-opus-4-6-v1", - "label": "Claude Opus 4.6 via Bedrock", - }, + {"id": CLAUDE_OPUS_48_MODEL_ID, "label": "Claude Opus 4.8"}, + {"id": GPT_55_MODEL_ID, "label": "GPT-5.5"}, {"id": "MiniMaxAI/MiniMax-M2.7", "label": "MiniMax M2.7"}, - {"id": "moonshotai/Kimi-K2.6", "label": "Kimi K2.6"}, + {"id": KIMI_K26_MODEL_ID, "label": "Kimi K2.6"}, {"id": "zai-org/GLM-5.1", "label": "GLM 5.1"}, {"id": "deepseek-ai/DeepSeek-V4-Pro:deepinfra", "label": "DeepSeek V4 Pro"}, ] _ROUTING_POLICIES = {"fastest", "cheapest", "preferred"} -_DIRECT_PREFIXES = ("anthropic/", "openai/", *LOCAL_MODEL_PREFIXES) _LOCAL_PROBE_TIMEOUT = 15.0 @@ -58,26 +56,25 @@ def is_valid_model_id(model_id: str) -> bool: """Loose format check — lets users pick any model id. Accepts: - • anthropic/ - • openai/ • ollama/, vllm/, lm_studio/, llamacpp/ • /[:] (HF router; tag = provider or policy) - • huggingface//[:] (same, accepts legacy prefix) + • huggingface//[:] (same, optional LiteLLM prefix) Actual availability is verified against the HF router catalog on switch, and by the provider on the probe's ping call. """ if not model_id: return False - if is_local_model_id(model_id): + normalized_model_id = strip_huggingface_model_prefix(model_id) or model_id + if is_local_model_id(normalized_model_id): return True - if is_reserved_local_model_id(model_id): + if is_reserved_local_model_id(normalized_model_id): return False - if any(model_id.startswith(prefix) for prefix in LOCAL_MODEL_PREFIXES): + if any(normalized_model_id.startswith(prefix) for prefix in LOCAL_MODEL_PREFIXES): return False - if "/" not in model_id: + if "/" not in normalized_model_id: return False - head = model_id.split(":", 1)[0] + head = normalized_model_id.split(":", 1)[0] parts = head.split("/") return len(parts) >= 2 and all(parts) @@ -88,10 +85,11 @@ def _print_hf_routing_info(model_id: str, console) -> bool: proceed with the switch, ``False`` to indicate a hard problem the user should notice before we fire the effort probe. - Anthropic / OpenAI ids return ``True`` without printing anything — - the probe below covers "does this model exist". + Local ids return ``True`` without printing anything. Router ids are checked + against the router catalog when possible; the probe below covers provider + availability for uncataloged ids. """ - if model_id.startswith(_DIRECT_PREFIXES): + if is_local_model_id(model_id): return True from agent.core import hf_router_catalog as cat @@ -162,7 +160,6 @@ def print_model_listing(config, console) -> None: console.print( "\n[dim]Paste any HF model id (e.g. 'MiniMaxAI/MiniMax-M2.7').\n" "Add ':fastest', ':cheapest', ':preferred', or ':' to override routing.\n" - "Use 'anthropic/' or 'openai/' for direct API access.\n" "Use 'ollama/', 'vllm/', 'lm_studio/', or " "'llamacpp/' for local OpenAI-compatible endpoints.[/dim]" ) @@ -173,8 +170,6 @@ def print_invalid_id(arg: str, console) -> None: console.print( "[dim]Expected:\n" " • /[:tag] (HF router — paste from huggingface.co)\n" - " • anthropic/\n" - " • openai/\n" " • ollama/ | vllm/ | lm_studio/ | llamacpp/[/dim]" ) diff --git a/agent/core/prompt_caching.py b/agent/core/prompt_caching.py deleted file mode 100644 index b30edd9fc4845738c08e972fdab712bf2ae3988d..0000000000000000000000000000000000000000 --- a/agent/core/prompt_caching.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Anthropic prompt caching breakpoints for outgoing LLM requests. - -Caching is GA on Anthropic's API and natively supported by litellm >=1.83 -via ``cache_control`` blocks. We apply two breakpoints (out of 4 allowed): - - 1. The tool block — caches all tool definitions as a single prefix. - 2. The system message — caches the rendered system prompt. - -Together these cover the ~4-5K static tokens that were being re-billed on -every turn. Subsequent turns within the 5-minute TTL hit cache_read pricing -(~10% of input cost) instead of full input. - -Non-Anthropic models (HF router, OpenAI) are passed through unchanged. -""" - -from typing import Any - - -def with_prompt_caching( - messages: list[Any], - tools: list[dict] | None, - model_name: str | None, -) -> tuple[list[Any], list[dict] | None]: - """Return (messages, tools) with cache_control breakpoints for Anthropic. - - No-op for non-Anthropic models. Original objects are not mutated; a fresh - list with replaced first message and last tool is returned, so callers - that share the underlying ``ContextManager.items`` list don't see their - persisted history rewritten. - """ - if not model_name or "anthropic" not in model_name: - return messages, tools - - if tools: - new_tools = list(tools) - last = dict(new_tools[-1]) - last["cache_control"] = {"type": "ephemeral"} - new_tools[-1] = last - tools = new_tools - - if messages: - first = messages[0] - role = ( - first.get("role") - if isinstance(first, dict) - else getattr(first, "role", None) - ) - if role == "system": - content = ( - first.get("content") - if isinstance(first, dict) - else getattr(first, "content", None) - ) - if isinstance(content, str) and content: - cached_block = [ - { - "type": "text", - "text": content, - "cache_control": {"type": "ephemeral"}, - } - ] - new_first = {"role": "system", "content": cached_block} - messages = [new_first] + list(messages[1:]) - - return messages, tools diff --git a/agent/core/redact.py b/agent/core/redact.py index 8978942c8a027b56e51acdd0f6485d4e9e0fbbf2..a91bcb280deec2a1dc69d68be4a218af147b4f73 100644 --- a/agent/core/redact.py +++ b/agent/core/redact.py @@ -18,10 +18,9 @@ from typing import Any _PATTERNS: list[tuple[re.Pattern, str]] = [ # Hugging Face tokens: hf_[A-Za-z0-9]{30,} (re.compile(r"hf_[A-Za-z0-9]{30,}"), "[REDACTED_HF_TOKEN]"), - # Anthropic: sk-ant-[A-Za-z0-9_\-]{20,} - (re.compile(r"sk-ant-[A-Za-z0-9_\-]{20,}"), "[REDACTED_ANTHROPIC_KEY]"), - # OpenAI: sk-[A-Za-z0-9]{40,} (legacy + proj keys) - (re.compile(r"sk-(?!ant-)[A-Za-z0-9_\-]{40,}"), "[REDACTED_OPENAI_KEY]"), + # Provider API keys with common sk-* prefixes. + (re.compile(r"sk-ant-[A-Za-z0-9_\-]{20,}"), "[REDACTED_PROVIDER_API_KEY]"), + (re.compile(r"sk-(?!ant-)[A-Za-z0-9_\-]{40,}"), "[REDACTED_PROVIDER_API_KEY]"), # GitHub classic PATs: ghp_, gho_, ghu_, ghs_, ghr_ followed by 36+ chars (re.compile(r"gh[pousr]_[A-Za-z0-9]{36,}"), "[REDACTED_GITHUB_TOKEN]"), # GitHub fine-grained PATs: github_pat_ @@ -36,8 +35,7 @@ _PATTERNS: list[tuple[re.Pattern, str]] = [ # still see which secret was referenced. Covers `KEY=value` and `KEY: value` # when the key looks secret-y. _SECRETY_NAMES = re.compile( - r"(?i)\b(HF_TOKEN|HUGGINGFACEHUB_API_TOKEN|ANTHROPIC_API_KEY|OPENAI_API_KEY|" - r"GITHUB_TOKEN|AWS_SECRET_ACCESS_KEY|AWS_ACCESS_KEY_ID|PASSWORD|SECRET|API_KEY)" + r"(?i)\b([A-Z0-9_]*(?:TOKEN|API_KEY|SECRET|PASSWORD|ACCESS_KEY_ID))" r"\s*[:=]\s*([^\s\"']+)" ) diff --git a/agent/core/session.py b/agent/core/session.py index 8999b81811ce5c0a2f2cf2b226d03a68282e802c..3ea4702c0d6a9094380b806b459ff35c500fe998 100644 --- a/agent/core/session.py +++ b/agent/core/session.py @@ -11,6 +11,8 @@ from enum import Enum from pathlib import Path from typing import Any, Optional +from litellm import Message + from agent.config import Config from agent.context_manager.manager import ContextManager from agent.messaging.gateway import NotificationGateway @@ -27,12 +29,10 @@ DEFAULT_SESSION_LOG_DIR = Path("session_logs") def _get_max_tokens_safe(model_name: str) -> int: """Return the max input-context tokens for a model. - Primary source: ``litellm.get_model_info(model)['max_input_tokens']`` — - LiteLLM maintains an upstream catalog that knows Claude Opus 4.6 is - 1M, GPT-5 is 272k, Sonnet 4.5 is 200k, and so on. Strips any HF routing - suffix / huggingface/ prefix so tagged ids ('moonshotai/Kimi-K2.6:cheapest') - look up the bare model. Falls back to a conservative 200k default for - models not in the catalog (typically HF-router-only models). + Primary source: ``litellm.get_model_info(model)['max_input_tokens']``. + Strips any HF routing suffix / huggingface/ prefix so tagged ids + ('moonshotai/Kimi-K2.6:cheapest') look up the bare model. Falls back to a + conservative 200k default for models not in the catalog. """ from litellm import get_model_info @@ -62,6 +62,7 @@ class OpType(Enum): INTERRUPT = "interrupt" UNDO = "undo" COMPACT = "compact" + NEW = "new" RESUME = "resume" SHUTDOWN = "shutdown" @@ -118,6 +119,11 @@ class Session: self.session_id = session_id or str(uuid.uuid4()) self.config = config self.is_running = True + # Billing mode for premium HF Router usage. The backend quota gate + # flips this on once the user is past their subsidized daily allowance, + # so the LLM call bills the user's own HF token instead of the Space. + # Persisted with the session so it survives idle-reclaim. + self.premium_user_billed: bool = False self.current_plan: list[dict[str, str]] = [] self._cancelled = asyncio.Event() self.pending_approval: Optional[dict[str, Any]] = None @@ -322,8 +328,11 @@ class Session: def update_model(self, model_name: str) -> None: """Switch the active model and update the context window limit.""" - self.config.model_name = model_name - self.context_manager.model_max_tokens = _get_max_tokens_safe(model_name) + from agent.core.model_ids import strip_huggingface_model_prefix + + normalized = strip_huggingface_model_prefix(model_name) or model_name + self.config.model_name = normalized + self.context_manager.model_max_tokens = _get_max_tokens_safe(normalized) def set_auto_approval_policy( self, *, enabled: bool, cost_cap_usd: float | None @@ -376,6 +385,82 @@ class Session: """Increment turn counter (called after each user interaction)""" self.turn_count += 1 + def start_new_conversation(self) -> dict[str, Any]: + """Rotate this runtime into a fresh conversation. + + The tool router, model/config choices, user identity, and external + resources stay attached to the CLI process. Conversation-specific state + gets reset so later saves do not merge with the prior chat. Warm runtime + resources such as the sandbox, in-flight job tracking, and probed + model-effort cache are deliberately preserved. + """ + previous_session_id = self.session_id + previous_turn_count = self.turn_count + previous_message_count = len(self.context_manager.items) + previous_non_system_count = sum( + 1 + for item in self.context_manager.items + if getattr(item, "role", None) != "system" + ) + + saved_path: str | None = None + if self.config.save_sessions and previous_non_system_count: + saved_path = self.save_and_upload_detached(self.config.session_dataset_repo) + + from agent.tools.plan_tool import reset_current_plan + + self.current_plan = [] + reset_current_plan() + + system_msg = self._fresh_system_message() + self.context_manager.items = [system_msg] if system_msg is not None else [] + self.context_manager.running_context_usage = 0 + + self.session_id = str(uuid.uuid4()) + self.session_start_time = datetime.now().isoformat() + self.turn_count = 0 + self.last_auto_save_turn = 0 + self.logged_events = [] + self._local_save_path = None + self._last_heartbeat_ts = None + self.pending_approval = None + self.auto_approval_estimated_spend_usd = 0.0 + self.reset_cancel() + + # Previous-session metadata is intentionally included for event + # consumers and telemetry, even though the CLI currently prints only + # the optional save path. + return { + "session_id": self.session_id, + "previous_session_id": previous_session_id, + "previous_turn_count": previous_turn_count, + "previous_message_count": previous_message_count, + "saved_path": saved_path, + } + + def _fresh_system_message(self) -> Message | None: + existing = ( + self.context_manager.items[0] + if self.context_manager.items + and getattr(self.context_manager.items[0], "role", None) == "system" + else None + ) + refresh = getattr(self.context_manager, "refresh_system_prompt", None) + if refresh is None: + return existing + try: + tool_specs = ( + self.tool_router.get_tool_specs_for_llm() if self.tool_router else [] + ) + return refresh( + tool_specs=tool_specs, + hf_token=self.hf_token, + local_mode=self.local_mode, + ) + except Exception as e: + logger.warning("Failed to refresh system prompt for new chat: %s", e) + return existing + async def auto_save_if_needed(self) -> None: """Check if auto-save should trigger and save if so (completely non-blocking)""" if not self.config.save_sessions: diff --git a/agent/core/session_persistence.py b/agent/core/session_persistence.py index e12467211b16fe12ec75fbf5b60edb5ee54f4072..97b8552ab130e1fc89ba29dc95f447c8013c7501 100644 --- a/agent/core/session_persistence.py +++ b/agent/core/session_persistence.py @@ -175,6 +175,7 @@ class MongoSessionStore(NoopSessionStore): turn_count: int = 0, pending_approval: list[dict[str, Any]] | None = None, claude_counted: bool = False, + premium_user_billed: bool = False, notification_destinations: list[str] | None = None, auto_approval_enabled: bool = False, auto_approval_cost_cap_usd: float | None = None, @@ -206,6 +207,7 @@ class MongoSessionStore(NoopSessionStore): "turn_count": turn_count, "pending_approval": pending_approval or [], "claude_counted": claude_counted, + "premium_user_billed": premium_user_billed, "notification_destinations": notification_destinations or [], "auto_approval_enabled": auto_approval_enabled, "auto_approval_cost_cap_usd": auto_approval_cost_cap_usd, @@ -228,13 +230,17 @@ class MongoSessionStore(NoopSessionStore): turn_count: int = 0, pending_approval: list[dict[str, Any]] | None = None, claude_counted: bool = False, + premium_user_billed: bool = False, created_at: datetime | None = None, notification_destinations: list[str] | None = None, auto_approval_enabled: bool = False, auto_approval_cost_cap_usd: float | None = None, auto_approval_estimated_spend_usd: float = 0.0, + raise_on_error: bool = False, ) -> None: if not self._ready(): + if raise_on_error: + raise RuntimeError("session store not ready") return now = _now() await self.upsert_session( @@ -249,6 +255,7 @@ class MongoSessionStore(NoopSessionStore): turn_count=turn_count, pending_approval=pending_approval, claude_counted=claude_counted, + premium_user_billed=premium_user_billed, notification_destinations=notification_destinations, auto_approval_enabled=auto_approval_enabled, auto_approval_cost_cap_usd=auto_approval_cost_cap_usd, @@ -278,6 +285,11 @@ class MongoSessionStore(NoopSessionStore): if ops: await self.db.session_messages.bulk_write(ops, ordered=False) except PyMongoError as e: + # Best-effort by default, but the reaper passes raise_on_error so a + # silent message-write failure doesn't let it evict a session whose + # latest messages never made it to Mongo. + if raise_on_error: + raise logger.warning("Failed to persist session %s snapshot: %s", session_id, e) async def load_session( diff --git a/agent/core/session_resume.py b/agent/core/session_resume.py index 941c426b7b216e099de204054843953eb70fd697..ac7d335f6c69b63c99f06519afb536ce4f4455b5 100644 --- a/agent/core/session_resume.py +++ b/agent/core/session_resume.py @@ -12,6 +12,7 @@ from typing import Any from litellm import Message +from agent.core.model_ids import strip_huggingface_model_prefix from agent.core.model_switcher import is_valid_model_id from agent.core.session import DEFAULT_SESSION_LOG_DIR @@ -224,8 +225,9 @@ def restore_session_from_log(session: Any, path: Path) -> dict[str, Any]: saved_model = data.get("model_name") invalid_saved_model: str | None = None if isinstance(saved_model, str) and saved_model: - if is_valid_model_id(saved_model): - session.update_model(saved_model) + normalized_model = strip_huggingface_model_prefix(saved_model) + if normalized_model and is_valid_model_id(normalized_model): + session.update_model(normalized_model) else: invalid_saved_model = saved_model logger.warning( diff --git a/agent/core/session_uploader.py b/agent/core/session_uploader.py index 404fd224563cdae3d91c2b93e05e8306ee91fb7e..ed2205202acb1f6284ef3c59daaa2137333a4ee0 100644 --- a/agent/core/session_uploader.py +++ b/agent/core/session_uploader.py @@ -396,7 +396,7 @@ sessions/YYYY-MM-DD/.jsonl **WARNING: no comprehensive redaction or human review has been performed for this dataset.** ML Intern applies automated best-effort scrubbing for common secret patterns -such as Hugging Face, Anthropic, OpenAI, GitHub, and AWS tokens before upload. +such as Hugging Face, GitHub, AWS, and provider API tokens before upload. This is not a privacy guarantee. These traces may contain sensitive information, including prompts, code, diff --git a/agent/core/telemetry.py b/agent/core/telemetry.py index 38d2bbe761fee99d7c8051d6788fc849df8a8fae..8f34ec95a860e3a859b8982264c960eee1f0a004 100644 --- a/agent/core/telemetry.py +++ b/agent/core/telemetry.py @@ -30,10 +30,8 @@ logger = logging.getLogger(__name__) def extract_usage(response_or_chunk: Any) -> dict: """Flat usage dict from a litellm response or final-chunk usage object. - Normalizes across providers: Anthropic exposes cache tokens as - ``cache_read_input_tokens`` / ``cache_creation_input_tokens``; OpenAI uses - ``prompt_tokens_details.cached_tokens``. Exposed under the stable keys - ``cache_read_tokens`` / ``cache_creation_tokens``. + Normalizes cache-token details across provider response shapes. Exposed + under the stable keys ``cache_read_tokens`` / ``cache_creation_tokens``. """ u = getattr(response_or_chunk, "usage", None) if u is None and isinstance(response_or_chunk, dict): @@ -97,11 +95,10 @@ async def record_llm_call( Pre-2026-04-29 only ``main`` calls were instrumented; observed gap on Cost Explorer was ~67%, with the other 5 call sites accounting for the rest. Tagging lets us split the dataset's ``total_cost_usd`` by - category and validate against AWS billing. + category and validate against billing data. - The ``/title`` (HF Router, not Bedrock) and ``/health/llm`` (diagnostic - endpoint, no session context) call sites are intentionally not - instrumented — together they're <1% of spend. + The ``/title`` and ``/health/llm`` diagnostic call sites are intentionally + not instrumented because they have no session context and are tiny. """ usage = extract_usage(response) if response is not None else {} cost_usd = 0.0 diff --git a/agent/main.py b/agent/main.py index 9d7d4399df1b8a8e59c048476029a3208ee1c33e..d7ae4128bf32ca037d9a8682b149f98dd956148e 100644 --- a/agent/main.py +++ b/agent/main.py @@ -12,6 +12,7 @@ import json import logging import os import signal +import subprocess import sys import time from dataclasses import dataclass @@ -27,6 +28,7 @@ from agent.core.agent_loop import submission_loop from agent.core import model_switcher from agent.core.hf_tokens import resolve_hf_token from agent.core.local_models import is_local_model_id +from agent.core.model_ids import strip_huggingface_model_prefix from agent.core.session import OpType from agent.core.tools import ToolRouter from agent.messaging.gateway import NotificationGateway @@ -73,6 +75,21 @@ def _tool_runtime_label(local_mode: bool) -> str: return "local filesystem" if local_mode else "HF sandbox" +def _normalize_config_model(config: Any) -> None: + normalized = strip_huggingface_model_prefix(getattr(config, "model_name", None)) + if normalized: + config.model_name = normalized + + +def _validate_cli_model_override(model: str) -> str: + if not model_switcher.is_valid_model_id(model): + raise ValueError( + "Invalid model id. Use an HF Router id like " + "'anthropic/claude-opus-4.8:fal-ai' or a supported local prefix." + ) + return model.removeprefix("huggingface/") + + async def _wait_for_initial_sandbox_preload(session_holder: list | None) -> None: session = session_holder[0] if session_holder else None task = getattr(session, "sandbox_preload_task", None) @@ -195,6 +212,14 @@ def _create_rich_console(): return get_console() +def _clear_terminal() -> None: + command = ["cmd", "/c", "cls"] if os.name == "nt" else ["clear"] + try: + subprocess.run(command, check=False) + except OSError: + pass + + class _ThinkingShimmer: """Animated shiny/shimmer thinking indicator — a bright gradient sweeps across the text.""" @@ -398,6 +423,18 @@ async def event_listener( elif event.event_type == "undo_complete": console.print("[dim]Undone.[/dim]") turn_complete_event.set() + elif event.event_type == "new_complete": + data = event.data or {} + if data.get("clear_screen"): + _clear_terminal() + saved_path = data.get("saved_path") + if saved_path: + console.print( + f"[dim]Started new chat. Prior chat saved to {saved_path}.[/dim]" + ) + else: + console.print("[dim]Started new chat.[/dim]") + turn_complete_event.set() elif event.event_type == "resume_complete": data = event.data or {} path = data.get("path", "?") @@ -902,6 +939,20 @@ async def _handle_slash_command( operation=Operation(op_type=OpType.COMPACT), ) + if command in {"/new", "/clear"}: + session = session_holder[0] if session_holder else None + if session is None: + get_console().print("[bold red]No active session to reset.[/bold red]") + return None + submission_id[0] += 1 + return Submission( + id=f"sub_{submission_id[0]}", + operation=Operation( + op_type=OpType.NEW, + data={"clear_screen": command == "/clear"}, + ), + ) + if command == "/resume": session = session_holder[0] if session_holder else None if session is None: @@ -958,9 +1009,9 @@ async def _handle_slash_command( console.print(f" [dim]{m}: {eff or 'off'}[/dim]") console.print( "[dim]Set with '/effort minimal|low|medium|high|xhigh|max|off'. " - "'max' is Anthropic-only; 'xhigh' is also supported by current " - "OpenAI GPT-5 models. The cascade falls back to whatever the " - "model actually accepts.[/dim]" + "HF Router accepts low|medium|high generically; higher preferences " + "are probed and the cascade falls back to whatever the selected " + "provider accepts.[/dim]" ) return None level = arg.lower() @@ -1109,14 +1160,15 @@ async def main(model: str | None = None, sandbox_tools: bool = False): """Interactive chat with the agent""" # Clear screen - os.system("clear" if os.name != "nt" else "cls") + _clear_terminal() # Create prompt session for input (needed early for token prompt) prompt_session = PromptSession() config = load_config(CLI_CONFIG_PATH, include_user_defaults=True) + _normalize_config_model(config) if model: - config.model_name = model + config.model_name = _validate_cli_model_override(model) _apply_tool_runtime_override(config, sandbox_tools=sandbox_tools) local_mode = _is_local_tool_runtime(config) @@ -1359,10 +1411,15 @@ async def headless_main( _configure_runtime_logging() config = load_config(CLI_CONFIG_PATH, include_user_defaults=True) + _normalize_config_model(config) config.yolo_mode = True # Auto-approve everything in headless mode if model: - config.model_name = model + try: + config.model_name = _validate_cli_model_override(model) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) _apply_tool_runtime_override(config, sandbox_tools=sandbox_tools) local_mode = _is_local_tool_runtime(config) diff --git a/agent/prompts/system_prompt_v3.yaml b/agent/prompts/system_prompt_v3.yaml index c4eb6e0d5337db0907dd5208160e8642f6c2bad6..ab2cfa2a4546226918c2e449b3ec2780ec2a292d 100644 --- a/agent/prompts/system_prompt_v3.yaml +++ b/agent/prompts/system_prompt_v3.yaml @@ -136,6 +136,8 @@ system_prompt: | Do NOT call sandbox_create before normal CPU work. Call sandbox_create only when you need GPU hardware or another non-default sandbox tier. + The sandbox filesystem does not survive session resumption. If a session is resumed, any files, installed packages, or running processes from earlier are gone — recreate what you need before relying on the sandbox. + Use a GPU sandbox (t4-small minimum) when testing code that uses CUDA, bf16/fp16, quantization, flash attention, torch.compile, or model loading. CPU sandboxes cannot test GPU code paths. If the available sandbox tiers cannot fit the full model path, test the largest useful smoke path, state what was not covered, and submit one HF job first. diff --git a/agent/tools/plan_tool.py b/agent/tools/plan_tool.py index b85ae22c560098cac787709c410746b34e817c65..218ec9b68bb8600e49a88a1bdbf327e6d65b8eac 100644 --- a/agent/tools/plan_tool.py +++ b/agent/tools/plan_tool.py @@ -9,6 +9,13 @@ from .types import ToolResult _current_plan: List[Dict[str, str]] = [] +def reset_current_plan() -> None: + """Clear the CLI-visible in-memory plan.""" + global _current_plan + + _current_plan = [] + + class PlanTool: """Tool for managing a list of todos with status tracking.""" diff --git a/agent/tools/research_tool.py b/agent/tools/research_tool.py index f5815be8332ef371d3e863652bfc6cdd5127bbc2..bcb4070d4a935a21b8447fe188195f91288026ff 100644 --- a/agent/tools/research_tool.py +++ b/agent/tools/research_tool.py @@ -17,7 +17,7 @@ from litellm import Message, acompletion from agent.core import telemetry from agent.core.doom_loop import check_for_doom_loop from agent.core.llm_params import _resolve_llm_params -from agent.core.prompt_caching import with_prompt_caching +from agent.core.model_ids import strip_huggingface_model_prefix from agent.core.session import Event logger = logging.getLogger(__name__) @@ -220,13 +220,8 @@ RESEARCH_TOOL_SPEC = { def _get_research_model(main_model: str) -> str: - """Pick a cheaper model for research based on the main model.""" - if main_model.startswith("anthropic/"): - return "anthropic/claude-sonnet-4-6" - if main_model.startswith("bedrock/") and "anthropic" in main_model: - return "bedrock/us.anthropic.claude-sonnet-4-6" - # For non-Anthropic models (HF router etc.), use the same model - return main_model + """Normalize the main model id for the research sub-call.""" + return strip_huggingface_model_prefix(main_model) or main_model async def research_handler( @@ -251,19 +246,18 @@ async def research_handler( user_content = f"Context: {context}\n\n{user_content}" messages.append(Message(role="user", content=user_content)) - # Use a cheaper/faster model for research + # Use the normalized router model for research main_model = session.config.model_name research_model = _get_research_model(main_model) - # Research is a cheap sub-call — cap the main session's effort at "high" - # so a user preference of ``max``/``xhigh`` (valid for Opus 4.6/4.7) doesn't - # propagate to a Sonnet research model that may not accept those levels. - # We also haven't probed this sub-model so we don't know its ceiling. + # Research is a cheap sub-call — cap the main session's effort at "high". + # We also haven't probed this sub-call's model so we don't know its ceiling. _pref = getattr(session.config, "reasoning_effort", None) _capped = "high" if _pref in ("max", "xhigh") else _pref llm_params = _resolve_llm_params( research_model, getattr(session, "hf_token", None), reasoning_effort=_capped, + bill_to_user=getattr(session, "premium_user_billed", False), ) # Get read-only tool specs from the session's tool router @@ -341,10 +335,9 @@ async def research_handler( ) ) try: - _msgs, _ = with_prompt_caching(messages, None, llm_params.get("model")) _t0 = time.monotonic() response = await acompletion( - messages=_msgs, + messages=messages, tools=None, # no tools — force text response stream=False, timeout=120, @@ -389,13 +382,10 @@ async def research_handler( ) try: - _msgs, _tools = with_prompt_caching( - messages, tool_specs if tool_specs else None, llm_params.get("model") - ) _t0 = time.monotonic() response = await acompletion( - messages=_msgs, - tools=_tools, + messages=messages, + tools=tool_specs if tool_specs else None, tool_choice="auto", stream=False, timeout=120, @@ -508,10 +498,9 @@ async def research_handler( ) ) try: - _msgs, _ = with_prompt_caching(messages, None, llm_params.get("model")) _t0 = time.monotonic() response = await acompletion( - messages=_msgs, + messages=messages, tools=None, stream=False, timeout=120, diff --git a/agent/utils/terminal_display.py b/agent/utils/terminal_display.py index 01f4e42e7c9cff8b1dccebb0948ca0039e4dd175..45850e893653253673a4f87a2132de6957fc8155 100644 --- a/agent/utils/terminal_display.py +++ b/agent/utils/terminal_display.py @@ -454,6 +454,8 @@ def print_yolo_approve(count: int) -> None: HELP_ROWS: tuple[tuple[str, str, str], ...] = ( ("/help", "", "Show this help"), + ("/new", "", "Start a fresh chat"), + ("/clear", "", "Clear terminal and start fresh"), ("/undo", "", "Undo last turn"), ("/compact", "", "Compact context window"), ("/resume", "[index|id|path]", "Pick up from ./session_logs"), diff --git a/backend/dependencies.py b/backend/dependencies.py index 58e02b7ee9a108e586496516bbd076e85b735fa2..027181add402f70c324ae81ae4b56bfba6b39d56 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -298,9 +298,7 @@ def _extract_token(request: Request) -> str | None: async def require_huggingface_org_member(request: Request) -> bool: """Return True if the caller is a member of the ``huggingface`` org. - Used to gate endpoints that can push a session onto an Anthropic model - billed to the Space's ``ANTHROPIC_API_KEY``. Returns True unconditionally - in dev mode so local testing isn't blocked. + Returns True unconditionally in dev mode so local testing isn't blocked. """ if not AUTH_ENABLED: return True diff --git a/backend/models.py b/backend/models.py index e12794aa2727e4652c8627c2ff24e21201f5d6de..1eec6ac851c7a6a50cf15083cde341e511a7b984 100644 --- a/backend/models.py +++ b/backend/models.py @@ -105,6 +105,8 @@ class SessionInfo(BaseModel): auto_approval: SessionAutoApprovalInfo = Field( default_factory=SessionAutoApprovalInfo ) + premium_user_billed: bool = False + premium_quota_counted: bool = False class SessionNotificationsRequest(BaseModel): diff --git a/backend/routes/agent.py b/backend/routes/agent.py index 965192b5a19b92452849fea1c4aa266ec3f69caf..cff625529ed163354e285bd3c08eb5d53519d83a 100644 --- a/backend/routes/agent.py +++ b/backend/routes/agent.py @@ -52,71 +52,64 @@ from session_manager import ( import user_quotas from agent.core.hf_access import get_jobs_access -from agent.core.hf_tokens import resolve_hf_request_token, resolve_hf_router_token +from agent.core.hf_tokens import resolve_hf_request_token from agent.core.llm_params import _resolve_llm_params +from agent.core.model_ids import ( + DEEPSEEK_V4_PRO_MODEL_ID, + DEFAULT_MODEL_ID, + GLM_51_MODEL_ID, + GPT_55_MODEL_ID, + KIMI_K26_MODEL_ID, + MINIMAX_M27_MODEL_ID, + is_premium_model_id, +) logger = logging.getLogger(__name__) router = APIRouter(prefix="/api", tags=["agent"]) _background_teardown_tasks: set[asyncio.Task] = set() -DEFAULT_CLAUDE_MODEL_ID = "bedrock/us.anthropic.claude-opus-4-6-v1" -DEFAULT_FREE_MODEL_ID = "moonshotai/Kimi-K2.6" -PREMIUM_MODEL_IDS = { - DEFAULT_CLAUDE_MODEL_ID, - "openai/gpt-5.5", -} +DEFAULT_PREMIUM_MODEL_ID = DEFAULT_MODEL_ID +DEFAULT_GPT_MODEL_ID = GPT_55_MODEL_ID +DEFAULT_FREE_MODEL_ID = KIMI_K26_MODEL_ID DATASET_UPLOAD_MULTIPART_SLACK_BYTES = 1024 * 1024 -def _claude_picker_model_id() -> str: - """Return the model ID used by the Claude option in the UI. - - The frontend config sets ``session_manager.config.model_name`` from - ``ML_INTERN_CLAUDE_MODEL_ID`` when that env var is present, otherwise it - falls back to the production Bedrock Claude model. This function only - exposes that resolved config value for the Claude picker; non-Claude models - are listed separately in the model switcher. - """ - return session_manager.config.model_name - - def _available_models() -> list[dict[str, Any]]: models = [ { - "id": "moonshotai/Kimi-K2.6", - "label": "Kimi K2.6", + "id": DEFAULT_PREMIUM_MODEL_ID, + "label": "Claude Opus 4.8", "provider": "huggingface", - "tier": "free", - "recommended": True, - }, - { - "id": _claude_picker_model_id(), - "label": "Claude Opus 4.6", - "provider": "anthropic", "tier": "pro", "recommended": True, }, { - "id": "openai/gpt-5.5", + "id": DEFAULT_GPT_MODEL_ID, "label": "GPT-5.5", - "provider": "openai", + "provider": "huggingface", "tier": "pro", }, { - "id": "MiniMaxAI/MiniMax-M2.7", + "id": DEFAULT_FREE_MODEL_ID, + "label": "Kimi K2.6", + "provider": "huggingface", + "tier": "free", + }, + { + "id": MINIMAX_M27_MODEL_ID, "label": "MiniMax M2.7", "provider": "huggingface", "tier": "free", }, { - "id": "zai-org/GLM-5.1", + "id": GLM_51_MODEL_ID, "label": "GLM 5.1", "provider": "huggingface", "tier": "free", }, { - "id": "deepseek-ai/DeepSeek-V4-Pro:deepinfra", + "id": DEEPSEEK_V4_PRO_MODEL_ID, "label": "DeepSeek V4 Pro", "provider": "huggingface", "tier": "free", @@ -129,7 +122,11 @@ AVAILABLE_MODELS = _available_models() def _is_premium_model(model_id: str) -> bool: - return model_id in PREMIUM_MODEL_IDS + return is_premium_model_id(model_id) + + +def _is_user_billed(model_id: str) -> bool: + return _is_premium_model(model_id) async def _model_override_for_new_session( @@ -138,24 +135,26 @@ async def _model_override_for_new_session( ) -> str | None: """Return the model override to use when creating a new session. - Explicit premium model requests are allowed and charged at message-submit - time. Implicit default sessions are more forgiving: when the configured - default is premium, start them on the first free model instead of spending - premium quota accidentally. + Explicit model requests are allowed and charged at message-submit time + when premium. Empty requests use the configured default model. """ - resolved_model = requested_model or session_manager.config.model_name - if not _is_premium_model(resolved_model): - return requested_model - if requested_model: - return requested_model + return requested_model - logger.info( - "Default premium model %s would spend quota; " - "creating session with free fallback %s", - resolved_model, - DEFAULT_FREE_MODEL_ID, + +def _premium_cap_message(plan: str) -> str: + """Over-allowance message for a premium model that can't fall back to user + billing. Defensive: every current premium model is user-billable and + overflows to the user's own HF account instead of reaching this. + """ + if plan == "pro": + return ( + "Daily premium model limit reached. Use a free model and try premium " + "models again tomorrow." + ) + return ( + "Daily premium model limit reached. Upgrade to HF Pro for " + f"{user_quotas.CLAUDE_PRO_DAILY}/day or use a free model." ) - return DEFAULT_FREE_MODEL_ID async def _enforce_premium_model_quota( @@ -169,9 +168,11 @@ async def _enforce_premium_model_quota( ``claude_counted`` flag on ``AgentSession`` guards against re-counting the same session; the stored field name is kept for persistence compatibility. - No-ops when the session's current model isn't premium, or when this - session has already been charged. Raises 429 when the user has hit - their daily cap. + Subsidizes the daily allowance (free = 2, pro = 20), organization-billed + through the HF Router. Past the allowance, premium router models flip the + session to ``premium_user_billed`` so the call bills the user's own HF + token instead of blocking. No-ops when the model isn't premium or when this + session's billing has already been decided. """ if agent_session.claude_counted: return @@ -181,27 +182,21 @@ async def _enforce_premium_model_quota( user_id = user["user_id"] plan = user.get("plan", "free") cap = user_quotas.daily_cap_for(plan) - new_count = await user_quotas.try_increment_claude(user_id, cap) - if new_count is None: - if plan == "pro": - message = ( - "Daily premium model limit reached. Use a free model and try " - "premium models again tomorrow." - ) - else: - message = ( - "Daily premium model limit reached. Upgrade to HF Pro for " - f"{user_quotas.CLAUDE_PRO_DAILY}/day or use a free model." + within_allowance = await user_quotas.try_increment_claude(user_id, cap) is not None + if not within_allowance: + if not _is_user_billed(model_name): + raise HTTPException( + status_code=429, + detail={ + "error": "premium_model_daily_cap", + "plan": plan, + "cap": cap, + "message": _premium_cap_message(plan), + }, ) - raise HTTPException( - status_code=429, - detail={ - "error": "premium_model_daily_cap", - "plan": plan, - "cap": cap, - "message": message, - }, - ) + # Past the subsidized allowance on a user-billable model: bill the + # user's own HF (OAuth) token for this session instead of blocking. + agent_session.session.premium_user_billed = True agent_session.claude_counted = True await session_manager.persist_session_snapshot(agent_session) @@ -385,14 +380,13 @@ async def generate_title( reasoning model — reasoning_effort=low keeps the reasoning budget small so the 60-token output budget isn't consumed before the title is written. """ - api_key = resolve_hf_router_token(_user_hf_token(user)) try: + llm_params = _resolve_llm_params( + "openai/gpt-oss-120b:cerebras", + _user_hf_token(user), + reasoning_effort="low", + ) response = await acompletion( - # Double openai/ prefix: LiteLLM strips the first as its provider - # prefix, leaving the HF model id on the wire for the router. - model="openai/openai/gpt-oss-120b:cerebras", - api_base="https://router.huggingface.co/v1", - api_key=api_key, messages=[ { "role": "system", @@ -409,7 +403,7 @@ async def generate_title( max_tokens=60, temperature=0.3, timeout=10, - reasoning_effort="low", + **llm_params, ) title = response.choices[0].message.content.strip().strip('"').strip("'") title = title.translate(_TITLE_STRIP_CHARS).strip() @@ -470,8 +464,7 @@ async def create_session( if model and model not in valid_ids: raise HTTPException(status_code=400, detail=f"Unknown model: {model}") - # Explicit premium selections are allowed. If the implicit configured - # default is premium, start the session on a free model instead. + # Empty requests use the configured default, which may be premium. model = await _model_override_for_new_session(request, model) try: @@ -502,7 +495,8 @@ async def restore_session_summary( session's context as a user-role system note. Optional ``"model"`` in the body overrides the session's LLM. The - premium-model quota runs at message-submit time, not here. + premium-model quota runs before summarization because that call uses the + session model immediately. """ messages = body.get("messages") if not isinstance(messages, list) or not messages: @@ -528,6 +522,14 @@ async def restore_session_summary( except SessionCapacityError as e: raise HTTPException(status_code=503, detail=str(e)) + agent_session = await _check_session_access( + session_id, + user, + request, + preload_sandbox=False, + ) + await _enforce_premium_model_quota(user, agent_session) + try: summarized = await session_manager.seed_from_summary(session_id, messages) except ValueError as e: @@ -659,6 +661,7 @@ async def upload_session_dataset( agent_session.session.context_manager.add_message( Message(role="user", content=dataset_context_note(uploaded)) ) + session_manager._touch(agent_session) await session_manager.persist_session_snapshot(agent_session) logger.info( "Uploaded dataset file %s to %s for session %s", diff --git a/backend/session_manager.py b/backend/session_manager.py index 3c992c9c09d6334e23db7c2b06a2b6b1cd8e662e..6a77d57f39eb27e4d139e5da7aac9371c8ae91f9 100644 --- a/backend/session_manager.py +++ b/backend/session_manager.py @@ -6,12 +6,18 @@ import logging import os import uuid from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import Any, Optional from agent.config import load_config from agent.core.agent_loop import process_submission +from agent.core.model_ids import ( + DEFAULT_MODEL_ID, + KIMI_K26_MODEL_ID, + is_known_router_model_id, + strip_huggingface_model_prefix, +) from agent.core.session import Event, OpType, Session from agent.core.session_persistence import get_session_store from agent.core.tools import ToolRouter @@ -96,13 +102,19 @@ class AgentSession: hf_token: str | None = None # User's HF OAuth token for tool execution task: asyncio.Task | None = None created_at: datetime = field(default_factory=datetime.utcnow) + # Last genuine activity (submit/turn-start/turn-finish/direct user write). + # Drives the idle reaper. Defaults to load time so a freshly-restored but + # untouched session isn't reaped for a full idle window. + last_active_at: datetime = field(default_factory=datetime.utcnow) is_active: bool = True is_processing: bool = False # True while a submission is being executed + # Set under the lock by the reaper while tearing this session down. Blocks + # submit() from enqueueing onto a session that's being evicted. + is_reaping: bool = False broadcaster: Any = None title: str | None = None - # True once this session has been counted against the user's daily - # Claude quota. Guards double-counting when the user re-selects an - # Anthropic model mid-session. + # True once this session has been counted against the user's daily premium + # quota. The field name is kept for persistence compatibility. claude_counted: bool = False @@ -125,25 +137,52 @@ DEFAULT_YOLO_COST_CAP_USD: float = 5.0 SANDBOX_SHUTDOWN_CLEANUP_CONCURRENCY: int = 10 SANDBOX_SHUTDOWN_CLEANUP_TIMEOUT_S: float = 60.0 +# ── Idle-session reaper ───────────────────────────────────────────── +# A live session idle ≥ REAPER_IDLE_HOURS with no in-flight work has its +# sandbox + RAM released and is evicted from the live pool, while staying +# fully resumable from Mongo (it reappears as a normal chat, never "ended"). +# This frees both the global pool and the user's concurrent slots. +REAPER_IDLE_HOURS: float = float(os.environ.get("REAPER_IDLE_HOURS", "2")) +REAPER_INTERVAL_S: float = float(os.environ.get("REAPER_INTERVAL_S", "300")) +REAP_TEARDOWN_TIMEOUT_S: float = float(os.environ.get("REAP_TEARDOWN_TIMEOUT_S", "30")) +REAPER_IDLE = timedelta(hours=REAPER_IDLE_HOURS) + class SessionManager: """Manages multiple concurrent agent sessions.""" def __init__(self, config_path: str | None = None) -> None: self.config = load_config(config_path or DEFAULT_CONFIG_PATH) + normalized_default = strip_huggingface_model_prefix(self.config.model_name) + if normalized_default: + self.config.model_name = normalized_default self.messaging_gateway = NotificationGateway(self.config.messaging) self.sessions: dict[str, AgentSession] = {} self._lock = asyncio.Lock() self.persistence_store = None + # In-flight create_session calls that have passed the capacity check + # but not yet inserted their session. Counted alongside + # active_session_count to hard-cap the global pool against the + # check-then-create race (see create_session). + self._pending_creates: int = 0 + self._reaper_task: asyncio.Task | None = None async def start(self) -> None: """Start shared background resources.""" self.persistence_store = get_session_store() await self.persistence_store.init() await self.messaging_gateway.start() + self._reaper_task = asyncio.create_task(self._reaper_loop()) async def close(self) -> None: """Flush and close shared background resources.""" + if self._reaper_task is not None: + self._reaper_task.cancel() + try: + await self._reaper_task + except asyncio.CancelledError: + pass + self._reaper_task = None await self._cleanup_all_sandboxes_on_close() await self.messaging_gateway.close() if self.persistence_store is not None: @@ -160,6 +199,37 @@ class SessionManager: 1 for s in self.sessions.values() if s.user_id == user_id and s.is_active ) + @staticmethod + def _touch(agent_session: "AgentSession") -> None: + """Stamp genuine activity so the idle reaper's clock resets. + + Call on real user/agent activity (submit, turn start/finish, direct + user-initiated writes) — never on passive reads or hydration, which + would keep an otherwise-idle session alive forever. + """ + agent_session.last_active_at = datetime.utcnow() + + @staticmethod + def _model_from_saved_metadata( + model: str | None, + *, + premium_user_billed: bool, + claude_counted: bool, + ) -> tuple[str, bool, bool]: + normalized = strip_huggingface_model_prefix(model) + if normalized and is_known_router_model_id(normalized): + return normalized, premium_user_billed, claude_counted + + fallback_model = KIMI_K26_MODEL_ID if premium_user_billed else DEFAULT_MODEL_ID + logger.warning( + "Saved session model %r failed validation; using %r", + model, + fallback_model, + ) + if fallback_model == KIMI_K26_MODEL_ID: + return fallback_model, False, False + return fallback_model, premium_user_billed, claude_counted + def _create_session_sync( self, *, @@ -177,10 +247,11 @@ class SessionManager: t0 = _time.monotonic() tool_router = ToolRouter(self.config.mcpServers, hf_token=hf_token) # Deep-copy config so each session's model switches independently — - # tab A picking GLM doesn't flip tab B off Claude. + # tab A picking GLM doesn't flip tab B off the default model. session_config = self.config.model_copy(deep=True) - if model: - session_config.model_name = model + normalized_model = strip_huggingface_model_prefix(model) + if normalized_model: + session_config.model_name = normalized_model session = Session( event_queue=event_queue, config=session_config, @@ -500,10 +571,20 @@ class SessionManager: *, runtime_state: str | None = None, status: str = "active", + raise_on_error: bool = False, ) -> None: - """Persist the current runtime context snapshot.""" + """Persist the current runtime context snapshot. + + Best-effort by default: a disabled store is a no-op and write failures + are swallowed. Pass ``raise_on_error=True`` when the caller must know + the snapshot was durably written (e.g. the reaper, which only evicts a + session after confirming it stayed resumable) — then a disabled store + or a write failure raises instead of silently dropping the snapshot. + """ store = self._store() if not getattr(store, "enabled", False): + if raise_on_error: + raise RuntimeError("persistence store is disabled") return try: await store.save_snapshot( @@ -519,6 +600,9 @@ class SessionManager: agent_session.session ), claude_counted=agent_session.claude_counted, + premium_user_billed=getattr( + agent_session.session, "premium_user_billed", False + ), created_at=agent_session.created_at, notification_destinations=list( agent_session.session.notification_destinations @@ -537,8 +621,11 @@ class SessionManager: ) or 0.0 ), + raise_on_error=raise_on_error, ) except Exception as e: + if raise_on_error: + raise logger.warning( "Failed to persist snapshot for %s: %s", agent_session.session_id, @@ -604,7 +691,11 @@ class SessionManager: from litellm import Message - model = meta.get("model") or self.config.model_name + model, premium_user_billed, claude_counted = self._model_from_saved_metadata( + meta.get("model") or self.config.model_name, + premium_user_billed=bool(meta.get("premium_user_billed", False)), + claude_counted=bool(meta.get("claude_counted")), + ) event_queue: asyncio.Queue = asyncio.Queue() submission_queue: asyncio.Queue = asyncio.Queue() tool_router, session = await asyncio.to_thread( @@ -634,9 +725,36 @@ class SessionManager: *restored_messages, ] + # If this session ever had a sandbox, its container did not survive the + # resume (a fresh, empty one is lazily recreated). Tell the agent so it + # recreates files/packages instead of assuming /app/train.py et al. still + # exist. Gated on sandbox_status so pure Q&A chats get no note. Mirrors + # the seed_from_summary note convention. + # + # Skip it when an approval is pending: the restored context ends with an + # assistant tool-call message awaiting results, so injecting a user + # message here would sit between the tool_calls and their results. On + # approval the real results get appended after the note, leaving them + # orphaned (the context manager stubs the "missing" result right after + # the assistant message) — which the provider rejects. The agent still + # learns the sandbox is empty when the approved tool runs against it. + if meta.get("sandbox_status") and not meta.get("pending_approval"): + session.context_manager.items.append( + Message( + role="user", + content=( + "[SYSTEM: This session was resumed and its sandbox was " + "reset. Any files, installed packages, or running " + "processes from earlier are gone — recreate what you " + "need before using the sandbox.]" + ), + ) + ) + self._restore_pending_approval(session, meta.get("pending_approval") or []) session.turn_count = int(meta.get("turn_count") or 0) session.auto_approval_enabled = bool(meta.get("auto_approval_enabled", False)) + session.premium_user_billed = premium_user_billed raw_cap = meta.get("auto_approval_cost_cap_usd") session.auto_approval_cost_cap_usd = ( float(raw_cap) if isinstance(raw_cap, int | float) else None @@ -660,7 +778,7 @@ class SessionManager: created_at=created_at, is_active=True, is_processing=False, - claude_counted=bool(meta.get("claude_counted")), + claude_counted=claude_counted, title=meta.get("title"), ) started = await self._start_agent_session( @@ -707,11 +825,17 @@ class SessionManager: maximum number of concurrent sessions. """ # ── Capacity checks ────────────────────────────────────────── + # Reserve a global slot under the lock so concurrent creates can't all + # pass the check then over-admit past MAX_SESSIONS (the build + insert + # happen later, outside the lock). active_session_count won't reflect + # this session until _start_agent_session inserts it, so we count + # _pending_creates alongside it to close that gap. async with self._lock: active_count = self.active_session_count - if active_count >= MAX_SESSIONS: + projected = active_count + self._pending_creates + if projected >= MAX_SESSIONS: raise SessionCapacityError( - f"Server is at capacity ({active_count}/{MAX_SESSIONS} sessions). " + f"Server is at capacity ({projected}/{MAX_SESSIONS} sessions). " "Please try again later.", error_type="global", ) @@ -723,6 +847,7 @@ class SessionManager: "concurrent sessions. Please close an existing session first.", error_type="per_user", ) + self._pending_creates += 1 session_id = str(uuid.uuid4()) @@ -730,41 +855,57 @@ class SessionManager: submission_queue: asyncio.Queue = asyncio.Queue() event_queue: asyncio.Queue = asyncio.Queue() - # Run blocking constructors in a thread to keep the event loop responsive. - tool_router, session = await asyncio.to_thread( - self._create_session_sync, - session_id=session_id, - user_id=user_id, - hf_username=hf_username, - hf_token=hf_token, - model=model, - event_queue=event_queue, - ) + reserved = True + try: + # Run blocking constructors in a thread to keep the event loop responsive. + tool_router, session = await asyncio.to_thread( + self._create_session_sync, + session_id=session_id, + user_id=user_id, + hf_username=hf_username, + hf_token=hf_token, + model=model, + event_queue=event_queue, + ) - # Create wrapper - agent_session = AgentSession( - session_id=session_id, - session=session, - tool_router=tool_router, - submission_queue=submission_queue, - user_id=user_id, - hf_username=hf_username, - hf_token=hf_token, - ) + # Create wrapper + agent_session = AgentSession( + session_id=session_id, + session=session, + tool_router=tool_router, + submission_queue=submission_queue, + user_id=user_id, + hf_username=hf_username, + hf_token=hf_token, + ) - await self._start_agent_session( - agent_session=agent_session, - event_queue=event_queue, - tool_router=tool_router, - ) - await self.persist_session_snapshot(agent_session, runtime_state="idle") - self._start_cpu_sandbox_preload(agent_session) + await self._start_agent_session( + agent_session=agent_session, + event_queue=event_queue, + tool_router=tool_router, + ) + # The session is now in self.sessions, so active_session_count + # reflects it — release the reservation before the slower (and + # non-capacity) persistence + preload work. + async with self._lock: + self._pending_creates -= 1 + reserved = False - if is_pro is not None and user_id and user_id != "dev": - await self._track_pro_status(agent_session, is_pro=is_pro) + await self.persist_session_snapshot(agent_session, runtime_state="idle") + self._start_cpu_sandbox_preload(agent_session) + + if is_pro is not None and user_id and user_id != "dev": + await self._track_pro_status(agent_session, is_pro=is_pro) - logger.info(f"Created session {session_id} for user {user_id}") - return session_id + logger.info(f"Created session {session_id} for user {user_id}") + return session_id + finally: + # Build/start failed before the session was inserted — always + # release the reservation so a failed create can't permanently + # shrink the pool. + if reserved: + async with self._lock: + self._pending_creates -= 1 async def _track_pro_status( self, agent_session: AgentSession, *, is_pro: bool @@ -824,9 +965,8 @@ class SessionManager: session = agent_session.session # Pass the real tool specs so the summarizer sees what the agent - # actually has — otherwise Anthropic's modify_params injects a - # dummy tool and the summarizer editorializes that the original - # tool calls were fabricated. + # actually has. Without them, the summarizer can editorialize that + # original tool calls were fabricated. tool_specs = None try: tool_specs = agent_session.tool_router.get_tool_specs_for_llm() @@ -856,6 +996,7 @@ class SessionManager: ), ) session.context_manager.items.append(seed) + self._touch(agent_session) await self.persist_session_snapshot(agent_session, runtime_state="idle") return len(parsed) @@ -907,6 +1048,156 @@ class SessionManager: SANDBOX_SHUTDOWN_CLEANUP_TIMEOUT_S, ) + async def _reaper_loop(self) -> None: + """Periodically release resources held by idle sessions. + + Modeled on EventBroadcaster.run: a long-lived task started in start() + and cancelled in close(). Per-sweep exceptions are swallowed so one bad + sweep never kills the loop. + """ + while True: + try: + await asyncio.sleep(REAPER_INTERVAL_S) + await self._reap_idle_sessions() + except asyncio.CancelledError: + break + except Exception as e: + logger.error("Idle-session reaper sweep failed: %s", e) + + async def _reap_idle_sessions(self) -> None: + """Select idle candidates under the lock, then tear each down. + + Candidates are non-dev sessions that are live, not processing, not + awaiting tool approval (those are "approve later", not idle — reaping + would destroy the sandbox the approved tool needs), and untouched for + the idle window. We only snapshot IDs under the lock; the actual + teardown in _reap_one re-acquires it, because tearing a session down + while holding the lock would deadlock (the lock is non-reentrant). + """ + # Reaping is only safe when sessions stay resumable from Mongo. With no + # store, eviction would destroy non-dev chats outright, so don't reap. + if not getattr(self._store(), "enabled", False): + return + + cutoff = datetime.utcnow() - REAPER_IDLE + async with self._lock: + candidates = [ + agent_session.session_id + for agent_session in self.sessions.values() + if agent_session.is_active + and not agent_session.is_processing + and not agent_session.is_reaping + and agent_session.user_id != "dev" + and not agent_session.session.pending_approval + and agent_session.last_active_at <= cutoff + ] + if not candidates: + return + + reaped = 0 + for session_id in candidates: + try: + if await self._reap_one(session_id, cutoff): + reaped += 1 + except Exception as e: + logger.warning("Failed to reap idle session %s: %s", session_id, e) + if reaped: + logger.info("Reaped %d idle session(s)", reaped) + + async def _reap_one(self, session_id: str, cutoff: datetime) -> bool: + """Tear down one idle session, leaving it resumable from Mongo. + + Re-checks every idle condition under the lock (a user may have become + active in the gap since selection), marks the session reaping, persists + a resumable snapshot outside the lock, then does one final locked + re-check before eviction. The runtime task is cancelled *outside* the + lock: its own ``finally`` frees the sandbox, and its identity-gated + persist no-ops because the session is already popped — so it can't + overwrite our resumable snapshot with ``"ended"`` and there's no + deadlock. Returns True if the session was reaped. + """ + async with self._lock: + agent_session = self.sessions.get(session_id) + if ( + agent_session is None + or not agent_session.is_active + or agent_session.is_processing + or agent_session.is_reaping + or agent_session.session.pending_approval + or agent_session.last_active_at > cutoff + or not agent_session.submission_queue.empty() + ): + return False + agent_session.is_reaping = True + + # Persist a resumable snapshot *before* eviction so a concurrent reopen + # reloads clean state. status="active" (never "ended") keeps it a normal + # chat in the sidebar. Do this outside the manager lock: Mongo writes can + # take network round trips, and is_reaping=True is enough to block submit + # from enqueueing while the snapshot is in flight. + try: + await self.persist_session_snapshot( + agent_session, + runtime_state="idle", + status="active", + raise_on_error=True, + ) + except Exception as e: + async with self._lock: + if self.sessions.get(session_id) is agent_session: + agent_session.is_reaping = False + logger.warning( + "Skipping reap of %s: could not persist resumable snapshot: %s", + session_id, + e, + ) + return False + + async with self._lock: + current = self.sessions.get(session_id) + if current is not agent_session: + return False + if ( + not agent_session.is_active + or agent_session.is_processing + or agent_session.session.pending_approval + or agent_session.last_active_at > cutoff + or not agent_session.submission_queue.empty() + ): + agent_session.is_reaping = False + return False + self.sessions.pop(session_id, None) + task = agent_session.task + session = agent_session.session + + if task is not None and not task.done(): + task.cancel() + # Use asyncio.wait, not wait_for: wait_for re-raises the cancelled + # task's CancelledError, which we'd have to swallow — and that same + # bare except would also eat an *outer* cancel aimed at the reaper + # itself (close() cancelling _reaper_task), hanging shutdown. + # asyncio.wait returns the cancelled task in `done` and lets an + # outer cancel propagate cleanly. + done, _pending = await asyncio.wait({task}, timeout=REAP_TEARDOWN_TIMEOUT_S) + if not done: + logger.warning( + "Reaper teardown timed out after %.0fs for %s; orphan " + "sweeper will handle any sandbox straggler", + REAP_TEARDOWN_TIMEOUT_S, + session_id, + ) + elif not task.cancelled(): + # Surface (and retrieve, to avoid "exception never retrieved") + # any non-cancellation teardown error. + exc = task.exception() + if exc is not None: + logger.warning("Reaper teardown error for %s: %s", session_id, exc) + else: + # No live task to run the cleanup finally — free the sandbox here so + # a reaped session never leaves an orphaned Space behind. + await self._cleanup_sandbox(session) + return True + async def _run_session( self, session_id: str, @@ -941,12 +1232,17 @@ class SessionManager: submission_queue.get(), timeout=1.0 ) agent_session.is_processing = True + self._touch(agent_session) try: should_continue = await process_submission( session, submission ) finally: agent_session.is_processing = False + # Stamp on turn finish too: a turn that ran longer + # than the idle window would otherwise be reaped the + # instant it completes. + self._touch(agent_session) await self.persist_session_snapshot(agent_session) if not should_continue: break @@ -982,10 +1278,10 @@ class SessionManager: logger.warning(f"Final-flush failed for {session_id}: {e}") async with self._lock: - if session_id in self.sessions: - self.sessions[session_id].is_active = False + if self.sessions.get(session_id) is agent_session: + agent_session.is_active = False await self.persist_session_snapshot( - self.sessions[session_id], + agent_session, runtime_state="ended", status="ended", ) @@ -993,16 +1289,26 @@ class SessionManager: logger.info(f"Session {session_id} ended") async def submit(self, session_id: str, operation: Operation) -> bool: - """Submit an operation to a session.""" - async with self._lock: - agent_session = self.sessions.get(session_id) - - if not agent_session or not agent_session.is_active: - logger.warning(f"Session {session_id} not found or inactive") - return False + """Submit an operation to a session. + Enqueues under the lock and rejects sessions being reaped, so submit + and reap can't interleave: either the message is enqueued before the + reaper's empty() re-check (which then aborts the reap), or the session + is already popped (we return False and the caller reloads a fresh + runtime from Mongo). The queue is unbounded, so put_nowait never blocks. + """ submission = Submission(id=f"sub_{uuid.uuid4().hex[:8]}", operation=operation) - await agent_session.submission_queue.put(submission) + async with self._lock: + agent_session = self.sessions.get(session_id) + if ( + not agent_session + or not agent_session.is_active + or agent_session.is_reaping + ): + logger.warning(f"Session {session_id} not found or inactive") + return False + agent_session.submission_queue.put_nowait(submission) + self._touch(agent_session) return True async def submit_user_input(self, session_id: str, text: str) -> bool: @@ -1042,6 +1348,7 @@ class SessionManager: user_message_index ) if success: + self._touch(agent_session) await self.persist_session_snapshot(agent_session, runtime_state="idle") return success @@ -1115,6 +1422,7 @@ class SessionManager: if not agent_session or not agent_session.is_active: return False agent_session.session.update_model(model_id) + self._touch(agent_session) await self.persist_session_snapshot(agent_session, runtime_state="idle") return True @@ -1150,6 +1458,7 @@ class SessionManager: else: session.auto_approval_enabled = bool(enabled) session.auto_approval_cost_cap_usd = cost_cap_usd + self._touch(agent_session) await self.persist_session_snapshot(agent_session) return self._auto_approval_summary(session) @@ -1196,6 +1505,10 @@ class SessionManager: agent_session.session.notification_destinations ), "auto_approval": self._auto_approval_summary(agent_session.session), + "premium_user_billed": getattr( + agent_session.session, "premium_user_billed", False + ), + "premium_quota_counted": agent_session.claude_counted, } def set_notification_destinations( @@ -1222,6 +1535,7 @@ class SessionManager: seen.add(name) agent_session.session.set_notification_destinations(normalized) + self._touch(agent_session) return normalized async def list_sessions(self, user_id: str | None = None) -> list[dict[str, Any]]: @@ -1259,6 +1573,10 @@ class SessionManager: "pending_approval": pending or None, "model": row.get("model"), "title": row.get("title"), + "premium_user_billed": bool( + row.get("premium_user_billed", False) + ), + "premium_quota_counted": bool(row.get("claude_counted", False)), "notification_destinations": row.get( "notification_destinations" ) diff --git a/backend/user_quotas.py b/backend/user_quotas.py index 4da4a8d91b755b64e165f82ba8940b0e1b19ae38..752b25d971dda0b0e6194108d16781441af4d673 100644 --- a/backend/user_quotas.py +++ b/backend/user_quotas.py @@ -1,4 +1,4 @@ -"""Daily quota for premium model session creations. +"""Daily quota for subsidized premium model sessions. Tracks per-user premium model session starts against a daily cap derived from the user's HF plan. MongoDB is the source of truth when configured; the @@ -7,12 +7,13 @@ in-process dict remains the fallback for local/dev/test runs. The public names still say ``claude`` because this quota bucket originally only covered Claude and the persisted session field uses that name. -Unit: session *creations*, not messages. A user who sends with a premium model -in a new session consumes one quota point; switching an already-counted session -back to a premium model doesn't (`AgentSession.claude_counted` guards that). +Unit: first premium-model submit in a session, not raw messages. A user who +sends with a premium model in a new session consumes one quota point; switching +an already-counted session back to a premium model doesn't +(`AgentSession.claude_counted` guards that). Cap tiers: - free user → CLAUDE_FREE_DAILY (1) + free user → CLAUDE_FREE_DAILY (2) pro user → CLAUDE_PRO_DAILY (20) """ @@ -26,7 +27,7 @@ from agent.core.session_persistence import ( _reset_store_for_tests, ) -CLAUDE_FREE_DAILY: int = int(os.environ.get("CLAUDE_FREE_DAILY", "1")) +CLAUDE_FREE_DAILY: int = int(os.environ.get("CLAUDE_FREE_DAILY", "2")) CLAUDE_PRO_DAILY: int = int(os.environ.get("CLAUDE_PRO_DAILY", "20")) # user_id -> (day_utc_iso, count_for_that_day) diff --git a/configs/cli_agent_config.json b/configs/cli_agent_config.json index 48ba4dd84b86f75d3f9038c9c8a0e1104feedcc7..8538fc21bbc1faf4b5949a16729d7d8099ea7bbc 100644 --- a/configs/cli_agent_config.json +++ b/configs/cli_agent_config.json @@ -1,5 +1,5 @@ { - "model_name": "anthropic/claude-opus-4-6", + "model_name": "anthropic/claude-opus-4.8:fal-ai", "save_sessions": true, "session_dataset_repo": "smolagents/ml-intern-sessions", "share_traces": true, diff --git a/configs/frontend_agent_config.json b/configs/frontend_agent_config.json index c674a223b018967b7ab4482f3228b0b58d054dd3..d287fe7a204e72471af1263de550f969f79c2773 100644 --- a/configs/frontend_agent_config.json +++ b/configs/frontend_agent_config.json @@ -1,5 +1,5 @@ { - "model_name": "${ML_INTERN_CLAUDE_MODEL_ID:-bedrock/us.anthropic.claude-opus-4-6-v1}", + "model_name": "${ML_INTERN_DEFAULT_MODEL_ID:-anthropic/claude-opus-4.8:fal-ai}", "save_sessions": true, "session_dataset_repo": "smolagents/ml-intern-sessions", "share_traces": true, diff --git a/frontend/src/components/Chat/ActivityStatusBar.tsx b/frontend/src/components/Chat/ActivityStatusBar.tsx index 3dd0af534ec1b7fa5861116b865b388f76a42eaa..61e502d2d4038c6310420ee67c1a787e1585361c 100644 --- a/frontend/src/components/Chat/ActivityStatusBar.tsx +++ b/frontend/src/components/Chat/ActivityStatusBar.tsx @@ -25,7 +25,7 @@ function formatResearchStatus(raw: string): string { const s = raw.replace(/^▸\s*/, ''); const jsonStart = s.indexOf('{'); const toolName = jsonStart > 0 ? s.slice(0, jsonStart).trim() : s.trim(); - let args: Record = {}; + const args: Record = {}; if (jsonStart > 0) { const jsonStr = s.slice(jsonStart); try { diff --git a/frontend/src/components/Chat/BillingBanner.tsx b/frontend/src/components/Chat/BillingBanner.tsx new file mode 100644 index 0000000000000000000000000000000000000000..4bb03cd1211a1db8020ca898d8bb571c0e7addec --- /dev/null +++ b/frontend/src/components/Chat/BillingBanner.tsx @@ -0,0 +1,81 @@ +/** + * Shown above the composer when the active session's premium usage is being + * billed to the user's own HF account — i.e. they're past the subsidized daily + * allowance and the session flipped to `premium_user_billed`. Dismissible once + * per day so we stay transparent about billing without nagging every session. + */ +import { useState } from 'react'; +import { Box, Link, Typography } from '@mui/material'; + +const DISMISS_KEY = 'ml-intern:billing-banner-dismissed'; +const today = () => new Date().toISOString().slice(0, 10); + +export default function BillingBanner() { + const [dismissed, setDismissed] = useState(() => { + try { + return localStorage.getItem(DISMISS_KEY) === today(); + } catch { + return false; + } + }); + + if (dismissed) return null; + + const dismiss = () => { + try { + localStorage.setItem(DISMISS_KEY, today()); + } catch { + /* ignore storage failures */ + } + setDismissed(true); + }; + + return ( + + + + You've used today's subsidized premium sessions — this session's usage is billed to your{' '} + + Hugging Face account + {' '} + through Hugging Face Inference Providers. + + + ✕ + + + + ); +} diff --git a/frontend/src/components/Chat/ChatErrorBanner.tsx b/frontend/src/components/Chat/ChatErrorBanner.tsx new file mode 100644 index 0000000000000000000000000000000000000000..a2cf6be876bd4cb2d6b12c36856ce76d3ffb743a --- /dev/null +++ b/frontend/src/components/Chat/ChatErrorBanner.tsx @@ -0,0 +1,92 @@ +import { useEffect, useMemo, useState } from 'react'; +import { Alert, AlertTitle, Box, Button, Typography } from '@mui/material'; + +interface ChatErrorBannerProps { + error: string; + sessionId: string; + model?: string | null; + onDismiss: () => void; +} + +export default function ChatErrorBanner({ error, sessionId, model, onDismiss }: ChatErrorBannerProps) { + const [copied, setCopied] = useState(false); + const [reportedAt, setReportedAt] = useState(() => new Date().toISOString()); + + useEffect(() => { + setReportedAt(new Date().toISOString()); + setCopied(false); + }, [error]); + + const details = useMemo( + () => [ + 'ML Intern message failure', + `time: ${reportedAt}`, + `session: ${sessionId}`, + `model: ${model || 'unknown'}`, + `error: ${error}`, + ].join('\n'), + [error, model, reportedAt, sessionId], + ); + + const copyDetails = async () => { + try { + await navigator.clipboard.writeText(details); + setCopied(true); + } catch { + setCopied(false); + } + }; + + return ( + + + + + + } + > + + Message failed + + + The backend could not process the last message. Retry after a moment. If it keeps + happening, raise an issue with the copied details. + + + {error} + + + + ); +} diff --git a/frontend/src/components/Chat/ChatInput.tsx b/frontend/src/components/Chat/ChatInput.tsx index dc8c9ca79ee35716d0f7877a9b5bacbb0f23cef1..293b86f64c335c56ba16802d7a710b345b05b5dd 100644 --- a/frontend/src/components/Chat/ChatInput.tsx +++ b/frontend/src/components/Chat/ChatInput.tsx @@ -20,14 +20,12 @@ import ArrowDropDownIcon from '@mui/icons-material/ArrowDropDown'; import StopIcon from '@mui/icons-material/Stop'; import AddIcon from '@mui/icons-material/Add'; import { apiFetch, apiUpload } from '@/utils/api'; -import { useUserQuota } from '@/hooks/useUserQuota'; -import ClaudeCapDialog from '@/components/ClaudeCapDialog'; +import type { UserQuota } from '@/hooks/useUserQuota'; import JobsUpgradeDialog from '@/components/JobsUpgradeDialog'; import { useAgentStore } from '@/store/agentStore'; import { useSessionStore } from '@/store/sessionStore'; import { CLAUDE_MODEL_PATH, - FIRST_FREE_MODEL_PATH, GPT_55_MODEL_PATH, isClaudePath, isPremiumPath, @@ -50,57 +48,97 @@ const getHfAvatarUrl = (modelId: string) => { const DEFAULT_MODEL_OPTIONS: ModelOption[] = [ { - id: 'kimi-k2.6', - name: 'Kimi K2.6', - description: 'Novita', - modelPath: 'moonshotai/Kimi-K2.6', - avatarUrl: getHfAvatarUrl('moonshotai/Kimi-K2.6'), - recommended: true, - }, - { - id: 'claude-opus', - name: 'Claude Opus 4.6', - description: 'Anthropic', + id: 'claude-opus-4-8', + name: 'Claude Opus 4.8', + description: 'Hugging Face', modelPath: CLAUDE_MODEL_PATH, - avatarUrl: 'https://huggingface.co/api/avatars/Anthropic', + avatarUrl: getHfAvatarUrl(CLAUDE_MODEL_PATH), recommended: true, }, { id: 'gpt-5.5', name: 'GPT-5.5', - description: 'OpenAI', + description: 'Hugging Face', modelPath: GPT_55_MODEL_PATH, - avatarUrl: 'https://huggingface.co/api/avatars/openai', + avatarUrl: getHfAvatarUrl(GPT_55_MODEL_PATH), + }, + { + id: 'kimi-k2.6', + name: 'Kimi K2.6', + description: 'Hugging Face', + modelPath: 'moonshotai/Kimi-K2.6', + avatarUrl: getHfAvatarUrl('moonshotai/Kimi-K2.6'), }, { id: 'minimax-m2.7', name: 'MiniMax M2.7', - description: 'Novita', + description: 'Hugging Face', modelPath: 'MiniMaxAI/MiniMax-M2.7', avatarUrl: getHfAvatarUrl('MiniMaxAI/MiniMax-M2.7'), }, { id: 'glm-5.1', name: 'GLM 5.1', - description: 'Together', + description: 'Hugging Face', modelPath: 'zai-org/GLM-5.1', avatarUrl: getHfAvatarUrl('zai-org/GLM-5.1'), }, { id: 'deepseek-v4-pro', name: 'DeepSeek V4 Pro', - description: 'DeepInfra', + description: 'Hugging Face', modelPath: 'deepseek-ai/DeepSeek-V4-Pro:deepinfra', avatarUrl: getHfAvatarUrl('deepseek-ai/DeepSeek-V4-Pro'), }, ]; +const normalizeModelPath = (path: string | undefined) => ( + (path ?? '') + .toLowerCase() + .replace(/^huggingface\//, '') + .replace(/claude-opus-4\.(\d)/g, 'claude-opus-4-$1') +); + const findModelByPath = (path: string, options: ModelOption[]): ModelOption | undefined => { + const normalizedPath = normalizeModelPath(path); + const matched = options.find((m) => { + const normalizedModelPath = normalizeModelPath(m.modelPath); + const normalizedId = normalizeModelPath(m.id); + return ( + m.modelPath === path || + normalizedModelPath === normalizedPath || + normalizedPath.includes(normalizedId) + ); + }); + if (matched) return matched; if (isClaudePath(path)) { const claude = options.find(isClaudeModel); if (claude) return claude; } - return options.find(m => m.modelPath === path || path?.includes(m.id)); + return undefined; +}; + +const modelOptionId = (modelPath: string) => ( + normalizeModelPath(modelPath) + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') +); + +const modelOptionFromApi = (model: { + id?: string; + label?: string; + provider?: string; + recommended?: boolean; +}): ModelOption | null => { + if (!model.id) return null; + return { + id: modelOptionId(model.id), + name: model.label ?? model.id, + description: 'Hugging Face', + modelPath: model.id, + avatarUrl: getHfAvatarUrl(model.id.replace(/^huggingface\//, '')), + recommended: Boolean(model.recommended), + }; }; const readApiErrorMessage = async (res: Response, fallback: string): Promise => { @@ -125,6 +163,8 @@ interface ChatInputProps { isProcessing?: boolean; disabled?: boolean; placeholder?: string; + quota: UserQuota | null; + refreshQuota: () => Promise | void; } interface DatasetUploadResponse { @@ -148,7 +188,6 @@ const DATASET_UPLOAD_EXTENSIONS = new Set(['csv', 'json', 'jsonl']); const isClaudeModel = (m: ModelOption) => isClaudePath(m.modelPath); const isPremiumModel = (m: ModelOption) => isPremiumPath(m.modelPath); -const firstFreeModel = (options: ModelOption[]) => options.find(m => !isPremiumModel(m)) ?? options[0]; const formatBytes = (bytes: number) => { if (bytes < 1024) return `${bytes} B`; @@ -160,7 +199,7 @@ const datasetRepoUrl = (repoId: string) => ( `https://huggingface.co/datasets/${repoId.split('/').map(encodeURIComponent).join('/')}` ); -export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, onDatasetUploaded, isProcessing = false, disabled = false, placeholder = 'Ask anything...' }: ChatInputProps) { +export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, onDatasetUploaded, isProcessing = false, disabled = false, placeholder = 'Ask anything...', quota, refreshQuota }: ChatInputProps) { const [input, setInput] = useState(''); const inputRef = useRef(null); const fileInputRef = useRef(null); @@ -171,14 +210,6 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, () => findModelByPath(initialModelPath ?? '', DEFAULT_MODEL_OPTIONS)?.id ?? DEFAULT_MODEL_OPTIONS[0].id, ); const [modelAnchorEl, setModelAnchorEl] = useState(null); - const { quota, refresh: refreshQuota } = useUserQuota(); - // The daily-cap dialog is triggered from two places: (a) a 429 returned - // from the chat transport when the user tries to send on a premium model over cap — - // surfaced via the agent-store flag — and (b) nothing else right now - // (switching models is free). Keeping the open state in the store means - // the hook layer can flip it without threading props through. - const claudeQuotaExhausted = useAgentStore((s) => s.claudeQuotaExhausted); - const setClaudeQuotaExhausted = useAgentStore((s) => s.setClaudeQuotaExhausted); const jobsUpgradeRequired = useAgentStore((s) => s.jobsUpgradeRequired); const setJobsUpgradeRequired = useAgentStore((s) => s.setJobsUpgradeRequired); const updateSessionModel = useSessionStore((s) => s.updateSessionModel); @@ -189,7 +220,6 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, const [uploadedDatasets, setUploadedDatasets] = useState([]); const [isUploadingDataset, setIsUploadingDataset] = useState(false); const [datasetUploadProgress, setDatasetUploadProgress] = useState(null); - const lastSentRef = useRef(''); useEffect(() => { modelOptionsRef.current = modelOptions; @@ -205,16 +235,10 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, .then((res) => (res.ok ? res.json() : null)) .then((data) => { if (cancelled || !data?.available) return; - const claude = data.available.find((m: { provider?: string; id?: string }) => ( - m.provider === 'anthropic' && m.id - )); - if (!claude?.id) return; - - const next = DEFAULT_MODEL_OPTIONS.map((option) => ( - isClaudeModel(option) - ? { ...option, modelPath: claude.id, name: claude.label ?? option.name } - : option - )); + const next = data.available + .map(modelOptionFromApi) + .filter((model: ModelOption | null): model is ModelOption => model !== null); + if (!next.length) return; modelOptionsRef.current = next; setModelOptions(next); if (!sessionIdRef.current) { @@ -256,7 +280,6 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, const handleSend = useCallback(() => { if (input.trim() && !disabled && !isUploadingDataset) { - lastSentRef.current = input; onSend(input); setInput(''); } @@ -325,14 +348,6 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, [sessionId, onDatasetUploaded], ); - // When the chat transport reports a premium-model quota 429, restore the typed - // text so the user doesn't lose their message. - useEffect(() => { - if (claudeQuotaExhausted && lastSentRef.current) { - setInput(lastSentRef.current); - } - }, [claudeQuotaExhausted]); - useEffect(() => { if (!datasetUploadError) return; const timeout = window.setTimeout(() => setDatasetUploadError(null), 7000); @@ -364,6 +379,7 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, const handleModelClick = (event: React.MouseEvent) => { setModelAnchorEl(event.currentTarget); + void refreshQuota(); }; const handleModelClose = () => { @@ -390,48 +406,6 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, } }; - // Dialog close: just clear the flag. The typed text is already restored. - const handleCapDialogClose = useCallback(() => { - setClaudeQuotaExhausted(false); - }, [setClaudeQuotaExhausted]); - - // "Use a free model" — switch the current session to Kimi (or the first - // non-premium option) and auto-retry the send that tripped the cap. - const handleUseFreeModel = useCallback(async () => { - setClaudeQuotaExhausted(false); - if (!sessionId) return; - const free = modelOptions.find(m => m.modelPath === FIRST_FREE_MODEL_PATH) - ?? firstFreeModel(modelOptions); - try { - const res = await apiFetch(`/api/session/${sessionId}/model`, { - method: 'POST', - body: JSON.stringify({ model: free.modelPath }), - }); - if (res.ok) { - setSelectedModelId(free.id); - updateSessionModel(sessionId, free.modelPath); - const retryText = lastSentRef.current; - if (retryText) { - onSend(retryText); - setInput(''); - lastSentRef.current = ''; - } - } - } catch { /* ignore */ } - }, [sessionId, onSend, setClaudeQuotaExhausted, modelOptions, updateSessionModel]); - - const handlePremiumUpgradeClick = useCallback(async () => { - if (!sessionId) return; - try { - await apiFetch(`/api/pro-click/${sessionId}`, { - method: 'POST', - body: JSON.stringify({ source: 'premium_cap_dialog', target: 'pro_pricing' }), - }); - } catch { - /* tracking is best-effort */ - } - }, [sessionId]); - const handleJobsUpgradeClose = useCallback(() => { setJobsUpgradeRequired(null); setAwaitingTopUp(false); @@ -473,14 +447,14 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, return () => document.removeEventListener('visibilitychange', onVisible); }, [awaitingTopUp, jobsUpgradeRequired, handleJobsRetry]); - // Hide the chip until the user has actually burned quota; opening a - // premium-model session without sending should not populate a counter. + // Show the remaining subsidized premium-session allowance for today. const premiumChip = (() => { - if (!quota || quota.premiumUsedToday === 0) return null; - if (quota.plan === 'free') { - return quota.premiumRemaining > 0 ? 'Free today' : 'Pro only'; + if (!quota) return null; + const remaining = Math.max(0, quota.premiumRemaining); + if (remaining === 0) { + return quota.plan === 'pro' ? '0 left – using HF billing' : '0 left – enable billing'; } - return `${quota.premiumUsedToday}/${quota.premiumDailyCap} today`; + return `${remaining} left today`; })(); return ( @@ -804,14 +778,6 @@ export default function ChatInput({ sessionId, initialModelPath, onSend, onStop, ))} - void; - onUseFreeModel: () => void; - onUpgrade: () => void; -} - -export default function ClaudeCapDialog({ - open, - plan, - cap, - onClose, - onUseFreeModel, - onUpgrade, -}: ClaudeCapDialogProps) { - const isFreePlan = plan === 'free'; - - return ( - - - You've hit your premium model limit - - - - Opus and GPT-5.5 are expensive to run, so we cap premium models at {cap}{' '} - {cap === 1 ? 'session' : 'sessions'} a day. {isFreePlan - ? 'HF Pro raises the daily premium-model limit.' - : 'Your plan has used today’s premium-model allowance.'}{' '} - Give Kimi, MiniMax, GLM, or DeepSeek a spin instead. - - {isFreePlan && ( - - - HF Pro ($9/mo) — more premium model sessions - - - {PRO_CAP} premium model sessions/day here, 20× HF Inference credits, - ZeroGPU access, and priority on Spaces hardware. - - - )} - - - {isFreePlan && ( - - )} - - - - ); -} diff --git a/frontend/src/components/SessionChat.tsx b/frontend/src/components/SessionChat.tsx index f771e254b7b2c35b79c89684e2da292e09b8f564..1aff70097988d087ec52d2cd6ce7108c0d9c7c8d 100644 --- a/frontend/src/components/SessionChat.tsx +++ b/frontend/src/components/SessionChat.tsx @@ -5,13 +5,17 @@ * runs — processing events — but only the active session renders visible * UI (MessageList + ChatInput). */ -import { useCallback, useEffect } from 'react'; +import { useCallback, useEffect, useState } from 'react'; import { useAgentChat } from '@/hooks/useAgentChat'; import { useAgentStore } from '@/store/agentStore'; import { useSessionStore } from '@/store/sessionStore'; import MessageList from '@/components/Chat/MessageList'; import ChatInput from '@/components/Chat/ChatInput'; import ExpiredBanner from '@/components/Chat/ExpiredBanner'; +import BillingBanner from '@/components/Chat/BillingBanner'; +import ChatErrorBanner from '@/components/Chat/ChatErrorBanner'; +import { useUserQuota } from '@/hooks/useUserQuota'; +import { isPremiumPath } from '@/utils/model'; import { apiFetch } from '@/utils/api'; import { logger } from '@/utils/logger'; @@ -23,9 +27,10 @@ interface SessionChatProps { export default function SessionChat({ sessionId, isActive, onSessionDead }: SessionChatProps) { const { isConnected, isProcessing, activityStatus, updateSession } = useAgentStore(); - const { updateSessionTitle, sessions } = useSessionStore(); + const { updateSessionTitle, sessions, setSessionProcessing } = useSessionStore(); const sessionMeta = sessions.find((s) => s.id === sessionId); const isExpired = sessionMeta?.expired === true; + const [chatError, setChatError] = useState(null); const { messages, @@ -39,8 +44,15 @@ export default function SessionChat({ sessionId, isActive, onSessionDead }: Sess } = useAgentChat({ sessionId, isActive, + // A backgrounded session that the backend reports mid-turn still needs to + // mount its live subscription; idle backgrounded sessions do not (that's + // what stops app load from reactivating every historical runtime). + isProcessing: sessionMeta?.isProcessing ?? false, onReady: () => logger.log(`Session ${sessionId} ready`), - onError: (error) => logger.error(`Session ${sessionId} error:`, error), + onError: (error) => { + logger.error(`Session ${sessionId} error:`, error); + setChatError(error); + }, onSessionDead, }); @@ -76,12 +88,54 @@ export default function SessionChat({ sessionId, isActive, onSessionDead }: Sess // SDK status is the ground truth — if it's streaming/submitted, agent is busy const sdkBusy = status === 'streaming' || status === 'submitted'; const busy = isProcessing || sdkBusy; + const { quota, refresh: refreshQuota } = useUserQuota({ enabled: isActive }); + + // Whether this session's premium usage is being billed to the user's own HF + // account (past the subsidized daily allowance). Re-read after each turn, + // since the backend flips it at submit time. Only premium-model sessions can + // ever be user-billed, so skip the fetch for free models. + const [premiumBilled, setPremiumBilled] = useState(false); + const [premiumQuotaCounted, setPremiumQuotaCounted] = useState(false); + const onPremiumModel = isPremiumPath(sessionMeta?.model ?? undefined); + useEffect(() => { + if (!isActive || !onPremiumModel) { + setPremiumBilled(false); + setPremiumQuotaCounted(false); + return; + } + if (busy) return; + let cancelled = false; + apiFetch(`/api/session/${sessionId}`) + .then((r) => (r.ok ? r.json() : null)) + .then((d) => { + if (!cancelled && d) { + setPremiumBilled(Boolean(d.premium_user_billed)); + setPremiumQuotaCounted(Boolean(d.premium_quota_counted)); + } + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, [busy, isActive, onPremiumModel, sessionId]); + + const sessionPremiumBilled = premiumBilled || Boolean(sessionMeta?.premiumUserBilled); + const sessionPremiumQuotaCounted = + premiumQuotaCounted || Boolean(sessionMeta?.premiumQuotaCounted); + const premiumBillingNotice = + sessionPremiumBilled || + (isActive && + onPremiumModel && + quota?.premiumRemaining === 0 && + !sessionPremiumQuotaCounted); const handleSendMessage = useCallback( async (text: string) => { if (!text.trim() || busy) return; + setChatError(null); updateSession(sessionId, { isProcessing: true, activityStatus: { type: 'thinking' } }); + setSessionProcessing(sessionId, true); sendMessage({ text: text.trim(), metadata: { createdAt: new Date().toISOString() } }); // Auto-title the session from the first user message @@ -101,7 +155,7 @@ export default function SessionChat({ sessionId, isActive, onSessionDead }: Sess }); } }, - [sessionId, sendMessage, messages, updateSessionTitle, busy, updateSession], + [sessionId, sendMessage, messages, updateSessionTitle, busy, updateSession, setSessionProcessing], ); // Don't render UI for background sessions — hooks still run @@ -120,20 +174,33 @@ export default function SessionChat({ sessionId, isActive, onSessionDead }: Sess {isExpired ? ( ) : ( - + <> + {premiumBillingNotice && } + {chatError && ( + setChatError(null)} + /> + )} + + )} ); diff --git a/frontend/src/hooks/useAgentChat.ts b/frontend/src/hooks/useAgentChat.ts index 34cb4b45f907a7584b0a063083372d67a7a07898..2ef2b71b3bd76f76daa598618ebcaa207d55f928 100644 --- a/frontend/src/hooks/useAgentChat.ts +++ b/frontend/src/hooks/useAgentChat.ts @@ -24,28 +24,55 @@ import { logger } from '@/utils/logger'; interface UseAgentChatOptions { sessionId: string; isActive: boolean; + /** Backend reports this session is mid-turn (from the GET /sessions list). */ + isProcessing?: boolean; onReady?: () => void; onError?: (error: string) => void; onSessionDead?: (sessionId: string) => void; } -export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionDead }: UseAgentChatOptions) { +export function useAgentChat({ sessionId, isActive, isProcessing = false, onReady, onError, onSessionDead }: UseAgentChatOptions) { const callbacksRef = useRef({ onReady, onError, onSessionDead }); callbacksRef.current = { onReady, onError, onSessionDead }; const isActiveRef = useRef(isActive); isActiveRef.current = isActive; + // Only the active tab — or a session the backend says is mid-turn — gets a + // reactivating hydration (the /messages + /session fetch and the SDK's + // resume reconnect both call ensure_session_loaded, which spins a runtime + + // sandbox back up for any not-currently-live session). Gating on this keeps + // app load from reactivating every historical session and refilling the + // global active-session pool. A processing session is already live, so + // hydrating it returns the existing object without inflating the pool. + const shouldReactivate = isActive || isProcessing; + const shouldReactivateRef = useRef(shouldReactivate); + shouldReactivateRef.current = shouldReactivate; + const { setNeedsAttention, updateSessionYolo } = useSessionStore(); // Helper: update this session's state (mirrors to globals if active) const updateSession = useAgentStore.getState().updateSession; + const setProcessingState = useCallback( + ( + next: boolean, + updates: Partial = {}, + ) => { + updateSession(sessionId, { ...updates, isProcessing: next }); + useSessionStore.getState().setSessionProcessing(sessionId, next); + }, + [sessionId, updateSession], + ); // -- Build side-channel callbacks (stable ref) -------------------------- const sideChannel = useMemo( () => ({ onReady: () => { - updateSession(sessionId, { isProcessing: false }); + // Mirror to the sidebar store too: agentStore drives the live UI, but + // SessionMeta.isProcessing (sessionStore) is what feeds shouldReactivate. + // Only mergeServerSessions sets it, so without clearing it here a + // finished background task keeps reactivating until the next list fetch. + setProcessingState(false); if (isActiveRef.current) { useAgentStore.getState().setConnected(true); } @@ -53,26 +80,25 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD callbacksRef.current.onReady?.(); }, onShutdown: () => { - updateSession(sessionId, { isProcessing: false }); + setProcessingState(false); if (isActiveRef.current) { useAgentStore.getState().setConnected(false); } }, onError: (error: string) => { - updateSession(sessionId, { isProcessing: false }); + setProcessingState(false); callbacksRef.current.onError?.(error); }, onProcessing: () => { - updateSession(sessionId, { - isProcessing: true, + setProcessingState(true, { activityStatus: { type: 'thinking' }, }); }, onProcessingDone: () => { - updateSession(sessionId, { isProcessing: false }); + setProcessingState(false); }, onUndoComplete: () => { - updateSession(sessionId, { isProcessing: false }); + setProcessingState(false); }, onCompacted: (oldTokens: number, newTokens: number) => { logger.log(`Context compacted: ${oldTokens} -> ${newTokens} tokens`); @@ -197,7 +223,13 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD ); } - updateSession(sessionId, { activityStatus: { type: 'waiting-approval' } }); + // Approval pauses the turn: the backend has returned and set + // is_processing=False, and no turn_complete/onProcessingDone fires on + // this path. Clear processing in both stores (sessionStore.isProcessing + // feeds shouldReactivate) so a backgrounded waiting-approval session + // doesn't stay "processing" until the next /sessions merge — + // activityStatus still surfaces the waiting-approval state. + setProcessingState(false, { activityStatus: { type: 'waiting-approval' } }); // Build panel data for this session's pending approval const firstTool = tools[0]; @@ -307,7 +339,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD onInterrupted: () => { /* no-op — handled by stop() caller */ }, }), // eslint-disable-next-line react-hooks/exhaustive-deps - [sessionId], + [sessionId, setProcessingState], ); // -- Create transport (one per session, stable for lifetime) ------------ @@ -349,23 +381,20 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD experimental_throttle: 80, // On mount, the SDK calls transport.reconnectToStream() which checks // is_processing and subscribes to the live event stream if the agent - // is mid-turn. Without this, page refresh kills live updates. - resume: true, + // is mid-turn. Without this, page refresh kills live updates. Gated on + // shouldReactivate so an idle backgrounded session isn't reactivated on + // app load just to find there's nothing to resume. + resume: shouldReactivate, // After all approval responses are set, auto-send to continue the agent loop. // Without this, addToolApprovalResponse only updates the UI — it won't trigger // sendMessages on the transport. sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithApprovalResponses, onError: (error) => { - updateSession(sessionId, { isProcessing: false }); - // Premium-model daily cap: open the cap dialog instead of the generic error - // banner. Transport marks the error with this sentinel. - if (error.message === 'CLAUDE_QUOTA_EXHAUSTED') { - if (isActiveRef.current) { - useAgentStore.getState().setClaudeQuotaExhausted(true); - } - return; - } + setProcessingState(false); logger.error('useChat error:', error); + callbacksRef.current.onError?.( + error instanceof Error ? error.message : String(error), + ); }, }); @@ -374,7 +403,13 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD chatActionsRef.current.messages = chat.messages; // -- Hydrate from backend on mount (page refresh recovery) -------------- + // Gated on shouldReactivate: an idle backgrounded session renders from its + // localStorage message cache + the sidebar list payload, with no per-session + // fetch that would reactivate its runtime/sandbox. Re-runs when a session + // becomes active (user selects it) or the list reports it processing, which + // is exactly when reactivation is wanted. useEffect(() => { + if (!shouldReactivate) return; let cancelled = false; (async () => { try { @@ -429,8 +464,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD // atomic update so the UI never sees isProcessing=false with stale // tool states (which would coerce them to 'output-available'). const savedResearch = loadResearch(sessionId); - updateSession(sessionId, { - isProcessing: true, + setProcessingState(true, { activityStatus: savedResearch?.stats.startedAt ? { type: 'tool', toolName: 'research', description: 'Resuming research...' } : { type: 'thinking' }, @@ -440,9 +474,10 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD }), }); } else if (pendingIds && pendingIds.size > 0) { - updateSession(sessionId, { activityStatus: { type: 'waiting-approval' } }); + setProcessingState(false, { activityStatus: { type: 'waiting-approval' } }); clearResearch(sessionId); } else { + setProcessingState(false); clearResearch(sessionId); } } catch { @@ -450,7 +485,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD } })(); return () => { cancelled = true; }; - }, [sessionId]); // eslint-disable-line react-hooks/exhaustive-deps + }, [sessionId, shouldReactivate, setProcessingState]); // eslint-disable-line react-hooks/exhaustive-deps // -- Re-hydrate + reconnect on wake from sleep ---------------------------- // The Vercel AI SDK only calls reconnectToStream() on mount, NOT on @@ -627,6 +662,10 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD const onVisible = async () => { if (document.visibilityState !== 'visible') return; + // Idle backgrounded sessions stay dormant on tab refocus too — otherwise + // every refocus would re-hydrate (and reactivate) every session, refilling + // the pool the reaper just drained. Only the active/processing session wakes. + if (!shouldReactivateRef.current) return; // Always re-hydrate messages on wake const result = await hydrateMessages(); @@ -641,7 +680,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD // If the backend is still processing, reconnect to the live event stream if (info?.is_processing) { - updateSession(sessionId, { isProcessing: true, activityStatus: { type: 'thinking' } }); + setProcessingState(true, { activityStatus: { type: 'thinking' } }); // Stop any previous reconnection stopReconnect(); @@ -666,7 +705,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD // If backend stopped processing, clean up if (fresh.info && !fresh.info.is_processing) { - updateSession(sessionId, { isProcessing: false }); + setProcessingState(false); stopReconnect(); } }, 3000); @@ -678,7 +717,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD document.removeEventListener('visibilitychange', onVisible); stopReconnect(); }; - }, [sessionId]); // eslint-disable-line react-hooks/exhaustive-deps + }, [sessionId, setProcessingState]); // eslint-disable-line react-hooks/exhaustive-deps // -- Persist messages --------------------------------------------------- const prevLenRef = useRef(initialMessages.length); @@ -713,11 +752,11 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD setMsgs(updated); saveMessages(sessionId, updated); } - updateSession(sessionId, { isProcessing: false }); + setProcessingState(false); } catch (e) { logger.error('Undo failed:', e); } - }, [sessionId, updateSession]); + }, [sessionId, setProcessingState]); // -- Approve tools ------------------------------------------------------ const approveTools = useCallback( @@ -741,8 +780,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD setNeedsAttention(sessionId, false); const hasApproved = approvals.some(a => a.approved); if (hasApproved) { - updateSession(sessionId, { - isProcessing: true, + setProcessingState(true, { activityStatus: { type: 'thinking' }, }); } @@ -753,7 +791,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD return true; }, - [sessionId, chat, updateSession, setNeedsAttention], + [sessionId, chat, setProcessingState, setNeedsAttention], ); // -- Stop (interrupt backend agent loop, keep SSE open for events) -------- @@ -761,9 +799,9 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD // Don't call chat.stop() — keep the SSE stream open so the backend's // tool_state_change(cancelled) and interrupted events reach the frontend. // The stream closes naturally when the backend sends finish events. - updateSession(sessionId, { isProcessing: false }); + setProcessingState(false); apiFetch(`/api/interrupt/${sessionId}`, { method: 'POST' }).catch(() => {}); - }, [sessionId, updateSession]); + }, [sessionId, setProcessingState]); // -- Edit message + regenerate from that point ---------------------------- const editAndRegenerate = useCallback(async (messageId: string, newText: string) => { diff --git a/frontend/src/hooks/useUserQuota.ts b/frontend/src/hooks/useUserQuota.ts index ed3371e8e84906e5ba82a8296c91423af6741dfc..30dabab4a28fb2c5b94133c8788a912e133763c4 100644 --- a/frontend/src/hooks/useUserQuota.ts +++ b/frontend/src/hooks/useUserQuota.ts @@ -18,13 +18,13 @@ export interface UserQuota { premiumRemaining: number; } -export function useUserQuota() { +export function useUserQuota({ enabled = true }: { enabled?: boolean } = {}) { const user = useAgentStore((s) => s.user); const [quota, setQuota] = useState(null); const [loading, setLoading] = useState(false); const refresh = useCallback(async () => { - if (!user?.authenticated) return; + if (!enabled || !user?.authenticated) return; setLoading(true); try { const res = await apiFetch('/api/user/quota'); @@ -41,7 +41,7 @@ export function useUserQuota() { } finally { setLoading(false); } - }, [user?.authenticated]); + }, [enabled, user?.authenticated]); useEffect(() => { refresh(); diff --git a/frontend/src/lib/sse-chat-transport.ts b/frontend/src/lib/sse-chat-transport.ts index 9bcdda581bcb7ab226e8c81c24216a5383b80880..e4da13c2f9153f3c74a25c1872644cde6a849889 100644 --- a/frontend/src/lib/sse-chat-transport.ts +++ b/frontend/src/lib/sse-chat-transport.ts @@ -54,6 +54,21 @@ function lastEventKey(sessionId: string): string { return `hf-agent-last-event:${sessionId}`; } +async function readErrorResponse(response: Response): Promise { + const raw = await response.text().catch(() => ''); + if (!raw) return response.statusText || 'Request failed'; + try { + const data = JSON.parse(raw); + const detail = data?.detail; + if (typeof detail === 'string') return detail; + if (detail?.message && typeof detail.message === 'string') return detail.message; + if (detail?.error && typeof detail.error === 'string') return detail.error; + return JSON.stringify(detail ?? data); + } catch { + return raw; + } +} + /** Parse an SSE text stream into AgentEvent objects. */ function createSSEParserStream(sessionId: string): TransformStream { let buffer = ''; @@ -413,15 +428,9 @@ export class SSEChatTransport implements ChatTransport { // it can flag the session for the catch-up banner. this.sideChannel.onSessionDead(sessionId); } - if (response.status === 429) { - // Premium-model daily quota gate tripped. The prefix is the detection marker - // for useAgentChat's onError handler, which surfaces the cap dialog - // instead of a generic error banner. - throw new Error('CLAUDE_QUOTA_EXHAUSTED'); - } if (!response.ok) { - const errorText = await response.text().catch(() => 'Request failed'); - throw new Error(`Chat request failed: ${response.status} ${errorText}`); + const errorText = await readErrorResponse(response); + throw new Error(`Chat request failed (${response.status}): ${errorText}`); } if (!response.body) { diff --git a/frontend/src/store/agentStore.ts b/frontend/src/store/agentStore.ts index 14bc35fc6960294f55dcb441965a3ffe683b7bcd..fd307261d210ee3b50f2a97718d0b55da9e7c0b5 100644 --- a/frontend/src/store/agentStore.ts +++ b/frontend/src/store/agentStore.ts @@ -118,8 +118,6 @@ interface AgentStore { activityStatus: ActivityStatus; user: User | null; llmHealthError: LLMHealthError | null; - /** Set when a premium-model send hits the daily quota; ChatInput opens the cap dialog. */ - claudeQuotaExhausted: boolean; jobsUpgradeRequired: JobsUpgradeState | null; // Right panel (single-artifact pattern) @@ -173,7 +171,6 @@ interface AgentStore { setActivityStatus: (status: ActivityStatus) => void; setUser: (user: User | null) => void; setLlmHealthError: (error: LLMHealthError | null) => void; - setClaudeQuotaExhausted: (exhausted: boolean) => void; setJobsUpgradeRequired: (state: JobsUpgradeState | null) => void; setPanel: (data: PanelData, view?: PanelView, editable?: boolean) => void; @@ -294,7 +291,6 @@ export const useAgentStore = create()((set, get) => ({ activityStatus: { type: 'idle' }, user: null, llmHealthError: null, - claudeQuotaExhausted: false, jobsUpgradeRequired: null, panelData: null, @@ -407,7 +403,6 @@ export const useAgentStore = create()((set, get) => ({ setActivityStatus: (status) => set({ activityStatus: status }), setUser: (user) => set({ user }), setLlmHealthError: (error) => set({ llmHealthError: error }), - setClaudeQuotaExhausted: (exhausted) => set({ claudeQuotaExhausted: exhausted }), setJobsUpgradeRequired: (state) => set({ jobsUpgradeRequired: state }), // ── Panel (single-artifact) ─────────────────────────────────────── diff --git a/frontend/src/store/sessionStore.ts b/frontend/src/store/sessionStore.ts index e4129e51a3927087d6f9dcc7028faf3ea9fcdae6..d6e9371edbd008f51bb0cbb5da2caede85462351 100644 --- a/frontend/src/store/sessionStore.ts +++ b/frontend/src/store/sessionStore.ts @@ -13,6 +13,10 @@ interface SessionStore { deleteSession: (id: string) => void; switchSession: (id: string) => void; setSessionActive: (id: string, isActive: boolean) => void; + /** Track whether this session is mid-turn. Set false on terminal events so a + * finished background task stops looking "processing" (which would otherwise + * keep reactivating it until the next GET /sessions merge). */ + setSessionProcessing: (id: string, isProcessing: boolean) => void; updateSessionTitle: (id: string, title: string) => void; updateSessionModel: (id: string, model: string | null) => void; setNeedsAttention: (id: string, needs: boolean) => void; @@ -27,7 +31,10 @@ interface SessionStore { title?: string | null; created_at: string; is_active?: boolean; + is_processing?: boolean; model?: string | null; + premium_user_billed?: boolean; + premium_quota_counted?: boolean; pending_approval?: unknown[] | null; auto_approval?: { enabled?: boolean; @@ -66,6 +73,8 @@ export const useSessionStore = create()( autoApprovalCostCapUsd: null, autoApprovalEstimatedSpendUsd: 0, autoApprovalRemainingUsd: null, + premiumUserBilled: false, + premiumQuotaCounted: false, }; set((state) => ({ sessions: [...state.sessions, newSession], @@ -117,7 +126,10 @@ export const useSessionStore = create()( ...existing, title: server.title || existing.title, isActive: server.is_active ?? existing.isActive, + isProcessing: Boolean(server.is_processing), model: server.model ?? existing.model ?? null, + premiumUserBilled: Boolean(server.premium_user_billed), + premiumQuotaCounted: Boolean(server.premium_quota_counted), needsAttention: Boolean(server.pending_approval?.length) || existing.needsAttention, expired: false, ...(auto @@ -139,6 +151,7 @@ export const useSessionStore = create()( title: server.title || `Chat ${merged.length + 1}`, createdAt: server.created_at || new Date().toISOString(), isActive: server.is_active ?? true, + isProcessing: Boolean(server.is_processing), needsAttention: Boolean(server.pending_approval?.length), model: server.model ?? null, expired: false, @@ -146,6 +159,8 @@ export const useSessionStore = create()( autoApprovalCostCapUsd: server.auto_approval?.cost_cap_usd ?? null, autoApprovalEstimatedSpendUsd: server.auto_approval?.estimated_spend_usd ?? 0, autoApprovalRemainingUsd: server.auto_approval?.remaining_usd ?? null, + premiumUserBilled: Boolean(server.premium_user_billed), + premiumQuotaCounted: Boolean(server.premium_quota_counted), }; merged.push(newSession); byId.set(id, newSession); @@ -202,6 +217,14 @@ export const useSessionStore = create()( })); }, + setSessionProcessing: (id: string, isProcessing: boolean) => { + set((state) => ({ + sessions: state.sessions.map((s) => + s.id === id ? { ...s, isProcessing } : s + ), + })); + }, + updateSessionTitle: (id: string, title: string) => { set((state) => ({ sessions: state.sessions.map((s) => @@ -229,7 +252,10 @@ export const useSessionStore = create()( { name: 'hf-agent-sessions', partialize: (state) => ({ - sessions: state.sessions, + // Reset the transient isProcessing flag so a stale `true` from a + // previous session can't trigger a reactivating hydration on the next + // cold load — it's always re-derived from the live GET /sessions list. + sessions: state.sessions.map((s) => ({ ...s, isProcessing: false })), activeSessionId: state.activeSessionId, }), } diff --git a/frontend/src/types/agent.ts b/frontend/src/types/agent.ts index 3847a9c6f41d72db54f2d7af8d97345e319ffafb..5ed5ccf846680f7ef88b8e7a52072b5f52e070d0 100644 --- a/frontend/src/types/agent.ts +++ b/frontend/src/types/agent.ts @@ -17,6 +17,13 @@ export interface SessionMeta { isActive: boolean; needsAttention: boolean; model?: string | null; + /** True when the backend reports this session is mid-turn (from + * GET /sessions). A processing session is already live in memory, so it + * keeps streaming in the background; idle sessions are NOT hydrated on app + * load, which is what keeps them from refilling the active-session pool. + * Transient — never persisted to localStorage (always re-derived from the + * server list). */ + isProcessing?: boolean; /** True when the backend no longer recognizes this session id (e.g. * after a backend restart). The UI shows a recovery banner and * disables input until the user chooses to restore-with-summary or @@ -26,6 +33,8 @@ export interface SessionMeta { autoApprovalCostCapUsd?: number | null; autoApprovalEstimatedSpendUsd?: number; autoApprovalRemainingUsd?: number | null; + premiumUserBilled?: boolean; + premiumQuotaCounted?: boolean; } export interface ToolApproval { diff --git a/frontend/src/utils/model.ts b/frontend/src/utils/model.ts index 84754f995ecf408d072448ffd062c35823c92b97..2708a969046706c4d4e68fbd7e41a4942e97bf8d 100644 --- a/frontend/src/utils/model.ts +++ b/frontend/src/utils/model.ts @@ -1,19 +1,24 @@ /** - * Shared model-id constants used by session-create call sites and the - * premium-model cap dialog "Use a free model" escape hatch. + * Shared model-id constants used by session-create call sites and the model + * picker. * * Keep in sync with MODEL_OPTIONS in components/Chat/ChatInput.tsx and * AVAILABLE_MODELS in backend/routes/agent.py. */ -export const CLAUDE_MODEL_PATH = 'bedrock/us.anthropic.claude-opus-4-6-v1'; -export const GPT_55_MODEL_PATH = 'openai/gpt-5.5'; -export const FIRST_FREE_MODEL_PATH = 'moonshotai/Kimi-K2.6'; +export const CLAUDE_OPUS_48_MODEL_PATH = 'anthropic/claude-opus-4.8:fal-ai'; +export const CLAUDE_MODEL_PATH = CLAUDE_OPUS_48_MODEL_PATH; +export const GPT_55_MODEL_PATH = 'openai/gpt-5.5:fal-ai'; + +const PREMIUM_MODEL_PATHS = new Set([ + CLAUDE_OPUS_48_MODEL_PATH, + GPT_55_MODEL_PATH, +]); export function isClaudePath(modelPath: string | undefined): boolean { return !!modelPath && modelPath.includes('anthropic'); } export function isPremiumPath(modelPath: string | undefined): boolean { - return modelPath === CLAUDE_MODEL_PATH || modelPath === GPT_55_MODEL_PATH; + return !!modelPath && PREMIUM_MODEL_PATHS.has(modelPath); } diff --git a/scripts/build_kpis.py b/scripts/build_kpis.py index 47a6515cef6339937f329e4007292d94b2a99b56..a4b2f4996f0eb4295cadc71b4f32db82b42445e3 100644 --- a/scripts/build_kpis.py +++ b/scripts/build_kpis.py @@ -58,7 +58,7 @@ re-running the same hour overwrites. tool_calls_per_turn_p50 / _p95 — calls / turns, among sessions with turns>0 tool_calls_by_name_json — JSON {tool: total_calls} (all tools seen) sessions_using_tool_json — JSON {tool: distinct_sessions_using} - sessions_by_model_json — JSON {model_name: count} (CLI vs Bedrock split) + sessions_by_model_json — JSON {model_name: count} (router/local split) ================================================================================ Usage @@ -560,7 +560,7 @@ def _aggregate(per_session: list[dict]) -> dict: "sessions_using_tool_json": json.dumps( dict(sessions_using_tool), sort_keys=True ), - # Surface split — answers "is research dropping on Bedrock specifically?". + # Surface split by selected model for dashboard drilldowns. "sessions_by_model_json": json.dumps(dict(sessions_by_model), sort_keys=True), } diff --git a/scripts/build_sft.py b/scripts/build_sft.py index 7a89989a1cc3e13a6638e612874e68ce01e5144c..7988b2239e4d17180aa53d9f23143050c0f80a91 100644 --- a/scripts/build_sft.py +++ b/scripts/build_sft.py @@ -13,7 +13,7 @@ Output row schema:: { "session_id": "...", - "model": "claude-opus-4-6", + "model": "anthropic/claude-opus-4.8:fal-ai", "timestamp": "2026-04-24T...", "tags": ["tool:hf_jobs", "gpu:a100", "hf_job:succeeded", ...], "messages": [...], # OpenAI / TRL SFTTrainer format diff --git a/scripts/prioritize_backlog.py b/scripts/prioritize_backlog.py index 0318c3bedd1c4cdafb442ec79546bdee5a03c01f..4145fd04255ff51b8f10e8e79d3f07638b4ba3aa 100644 --- a/scripts/prioritize_backlog.py +++ b/scripts/prioritize_backlog.py @@ -7,7 +7,7 @@ likely product impact. Usage: uv run python scripts/prioritize_backlog.py - uv run python scripts/prioritize_backlog.py --model openai/gpt-5.5 + uv run python scripts/prioritize_backlog.py --model openai/gpt-5.5:fal-ai Outputs: scratch/backlog-prioritization//sources.json @@ -1277,7 +1277,7 @@ def _response_content(response: Any) -> str: def _temperature_for_params(llm_params: dict[str, Any]) -> float: - # Anthropic requires temperature=1 when adaptive/extended thinking is active. + # Some providers require temperature=1 when adaptive reasoning is active. if llm_params.get("thinking") or llm_params.get("output_config"): return 1.0 return 0.2 diff --git a/tests/integration/test_live_thinking_models.py b/tests/integration/test_live_thinking_models.py index 391b260bfe566171433250299966972b0152c68d..81fb85db61880643ea74cfdf333ce821dbad558b 100644 --- a/tests/integration/test_live_thinking_models.py +++ b/tests/integration/test_live_thinking_models.py @@ -1,9 +1,8 @@ -"""Opt-in live provider checks for thinking metadata replay. +"""Opt-in live HF Router checks. These tests intentionally call paid model APIs and are skipped unless -``ML_INTERN_LIVE_LLM_TESTS=1`` plus the relevant provider key are set. -They cover the concrete model families involved in #87 without making -default CI depend on external credentials or provider availability. +``ML_INTERN_LIVE_LLM_TESTS=1`` plus ``HF_TOKEN`` are set. They verify the +router-only request path without requiring provider-specific API keys. """ from __future__ import annotations @@ -21,14 +20,13 @@ from agent.core.agent_loop import ( _call_llm_streaming, ) from agent.core.llm_params import _resolve_llm_params +from agent.core.model_ids import DEFAULT_MODEL_ID if env_file := os.environ.get("ML_INTERN_LIVE_ENV_FILE"): load_dotenv(Path(env_file)) LIVE_TESTS_ENABLED = os.environ.get("ML_INTERN_LIVE_LLM_TESTS") == "1" -OPUS_47_MODEL = "anthropic/claude-opus-4-7" -LATEST_GPT_MODEL = "openai/gpt-5.2" REPORT_RESULT_TOOL = [ { "type": "function", @@ -55,9 +53,9 @@ def _skip_without_live_flag() -> None: pytest.skip("set ML_INTERN_LIVE_LLM_TESTS=1 to run paid live LLM tests") -def _skip_without_env(name: str) -> None: - if not os.environ.get(name): - pytest.skip(f"set {name} to run this live provider test") +def _skip_without_hf_token() -> None: + if not os.environ.get("HF_TOKEN"): + pytest.skip("set HF_TOKEN to run this live router test") def _session(model_name: str): @@ -75,54 +73,16 @@ def _session(model_name: str): @pytest.mark.asyncio -async def test_live_opus_47_preserves_thinking_metadata_for_replay(): +async def test_live_default_router_model_does_not_replay_reasoning_metadata(): _skip_without_live_flag() - _skip_without_env("ANTHROPIC_API_KEY") + _skip_without_hf_token() - session = _session(OPUS_47_MODEL) + session = _session(DEFAULT_MODEL_ID) llm_params = _resolve_llm_params( - OPUS_47_MODEL, - reasoning_effort="high", - ) - - result = await _call_llm_streaming( - session, - messages=[ - Message( - role="user", - content=( - "Use careful reasoning for this small check. " - "If 17 * 19 = 323, call report_result with answer OPUS_OK." - ), - ) - ], - tools=REPORT_RESULT_TOOL, - llm_params=llm_params, - ) - - replay = _assistant_message_from_result( - result, - model_name=OPUS_47_MODEL, - ) - - assert result.content or result.tool_calls_acc - assert result.thinking_blocks, ( - "Opus returned no thinking_blocks with reasoning_effort='high' - " - "check that adaptive thinking params are being forwarded correctly" - ) - assert getattr(replay, "thinking_blocks", None) == result.thinking_blocks - assert getattr(replay, "reasoning_content", None) == result.reasoning_content - - -@pytest.mark.asyncio -async def test_live_latest_gpt_does_not_replay_reasoning_metadata(): - _skip_without_live_flag() - _skip_without_env("OPENAI_API_KEY") - - session = _session(LATEST_GPT_MODEL) - llm_params = _resolve_llm_params( - LATEST_GPT_MODEL, + DEFAULT_MODEL_ID, + os.environ["HF_TOKEN"], reasoning_effort="low", + bill_to_user=True, ) result = await _call_llm_streaming( @@ -130,21 +90,14 @@ async def test_live_latest_gpt_does_not_replay_reasoning_metadata(): messages=[ Message( role="user", - content="Call report_result with answer GPT_OK.", + content="Call report_result with answer ROUTER_OK.", ) ], tools=REPORT_RESULT_TOOL, llm_params=llm_params, ) - # Even if a GPT-family response carries provider reasoning internally, - # OpenAI-compatible history must not echo it back on the next tool turn. - # Force the non-None strip path when the live model omits reasoning details. - result.reasoning_content = result.reasoning_content or "synthetic-reasoning" - replay = _assistant_message_from_result( - result, - model_name=LATEST_GPT_MODEL, - ) + replay = _assistant_message_from_result(result) assert result.content or result.tool_calls_acc assert getattr(replay, "thinking_blocks", None) is None diff --git a/tests/unit/test_agent_model_gating.py b/tests/unit/test_agent_model_gating.py index 8a3f88d834ef1149636925f57bab2a690993ce1f..991ca5a1fd942e8de223897661965aaf69aa0b03 100644 --- a/tests/unit/test_agent_model_gating.py +++ b/tests/unit/test_agent_model_gating.py @@ -1,6 +1,5 @@ """Tests for premium model handling in backend/routes/agent.py.""" -import asyncio import sys from pathlib import Path from types import SimpleNamespace @@ -22,34 +21,26 @@ def _reset_quota_store(): agent.user_quotas._reset_for_tests() -def test_premium_model_predicate_includes_bedrock_claude_and_gpt55_only(): - assert agent._is_premium_model("bedrock/us.anthropic.claude-opus-4-6-v1") - assert agent._is_premium_model("openai/gpt-5.5") - assert not agent._is_premium_model("anthropic/claude-opus-4-6") - assert not agent._is_premium_model("moonshotai/Kimi-K2.6") - - -@pytest.mark.asyncio -async def test_default_premium_session_falls_back_to_free_model(monkeypatch): - monkeypatch.setattr( - agent.session_manager.config, - "model_name", - agent.DEFAULT_CLAUDE_MODEL_ID, +def _premium_session(model: str = agent.DEFAULT_PREMIUM_MODEL_ID): + return SimpleNamespace( + claude_counted=False, + session=SimpleNamespace( + config=SimpleNamespace(model_name=model), + premium_user_billed=False, + ), ) - model = await agent._model_override_for_new_session(None, None) - assert model == agent.DEFAULT_FREE_MODEL_ID +def test_premium_model_predicate_uses_router_ids_only(): + assert agent._is_premium_model(agent.DEFAULT_PREMIUM_MODEL_ID) + assert agent._is_premium_model(agent.DEFAULT_GPT_MODEL_ID) + assert agent._is_user_billed(agent.DEFAULT_PREMIUM_MODEL_ID) + assert not agent._is_premium_model("moonshotai/Kimi-K2.6") + assert not agent._is_premium_model("unsupported/model") @pytest.mark.asyncio -async def test_default_free_session_keeps_config_default(monkeypatch): - monkeypatch.setattr( - agent.session_manager.config, - "model_name", - agent.DEFAULT_FREE_MODEL_ID, - ) - +async def test_default_session_uses_configured_default_model(): model = await agent._model_override_for_new_session(None, None) assert model is None @@ -59,10 +50,10 @@ async def test_default_free_session_keeps_config_default(monkeypatch): async def test_explicit_premium_session_allowed_for_authenticated_user(): model = await agent._model_override_for_new_session( None, - agent.DEFAULT_CLAUDE_MODEL_ID, + agent.DEFAULT_PREMIUM_MODEL_ID, ) - assert model == agent.DEFAULT_CLAUDE_MODEL_ID + assert model == agent.DEFAULT_PREMIUM_MODEL_ID @pytest.mark.asyncio @@ -88,17 +79,36 @@ async def test_switching_to_premium_model_is_allowed_for_authenticated_user( response = await agent.set_session_model( "s1", - {"model": "openai/gpt-5.5"}, + {"model": agent.DEFAULT_GPT_MODEL_ID}, request=None, user={"user_id": "u1", "plan": "free"}, ) - assert response == {"session_id": "s1", "model": "openai/gpt-5.5"} - assert updated == [("s1", "openai/gpt-5.5")] + assert response == {"session_id": "s1", "model": agent.DEFAULT_GPT_MODEL_ID} + assert updated == [("s1", agent.DEFAULT_GPT_MODEL_ID)] @pytest.mark.asyncio -async def test_premium_quota_charges_gpt55(monkeypatch): +async def test_switching_to_unknown_model_id_is_rejected(monkeypatch): + async def fake_check_session_access(session_id, user, request=None): + return SimpleNamespace(user_id=user["user_id"]) + + monkeypatch.setattr(agent, "_check_session_access", fake_check_session_access) + + with pytest.raises(HTTPException) as exc_info: + await agent.set_session_model( + "s1", + {"model": "unsupported/model"}, + request=None, + user={"user_id": "u1", "plan": "free"}, + ) + + assert exc_info.value.status_code == 400 + assert "Unknown model" in exc_info.value.detail + + +@pytest.mark.asyncio +async def test_premium_quota_charges_without_user_billing_inside_allowance(monkeypatch): persisted = [] async def fake_persist_session_snapshot(agent_session): @@ -110,12 +120,7 @@ async def test_premium_quota_charges_gpt55(monkeypatch): fake_persist_session_snapshot, ) - agent_session = SimpleNamespace( - claude_counted=False, - session=SimpleNamespace( - config=SimpleNamespace(model_name="openai/gpt-5.5"), - ), - ) + agent_session = _premium_session() await agent._enforce_premium_model_quota( {"user_id": "u1", "plan": "free"}, @@ -123,12 +128,15 @@ async def test_premium_quota_charges_gpt55(monkeypatch): ) assert agent_session.claude_counted is True + assert agent_session.session.premium_user_billed is False assert persisted == [agent_session] assert await agent.user_quotas.get_claude_used_today("u1") == 1 @pytest.mark.asyncio -async def test_free_user_premium_quota_rejects_second_session(monkeypatch): +async def test_free_user_gets_two_subsidized_premium_sessions_then_user_billing( + monkeypatch, +): async def fake_persist_session_snapshot(_agent_session): return None @@ -138,32 +146,40 @@ async def test_free_user_premium_quota_rejects_second_session(monkeypatch): fake_persist_session_snapshot, ) - first_session = SimpleNamespace( - claude_counted=False, - session=SimpleNamespace( - config=SimpleNamespace(model_name="openai/gpt-5.5"), - ), - ) - second_session = SimpleNamespace( - claude_counted=False, - session=SimpleNamespace( - config=SimpleNamespace(model_name="openai/gpt-5.5"), - ), + first = _premium_session() + await agent._enforce_premium_model_quota({"user_id": "g1", "plan": "free"}, first) + assert first.session.premium_user_billed is False + + second = _premium_session() + await agent._enforce_premium_model_quota({"user_id": "g1", "plan": "free"}, second) + assert second.session.premium_user_billed is False + + third = _premium_session() + await agent._enforce_premium_model_quota({"user_id": "g1", "plan": "free"}, third) + assert third.session.premium_user_billed is True + assert await agent.user_quotas.get_claude_used_today("g1") == 2 + + +@pytest.mark.asyncio +async def test_free_model_does_not_consume_premium_quota(monkeypatch): + async def fail_if_persisted(_agent_session): + raise AssertionError("free model should not consume premium quota") + + monkeypatch.setattr( + agent.session_manager, + "persist_session_snapshot", + fail_if_persisted, ) + agent_session = _premium_session("moonshotai/Kimi-K2.6") + await agent._enforce_premium_model_quota( - {"user_id": "free-user", "plan": "free"}, - first_session, + {"user_id": "u1", "plan": "free"}, + agent_session, ) - with pytest.raises(HTTPException) as exc_info: - await agent._enforce_premium_model_quota( - {"user_id": "free-user", "plan": "free"}, - second_session, - ) - assert exc_info.value.status_code == 429 - assert exc_info.value.detail["error"] == "premium_model_daily_cap" - assert exc_info.value.detail["plan"] == "free" + assert agent_session.claude_counted is False + assert await agent.user_quotas.get_claude_used_today("u1") == 0 @pytest.mark.asyncio @@ -178,84 +194,79 @@ async def test_pro_user_uses_pro_premium_quota(monkeypatch): ) for index in range(2): - agent_session = SimpleNamespace( - claude_counted=False, - session=SimpleNamespace( - config=SimpleNamespace(model_name="openai/gpt-5.5"), - ), - ) + agent_session = _premium_session() await agent._enforce_premium_model_quota( {"user_id": "pro-user", "plan": "pro"}, agent_session, ) assert agent_session.claude_counted is True + assert agent_session.session.premium_user_billed is False assert await agent.user_quotas.get_claude_used_today("pro-user") == index + 1 @pytest.mark.asyncio -async def test_org_plan_uses_free_premium_quota(monkeypatch): - async def fake_persist_session_snapshot(_agent_session): +async def test_pro_user_billable_overflow_also_bills_user(monkeypatch): + async def fake_persist(_agent_session): return None - monkeypatch.setattr( - agent.session_manager, - "persist_session_snapshot", - fake_persist_session_snapshot, - ) - - first_session = SimpleNamespace( - claude_counted=False, - session=SimpleNamespace( - config=SimpleNamespace(model_name="openai/gpt-5.5"), - ), - ) - second_session = SimpleNamespace( - claude_counted=False, - session=SimpleNamespace( - config=SimpleNamespace(model_name="openai/gpt-5.5"), - ), - ) + monkeypatch.setattr(agent.session_manager, "persist_session_snapshot", fake_persist) + monkeypatch.setattr(agent.user_quotas, "daily_cap_for", lambda plan: 1) await agent._enforce_premium_model_quota( - {"user_id": "org-user", "plan": "org"}, - first_session, + {"user_id": "p1", "plan": "pro"}, _premium_session() ) - with pytest.raises(HTTPException) as exc_info: - await agent._enforce_premium_model_quota( - {"user_id": "org-user", "plan": "org"}, - second_session, - ) - - assert exc_info.value.status_code == 429 - assert exc_info.value.detail["plan"] == "org" - assert "Upgrade to HF Pro" in exc_info.value.detail["message"] + over = _premium_session() + await agent._enforce_premium_model_quota({"user_id": "p1", "plan": "pro"}, over) + assert over.session.premium_user_billed is True @pytest.mark.asyncio -async def test_premium_quota_skips_direct_anthropic(monkeypatch): - async def fail_if_persisted(_agent_session): - raise AssertionError("direct Anthropic should not consume premium quota") +async def test_restore_summary_enforces_premium_quota_before_seed(monkeypatch): + events = [] + agent_session = _premium_session() - monkeypatch.setattr( - agent.session_manager, - "persist_session_snapshot", - fail_if_persisted, - ) + class Request: + headers = {} + cookies = {} - agent_session = SimpleNamespace( - claude_counted=False, - session=SimpleNamespace( - config=SimpleNamespace(model_name="anthropic/claude-opus-4-6"), - ), - ) + async def fake_create_session(**kwargs): + events.append(("create", kwargs["model"])) + return "s1" - await agent._enforce_premium_model_quota( + async def fake_check_session_access( + session_id, user, request, preload_sandbox=True + ): + events.append(("check", session_id, preload_sandbox)) + return agent_session + + async def fake_enforce_quota(user, session): + assert user["user_id"] == "u1" + assert session is agent_session + session.session.premium_user_billed = True + events.append(("quota", session.session.config.model_name)) + + async def fake_seed(session_id, messages): + events.append(("seed", session_id, agent_session.session.premium_user_billed)) + return len(messages) + + monkeypatch.setattr(agent.session_manager, "create_session", fake_create_session) + monkeypatch.setattr(agent, "_check_session_access", fake_check_session_access) + monkeypatch.setattr(agent, "_enforce_premium_model_quota", fake_enforce_quota) + monkeypatch.setattr(agent.session_manager, "seed_from_summary", fake_seed) + + response = await agent.restore_session_summary( + Request(), + {"messages": [{"role": "user", "content": "resume this"}]}, {"user_id": "u1", "plan": "free"}, - agent_session, ) - assert agent_session.claude_counted is False - assert await agent.user_quotas.get_claude_used_today("u1") == 0 + assert response.session_id == "s1" + assert events == [ + ("create", None), + ("check", "s1", False), + ("quota", agent.DEFAULT_PREMIUM_MODEL_ID), + ("seed", "s1", True), + ] @pytest.mark.asyncio @@ -320,60 +331,3 @@ async def test_set_session_yolo_calls_manager_with_cap_presence(monkeypatch): }, ) ] - - -@pytest.mark.asyncio -async def test_delete_session_access_check_skips_sandbox_preload(monkeypatch): - ensure_calls = [] - delete_calls = [] - - async def fake_ensure_session_loaded(session_id, user_id, **kwargs): - ensure_calls.append((session_id, user_id, kwargs)) - return SimpleNamespace(user_id=user_id) - - async def fake_delete_session(session_id): - delete_calls.append(session_id) - return True - - monkeypatch.setattr( - agent.session_manager, - "ensure_session_loaded", - fake_ensure_session_loaded, - ) - monkeypatch.setattr(agent.session_manager, "delete_session", fake_delete_session) - - response = await agent.delete_session("s1", {"user_id": "u1"}) - - assert response == {"status": "deleted", "session_id": "s1"} - assert delete_calls == ["s1"] - assert ensure_calls[0][2]["preload_sandbox"] is False - - -@pytest.mark.asyncio -async def test_teardown_session_access_check_skips_sandbox_preload(monkeypatch): - ensure_calls = [] - teardown_calls = [] - - async def fake_ensure_session_loaded(session_id, user_id, **kwargs): - ensure_calls.append((session_id, user_id, kwargs)) - return SimpleNamespace(user_id=user_id) - - async def fake_teardown_sandbox(session_id): - teardown_calls.append(session_id) - return True - - monkeypatch.setattr( - agent.session_manager, - "ensure_session_loaded", - fake_ensure_session_loaded, - ) - monkeypatch.setattr( - agent.session_manager, "teardown_sandbox", fake_teardown_sandbox - ) - - response = await agent.teardown_session_sandbox("s1", {"user_id": "u1"}) - await asyncio.sleep(0) - - assert response == {"status": "teardown_requested", "session_id": "s1"} - assert teardown_calls == ["s1"] - assert ensure_calls[0][2]["preload_sandbox"] is False diff --git a/tests/unit/test_build_kpis.py b/tests/unit/test_build_kpis.py index e792e5b4a6b9a05b46d570a0c98cbf3fa63b2118..bb73e3a9cb9c3980f781d3b38ea2bd0afca209ca 100644 --- a/tests/unit/test_build_kpis.py +++ b/tests/unit/test_build_kpis.py @@ -28,7 +28,7 @@ def _session(events, user_id="u1", start="2026-04-24T09:59:00"): "session_id": "sess-" + user_id, "session_start_time": start, "session_end_time": "2026-04-24T10:05:00", - "model_name": "claude-opus-4-6", + "model_name": "anthropic/claude-opus-4.8:fal-ai", "messages": [{"role": "user", "content": "hi"}], "events": events, "user_id": user_id, @@ -213,7 +213,7 @@ def test_per_tool_counts_in_session_metrics(): assert m["_research_calls"] == 1 assert m["_distinct_tools_used"] == 3 assert m["_total_named_tool_calls"] == 4 - assert m["_model_name"] == "claude-opus-4-6" + assert m["_model_name"] == "anthropic/claude-opus-4.8:fal-ai" def test_aggregate_research_kpis_only_count_doer_sessions(): @@ -366,22 +366,22 @@ def test_aggregate_sessions_by_model_split(): import json as _json mod = _load() - s_anthropic = _session([], user_id="a") - s_anthropic["model_name"] = "anthropic/claude-opus-4-6" - s_bedrock = _session([], user_id="b") - s_bedrock["model_name"] = "bedrock/us.anthropic.claude-opus-4-6-v1" - s_bedrock2 = _session([], user_id="c") - s_bedrock2["model_name"] = "bedrock/us.anthropic.claude-opus-4-6-v1" + s_claude = _session([], user_id="a") + s_claude["model_name"] = "anthropic/claude-opus-4.8:fal-ai" + s_kimi = _session([], user_id="b") + s_kimi["model_name"] = "moonshotai/Kimi-K2.6" + s_kimi2 = _session([], user_id="c") + s_kimi2["model_name"] = "moonshotai/Kimi-K2.6" row = mod._aggregate( [ - mod._session_metrics(s_anthropic), - mod._session_metrics(s_bedrock), - mod._session_metrics(s_bedrock2), + mod._session_metrics(s_claude), + mod._session_metrics(s_kimi), + mod._session_metrics(s_kimi2), ] ) assert _json.loads(row["sessions_by_model_json"]) == { - "anthropic/claude-opus-4-6": 1, - "bedrock/us.anthropic.claude-opus-4-6-v1": 2, + "anthropic/claude-opus-4.8:fal-ai": 1, + "moonshotai/Kimi-K2.6": 2, } diff --git a/tests/unit/test_build_sft.py b/tests/unit/test_build_sft.py index ab24ec5dc5589ac1f11413a43a9ac640427b0250..e9d1aa6468d0e2dc5ebfed7fd9ba9fb93baf0fa7 100644 --- a/tests/unit/test_build_sft.py +++ b/tests/unit/test_build_sft.py @@ -19,7 +19,7 @@ def _session_row(): "session_id": "abc", "session_start_time": "2026-04-24T10:00:00", "session_end_time": "2026-04-24T10:05:00", - "model_name": "claude-opus-4-6", + "model_name": "anthropic/claude-opus-4.8:fal-ai", "messages": [ {"role": "system", "content": "You are an agent"}, {"role": "user", "content": "fine-tune llama"}, @@ -77,7 +77,7 @@ def test_reshape_preserves_messages_and_tools_and_adds_tags(): mod = _load() row = mod._reshape_to_sft(_session_row()) assert row["session_id"] == "abc" - assert row["model"] == "claude-opus-4-6" + assert row["model"] == "anthropic/claude-opus-4.8:fal-ai" assert row["timestamp"] == "2026-04-24T10:00:00" # Messages preserved verbatim, in order, with tool_calls + tool role rows. assert len(row["messages"]) == 5 diff --git a/tests/unit/test_cli_local_models.py b/tests/unit/test_cli_local_models.py index 836fb3fdfd9f8dc99d4aac56b8b48d04e600ed6f..fd694e5d6f9a04e5389a388d07d78534a13c8427 100644 --- a/tests/unit/test_cli_local_models.py +++ b/tests/unit/test_cli_local_models.py @@ -30,6 +30,19 @@ def test_openai_compat_prefix_is_not_supported(): assert not model_switcher.is_valid_model_id("openai-compat/custom-model") +def test_suggested_models_include_router_opus48_and_no_native_ids(): + ids = {m["id"] for m in model_switcher.SUGGESTED_MODELS} + + assert "anthropic/claude-opus-4.8:fal-ai" in ids + assert all(model_id.count("/") >= 1 for model_id in ids) + + +def test_model_switcher_accepts_router_model_ids(): + assert model_switcher.is_valid_model_id("openai/gpt-5.5:fal-ai") + assert model_switcher.is_valid_model_id("openai/gpt-oss-120b") + assert model_switcher.is_valid_model_id("moonshotai/Kimi-K2.6") + + def test_local_models_skip_hf_router_catalog_output(): class NoPrintConsole: def print(self, *args, **kwargs): @@ -52,8 +65,8 @@ async def test_probe_and_switch_local_model_uses_no_effort(monkeypatch): monkeypatch.setattr(model_switcher, "acompletion", fake_acompletion) class Config: - model_name = "openai/gpt-5.5" - reasoning_effort = "max" + model_name = "anthropic/claude-opus-4.8:fal-ai" + reasoning_effort = "high" class Session: def __init__(self): @@ -91,7 +104,7 @@ async def test_probe_and_switch_local_model_rejects_probe_errors(monkeypatch): monkeypatch.setattr(model_switcher, "acompletion", failing_acompletion) class Config: - model_name = "openai/gpt-5.5" + model_name = "anthropic/claude-opus-4.8:fal-ai" reasoning_effort = None class Session: @@ -116,6 +129,6 @@ async def test_probe_and_switch_local_model_rejects_probe_errors(monkeypatch): hf_token=None, ) - assert config.model_name == "openai/gpt-5.5" + assert config.model_name == "anthropic/claude-opus-4.8:fal-ai" assert session.model_id is None assert "ollama/llama3.1:8b" not in session.model_effective_effort diff --git a/tests/unit/test_cli_new_clear.py b/tests/unit/test_cli_new_clear.py new file mode 100644 index 0000000000000000000000000000000000000000..963083dea051a9e24e3313eb2581cf6a99ef2b26 --- /dev/null +++ b/tests/unit/test_cli_new_clear.py @@ -0,0 +1,147 @@ +import asyncio +from types import SimpleNamespace + +import pytest +from litellm import Message + +import agent.main as main_mod +from agent.core.agent_loop import process_submission +from agent.core.session import Event, OpType, Session +from agent.tools import plan_tool + + +class _FakeConfig: + model_name = "openai/gpt-5.5:fal-ai" + save_sessions = False + session_dataset_repo = "fake/repo" + auto_save_interval = 1 + heartbeat_interval_s = 60 + max_iterations = 10 + yolo_mode = False + confirm_cpu_jobs = False + auto_file_upload = False + reasoning_effort = None + share_traces = False + personal_trace_repo_template = None + mcpServers: dict = {} + + +class _FakeContext: + def __init__(self) -> None: + self.items = [ + Message(role="system", content="system prompt"), + Message(role="user", content="old task"), + Message(role="assistant", content="old answer"), + ] + self.model_max_tokens = 200_000 + self.running_context_usage = 123 + self.on_message_added = None + + +def _make_session() -> Session: + return Session( + event_queue=asyncio.Queue(), + config=_FakeConfig(), + tool_router=None, + context_manager=_FakeContext(), + hf_token=None, + user_id="user-a", + local_mode=True, + ) + + +def test_start_new_conversation_rotates_session_state(monkeypatch): + plan_tool.reset_current_plan() + session = _make_session() + session.config.save_sessions = True + session.turn_count = 2 + session.last_auto_save_turn = 2 + session.logged_events = [{"event_type": "turn_complete", "data": {}}] + session._local_save_path = "session_logs/old.json" + session.pending_approval = {"tool_calls": ["pending"]} + session.auto_approval_estimated_spend_usd = 1.25 + session.current_plan = [ + {"id": "old-session", "content": "old session item", "status": "pending"} + ] + monkeypatch.setattr( + plan_tool, + "_current_plan", + [{"id": "global", "content": "global item", "status": "in_progress"}], + ) + old_session_id = session.session_id + uploads: list[str] = [] + + def fake_upload(repo_id: str) -> str: + uploads.append(repo_id) + return "session_logs/saved.json" + + monkeypatch.setattr(session, "save_and_upload_detached", fake_upload) + + result = session.start_new_conversation() + + assert uploads == ["fake/repo"] + assert result["previous_session_id"] == old_session_id + assert result["previous_turn_count"] == 2 + assert result["saved_path"] == "session_logs/saved.json" + assert session.session_id != old_session_id + assert [msg.role for msg in session.context_manager.items] == ["system"] + assert session.context_manager.items[0].content == "system prompt" + assert session.context_manager.running_context_usage == 0 + assert session.turn_count == 0 + assert session.last_auto_save_turn == 0 + assert session.logged_events == [] + assert session._local_save_path is None + assert session.pending_approval is None + assert session.auto_approval_estimated_spend_usd == 0.0 + assert session.current_plan == [] + assert plan_tool.get_current_plan() == [] + + +@pytest.mark.asyncio +async def test_new_submission_resets_context_and_reports_clear_flag(): + session = _make_session() + submission = SimpleNamespace( + operation=SimpleNamespace( + op_type=OpType.NEW, + data={"clear_screen": True}, + ) + ) + + should_continue = await process_submission(session, submission) + + event: Event = await session.event_queue.get() + assert should_continue is True + assert event.event_type == "new_complete" + assert event.data["clear_screen"] is True + assert event.data["session_id"] == session.session_id + assert [msg.role for msg in session.context_manager.items] == ["system"] + assert [e["event_type"] for e in session.logged_events] == ["new_complete"] + + +@pytest.mark.asyncio +async def test_new_and_clear_slash_commands_share_operation_with_distinct_clear_flags(): + submission_id = [0] + holder = [object()] + + new_submission = await main_mod._handle_slash_command( + "/new", + _FakeConfig(), + holder, + asyncio.Queue(), + submission_id, + ) + clear_submission = await main_mod._handle_slash_command( + "/clear", + _FakeConfig(), + holder, + asyncio.Queue(), + submission_id, + ) + + assert new_submission is not None + assert new_submission.operation.op_type == OpType.NEW + assert new_submission.operation.data == {"clear_screen": False} + assert clear_submission is not None + assert clear_submission.operation.op_type == OpType.NEW + assert clear_submission.operation.data == {"clear_screen": True} + assert submission_id == [2] diff --git a/tests/unit/test_cli_rendering.py b/tests/unit/test_cli_rendering.py index f9228cec8895aa365fdcef52ffcd3adcc5f088b7..fdbff93bc9f8e1c66bee8f9d42e36a8c72bb9646 100644 --- a/tests/unit/test_cli_rendering.py +++ b/tests/unit/test_cli_rendering.py @@ -13,22 +13,29 @@ from agent.tools.research_tool import _get_research_model from agent.utils import terminal_display -def test_direct_anthropic_research_model_stays_off_bedrock(): +def test_router_anthropic_research_model_is_unchanged(): assert ( - _get_research_model("anthropic/claude-opus-4-6") - == "anthropic/claude-sonnet-4-6" + _get_research_model("anthropic/claude-opus-4.8:fal-ai") + == "anthropic/claude-opus-4.8:fal-ai" ) -def test_bedrock_anthropic_research_model_stays_on_bedrock(): +def test_non_anthropic_research_model_is_unchanged(): + assert _get_research_model("openai/gpt-oss-120b") == "openai/gpt-oss-120b" + + +def test_huggingface_prefix_research_model_strips_prefix(): assert ( - _get_research_model("bedrock/us.anthropic.claude-opus-4-6-v1") - == "bedrock/us.anthropic.claude-sonnet-4-6" + _get_research_model("huggingface/anthropic/claude-opus-4.8:fal-ai") + == "anthropic/claude-opus-4.8:fal-ai" ) -def test_non_anthropic_research_model_is_unchanged(): - assert _get_research_model("openai/gpt-5.4") == "openai/gpt-5.4" +def test_hf_router_openai_research_model_is_unchanged(): + assert ( + _get_research_model("huggingface/openai/gpt-5.5:fal-ai") + == "openai/gpt-5.5:fal-ai" + ) def test_help_output_keeps_descriptions_aligned(monkeypatch): @@ -104,12 +111,12 @@ def test_cli_forwards_model_flag_to_interactive_main(monkeypatch): seen["model"] = model seen["sandbox_tools"] = sandbox_tools - monkeypatch.setattr(sys, "argv", ["ml-intern", "--model", "openai/gpt-5.5"]) + monkeypatch.setattr(sys, "argv", ["ml-intern", "--model", "openai/gpt-5.5:fal-ai"]) monkeypatch.setattr(main_mod, "main", fake_main) main_mod.cli() - assert seen["model"] == "openai/gpt-5.5" + assert seen["model"] == "openai/gpt-5.5:fal-ai" assert seen["sandbox_tools"] is False @@ -173,7 +180,7 @@ async def test_interactive_main_applies_model_override_before_banner(monkeypatch pass def fake_banner(*, model=None, hf_user=None, tool_runtime=None): - assert model == "openai/gpt-5.5" + assert model == "openai/gpt-5.5:fal-ai" assert hf_user == "tester" assert tool_runtime == "local filesystem" raise StopAfterBanner @@ -194,7 +201,7 @@ async def test_interactive_main_applies_model_override_before_banner(monkeypatch monkeypatch.setattr(main_mod, "print_banner", fake_banner) with pytest.raises(StopAfterBanner): - await main_mod.main(model="openai/gpt-5.5") + await main_mod.main(model="openai/gpt-5.5:fal-ai") @pytest.mark.asyncio diff --git a/tests/unit/test_compaction_loop_break.py b/tests/unit/test_compaction_loop_break.py index 2ce5ead521c2539b5595bee7bf5d61767919f192..8f79f52a328962ee25e32ddedc87c7857fe67b8f 100644 --- a/tests/unit/test_compaction_loop_break.py +++ b/tests/unit/test_compaction_loop_break.py @@ -38,6 +38,8 @@ from agent.context_manager.manager import ( _MAX_TOKENS_PER_MESSAGE, ) +ROUTER_CLAUDE = "anthropic/claude-opus-4.8:fal-ai" + # ── helpers ──────────────────────────────────────────────────────────── @@ -70,7 +72,7 @@ def test_truncate_oversized_skips_messages_below_threshold(): cm = _make_cm() msgs = [_msg("user", "small content")] with patch("litellm.token_counter", return_value=100): - out = cm._truncate_oversized(msgs, "anthropic/claude-opus-4-6") + out = cm._truncate_oversized(msgs, ROUTER_CLAUDE) assert out == msgs # unchanged @@ -80,7 +82,7 @@ def test_truncate_oversized_replaces_content_above_threshold(): msgs = [_msg("user", big)] # token_counter returns the simulated big size for any message in this test with patch("litellm.token_counter", return_value=_MAX_TOKENS_PER_MESSAGE * 2): - out = cm._truncate_oversized(msgs, "anthropic/claude-opus-4-6") + out = cm._truncate_oversized(msgs, ROUTER_CLAUDE) assert len(out) == 1 assert out[0].content != big assert "[truncated for compaction" in out[0].content @@ -88,10 +90,7 @@ def test_truncate_oversized_replaces_content_above_threshold(): def test_truncate_oversized_preserves_thinking_blocks(): - """Anthropic extended-thinking models reject the next request with - ``Invalid signature in thinking block`` if a prior assistant message - drops thinking_blocks. Truncation must keep this metadata. - """ + """Truncation must preserve assistant metadata attached by providers.""" cm = _make_cm() big = "x" * (_MAX_TOKENS_PER_MESSAGE * 5) thinking = [{"type": "thinking", "thinking": "...", "signature": "abc123"}] @@ -99,7 +98,7 @@ def test_truncate_oversized_preserves_thinking_blocks(): msg.thinking_blocks = thinking msg.reasoning_content = "deep thought" with patch("litellm.token_counter", return_value=_MAX_TOKENS_PER_MESSAGE * 2): - out = cm._truncate_oversized([msg], "anthropic/claude-opus-4-6") + out = cm._truncate_oversized([msg], ROUTER_CLAUDE) assert getattr(out[0], "thinking_blocks", None) == thinking assert getattr(out[0], "reasoning_content", None) == "deep thought" @@ -117,7 +116,7 @@ def test_truncate_oversized_never_touches_system_message(): huge_system = "x" * (_MAX_TOKENS_PER_MESSAGE * 5) msgs = [_msg("system", huge_system)] with patch("litellm.token_counter", return_value=_MAX_TOKENS_PER_MESSAGE * 2): - out = cm._truncate_oversized(msgs, "anthropic/claude-opus-4-6") + out = cm._truncate_oversized(msgs, ROUTER_CLAUDE) assert out[0].content == huge_system, "system message must never be truncated" @@ -129,7 +128,7 @@ def test_truncate_oversized_resilient_to_token_counter_failure(): cm = _make_cm() msgs = [_msg("user", "anything")] with patch("litellm.token_counter", side_effect=Exception("counter blew up")): - out = cm._truncate_oversized(msgs, "anthropic/claude-opus-4-6") + out = cm._truncate_oversized(msgs, ROUTER_CLAUDE) assert out == msgs @@ -174,7 +173,7 @@ async def test_compact_raises_when_post_compact_still_over_threshold(): ): with pytest.raises(CompactionFailedError): await cm.compact( - model_name="anthropic/claude-opus-4-6", + model_name=ROUTER_CLAUDE, tool_specs=None, hf_token=None, session=None, @@ -191,7 +190,7 @@ async def test_compact_does_not_duplicate_system_when_idx_is_zero(): walk-back ``while idx > 1`` loop is a no-op. Without an explicit clamp ``if idx < 1: idx = 1``, ``recent_messages = items[0:]`` starts at the system message, and the rebuild duplicates system + - first-user. Anthropic API rejects two system messages. + first-user. Chat providers can reject duplicate system messages. """ cm = _make_cm(model_max_tokens=100_000, untouched_messages=5) cm.items = [ @@ -220,7 +219,7 @@ async def test_compact_does_not_duplicate_system_when_idx_is_zero(): patch("litellm.token_counter", return_value=100), ): await cm.compact( - model_name="anthropic/claude-opus-4-6", + model_name=ROUTER_CLAUDE, tool_specs=None, hf_token=None, session=None, @@ -236,7 +235,7 @@ async def test_compact_does_not_duplicate_system_when_idx_is_zero(): # Bot review on PR #213 caught a follow-up bug: clamping idx=1 # excludes the system but still overlaps with first_user_idx (also 1), # so first_user_msg ends up in BOTH head and recent_messages → - # duplicate user message → Anthropic 400 (two consecutive user roles). + # duplicate user message → provider 400 (two consecutive user roles). task_count = sum( 1 for m in cm.items if m.role == "user" and (m.content or "") == "task" ) @@ -244,13 +243,13 @@ async def test_compact_does_not_duplicate_system_when_idx_is_zero(): f"Expected exactly 1 'task' user message, found {task_count}. " f"Roles+content: {[(m.role, (m.content or '')[:20]) for m in cm.items]}" ) - # Defense in depth: no two consecutive same-role messages (Anthropic - # API contract). System counts separately. + # Defense in depth: no two consecutive same-role messages. System counts + # separately. non_system = [m for m in cm.items if m.role != "system"] for i in range(1, len(non_system)): assert non_system[i].role != non_system[i - 1].role, ( f"Two consecutive {non_system[i].role} messages at non-system " - f"position {i - 1},{i} — Anthropic API rejects this. " + f"position {i - 1},{i} — chat providers reject this. " f"Roles: {[m.role for m in cm.items]}" ) @@ -285,7 +284,7 @@ async def test_compact_succeeds_when_post_compact_under_threshold(): patch("litellm.token_counter", return_value=100), ): await cm.compact( - model_name="anthropic/claude-opus-4-6", + model_name=ROUTER_CLAUDE, tool_specs=None, hf_token=None, session=None, @@ -308,7 +307,7 @@ async def test_compact_and_notify_terminates_session_on_failure(): session = MagicMock() session.session_id = "sess-123" session.is_running = True - session.config.model_name = "anthropic/claude-opus-4-6" + session.config.model_name = ROUTER_CLAUDE session.hf_token = None session.tool_router.get_tool_specs_for_llm.return_value = [] session.send_event = AsyncMock() @@ -344,7 +343,7 @@ async def test_compact_and_notify_passes_through_on_success(): session = MagicMock() session.session_id = "sess-456" session.is_running = True - session.config.model_name = "anthropic/claude-opus-4-6" + session.config.model_name = ROUTER_CLAUDE session.hf_token = None session.tool_router.get_tool_specs_for_llm.return_value = [] session.send_event = AsyncMock() diff --git a/tests/unit/test_dangling_tool_calls.py b/tests/unit/test_dangling_tool_calls.py index b4215f173b30deb947095ea9d8ac6a0eca58fe35..8901f99df0bc3ec4ff41fe9b9493033bc52896cb 100644 --- a/tests/unit/test_dangling_tool_calls.py +++ b/tests/unit/test_dangling_tool_calls.py @@ -2,9 +2,8 @@ Reproduces the failure mode behind observatory sessions 8dd2ce30 and 59c9e678 (2026-04-25): a tool call cancelled mid-execution leaves an -orphan ``tool_use`` in history; the user types a follow-up; Bedrock -rejects the next request with HTTP 400 ``messages.N: tool_use ids were -found without tool_result blocks immediately after``. +orphan tool call in history; the user types a follow-up; providers reject the +next request with HTTP 400 because tool calls are missing matching tool results. """ from litellm import ChatCompletionMessageToolCall, Message @@ -105,7 +104,7 @@ def test_orphan_in_earlier_turn_still_gets_patched(): """Two-turn history where the FIRST turn was interrupted. Old patcher stopped at the first user msg encountered while scanning - backwards, so this case never got fixed and Bedrock rejected. + backwards, so this case never got fixed and providers rejected the request. """ cm = _make_cm() cm.items.extend( diff --git a/tests/unit/test_heartbeat.py b/tests/unit/test_heartbeat.py index 56161be801d97fcdacff96cac297e696251a9406..f98ca660bf8929d69b91d2493b722c762a9f6c70 100644 --- a/tests/unit/test_heartbeat.py +++ b/tests/unit/test_heartbeat.py @@ -14,7 +14,7 @@ from agent.core.session import Event, Session class _FakeConfig: - model_name = "claude-opus-4-6" + model_name = "anthropic/claude-opus-4.8:fal-ai" save_sessions = True session_dataset_repo = "fake/repo" auto_save_interval = 1 diff --git a/tests/unit/test_llm_error_classification.py b/tests/unit/test_llm_error_classification.py index 8bcd54fd20119e614a51cffe62ea3896186cb19d..a90089680208c1bdfe0637882554f1073356e108 100644 --- a/tests/unit/test_llm_error_classification.py +++ b/tests/unit/test_llm_error_classification.py @@ -2,12 +2,12 @@ Covers two regressions on 2026-04-25: -1. Non-Anthropic context overflow (Kimi 365k > 262k) was not classified as +1. Router context overflow (Kimi 365k > 262k) was not classified as ``_is_context_overflow_error``, so the recovery path didn't fire and session 62ccfdcb died with 68 wasted compaction events. -2. Bedrock TPM rate limit (`Too many tokens, please wait before trying - again.`) needs the longer rate-limit retry schedule. The old schedule +2. Provider token-bucket rate limits (`Too many tokens, please wait before + trying again.`) need the longer rate-limit retry schedule. The old schedule ([5, 15, 30] = 50s) burned through 6 sessions costing >$2,400 combined on the same day. """ @@ -48,10 +48,10 @@ def test_random_error_is_not_context_overflow(): # ── rate limit ────────────────────────────────────────────────────────── -def test_bedrock_too_many_tokens_is_rate_limit(): +def test_provider_too_many_tokens_is_rate_limit(): # Verbatim from sessions b37a3823, c4d7a831, b63c4933 (2026-04-25). err = Exception( - 'litellm.RateLimitError: BedrockException - {"message":"Too many ' + 'litellm.RateLimitError: ProviderException - {"message":"Too many ' 'tokens, please wait before trying again."}' ) assert _is_rate_limit_error(err) @@ -95,8 +95,8 @@ def test_non_transient_returns_none(): assert _retry_delay_for(err, 0) is None -def test_rate_limit_total_budget_covers_bedrock_bucket_recovery(): +def test_rate_limit_total_budget_covers_token_bucket_recovery(): """The whole point of the rate-limit schedule: total wait time should - exceed the ~60s Bedrock TPM bucket recovery window.""" + exceed a typical ~60s provider token-bucket recovery window.""" assert len(_LLM_RATE_LIMIT_RETRY_DELAYS) == _MAX_LLM_RETRIES - 1 assert sum(_LLM_RATE_LIMIT_RETRY_DELAYS) > 60 diff --git a/tests/unit/test_llm_params.py b/tests/unit/test_llm_params.py index a7c7b4cd7df83e4a5427b1f35dad90aeb3b7beaa..3f38a5a7cb0c798f4175c6c2145689e5072263f5 100644 --- a/tests/unit/test_llm_params.py +++ b/tests/unit/test_llm_params.py @@ -6,30 +6,107 @@ from agent.core.llm_params import ( _resolve_hf_router_token, _resolve_llm_params, ) +from agent.core.model_ids import HF_ROUTER_BASE_URL -def test_openai_xhigh_effort_is_forwarded(): +def test_hf_router_params_for_default_premium_model(monkeypatch): + monkeypatch.setenv("INFERENCE_TOKEN", "inference-token") + monkeypatch.setenv("HF_BILL_TO", "smolagents") + params = _resolve_llm_params( - "openai/gpt-5.5", - reasoning_effort="xhigh", + "anthropic/claude-opus-4.8:fal-ai", + "session-token", + reasoning_effort="high", strict=True, ) - assert params["model"] == "openai/gpt-5.5" - assert params["reasoning_effort"] == "xhigh" + assert params == { + "model": "openai/anthropic/claude-opus-4.8:fal-ai", + "api_base": HF_ROUTER_BASE_URL, + "api_key": "inference-token", + "extra_headers": {"X-HF-Bill-To": "smolagents"}, + "extra_body": {"reasoning_effort": "high"}, + } -def test_openai_max_effort_is_still_rejected(): - try: +def test_hf_router_rejects_max_effort_in_strict_mode(): + with pytest.raises(UnsupportedEffortError, match="HF Router"): _resolve_llm_params( - "openai/gpt-5.4", + "anthropic/claude-opus-4.8:fal-ai", reasoning_effort="max", strict=True, ) - except UnsupportedEffortError as exc: - assert "OpenAI doesn't accept effort='max'" in str(exc) - else: - raise AssertionError("Expected UnsupportedEffortError for max effort") + + +def test_hf_router_drops_unsupported_effort_in_non_strict_mode(monkeypatch): + monkeypatch.setenv("HF_TOKEN", "hf-token") + + params = _resolve_llm_params( + "anthropic/claude-opus-4.8:fal-ai", + reasoning_effort="max", + strict=False, + ) + + assert params["api_base"] == HF_ROUTER_BASE_URL + assert params["api_key"] == "hf-token" + assert "extra_body" not in params + + +def test_user_billed_premium_uses_session_token_without_bill_to(monkeypatch): + monkeypatch.setenv("INFERENCE_TOKEN", "inference-token") + monkeypatch.setenv("HF_BILL_TO", "smolagents") + + params = _resolve_llm_params( + "anthropic/claude-opus-4.8:fal-ai", + "session-token", + reasoning_effort="high", + strict=True, + bill_to_user=True, + ) + + assert params["model"] == "openai/anthropic/claude-opus-4.8:fal-ai" + assert params["api_base"] == HF_ROUTER_BASE_URL + assert params["api_key"] == "session-token" + assert "extra_headers" not in params + assert params["extra_body"] == {"reasoning_effort": "high"} + + +def test_user_billed_premium_does_not_fall_back_to_cached_token(monkeypatch): + import huggingface_hub + + monkeypatch.setenv("INFERENCE_TOKEN", "inference-token") + monkeypatch.setenv("HF_TOKEN", "server-token") + monkeypatch.setattr(huggingface_hub, "get_token", lambda: "cached-token") + + params = _resolve_llm_params( + "anthropic/claude-opus-4.8:fal-ai", + None, + bill_to_user=True, + ) + + assert params["api_key"] is None + assert "extra_headers" not in params + + +def test_bill_to_user_ignored_for_free_models(monkeypatch): + monkeypatch.setenv("INFERENCE_TOKEN", "inference-token") + monkeypatch.setenv("HF_BILL_TO", "smolagents") + + params = _resolve_llm_params( + "moonshotai/Kimi-K2.6", "session-token", bill_to_user=True + ) + + assert params["api_key"] == "inference-token" + assert params["extra_headers"] == {"X-HF-Bill-To": "smolagents"} + + +def test_huggingface_prefix_is_stripped_for_router_calls(monkeypatch): + monkeypatch.setenv("INFERENCE_TOKEN", "inference-token") + + params = _resolve_llm_params("huggingface/openai/gpt-5.5:fal-ai") + + assert params["model"] == "openai/openai/gpt-5.5:fal-ai" + assert params["api_base"] == HF_ROUTER_BASE_URL def test_resolve_ollama_params_adds_v1_and_uses_default_key(monkeypatch): diff --git a/tests/unit/test_prioritize_backlog.py b/tests/unit/test_prioritize_backlog.py index 9a8fd316795d3ac6e784259af99f44bbc2e83cd3..2eaf55450e2d7b072a2cf148fed22062c5726f9b 100644 --- a/tests/unit/test_prioritize_backlog.py +++ b/tests/unit/test_prioritize_backlog.py @@ -691,7 +691,7 @@ def test_render_markdown_report_from_sample_ranking(): ranking, records, generated_at="2026-05-04T10:00:00+00:00", - model="openai/gpt-5.5", + model="openai/gpt-5.5:fal-ai", ) assert "# ML Intern Backlog Prioritization" in report diff --git a/tests/unit/test_redact.py b/tests/unit/test_redact.py index 24c14253633b4751eaf3e16bd6cf06a900773641..6ccd25cf82d8dfef56ae39e871b5433effa8d48e 100644 --- a/tests/unit/test_redact.py +++ b/tests/unit/test_redact.py @@ -10,7 +10,7 @@ def test_hf_token(): assert "[REDACTED_HF_TOKEN]" in out -def test_anthropic_key(): +def test_provider_key(): s = "key=sk-ant-api03_" + "a" * 40 out = scrub_string(s) # The env-var name prefix matches too; just verify we don't leave the body. diff --git a/tests/unit/test_sandbox_auto_start.py b/tests/unit/test_sandbox_auto_start.py index c6051c177f740f366a02c77fea1242593ddf82cb..d92a5fbdb39bd1c25cd241436775cfc5b149f5e3 100644 --- a/tests/unit/test_sandbox_auto_start.py +++ b/tests/unit/test_sandbox_auto_start.py @@ -112,7 +112,7 @@ async def test_cli_sandbox_runtime_preloads_and_tears_down_sandbox(monkeypatch): event_queue = asyncio.Queue() session_holder = [None] config = Config.model_validate( - {"model_name": "openai/gpt-5.5", "save_sessions": False} + {"model_name": "openai/gpt-5.5:fal-ai", "save_sessions": False} ) task = asyncio.create_task( diff --git a/tests/unit/test_session_manager_persistence.py b/tests/unit/test_session_manager_persistence.py index 59016eca816b484971600f6f2e6ee83a2bd107c9..55c149b074e5f2bbb87d5711ee7920c96b611dcb 100644 --- a/tests/unit/test_session_manager_persistence.py +++ b/tests/unit/test_session_manager_persistence.py @@ -16,6 +16,7 @@ _BACKEND_DIR = Path(__file__).resolve().parent.parent.parent / "backend" if str(_BACKEND_DIR) not in sys.path: sys.path.insert(0, str(_BACKEND_DIR)) +from agent.core.model_ids import DEFAULT_MODEL_ID, KIMI_K26_MODEL_ID # noqa: E402 from agent.core.session_persistence import NoopSessionStore # noqa: E402 from session_manager import AgentSession, SessionManager # noqa: E402 @@ -105,6 +106,8 @@ def _manager_with_store(store: NoopSessionStore) -> SessionManager: manager._lock = asyncio.Lock() manager.persistence_store = store manager.messaging_gateway = CloseableResource() + manager._pending_creates = 0 + manager._reaper_task = None return manager @@ -125,6 +128,34 @@ def _runtime_agent_session( ) +def test_unknown_saved_model_defaults_to_claude(): + model, premium_user_billed, claude_counted = ( + SessionManager._model_from_saved_metadata( + "unsupported/model", + premium_user_billed=False, + claude_counted=False, + ) + ) + + assert model == DEFAULT_MODEL_ID + assert premium_user_billed is False + assert claude_counted is False + + +def test_unknown_saved_user_billed_model_defaults_to_free_model(): + model, premium_user_billed, claude_counted = ( + SessionManager._model_from_saved_metadata( + "unsupported/model", + premium_user_billed=True, + claude_counted=True, + ) + ) + + assert model == KIMI_K26_MODEL_ID + assert premium_user_billed is False + assert claude_counted is False + + @pytest.mark.asyncio async def test_update_session_auto_approval_defaults_to_five_dollars(): manager = _manager_with_store(NoopSessionStore()) @@ -626,6 +657,87 @@ async def test_lazy_restore_preserves_auto_approval_policy(): await _cancel_runtime_tasks(manager) +@pytest.mark.asyncio +async def test_lazy_restore_injects_sandbox_reset_note_when_session_had_sandbox(): + store = RestoreStore( + metadata={ + "session_id": "had-sandbox", + "user_id": "owner", + "model": "test-model", + "sandbox_status": "destroyed", + } + ) + manager = _manager_with_store(store) + stop = _install_fake_runtime(manager) + + try: + restored = await manager.ensure_session_loaded("had-sandbox", user_id="owner") + + assert restored is not None + items = restored.session.context_manager.items + assert len(items) == 1 + assert "sandbox was reset" in items[0].content + finally: + stop.set() + await _cancel_runtime_tasks(manager) + + +@pytest.mark.asyncio +async def test_lazy_restore_skips_sandbox_reset_note_when_no_sandbox(): + store = RestoreStore( + metadata={ + "session_id": "no-sandbox", + "user_id": "owner", + "model": "test-model", + } + ) + manager = _manager_with_store(store) + stop = _install_fake_runtime(manager) + + try: + restored = await manager.ensure_session_loaded("no-sandbox", user_id="owner") + + assert restored is not None + assert restored.session.context_manager.items == [] + finally: + stop.set() + await _cancel_runtime_tasks(manager) + + +@pytest.mark.asyncio +async def test_lazy_restore_skips_sandbox_note_when_pending_approval(): + """The sandbox-reset note must be skipped when an approval is pending: it + would land between the restored assistant tool-calls and their results, + orphaning the tool results on approval (the provider rejects the ordering).""" + store = RestoreStore( + metadata={ + "session_id": "had-sandbox-pending", + "user_id": "owner", + "model": "test-model", + "sandbox_status": "destroyed", + "pending_approval": [ + {"tool": "bash", "tool_call_id": "call_1", "arguments": {}} + ], + } + ) + manager = _manager_with_store(store) + stop = _install_fake_runtime(manager) + + try: + restored = await manager.ensure_session_loaded( + "had-sandbox-pending", user_id="owner" + ) + + assert restored is not None + items = restored.session.context_manager.items + assert not any("sandbox was reset" in getattr(m, "content", "") for m in items) + # The pending approval itself is still restored. + assert restored.session.pending_approval is not None + finally: + stop.set() + await _cancel_runtime_tasks(manager) + + @pytest.mark.asyncio async def test_list_sessions_dev_uses_store_dev_visibility(): class ListStore(NoopSessionStore): @@ -643,6 +755,8 @@ async def test_list_sessions_dev_uses_store_dev_visibility(): "user_id": "alice", "model": "m", "created_at": datetime.now(UTC), + "premium_user_billed": True, + "claude_counted": True, "auto_approval_enabled": True, "auto_approval_cost_cap_usd": 5.0, "auto_approval_estimated_spend_usd": 2.0, @@ -664,6 +778,8 @@ async def test_list_sessions_dev_uses_store_dev_visibility(): assert store.seen_user_id == "dev" assert {session["session_id"] for session in sessions} == {"s1", "s2"} yolo = next(session for session in sessions if session["session_id"] == "s1") + assert yolo["premium_user_billed"] is True + assert yolo["premium_quota_counted"] is True assert yolo["auto_approval"] == { "enabled": True, "cost_cap_usd": 5.0, diff --git a/tests/unit/test_session_reaper.py b/tests/unit/test_session_reaper.py new file mode 100644 index 0000000000000000000000000000000000000000..0af8c3248beeef2b2e46d1e8ffd521188fb61406 --- /dev/null +++ b/tests/unit/test_session_reaper.py @@ -0,0 +1,551 @@ +"""Tests for the idle-session reaper and the global create-slot reservation. + +Covers Parts B (hard global cap), C (idle reaper + activity stamps + safe +teardown + submit/reap race), and E (per-user concurrent cap interacts with +reaping) of the session-limit fix. +""" + +from __future__ import annotations + +import asyncio +import sys +from datetime import datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +_BACKEND_DIR = Path(__file__).resolve().parent.parent.parent / "backend" +if str(_BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(_BACKEND_DIR)) + +import session_manager as sm # noqa: E402 +from agent.core.session_persistence import NoopSessionStore # noqa: E402 +from session_manager import ( # noqa: E402 + AgentSession, + Operation, + SessionCapacityError, + SessionManager, +) +from agent.core.session import OpType # noqa: E402 + + +class RecordingStore(NoopSessionStore): + """Captures every save_snapshot call so tests can assert persistence.""" + + enabled = True + + def __init__(self) -> None: + self.snapshots: list[dict[str, Any]] = [] + + async def save_snapshot(self, **kwargs: Any) -> None: + self.snapshots.append(kwargs) + + def snapshots_for(self, session_id: str) -> list[dict[str, Any]]: + return [s for s in self.snapshots if s.get("session_id") == session_id] + + +class FakeSession: + """Minimal Session stand-in supporting both persistence and _run_session.""" + + def __init__(self, *, hf_token: str | None = "token") -> None: + self.hf_token = hf_token + self.context_manager = SimpleNamespace(items=[]) + self.pending_approval: Any = None + self.turn_count = 0 + self.config = SimpleNamespace(model_name="test-model", save_sessions=False) + self.notification_destinations: list[str] = [] + self.auto_approval_enabled = False + self.auto_approval_cost_cap_usd = None + self.auto_approval_estimated_spend_usd = 0.0 + self.is_running = True + + async def send_event(self, event: Any) -> None: + return None + + +class FakeToolRouter: + async def __aenter__(self) -> "FakeToolRouter": + return self + + async def __aexit__(self, *exc: Any) -> bool: + return False + + +def _manager() -> SessionManager: + manager = object.__new__(SessionManager) + manager.config = SimpleNamespace(model_name="test-model") + manager.sessions = {} + manager._lock = asyncio.Lock() + manager.persistence_store = RecordingStore() + manager.messaging_gateway = SimpleNamespace() + manager._pending_creates = 0 + manager._reaper_task = None + return manager + + +def _make_agent_session( + session_id: str, + *, + user_id: str = "owner", + last_active_at: datetime | None = None, + is_processing: bool = False, + pending_approval: Any = None, +) -> AgentSession: + session = FakeSession() + session.pending_approval = pending_approval + agent_session = AgentSession( + session_id=session_id, + session=session, # type: ignore[arg-type] + tool_router=FakeToolRouter(), # type: ignore[arg-type] + submission_queue=asyncio.Queue(), + user_id=user_id, + hf_token="token", + ) + agent_session.is_processing = is_processing + if last_active_at is not None: + agent_session.last_active_at = last_active_at + return agent_session + + +async def _start_real_run_session( + manager: SessionManager, + session_id: str, + *, + user_id: str = "owner", + last_active_at: datetime | None = None, +) -> AgentSession: + """Insert a session and start the REAL _run_session task for it.""" + agent_session = _make_agent_session( + session_id, user_id=user_id, last_active_at=last_active_at + ) + event_queue: asyncio.Queue = asyncio.Queue() + await manager._start_agent_session( + agent_session=agent_session, + event_queue=event_queue, + tool_router=agent_session.tool_router, + ) + await asyncio.sleep(0) # let the run loop reach its queue wait + return agent_session + + +def _install_fake_create(manager: SessionManager) -> asyncio.Event: + """Replace blocking constructors + run loop with fakes for create tests.""" + stop = asyncio.Event() + + def fake_create_session_sync(**kwargs: Any): + return object(), FakeSession(hf_token=kwargs.get("hf_token")) + + async def fake_run_session(*_: Any) -> None: + await stop.wait() + + manager._create_session_sync = fake_create_session_sync # type: ignore[method-assign] + manager._run_session = fake_run_session # type: ignore[method-assign] + manager._start_cpu_sandbox_preload = lambda _agent_session: None # type: ignore[method-assign] + return stop + + +async def _cancel_tasks(manager: SessionManager) -> None: + tasks = [ + agent_session.task + for agent_session in manager.sessions.values() + if agent_session.task and not agent_session.task.done() + ] + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + +# ── Reaper happy path ─────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_reaper_evicts_idle_session_as_resumable(): + manager = _manager() + cleaned: list[Any] = [] + + async def fake_cleanup(session: Any) -> None: + cleaned.append(session) + + manager._cleanup_sandbox = fake_cleanup # type: ignore[method-assign] + + agent_session = await _start_real_run_session( + manager, + "stale", + last_active_at=datetime.utcnow() - timedelta(hours=3), + ) + session = agent_session.session + + await manager._reap_idle_sessions() + + # Evicted from the live pool, sandbox torn down by the task's finally. + assert "stale" not in manager.sessions + assert cleaned == [session] + + # Persisted resumable (status="active", runtime_state="idle"), never "ended". + store = manager.persistence_store + snapshots = store.snapshots_for("stale") + assert snapshots, "reaper should persist a snapshot" + assert all(s["status"] == "active" for s in snapshots) + assert snapshots[-1]["runtime_state"] == "idle" + assert not any(s["status"] == "ended" for s in snapshots) + + +# ── Spared sessions ───────────────────────────────────────────────────── + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kwargs", + [ + # Fresh: touched just now. + {"last_active_at": None}, + # Currently processing a turn. + { + "last_active_at": datetime.utcnow() - timedelta(hours=5), + "is_processing": True, + }, + # Awaiting tool approval ("approve later", not idle). + { + "last_active_at": datetime.utcnow() - timedelta(hours=5), + "pending_approval": {"tool_calls": [object()]}, + }, + # Dev sessions are never reaped. + {"last_active_at": datetime.utcnow() - timedelta(hours=5), "user_id": "dev"}, + ], + ids=["fresh", "processing", "pending_approval", "dev"], +) +async def test_reaper_spares(kwargs): + manager = _manager() + agent_session = _make_agent_session("spared", **kwargs) + manager.sessions["spared"] = agent_session + + await manager._reap_idle_sessions() + + assert "spared" in manager.sessions + assert agent_session.is_reaping is False + + +# ── Submit / reap race ────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_reap_aborts_when_message_enqueued_first(): + """A message enqueued before teardown makes the queue non-empty, so the + reaper's re-check aborts — the message is never lost.""" + manager = _manager() + agent_session = _make_agent_session( + "racing", last_active_at=datetime.utcnow() - timedelta(hours=3) + ) + manager.sessions["racing"] = agent_session + agent_session.submission_queue.put_nowait(object()) + + cutoff = datetime.utcnow() - sm.REAPER_IDLE + reaped = await manager._reap_one("racing", cutoff) + + assert reaped is False + assert "racing" in manager.sessions + assert agent_session.is_reaping is False + assert agent_session.submission_queue.qsize() == 1 + + +@pytest.mark.asyncio +async def test_submit_rejected_while_reaping(): + """submit() refuses a session being reaped instead of silently enqueuing + onto a dying runtime (the caller then reloads a fresh one).""" + manager = _manager() + agent_session = _make_agent_session("reaping") + agent_session.is_reaping = True + manager.sessions["reaping"] = agent_session + + ok = await manager.submit("reaping", Operation(op_type=OpType.USER_INPUT, data={})) + + assert ok is False + assert agent_session.submission_queue.empty() + + +# ── Turn-finish timestamp ─────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_turn_finish_restamps_activity(monkeypatch): + """A turn that runs longer than the idle window must not be reaped the + instant it finishes — the turn-finish stamp resets the idle clock.""" + manager = _manager() + + async def fake_cleanup(session: Any) -> None: + return None + + manager._cleanup_sandbox = fake_cleanup # type: ignore[method-assign] + + agent_session = await _start_real_run_session( + manager, "longturn", last_active_at=datetime.utcnow() + ) + + async def fake_process(session: Any, submission: Any) -> bool: + # Simulate a turn that has been running far longer than the idle + # window before it completes. + agent_session.last_active_at = datetime.utcnow() - timedelta(hours=3) + return True + + monkeypatch.setattr(sm, "process_submission", fake_process) + + agent_session.submission_queue.put_nowait( + sm.Submission(id="s1", operation=Operation(op_type=OpType.USER_INPUT, data={})) + ) + + # Wait for the turn-finish stamp to land. + for _ in range(200): + await asyncio.sleep(0.01) + if datetime.utcnow() - agent_session.last_active_at < timedelta(minutes=1): + break + + assert datetime.utcnow() - agent_session.last_active_at < timedelta(minutes=1) + + await _cancel_tasks(manager) + + +# ── Global reservation race (Part B) ──────────────────────────────────── + + +@pytest.mark.asyncio +async def test_concurrent_creates_cannot_exceed_global_cap(monkeypatch): + manager = _manager() + stop = _install_fake_create(manager) + monkeypatch.setattr(sm, "MAX_SESSIONS", 3) + + try: + results = await asyncio.gather( + *[manager.create_session(user_id="owner") for _ in range(10)], + return_exceptions=True, + ) + created = [r for r in results if isinstance(r, str)] + errors = [r for r in results if isinstance(r, SessionCapacityError)] + + assert len(created) == 3 + assert len(errors) == 7 + assert all(e.error_type == "global" for e in errors) + assert len(manager.sessions) == 3 + assert manager._pending_creates == 0 + finally: + stop.set() + await _cancel_tasks(manager) + + +@pytest.mark.asyncio +async def test_failed_build_releases_reservation(monkeypatch): + manager = _manager() + _install_fake_create(manager) + monkeypatch.setattr(sm, "MAX_SESSIONS", 5) + + def boom(**_: Any): + raise RuntimeError("build failed") + + manager._create_session_sync = boom # type: ignore[method-assign] + + with pytest.raises(RuntimeError): + await manager.create_session(user_id="owner") + + # The reservation must be released so a failed create can't shrink the pool. + assert manager._pending_creates == 0 + assert manager.sessions == {} + + +# ── Per-user concurrent cap interacts with reclaimed slots (Part E) ────── + + +@pytest.mark.asyncio +async def test_per_user_cap_frees_up_after_slot_reclaimed(): + manager = _manager() + stop = _install_fake_create(manager) + + try: + for i in range(sm.MAX_SESSIONS_PER_USER): + manager.sessions[f"owner-{i}"] = _make_agent_session( + f"owner-{i}", user_id="owner" + ) + + # At the concurrent cap → rejected. + with pytest.raises(SessionCapacityError) as exc: + await manager.create_session(user_id="owner") + assert exc.value.error_type == "per_user" + + # Reclaiming a slot (the reaper evicts an idle session) frees capacity. + manager.sessions.pop("owner-0") + new_id = await manager.create_session(user_id="owner") + + assert isinstance(new_id, str) + assert manager._count_user_sessions("owner") == sm.MAX_SESSIONS_PER_USER + finally: + stop.set() + await _cancel_tasks(manager) + + +# ── Persistence safety (resumability invariant) ────────────────────────── + + +@pytest.mark.asyncio +async def test_reaper_skips_when_persistence_disabled(): + """With no usable store a reaped session couldn't be restored, so eviction + would destroy non-dev chats outright. The sweep must be a no-op.""" + manager = _manager() + manager.persistence_store = NoopSessionStore() # enabled = False + agent_session = _make_agent_session( + "idle", last_active_at=datetime.utcnow() - timedelta(hours=5) + ) + manager.sessions["idle"] = agent_session + + await manager._reap_idle_sessions() + + assert "idle" in manager.sessions + assert agent_session.is_reaping is False + + +@pytest.mark.asyncio +async def test_reap_aborts_when_snapshot_write_fails(): + """If the resumable snapshot can't be written (e.g. a transient Mongo + error), abort rather than evict unrecoverable state — leave it live.""" + manager = _manager() + + class FailingStore(RecordingStore): + async def save_snapshot(self, **kwargs: Any) -> None: + raise RuntimeError("mongo write failed") + + manager.persistence_store = FailingStore() + agent_session = _make_agent_session( + "idle", last_active_at=datetime.utcnow() - timedelta(hours=5) + ) + manager.sessions["idle"] = agent_session + + cutoff = datetime.utcnow() - sm.REAPER_IDLE + reaped = await manager._reap_one("idle", cutoff) + + assert reaped is False + assert "idle" in manager.sessions + assert agent_session.is_reaping is False + + +@pytest.mark.asyncio +async def test_reap_aborts_when_message_write_fails_silently(): + """The real store swallows message bulk_write errors for best-effort callers + and only surfaces them when asked to be strict. The reaper must request + strict mode, so a silent message-write failure still aborts the reap (not + only metadata/connection failures that already make save_snapshot raise).""" + manager = _manager() + + class StrictModeStore(RecordingStore): + # Mirrors MongoSessionStore: message-write failure is swallowed unless + # the caller passes raise_on_error (which the reaper does). + async def save_snapshot( + self, *, raise_on_error: bool = False, **kwargs: Any + ) -> None: + if raise_on_error: + raise RuntimeError("message bulk_write failed") + + manager.persistence_store = StrictModeStore() + agent_session = _make_agent_session( + "idle", last_active_at=datetime.utcnow() - timedelta(hours=5) + ) + manager.sessions["idle"] = agent_session + + cutoff = datetime.utcnow() - sm.REAPER_IDLE + reaped = await manager._reap_one("idle", cutoff) + + assert reaped is False + assert "idle" in manager.sessions + assert agent_session.is_reaping is False + + +# ── Reap / restore race ────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_old_reaped_task_does_not_end_freshly_restored_session(): + """If a user reopens after the reaper pops the old wrapper but before the + old task finishes sandbox cleanup, the old finally must not mark the fresh + wrapper ended.""" + manager = _manager() + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + + async def slow_cleanup(session: Any) -> None: + cleanup_started.set() + await release_cleanup.wait() + + manager._cleanup_sandbox = slow_cleanup # type: ignore[method-assign] + + old = await _start_real_run_session( + manager, + "restore-race", + last_active_at=datetime.utcnow() - timedelta(hours=3), + ) + cutoff = datetime.utcnow() - sm.REAPER_IDLE + reap_task = asyncio.create_task(manager._reap_one("restore-race", cutoff)) + + for _ in range(100): + await asyncio.sleep(0.01) + if "restore-race" not in manager.sessions and cleanup_started.is_set(): + break + + assert "restore-race" not in manager.sessions + assert cleanup_started.is_set() + + fresh = _make_agent_session("restore-race") + async with manager._lock: + manager.sessions["restore-race"] = fresh + + release_cleanup.set() + assert await reap_task is True + if old.task is not None: + assert old.task.done() + + assert manager.sessions["restore-race"] is fresh + assert fresh.is_active is True + assert all( + snapshot.get("status") != "ended" + for snapshot in manager.persistence_store.snapshots_for("restore-race") + ) + + +# ── Shutdown safety (cancellation must propagate) ──────────────────────── + + +@pytest.mark.asyncio +async def test_reaper_teardown_propagates_outer_cancellation(): + """Cancelling the reaper while it awaits a slow teardown must propagate, so + close() can't hang. Regression for the CancelledError-conflation bug: the + old wait_for + bare-except swallowed the reaper's own cancellation.""" + manager = _manager() + + release = asyncio.Event() + + async def slow_cleanup(session: Any) -> None: + await release.wait() # block teardown so _reap_one parks in the wait + + manager._cleanup_sandbox = slow_cleanup # type: ignore[method-assign] + + agent_session = await _start_real_run_session( + manager, "slow", last_active_at=datetime.utcnow() - timedelta(hours=3) + ) + cutoff = datetime.utcnow() - sm.REAPER_IDLE + + reap_task = asyncio.create_task(manager._reap_one("slow", cutoff)) + # Let _reap_one persist + pop, then enter the teardown wait (the session + # task is stuck in slow_cleanup, so the wait won't complete on its own). + for _ in range(100): + await asyncio.sleep(0.01) + if "slow" not in manager.sessions: + break + await asyncio.sleep(0.02) + + # Simulate close() cancelling the reaper mid-teardown. + reap_task.cancel() + with pytest.raises(asyncio.CancelledError): + await reap_task + + # Unblock the stuck teardown and reap the orphaned session task. + release.set() + if agent_session.task is not None: + await asyncio.gather(agent_session.task, return_exceptions=True) diff --git a/tests/unit/test_session_resume.py b/tests/unit/test_session_resume.py index 6bb8c673de988179f883461683d0f47565282a5b..1a33f5ae2eda2b28e3084473bab158f789f6ad51 100644 --- a/tests/unit/test_session_resume.py +++ b/tests/unit/test_session_resume.py @@ -10,6 +10,8 @@ from litellm import Message from agent.core import session_resume +ROUTER_GPT_55 = "openai/gpt-5.5:fal-ai" + def _write_session_log( directory: Path, @@ -29,7 +31,7 @@ def _write_session_log( "user_id": user_id, "session_start_time": "2026-01-01T00:00:00", "session_end_time": "2026-01-01T00:05:00", - "model_name": "openai/gpt-5.5", + "model_name": ROUTER_GPT_55, "messages": [ {"role": "system", "content": "old system"}, {"role": "user", "content": content}, @@ -114,10 +116,10 @@ def test_restore_continues_when_user_id_matches(tmp_path): assert result["restored_count"] == 1 assert result["dropped_count"] == 0 assert result["forked"] is False - assert result["model_name"] == "openai/gpt-5.5" + assert result["model_name"] == ROUTER_GPT_55 assert result["had_redacted_content"] is False assert result["invalid_saved_model"] is None - assert session.config.model_name == "openai/gpt-5.5" + assert session.config.model_name == ROUTER_GPT_55 assert session.session_id == "saved-session" # Source log path is never reused: future heartbeat saves write to a # fresh file so the snapshot stays intact (regression: see source-log @@ -130,7 +132,7 @@ def test_restore_continues_when_user_id_matches(tmp_path): assert session.context_manager.items[0].content == "current system" assert session.context_manager.items[1].content == "continue this work" assert session.context_manager.running_context_usage == 123 - assert session.context_manager.recompute_calls == ["openai/gpt-5.5"] + assert session.context_manager.recompute_calls == [ROUTER_GPT_55] assert len(session.logged_events) == 1 marker = session.logged_events[0] assert marker["event_type"] == "resumed_from" @@ -240,7 +242,7 @@ def test_restore_counts_dropped_messages(tmp_path): { "session_id": "saved", "user_id": "user-a", - "model_name": "openai/gpt-5.5", + "model_name": ROUTER_GPT_55, "messages": [ {"role": "user", "content": "hi"}, {"role": "user", "content": 12345}, # invalid content type @@ -279,7 +281,7 @@ def test_restore_does_not_overwrite_source_log_on_save(tmp_path, monkeypatch): "user_id": "user-a", "session_start_time": "2026-01-01T00:00:00", "session_end_time": "2026-01-01T00:05:00", - "model_name": "openai/gpt-5.5", + "model_name": ROUTER_GPT_55, "messages": [ {"role": "system", "content": "old system"}, {"role": "user", "content": "earlier work"}, @@ -293,7 +295,7 @@ def test_restore_does_not_overwrite_source_log_on_save(tmp_path, monkeypatch): src_bytes_before = src_path.read_bytes() class _Cfg: - model_name = "openai/gpt-5.5" + model_name = ROUTER_GPT_55 save_sessions = True session_dataset_repo = None auto_save_interval = 1 diff --git a/tests/unit/test_session_uploader.py b/tests/unit/test_session_uploader.py index 82f5503b575f6581331c782d97aaa008bbd148fd..a8bf0b367223f656eaa90ba13818a15e0c29f097 100644 --- a/tests/unit/test_session_uploader.py +++ b/tests/unit/test_session_uploader.py @@ -12,7 +12,7 @@ from agent.core.session_uploader import ( ) HF_SECRET = "hf_" + "a" * 30 -ANTHROPIC_SECRET = "sk-ant-" + "b" * 24 +PROVIDER_SECRET = "sk-ant-" + "b" * 24 GITHUB_SECRET = "ghp_" + "c" * 36 @@ -103,7 +103,7 @@ def test_claude_code_jsonl_uses_message_timestamps(): events = to_claude_code_jsonl( { "session_id": "session-123", - "model_name": "anthropic/claude-opus-4-6", + "model_name": "anthropic/claude-opus-4.8:fal-ai", "session_start_time": "2026-01-01T00:00:00", "messages": [ { @@ -140,10 +140,10 @@ def test_row_payload_scrubs_messages_events_and_tools(tmp_path): "user_id": "lewtun", "session_start_time": "2026-01-01T00:00:00", "session_end_time": "2026-01-01T00:00:03", - "model_name": "anthropic/claude-opus-4-6", + "model_name": "anthropic/claude-opus-4.8:fal-ai", "total_cost_usd": 0.01, "messages": [{"role": "user", "content": f"token {HF_SECRET}"}], - "events": [{"type": "debug", "content": f"key {ANTHROPIC_SECRET}"}], + "events": [{"type": "debug", "content": f"key {PROVIDER_SECRET}"}], "tools": [{"name": "bash", "env": f"GITHUB_TOKEN={GITHUB_SECRET}"}], } @@ -151,10 +151,10 @@ def test_row_payload_scrubs_messages_events_and_tools(tmp_path): payload = tmp_file.read_text() assert HF_SECRET not in payload - assert ANTHROPIC_SECRET not in payload + assert PROVIDER_SECRET not in payload assert GITHUB_SECRET not in payload assert "[REDACTED_HF_TOKEN]" in payload - assert "[REDACTED_ANTHROPIC_KEY]" in payload + assert "[REDACTED_PROVIDER_API_KEY]" in payload assert "GITHUB_TOKEN=[REDACTED]" in payload @@ -162,7 +162,7 @@ def test_claude_code_payload_scrubs_messages_before_conversion(tmp_path): tmp_file = tmp_path / "claude_code.jsonl" data = { "session_id": "session-123", - "model_name": "anthropic/claude-opus-4-6", + "model_name": "anthropic/claude-opus-4.8:fal-ai", "session_start_time": "2026-01-01T00:00:00", "messages": [ { @@ -178,7 +178,7 @@ def test_claude_code_payload_scrubs_messages_before_conversion(tmp_path): "id": "call-1", "function": { "name": "bash", - "arguments": json.dumps({"key": ANTHROPIC_SECRET}), + "arguments": json.dumps({"key": PROVIDER_SECRET}), }, } ], @@ -197,8 +197,8 @@ def test_claude_code_payload_scrubs_messages_before_conversion(tmp_path): payload = tmp_file.read_text() assert HF_SECRET not in payload - assert ANTHROPIC_SECRET not in payload + assert PROVIDER_SECRET not in payload assert GITHUB_SECRET not in payload assert "[REDACTED_HF_TOKEN]" in payload - assert "[REDACTED_ANTHROPIC_KEY]" in payload + assert "[REDACTED_PROVIDER_API_KEY]" in payload assert "GITHUB_TOKEN=[REDACTED]" in payload diff --git a/tests/unit/test_sft_tagger.py b/tests/unit/test_sft_tagger.py index cf02e7b8a35a4fef3cc476bf2eed18c87f75fbdc..8d28782d09ebfbefb384334cfd42f2e5d9625e54 100644 --- a/tests/unit/test_sft_tagger.py +++ b/tests/unit/test_sft_tagger.py @@ -7,7 +7,7 @@ def _ev(event_type, data=None, ts="2026-04-24T10:00:00"): return {"timestamp": ts, "event_type": event_type, "data": data or {}} -def _traj(events=None, messages=None, model="claude-opus-4-6"): +def _traj(events=None, messages=None, model="anthropic/claude-opus-4.8:fal-ai"): return { "session_id": "sess-1", "model_name": model, @@ -20,8 +20,7 @@ def _traj(events=None, messages=None, model="claude-opus-4-6"): def test_model_family(): - assert "model:opus" in tag_session(_traj(model="claude-opus-4-6")) - assert "model:sonnet" in tag_session(_traj(model="bedrock/claude-sonnet-4-5")) + assert "model:opus" in tag_session(_traj(model="anthropic/claude-opus-4.8:fal-ai")) assert "model:kimi" in tag_session(_traj(model="moonshotai/Kimi-K2.6")) assert "model:other" in tag_session(_traj(model="unknown-model-xyz")) diff --git a/tests/unit/test_thinking_history.py b/tests/unit/test_thinking_history.py index 6ec92958e18d44b602e779cfeaea55c8f0e8ea5a..525533bc5f6eb24c5c117eccb25a332b6e26e197 100644 --- a/tests/unit/test_thinking_history.py +++ b/tests/unit/test_thinking_history.py @@ -3,264 +3,67 @@ from types import SimpleNamespace import pytest from litellm import ChatCompletionMessageToolCall, Message -from agent.core import agent_loop from agent.core.agent_loop import ( LLMResult, - _call_llm_streaming, _assistant_message_from_result, - _extract_thinking_state, + _call_llm_streaming, + _strip_thinking_state_from_messages, ) -def test_extract_thinking_state_from_litellm_message(): - message = Message( - role="assistant", - content="working", - thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], - reasoning_content="reasoned", - ) - - thinking_blocks, reasoning_content = _extract_thinking_state(message) - - assert thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] - assert reasoning_content == "reasoned" - - -def test_extract_thinking_state_from_provider_fields(): - message = SimpleNamespace( - provider_specific_fields={ - "thinking_blocks": [{"type": "thinking", "thinking": "reasoned"}], - "reasoning_content": "reasoned", - }, - ) - - thinking_blocks, reasoning_content = _extract_thinking_state(message) - - assert thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] - assert reasoning_content == "reasoned" - - -def test_assistant_message_from_result_preserves_thinking_with_tool_calls(): +def test_assistant_message_from_result_keeps_content_and_tool_calls(): tool_call = ChatCompletionMessageToolCall( id="call_1", type="function", function={"name": "bash", "arguments": '{"command": "date"}'}, ) result = LLMResult( - content=None, + content="working", tool_calls_acc={}, token_count=12, finish_reason="tool_calls", - thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], - reasoning_content="reasoned", ) - message = _assistant_message_from_result( - result, - model_name="anthropic/claude-opus-4-6", - tool_calls=[tool_call], - ) + message = _assistant_message_from_result(result, tool_calls=[tool_call]) + assert message.content == "working" assert message.tool_calls == [tool_call] - assert message.thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] - assert message.reasoning_content == "reasoned" - - -def test_assistant_message_from_result_strips_non_anthropic_reasoning_content(): - result = LLMResult( - content=None, - tool_calls_acc={}, - token_count=12, - finish_reason="tool_calls", - thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], - reasoning_content="reasoned", - ) - - message = _assistant_message_from_result( - result, - model_name="openai/Qwen/Qwen3-Next-80B-A3B-Instruct", - ) - - assert getattr(message, "thinking_blocks", None) is None - assert getattr(message, "reasoning_content", None) is None - - -def test_assistant_message_from_result_omits_absent_thinking_fields(): - result = LLMResult( - content="done", - tool_calls_acc={}, - token_count=12, - finish_reason="stop", - ) - - message = _assistant_message_from_result( - result, - model_name="anthropic/claude-opus-4-6", - ) - - assert message.content == "done" assert getattr(message, "thinking_blocks", None) is None assert getattr(message, "reasoning_content", None) is None -@pytest.mark.asyncio -async def test_streaming_call_rebuilds_anthropic_thinking_state(monkeypatch): - async def fake_stream(): - yield SimpleNamespace( - choices=[ - SimpleNamespace( - delta=SimpleNamespace(content="done", tool_calls=None), - finish_reason="stop", - ) - ], - ) - yield SimpleNamespace(choices=[], usage=SimpleNamespace(total_tokens=3)) - - async def fake_acompletion(**_kwargs): - return fake_stream() - - def fake_chunk_builder(chunks, **_kwargs): - assert len(chunks) == 2 - return SimpleNamespace( - choices=[ - SimpleNamespace( - message=Message( - role="assistant", - content="done", - thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], - reasoning_content="reasoned", - ) - ) - ] - ) - - events = [] - - async def send_event(event): - events.append(event) - - session = SimpleNamespace( - config=SimpleNamespace(model_name="anthropic/claude-opus-4-6"), - is_cancelled=False, - send_event=send_event, - ) - monkeypatch.setattr(agent_loop, "acompletion", fake_acompletion) - monkeypatch.setattr(agent_loop, "stream_chunk_builder", fake_chunk_builder) - - result = await _call_llm_streaming( - session, - messages=[Message(role="user", content="hi")], - tools=[], - llm_params={"model": "anthropic/claude-opus-4-6"}, - ) - - assert result.content == "done" - assert result.thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] - assert result.reasoning_content == "reasoned" - - -@pytest.mark.asyncio -async def test_streaming_call_rebuilds_anthropic_delta_thinking_state(monkeypatch): - async def fake_stream(): - yield SimpleNamespace( - choices=[ - SimpleNamespace( - delta=SimpleNamespace( - content=None, - tool_calls=None, - thinking_blocks=[ - { - "type": "thinking", - "thinking": "reasoned", - "signature": "", - } - ], - ), - finish_reason=None, - ) - ], - ) - yield SimpleNamespace( - choices=[ - SimpleNamespace( - delta=SimpleNamespace( - content=None, - tool_calls=None, - thinking_blocks=[ - { - "type": "thinking", - "thinking": "", - "signature": "signed", - } - ], - ), - finish_reason=None, - ) - ], - ) - yield SimpleNamespace( - choices=[ - SimpleNamespace( - delta=SimpleNamespace(content="done", tool_calls=None), - finish_reason="stop", - ) +def test_strip_thinking_state_from_saved_messages(): + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "stale"}, + {"type": "text", "text": "done"}, ], - ) - yield SimpleNamespace(choices=[], usage=SimpleNamespace(total_tokens=3)) - - async def fake_acompletion(**_kwargs): - return fake_stream() - - def fake_chunk_builder(chunks, **_kwargs): - assert len(chunks) == 4 - return SimpleNamespace( - choices=[ - SimpleNamespace( - message=Message( - role="assistant", - content="done", - thinking_blocks=[ - { - "type": "thinking", - "thinking": "reasoned", - "signature": "signed", - } - ], - reasoning_content="reasoned", - ) - ) - ] - ) - - events = [] - - async def send_event(event): - events.append(event) + "thinking_blocks": [{"type": "thinking", "thinking": "stale"}], + "reasoning_content": "stale", + "provider_specific_fields": { + "thinking_blocks": [{"type": "thinking", "thinking": "stale"}], + "reasoning_content": "stale", + "other": "kept", + }, + } + ] - session = SimpleNamespace( - config=SimpleNamespace(model_name="anthropic/claude-opus-4-7"), - is_cancelled=False, - send_event=send_event, - ) - monkeypatch.setattr(agent_loop, "acompletion", fake_acompletion) - monkeypatch.setattr(agent_loop, "stream_chunk_builder", fake_chunk_builder) + stripped = _strip_thinking_state_from_messages(messages) - result = await _call_llm_streaming( - session, - messages=[Message(role="user", content="hi")], - tools=[], - llm_params={"model": "anthropic/claude-opus-4-7"}, - ) - - assert result.content == "done" - assert result.thinking_blocks == [ - {"type": "thinking", "thinking": "reasoned", "signature": "signed"} + assert stripped == 5 + assert messages == [ + { + "role": "assistant", + "content": [{"type": "text", "text": "done"}], + "provider_specific_fields": {"other": "kept"}, + } ] - assert result.reasoning_content == "reasoned" @pytest.mark.asyncio -async def test_streaming_call_skips_chunk_rebuild_for_non_anthropic(monkeypatch): +async def test_streaming_call_returns_wire_safe_result(monkeypatch): async def fake_stream(): yield SimpleNamespace( choices=[ @@ -270,33 +73,29 @@ async def test_streaming_call_skips_chunk_rebuild_for_non_anthropic(monkeypatch) ) ], ) + yield SimpleNamespace(choices=[], usage=SimpleNamespace(total_tokens=3)) async def fake_acompletion(**_kwargs): return fake_stream() - def fail_chunk_builder(*_args, **_kwargs): - raise AssertionError("stream_chunk_builder should not run") - events = [] async def send_event(event): events.append(event) session = SimpleNamespace( - config=SimpleNamespace(model_name="openai/Qwen/Qwen3"), + config=SimpleNamespace(model_name="anthropic/claude-opus-4.8:fal-ai"), is_cancelled=False, send_event=send_event, ) - monkeypatch.setattr(agent_loop, "acompletion", fake_acompletion) - monkeypatch.setattr(agent_loop, "stream_chunk_builder", fail_chunk_builder) + monkeypatch.setattr("agent.core.agent_loop.acompletion", fake_acompletion) result = await _call_llm_streaming( session, messages=[Message(role="user", content="hi")], tools=[], - llm_params={"model": "openai/Qwen/Qwen3"}, + llm_params={"model": "openai/anthropic/claude-opus-4.8:fal-ai"}, ) assert result.content == "done" - assert result.thinking_blocks is None - assert result.reasoning_content is None + assert result.token_count == 3 diff --git a/tests/unit/test_user_quotas.py b/tests/unit/test_user_quotas.py index 840e442feed728a2678ebe57ada5b0f3217a00c0..e4af629c8c5fb51d6d17fc5291cd025b083c9851 100644 --- a/tests/unit/test_user_quotas.py +++ b/tests/unit/test_user_quotas.py @@ -1,4 +1,4 @@ -"""Tests for backend/user_quotas.py — the in-memory Claude daily-quota store.""" +"""Tests for backend/user_quotas.py — the in-memory premium quota store.""" import asyncio import sys @@ -121,11 +121,13 @@ async def test_refund_on_stale_day_resets_rather_than_underflow(): @pytest.mark.asyncio -async def test_free_user_cap_reached_at_one(): +async def test_free_user_cap_reached_at_two(): cap = user_quotas.daily_cap_for("free") + assert cap == 2 + assert await user_quotas.increment_claude("freebie") == 1 used = await user_quotas.increment_claude("freebie") - assert used == 1 - assert used >= cap # first bump exhausts the free tier (cap=1) + assert used == 2 + assert used >= cap @pytest.mark.asyncio