Fix HF output limits and requirements feedback routing

src/qa_platform/llm_runtime.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import json
 import os
 import socket
+import time
 from dataclasses import dataclass
 from importlib import import_module
 from pathlib import Path
@@ -18,7 +19,7 @@ class LLMRuntimeError(RuntimeError):
 
 HTTP_TIMEOUT_SECONDS = 55
 OLLAMA_DISCOVERY_TIMEOUT_SECONDS = 2
-HF_PROVIDER_MAX_ATTEMPTS = 1
+HF_OUTPUT_TOKEN_BUDGET = 8192
 HF_MODEL_FAMILY_MAP = {
     "Qwen 3 32B": "Qwen/Qwen3-32B",
     "Qwen3-30B-A3B": "Qwen/Qwen3-30B-A3B",
@@ -189,124 +190,89 @@ def _call_hf_structured_llm(
     schema: dict[str, Any],
     temperature: float,
 ) -> tuple[dict[str, Any], LLMCallMetadata]:
-    token = os.environ.get("HF_TOKEN", "").strip()
-    if not token:
-        raise LLMRuntimeError("HF_TOKEN is required for Hugging Face live LLM execution.")
-
+    endpoint, api_key = _resolve_endpoint_and_key(runtime_config)
+    model_id = resolve_live_model_id(runtime_config)
+    # Keep the selected provider strategy throughout retries, and share one budget.
+    deadline = time.monotonic() + max(0.1, runtime_config.timeout_seconds - 1)
     errors: list[str] = []
-    for model_id in _candidate_model_ids(runtime_config):
-        for attempt in range(1, HF_PROVIDER_MAX_ATTEMPTS + 1):
-            try:
-                content, usage = _post_hf_inference_chat_completion(
-                    api_key=token,
-                    model_id=model_id,
-                    messages=[
-                        {
-                            "role": "system",
-                            "content": (
-                                system_prompt.strip()
-                                + "\nReturn only valid JSON matching the requested schema. "
-                                + "Do not add markdown fences or commentary."
-                            ),
-                        },
-                        {
-                            "role": "user",
-                            "content": (
-                                user_prompt.strip()
-                                + "\nJSON schema to satisfy exactly:\n"
-                                + json.dumps(
-                                    {
-                                        "name": schema_name,
-                                        "schema": schema,
-                                    },
-                                    ensure_ascii=False,
-                                )
-                            ),
-                        },
-                    ],
-                    temperature=temperature,
-                )
-                return _extract_json_object(content), LLMCallMetadata(
-                    endpoint="huggingface_hub.InferenceClient.chat_completion",
-                    model_id=model_id,
-                    provider_strategy=runtime_config.provider_strategy,
-                    response_format_used=False,
-                    usage=usage,
-                )
-            except Exception as exc:  # noqa: BLE001
-                errors.append(
-                    f"{model_id} inference-client attempt {attempt}/{HF_PROVIDER_MAX_ATTEMPTS}: {exc}"
-                )
-                break
 
-    endpoint, api_key = _resolve_endpoint_and_key(runtime_config)
+    def remaining_timeout() -> float:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise LLMRuntimeError(
+                f"HF call budget exhausted for {runtime_config.agent_name}. "
+                f"Attempts: {' | '.join(errors)}"
+            )
+        return remaining
+
+    json_messages = [
+        {
+            "role": "system",
+            "content": (
+                system_prompt.strip()
+                + "\nReturn only valid JSON matching the requested schema. "
+                + "Do not add markdown fences or commentary."
+            ),
+        },
+        {
+            "role": "user",
+            "content": (
+                user_prompt.strip()
+                + "\nJSON schema to satisfy exactly:\n"
+                + json.dumps({"name": schema_name, "schema": schema}, ensure_ascii=False)
+            ),
+        },
+    ]
+    try:
+        content, usage = _post_hf_inference_chat_completion(
+            api_key=api_key,
+            model_id=model_id,
+            messages=json_messages,
+            temperature=temperature,
+            timeout_seconds=remaining_timeout(),
+        )
+        return _extract_json_object(content), LLMCallMetadata(
+            endpoint="huggingface_hub.InferenceClient.chat_completion",
+            model_id=model_id,
+            provider_strategy=runtime_config.provider_strategy,
+            response_format_used=False,
+            usage=usage,
+        )
+    except Exception as exc:  # noqa: BLE001
+        errors.append(f"{model_id} inference-client attempt: {exc}")
+
     response_format = {
         "type": "json_schema",
-        "json_schema": {
-            "name": schema_name,
-            "schema": schema,
-            "strict": True,
-        },
+        "json_schema": {"name": schema_name, "schema": schema, "strict": True},
     }
-    messages = [
-        {"role": "system", "content": system_prompt.strip()},
-        {"role": "user", "content": user_prompt.strip()},
-    ]
-    for model_id in _candidate_model_ids(runtime_config):
-        for attempt in range(1, HF_PROVIDER_MAX_ATTEMPTS + 1):
-            fallback_system_prompt = (
-                system_prompt.strip()
-                + "\nReturn only valid JSON matching the requested schema. Do not add markdown fences or commentary."
+    for format_value in (response_format, None):
+        timeout_seconds = remaining_timeout()
+        try:
+            payload = _post_chat_completion(
+                endpoint=endpoint,
+                api_key=api_key,
+                model_id=model_id,
+                messages=(
+                    [{"role": "system", "content": system_prompt.strip()},
+                     {"role": "user", "content": user_prompt.strip()}]
+                    if format_value else json_messages
+                ),
+                temperature=temperature,
+                response_format=format_value,
+                timeout_seconds=timeout_seconds,
+                max_tokens=HF_OUTPUT_TOKEN_BUDGET,
             )
-            fallback_user_prompt = (
-                user_prompt.strip()
-                + "\nJSON schema to satisfy exactly:\n"
-                + json.dumps(schema, ensure_ascii=False)
+            if (payload.get("choices") or [{}])[0].get("finish_reason") == "length":
+                raise LLMRuntimeError("HF response exhausted its 8192-token output budget.")
+            return _extract_json_object(_extract_message_content(payload)), LLMCallMetadata(
+                endpoint=endpoint,
+                model_id=model_id,
+                provider_strategy=runtime_config.provider_strategy,
+                response_format_used=bool(format_value),
+                usage=payload.get("usage") or {},
             )
-            try:
-                payload = _post_chat_completion(
-                    endpoint=endpoint,
-                    api_key=api_key,
-                    model_id=model_id,
-                    messages=messages,
-                    temperature=temperature,
-                    response_format=response_format,
-                )
-                content = _extract_message_content(payload)
-                return json.loads(content), LLMCallMetadata(
-                    endpoint=endpoint,
-                    model_id=model_id,
-                    provider_strategy=runtime_config.provider_strategy,
-                    response_format_used=True,
-                    usage=payload.get("usage") or {},
-                )
-            except (LLMRuntimeError, json.JSONDecodeError) as first_error:
-                errors.append(
-                    f"{model_id} router structured-output attempt {attempt}/{HF_PROVIDER_MAX_ATTEMPTS}: {first_error}"
-                )
-                try:
-                    fallback_payload = _post_chat_completion(
-                        endpoint=endpoint,
-                        api_key=api_key,
-                        model_id=model_id,
-                        messages=[
-                            {"role": "system", "content": fallback_system_prompt},
-                            {"role": "user", "content": fallback_user_prompt},
-                        ],
-                        temperature=temperature,
-                        response_format=None,
-                    )
-                    content = _extract_message_content(fallback_payload)
-                    return _extract_json_object(content), LLMCallMetadata(
-                        endpoint=endpoint,
-                        model_id=model_id,
-                        provider_strategy=runtime_config.provider_strategy,
-                        response_format_used=False,
-                        usage=fallback_payload.get("usage") or {},
-                    )
-                except Exception as second_error:  # noqa: BLE001
-                    errors.append(f"{model_id} router fallback attempt: {second_error}")
-                break
+        except Exception as exc:  # noqa: BLE001
+            errors.append(f"{model_id} router {'structured' if format_value else 'JSON'} attempt: {exc}")
 
     raise LLMRuntimeError(
         f"Live LLM call failed for {runtime_config.agent_name}. Attempts: {' | '.join(errors)}"
@@ -407,6 +373,7 @@ def _post_hf_inference_chat_completion(
     model_id: str,
     messages: list[dict[str, str]],
     temperature: float,
+    timeout_seconds: float = HTTP_TIMEOUT_SECONDS,
 ) -> tuple[str, dict[str, Any]]:
     try:
         huggingface_hub = import_module("huggingface_hub")
@@ -415,21 +382,23 @@ def _post_hf_inference_chat_completion(
             "huggingface_hub is not installed. Install it to use Hugging Face live execution."
         ) from exc
 
-    client = huggingface_hub.InferenceClient(model=model_id, token=api_key)
+    client = huggingface_hub.InferenceClient(model=model_id, token=api_key, timeout=timeout_seconds)
     try:
         response = client.chat_completion(
             messages=messages,
-            max_tokens=2000,
+            max_tokens=HF_OUTPUT_TOKEN_BUDGET,
             temperature=temperature,
         )
     except (TimeoutError, socket.timeout) as exc:
         raise LLMRuntimeError(
-            f"Hugging Face InferenceClient timed out after {HTTP_TIMEOUT_SECONDS} seconds."
+            f"Hugging Face InferenceClient timed out after {timeout_seconds:g} seconds."
         ) from exc
     except Exception as exc:  # noqa: BLE001
         raise LLMRuntimeError(str(exc)) from exc
 
     choice = response.choices[0]
+    if getattr(choice, "finish_reason", None) == "length":
+        raise LLMRuntimeError("HF response exhausted its 8192-token output budget.")
     message = getattr(choice, "message", None)
     content = None
     if message is not None:
@@ -455,6 +424,8 @@ def _post_chat_completion(
     messages: list[dict[str, str]],
     temperature: float,
     response_format: dict[str, Any] | None,
+    timeout_seconds: float = HTTP_TIMEOUT_SECONDS,
+    max_tokens: int | None = None,
 ) -> dict[str, Any]:
     body: dict[str, Any] = {
         "model": model_id,
@@ -462,6 +433,8 @@ def _post_chat_completion(
         "stream": False,
         "temperature": temperature,
     }
+    if max_tokens is not None:
+        body["max_tokens"] = max_tokens
     if response_format:
         body["response_format"] = response_format
 
@@ -474,7 +447,7 @@ def _post_chat_completion(
     encoded = json.dumps(body).encode("utf-8")
     req = request.Request(endpoint, data=encoded, headers=headers, method="POST")
     try:
-        with request.urlopen(req, timeout=HTTP_TIMEOUT_SECONDS) as response:
+        with request.urlopen(req, timeout=timeout_seconds) as response:
             return json.loads(response.read().decode("utf-8"))
     except error.HTTPError as exc:
         detail = exc.read().decode("utf-8", errors="replace")
@@ -485,7 +458,7 @@ def _post_chat_completion(
         raise LLMRuntimeError(f"Could not reach model endpoint {endpoint}: {exc.reason}") from exc
     except (TimeoutError, socket.timeout) as exc:
         raise LLMRuntimeError(
-            f"Model endpoint {endpoint} timed out after {HTTP_TIMEOUT_SECONDS} seconds."
+            f"Model endpoint {endpoint} timed out after {timeout_seconds:g} seconds."
         ) from exc
 
 

src/qa_platform/orchestrator.py

@@ -914,12 +914,11 @@ class OrchestratorAgent:
                 "feedback_messages": review.improvement_actions[:2]
                 or ["Strengthen oracle quality and expected results in the current test designs."],
             }
-        if "requirements" in routing_focus and "test_design" not in routing_focus:
+        if "requirements" in routing_focus:
             return {
                 "from_agent": "Review Agent",
                 "to_agent": "Requirements Analyst",
-                "feedback_messages": review.improvement_actions[:2]
-                or ["Clarify ambiguous requirements and reduce assumptions before redesign."],
+                "feedback_messages": self._requirements_review_feedback(review),
             }
 
         weak_oracle_findings = [finding for finding in review.findings if "weak oracle definition" in finding]
@@ -1007,6 +1006,25 @@ class OrchestratorAgent:
 
         return None
 
+    def _requirements_review_feedback(self, review) -> list[str]:
+        clarification_words = (
+            "clarify", "define", "specify", "document", "resolve",
+            "reduce assumptions", "remove assumptions",
+        )
+        # Prefer clarification requests over test actions that merely mention assumptions.
+        targeted = [
+            action for action in review.improvement_actions
+            if any(word in action.lower() for word in clarification_words)
+        ]
+        if targeted:
+            return targeted[:2]
+        requirement_words = ("requirement", "acceptance criteria", "assumption", "ambigu", "policy", "rules")
+        findings = [
+            finding for finding in review.findings
+            if any(word in finding.lower() for word in requirement_words)
+        ]
+        return findings[:2] or ["Clarify ambiguous requirements and reduce assumptions before redesign."]
+
     def _normalize_routing_focus(
         self,
         raw_focus: list[str],
@@ -1021,10 +1039,17 @@ class OrchestratorAgent:
                 focus.add("requirements")
             elif normalized in {"test_design", "test design", "designer", "test design agent"}:
                 focus.add("test_design")
+        # Explicit agent routing takes precedence over words in the explanation.
+        if focus:
+            return focus
         if any(
             keyword in combined
             for keyword in (
                 "assumption",
+                "ambiguity",
+                "ambiguities",
+                "undefined",
+                "unspecified",
                 "ambiguous requirement",
                 "unclear requirement",
                 "acceptance criteria",

tests/test_feedback_routing.py

@@ -0,0 +1,50 @@
+import sys
+import unittest
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
+from qa_platform.models import ReviewReport
+from qa_platform.orchestrator import OrchestratorAgent
+
+
+class FeedbackRoutingTests(unittest.TestCase):
+    def route(self, focus, finding, action):
+        orchestrator = OrchestratorAgent()
+        orchestrator.reviewer.last_execution = {'routing_focus': focus}
+        return orchestrator._select_review_backtrack_route([], ReviewReport(
+            approved=False, coverage_ratio=1.0, findings=[finding], improvement_actions=[action]))
+
+    def test_explicit_requirements_route_survives_oracle_wording(self):
+        route = self.route(['requirements'], 'Unclear requirement causes weak oracle.', 'Clarify requirement before revising test case oracle.')
+        self.assertEqual(route['to_agent'], 'Requirements Analyst')
+
+    def test_inferred_mixed_problems_fix_requirements_first(self):
+        route = self.route([], 'Ambiguous requirement causes weak oracle.', 'Clarify requirement before revising test case oracle.')
+        self.assertEqual(route['to_agent'], 'Requirements Analyst')
+
+    def test_explicit_design_route_is_not_overridden_by_explanatory_keywords(self):
+        route = self.route(['test_design'], 'Oracle fails to check acceptance criteria.', 'Revise the test case.')
+        self.assertEqual(route['to_agent'], 'Test Design Agent')
+
+    def test_explicit_mixed_route_fixes_requirements_first(self):
+        route = self.route(['test_design', 'requirements'], 'Unclear requirement and weak oracle.', 'Clarify requirement.')
+        self.assertEqual(route['to_agent'], 'Requirements Analyst')
+
+    def test_requirements_feedback_filters_out_earlier_test_actions(self):
+        orchestrator = OrchestratorAgent()
+        orchestrator.reviewer.last_execution = {'routing_focus': ['requirements', 'test_design']}
+        route = orchestrator._select_review_backtrack_route([], ReviewReport(
+            approved=False, coverage_ratio=1.0, findings=['Unclear requirement and weak oracle.'],
+            improvement_actions=['Enhance test oracle with server checks.', 'Clarify acceptance criteria for registration.']))
+        self.assertEqual(route['feedback_messages'], ['Clarify acceptance criteria for registration.'])
+
+    def test_undefined_policy_returns_to_requirements(self):
+        route = self.route([], 'Password policy is undefined and the oracle is weak.', 'Document the password rules.')
+        self.assertEqual(route['to_agent'], 'Requirements Analyst')
+
+    def test_test_action_mentioning_assumptions_does_not_take_clarification_slot(self):
+        orchestrator = OrchestratorAgent()
+        orchestrator.reviewer.last_execution = {'routing_focus': ['requirements', 'test_design']}
+        route = orchestrator._select_review_backtrack_route([], ReviewReport(
+            approved=False, coverage_ratio=1.0, findings=['Password policy is unclear.'],
+            improvement_actions=['Create a test covering security assumptions.', 'Clarify password policy in the requirements.']))
+        self.assertEqual(route['feedback_messages'], ['Clarify password policy in the requirements.'])

tests/test_hf_runtime.py

@@ -0,0 +1,60 @@
+import json
+import os
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
+from qa_platform import llm_runtime as runtime
+from qa_platform.agent_runtime import build_agent_runtime_configs
+
+
+class HFRuntimeTests(unittest.TestCase):
+    def test_sdk_has_output_space_and_network_timeout(self):
+        client = Mock()
+        client.chat_completion.return_value = SimpleNamespace(
+            choices=[SimpleNamespace(finish_reason='stop', message=SimpleNamespace(content='{"ok":true}'))],
+            usage=SimpleNamespace(prompt_tokens=10, completion_tokens=2400, total_tokens=2410),
+        )
+        hub = SimpleNamespace(InferenceClient=Mock(return_value=client))
+        with patch.object(runtime, 'import_module', return_value=hub):
+            content, usage = runtime._post_hf_inference_chat_completion('secret', 'model:cheapest', [], .2, timeout_seconds=90)
+        self.assertTrue(json.loads(content)['ok'])
+        self.assertEqual(usage['completion_tokens'], 2400)
+        self.assertEqual(hub.InferenceClient.call_args.kwargs['timeout'], 90)
+        self.assertEqual(client.chat_completion.call_args.kwargs['max_tokens'], 8192)
+
+    def test_truncated_response_is_reported_even_if_json_looks_valid(self):
+        client = Mock()
+        client.chat_completion.return_value = SimpleNamespace(
+            choices=[SimpleNamespace(finish_reason='length', message=SimpleNamespace(content='{"ok":true}'))],
+        )
+        with patch.object(runtime, 'import_module', return_value=SimpleNamespace(InferenceClient=Mock(return_value=client))):
+            with self.assertRaisesRegex(runtime.LLMRuntimeError, 'output budget'):
+                runtime._post_hf_inference_chat_completion('secret', 'model', [], .2)
+
+    def test_fallback_keeps_provider_and_shares_remaining_budget(self):
+        config = build_agent_runtime_configs()[1]
+        with patch.dict(os.environ, {'HF_TOKEN': 'test-token'}), \
+             patch.object(runtime, 'load_local_env'), \
+             patch.object(runtime.time, 'monotonic', side_effect=[0, 0, 100]), \
+             patch.object(runtime, '_post_hf_inference_chat_completion', side_effect=runtime.LLMRuntimeError('provider unavailable')) as sdk, \
+             patch.object(runtime, '_post_chat_completion', return_value={'choices': [{'finish_reason': 'stop', 'message': {'content': '{"ok":true}'}}]}) as router:
+            payload, _ = runtime.call_structured_llm(config, 'Return JSON', 'Test', 'schema', {'type': 'object'})
+        self.assertTrue(payload['ok'])
+        self.assertEqual(sdk.call_args.kwargs['timeout_seconds'], 149)
+        self.assertEqual(router.call_args.kwargs['timeout_seconds'], 49)
+        self.assertEqual(router.call_args.kwargs['model_id'], config.model_id)
+
+    def test_no_fallback_starts_after_budget_exhaustion(self):
+        config = build_agent_runtime_configs()[1]
+        with patch.dict(os.environ, {'HF_TOKEN': 'test-token'}), \
+             patch.object(runtime, 'load_local_env'), \
+             patch.object(runtime.time, 'monotonic', side_effect=[0, 0, 150]), \
+             patch.object(runtime, '_post_hf_inference_chat_completion', side_effect=runtime.LLMRuntimeError('timeout')), \
+             patch.object(runtime, '_post_chat_completion') as router:
+            with self.assertRaisesRegex(runtime.LLMRuntimeError, 'budget exhausted'):
+                runtime.call_structured_llm(config, 'Return JSON', 'Test', 'schema', {'type': 'object'})
+        router.assert_not_called()

tests/test_service_feedback.py

@@ -0,0 +1,54 @@
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import app
+
+
+class ServiceFeedbackTests(unittest.TestCase):
+    def test_service_review_returns_to_requirements_before_redesign(self):
+        calls = []
+        requirement_calls = []
+        reviews = 0
+
+        def invoke(**kwargs):
+            nonlocal reviews
+            key = kwargs['service_agent_key']
+            calls.append(key)
+            if key == 'requirements_analyst':
+                requirement_calls.append(kwargs['agent_input']['feedback_messages'])
+                output = {'requirements': [{
+                    'requirement_id': 'REQ-001', 'original_text': 'The user can log in.',
+                    'normalized_text': 'The user can log in.', 'priority': 'medium',
+                    'acceptance_criteria': ['Valid credentials authenticate the user.'], 'assumptions': [],
+                }]}
+            elif key == 'test_designer':
+                output = {'test_designs': [{
+                    'test_case_id': 'TC-001', 'requirement_ids': ['REQ-001'],
+                    'title': 'Valid login', 'test_type': 'functional',
+                    'steps': ['Submit valid credentials.'], 'expected_results': ['Authenticated.'],
+                    'oracle': {'description': 'Authenticated user session is established.'},
+                }]}
+            else:
+                reviews += 1
+                output = {'review': {
+                    'approved': reviews > 1, 'coverage_ratio': 1.0,
+                    'findings': [] if reviews > 1 else [{
+                        'severity': 'major', 'target_type': 'test_case', 'target_id': 'TC-001',
+                        'issue': 'Ambiguous requirement produces a weak oracle.',
+                        'recommendation': 'Clarify requirement before revising the test case.',
+                    }],
+                    'improvement_actions': [] if reviews > 1 else ['Clarify requirement before revising the test case oracle.'],
+                }}
+            return {'status': 'completed', 'output': output, 'memory_updates': {}, 'errors': []}
+
+        configs = app.build_backend_agent_runtime_configs('QA agent service', app.DEFAULT_QA_AGENT_SERVICE_BASE_URL)
+        with patch.object(app, '_invoke_qa_agent_service', side_effect=invoke):
+            result = app.build_orchestrator_for_backend('QA agent service', app.DEFAULT_QA_AGENT_SERVICE_BASE_URL).process(
+                'Login', 'The user can log in.', agent_configs=configs,
+                run_controls=app.RunControlConfig(max_rounds=10, max_feedback_messages=12, max_feedback_per_agent_pair=4))
+        self.assertTrue(result.review.approved)
+        self.assertEqual(calls, ['requirements_analyst', 'test_designer', 'reviewer'] * 2)
+        self.assertEqual(result.pair_feedback_counts, {'Review Agent -> Requirements Analyst': 1})
+        self.assertTrue(requirement_calls[1])