Update app.py

app.py

@@ -694,14 +694,39 @@ async def call_openai(client: AsyncOpenAI, system: str, user: str, max_tokens: i
                 temperature=0.3,
             )
             content = resp.choices[0].message.content or "{}"
+            finish  = getattr(resp.choices[0], "finish_reason", None)
             try:
                 result = json.loads(content)
                 if "error" in result and isinstance(result["error"], dict):
                     raise ValueError(f"OpenAI returned error: {result['error']}")
                 return result
             except json.JSONDecodeError:
+                # A truncated answer is not malformed JSON in the usual sense:
+                # it is a correct answer that ran out of room, and the two need
+                # opposite fixes (raise max_tokens vs. fix the prompt). The
+                # previous version re-parsed a brace-matched slice with no
+                # guard, so a truncation raised JSONDecodeError from INSIDE the
+                # JSONDecodeError handler, escaped through `except Exception:
+                # raise`, and reached the browser as a bare HTTP 500 with no
+                # explanation anywhere.
                 m = re.search(r"\{[\s\S]*\}", content)
-                return json.loads(m.group(0)) if m else {"error": "Failed to parse response"}
+                if m:
+                    try:
+                        return json.loads(m.group(0))
+                    except json.JSONDecodeError:
+                        pass
+                if finish == "length":
+                    logger.error(
+                        "[OpenAI] %s hit max_tokens=%s (%d chars returned) — response truncated",
+                        _model, max_tokens, len(content))
+                    return {"error": (
+                        f"The model's answer was cut off at the {max_tokens}-token "
+                        f"limit before it could be completed. Try a shorter abstract, "
+                        f"or ask for fewer journals.")}
+                logger.error(
+                    "[OpenAI] %s returned unparseable JSON (finish_reason=%s): %.300s",
+                    _model, finish, content)
+                return {"error": "The model returned a malformed response. Please try again."}
         except RateLimitError as e:
             err_msg = str(e).lower()
             # Quota exceeded = billing cap, not a rate spike — don't retry
@@ -1075,7 +1100,8 @@ async def analyze_journal(req: JournalRequest, request: Request):
         "IMPORTANT: Return EXACTLY 10 detailed journals in journals array and EXACTLY 20 quick matches in extended_list (30 total).\n\n"
         "Return JSON only."
     )
-    parsed = await call_openai(client, JOURNAL_SYSTEM_PROMPT, user_msg)
+    parsed = await call_openai(client, JOURNAL_SYSTEM_PROMPT, user_msg,
+                               max_tokens=16000)
     if "error" in parsed:
         raise HTTPException(500, parsed["error"])
 
@@ -1416,6 +1442,22 @@ async def browse_subject(req: SubjectRequest, request: Request):
                        area, req.subject)
         area = ""
 
+    # The user's own words, checked against the closed vocabulary directly.
+    # This depends on nothing the model got right, and it is what makes an AREA
+    # query work at all: "Chemistry" is one of the 27 SCImago areas (ASJC 1600)
+    # and is NOT a category — the nearest category name is "Chemistry
+    # (miscellaneous)". A model that echoes "chemistry" into scimago_category
+    # therefore produces an unlisted name, and the refusal below used to fire
+    # on a subject the taxonomy covers perfectly well.
+    raw_subject    = req.subject.strip().lower()
+    typed_category = raw_subject if is_real_category(raw_subject) else ""
+    typed_area     = raw_subject if is_real_area(raw_subject) else ""
+    if not category and typed_category:
+        logger.info("[BrowseSubject] recovered category %r from the user's input", typed_category)
+        category = typed_category
+    if not area and typed_area:
+        area = typed_area
+
     # Suggestions the user can act on: the model's nearest names, hard-filtered
     # to real ones, topped up deterministically. Never invented.
     def _suggestions() -> list:
@@ -1439,7 +1481,16 @@ async def browse_subject(req: SubjectRequest, request: Request):
     # Refusing and naming the nearest real categories is both the honest answer
     # and the more useful one: Law, Sociology and Political Science, and
     # Pathology and Forensic Medicine are where criminology research is indexed.
-    if bad_category or (not category and not area):
+    #
+    # The test for "did the user ask for an area" is the USER'S INPUT against
+    # the closed vocabulary — never the model's output, which is exactly what
+    # used to conflate "chemistry" (a real area, a real request) with "crime"
+    # (no category, no area, nothing to answer with). Keying the refusal on
+    # bad_category alone meant a model that echoed a valid area name into the
+    # category field could veto its own correct answer.
+    have_category = bool(category)
+    have_area     = bool(area) and bool(typed_area)
+    if not have_category and not have_area:
         usage.usage.record_gap(
             "journal_subject",
             "category_invented" if bad_category else "subject_unrecognised")