Spaces:
Sleeping
Sleeping
fix: Improve LLM prompt and JSON response parsing
Browse files
main.py
CHANGED
|
@@ -497,12 +497,19 @@ SYSTEM_PROMPT: str = (
|
|
| 497 |
"Start directly with { and end with }. Escape all internal double quotes using \\\". "
|
| 498 |
"No unescaped newlines inside string fields.\n\n"
|
| 499 |
"REQUIRED JSON SCHEMA:\n"
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 506 |
)
|
| 507 |
|
| 508 |
|
|
@@ -542,9 +549,19 @@ def validate_report(parsed: dict) -> dict:
|
|
| 542 |
Validate the LLM response against the required 4-key schema and normalise types.
|
| 543 |
Raises ValueError on any schema violation.
|
| 544 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 545 |
missing = set(REQUIRED_KEYS) - parsed.keys()
|
| 546 |
if missing:
|
| 547 |
-
raise ValueError(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 548 |
|
| 549 |
for key, expected in REQUIRED_KEYS.items():
|
| 550 |
val = parsed[key]
|
|
|
|
| 497 |
"Start directly with { and end with }. Escape all internal double quotes using \\\". "
|
| 498 |
"No unescaped newlines inside string fields.\n\n"
|
| 499 |
"REQUIRED JSON SCHEMA:\n"
|
| 500 |
+
"Return a JSON object with the following structure. Replace the placeholders with your actual evaluation data:\n"
|
| 501 |
+
"{\n"
|
| 502 |
+
" \"score\": 82.50,\n"
|
| 503 |
+
" \"language\": \"Detected language or stack\",\n"
|
| 504 |
+
" \"code_quality\": \"Detailed overall quality assessment, 3-5 sentences...\",\n"
|
| 505 |
+
" \"questions\": [\n"
|
| 506 |
+
" {\n"
|
| 507 |
+
" \"question\": \"The parsed question or requirement\",\n"
|
| 508 |
+
" \"is_correct\": true,\n"
|
| 509 |
+
" \"feedback\": \"Detailed explanation of why it is correct or incorrect, 2-4 sentences minimum...\"\n"
|
| 510 |
+
" }\n"
|
| 511 |
+
" ]\n"
|
| 512 |
+
"}"
|
| 513 |
)
|
| 514 |
|
| 515 |
|
|
|
|
| 549 |
Validate the LLM response against the required 4-key schema and normalise types.
|
| 550 |
Raises ValueError on any schema violation.
|
| 551 |
"""
|
| 552 |
+
# Unpack if the LLM nested the response inside a root key (e.g. {"evaluation": {...}})
|
| 553 |
+
if len(parsed) == 1 and isinstance(list(parsed.values())[0], dict):
|
| 554 |
+
inner = list(parsed.values())[0]
|
| 555 |
+
if "score" in inner or "questions" in inner:
|
| 556 |
+
parsed = inner
|
| 557 |
+
|
| 558 |
missing = set(REQUIRED_KEYS) - parsed.keys()
|
| 559 |
if missing:
|
| 560 |
+
raise ValueError(
|
| 561 |
+
f"LLM response missing required keys: {missing}. "
|
| 562 |
+
f"Parsed response keys: {list(parsed.keys())}. "
|
| 563 |
+
f"Raw: {json.dumps(parsed)[:300]}"
|
| 564 |
+
)
|
| 565 |
|
| 566 |
for key, expected in REQUIRED_KEYS.items():
|
| 567 |
val = parsed[key]
|