DhyandevP commited on
Commit
d0d14d2
·
1 Parent(s): a963267

fix: Improve LLM prompt and JSON response parsing

Browse files
Files changed (1) hide show
  1. main.py +24 -7
main.py CHANGED
@@ -497,12 +497,19 @@ SYSTEM_PROMPT: str = (
497
  "Start directly with { and end with }. Escape all internal double quotes using \\\". "
498
  "No unescaped newlines inside string fields.\n\n"
499
  "REQUIRED JSON SCHEMA:\n"
500
- '{\"score\": number (float, 2 decimal places, e.g. 82.50), '
501
- '\"language\": \"String (detected language or stack)\", '
502
- '\"code_quality\": \"String (detailed overall quality assessment, 3-5 sentences)\", '
503
- '\"questions\": [{\"question\": \"String (the parsed question/requirement)\", '
504
- '\"is_correct\": boolean, '
505
- '\"feedback\": \"String (detailed explanation, 2-4 sentences minimum)\"}]}'
 
 
 
 
 
 
 
506
  )
507
 
508
 
@@ -542,9 +549,19 @@ def validate_report(parsed: dict) -> dict:
542
  Validate the LLM response against the required 4-key schema and normalise types.
543
  Raises ValueError on any schema violation.
544
  """
 
 
 
 
 
 
545
  missing = set(REQUIRED_KEYS) - parsed.keys()
546
  if missing:
547
- raise ValueError(f"LLM response missing required keys: {missing}")
 
 
 
 
548
 
549
  for key, expected in REQUIRED_KEYS.items():
550
  val = parsed[key]
 
497
  "Start directly with { and end with }. Escape all internal double quotes using \\\". "
498
  "No unescaped newlines inside string fields.\n\n"
499
  "REQUIRED JSON SCHEMA:\n"
500
+ "Return a JSON object with the following structure. Replace the placeholders with your actual evaluation data:\n"
501
+ "{\n"
502
+ " \"score\": 82.50,\n"
503
+ " \"language\": \"Detected language or stack\",\n"
504
+ " \"code_quality\": \"Detailed overall quality assessment, 3-5 sentences...\",\n"
505
+ " \"questions\": [\n"
506
+ " {\n"
507
+ " \"question\": \"The parsed question or requirement\",\n"
508
+ " \"is_correct\": true,\n"
509
+ " \"feedback\": \"Detailed explanation of why it is correct or incorrect, 2-4 sentences minimum...\"\n"
510
+ " }\n"
511
+ " ]\n"
512
+ "}"
513
  )
514
 
515
 
 
549
  Validate the LLM response against the required 4-key schema and normalise types.
550
  Raises ValueError on any schema violation.
551
  """
552
+ # Unpack if the LLM nested the response inside a root key (e.g. {"evaluation": {...}})
553
+ if len(parsed) == 1 and isinstance(list(parsed.values())[0], dict):
554
+ inner = list(parsed.values())[0]
555
+ if "score" in inner or "questions" in inner:
556
+ parsed = inner
557
+
558
  missing = set(REQUIRED_KEYS) - parsed.keys()
559
  if missing:
560
+ raise ValueError(
561
+ f"LLM response missing required keys: {missing}. "
562
+ f"Parsed response keys: {list(parsed.keys())}. "
563
+ f"Raw: {json.dumps(parsed)[:300]}"
564
+ )
565
 
566
  for key, expected in REQUIRED_KEYS.items():
567
  val = parsed[key]