Test_Space / verifier.py
DarkSting's picture
Update verifier.py
6f78b1d verified
Raw History Blame Contribute Delete
14.3 kB
import base64
import json
import logging
import os
logger = logging.getLogger(__name__)
try:
from openai import OpenAI
CHAT_AVAILABLE = True
except ImportError:
CHAT_AVAILABLE = False
API_KEY = os.getenv("VALIDATE_KEY", "")
MODEL = os.getenv("MODEL", "")
TIMEOUT = 60
# Schema mirrors app.py's SYSTEM_PROMPT exactly β€” this module must never
# drift from what the local model outputs.
_SYSTEM_PROMPT = """You are a resume-parsing quality checker. You will be given a resume \
(as a PDF or as extracted text) and a JSON extraction of it produced by another model. Carefully \
verify the JSON against the actual resume content and correct any fields that are wrong, missing, \
or incomplete. Return ONLY a single valid JSON object β€” no markdown, no commentary, no code fences \
β€” using exactly this schema (use null for missing scalar fields and [] for missing lists):
{
"name": "Full Name or null",
"email": "email@example.com or null",
"phone": "phone number or null",
"skills": ["skill1", "skill2"],
"programming_languages": ["Python", "JavaScript"],
"experience": [
{
"job_title": "Job Title",
"company": "Company Name",
"start_date": "MM/YYYY or MM/DD/YYYY or null",
"end_date": "MM/YYYY or MM/DD/YYYY or Present or null",
"responsibilities": ["responsibility1", "responsibility2"]
}
],
"total_experience_years": 0.0,
"education": [
{
"degree": "Degree Name",
"field": "Field of Study or null",
"institution": "Institution Name or null",
"year": 2020
}
],
"certifications": ["cert1", "cert2"],
"projects": ["Project Name 1", "Project Name 2"],
"publications": ["publication1"],
"achievements": ["achievement1"],
"summary": "Brief professional summary or null"
}
Be thorough and accurate. Infer programming languages and technical skills even if not stated under an \
explicit "skills" heading. Do not invent information that isn't supported by the resume. If the given \
JSON is already correct and complete, return it unchanged."""
# ---------------------------------------------------------------------------
# Public entry point β€” resume-parsing verification
# ---------------------------------------------------------------------------
def verify(model_json: dict, resume_text: str = "", pdf_bytes: bytes = None) -> dict:
"""
Verify/correct the local model's extraction.
Parameters
----------
model_json : dict Raw JSON produced by the local Gemma model
resume_text : str Resume plain text (used when no PDF is available,
or alongside the PDF for extra context)
pdf_bytes : bytes Original PDF bytes, when available
check against the real document rather than just
the locally-extracted text
Returns
-------
dict Either the corrected JSON the original model_json unchanged
(tagged parser_source="model_only") if verification was
skipped or failed. Never raises.
"""
if not CHAT_AVAILABLE:
logger.warning("[Verifier] openai package not installed β€” skipping verification.")
return {**model_json, "parser_source": "model_only"}
if not CHAT_AVAILABLE:
logger.warning("[Verifier] API_KEY not set β€” skipping verification.")
return {**model_json, "parser_source": "model_only"}
try:
corrected = _call_API(model_json, resume_text=resume_text, pdf_bytes=pdf_bytes)
except Exception as e:
logger.error("[Verifier] Verification call failed: %s", e)
return {**model_json, "parser_source": "model_only"}
if corrected is None:
return {**model_json, "parser_source": "model_only"}
# Safety net β€” never let a worse/emptier result overwrite a
# good local-model result.
if _quality_score(corrected) < _quality_score(model_json):
logger.warning("[Verifier] Corrected result looked worse than model output β€” keeping model output.")
return {**model_json, "parser_source": "model_only"}
corrected["parser_source"] = "raw_model"
return corrected
# ---------------------------------------------------------------------------
# Internal helpers β€” resume-parsing verification
# ---------------------------------------------------------------------------
def _call_API(model_json: dict, resume_text: str, pdf_bytes: bytes = None) -> dict:
"""Returns corrected dict, or None if the call/parse failed."""
model_json_str = json.dumps(_strip_internal_keys(model_json), indent=2)
user_instruction = (
"Here is a resume and a JSON extraction of it produced by another model.\n\n"
"Carefully compare the JSON against the resume. If every field is correct and "
"complete, return the SAME JSON unchanged. If anything is wrong, missing, or "
"incomplete (wrong name, missed skills, wrong dates, missing education/projects/"
"certifications, etc.), return a CORRECTED version of the JSON with the accurate "
"values instead. Always return the full JSON object in the exact schema described, "
"never partial output.\n\n"
f"Extracted JSON to verify:\n{model_json_str}"
)
logger.info("[Verifier] Calling (%s) to verify/correct model extraction …", MODEL)
client = OpenAI(api_key=API_KEY, timeout=TIMEOUT)
if pdf_bytes:
b64_pdf = base64.b64encode(pdf_bytes).decode("utf-8")
response = client.responses.create(
model=MODEL,
temperature=0,
input=[
{"role": "system", "content": _SYSTEM_PROMPT},
{
"role": "user",
"content": [
{
"type": "input_file",
"filename": "resume.pdf",
"file_data": f"data:application/pdf;base64,{b64_pdf}",
},
{"type": "input_text", "text": user_instruction},
],
},
],
text={"format": {"type": "json_object"}},
)
content = response.output_text
else:
completion = client.chat.completions.create(
model=MODEL,
response_format={"type": "json_object"},
temperature=0,
messages=[
{"role": "system", "content": _SYSTEM_PROMPT},
{"role": "user", "content": f"Resume text:\n{resume_text}\n\n{user_instruction}"},
],
)
content = completion.choices[0].message.content
try:
raw = json.loads(content)
except json.JSONDecodeError as e:
logger.error("[Verifier] returned invalid JSON: %s", e)
return None
logger.info("[Verifier] Verification/correction pass complete.")
return raw
def _strip_internal_keys(data: dict) -> dict:
"""Drop any bookkeeping keys before showing the JSON."""
keys = (
"name", "email", "phone", "skills", "programming_languages",
"experience", "total_experience_years", "education",
"certifications", "projects", "publications", "achievements",
"summary",
)
return {k: data.get(k) for k in keys if k in data}
def _quality_score(data: dict) -> int:
"""Rough heuristic: how many meaningful fields are populated."""
score = 0
for key in ("name", "skills", "programming_languages", "experience",
"education", "certifications", "projects"):
if data.get(key):
score += 1
return score
# ---------------------------------------------------------------------------
# Public entry point β€” skill-graph inference verification
# ---------------------------------------------------------------------------
#
# Separate from the resume-parsing verify() above: this checks a GNN's
# skill-graph inferences (from gnn_skill_graph.py's SkillGraph.infer())
# rather than resume-extraction JSON. Kept in the same module since both
# are "second-opinion LLM check" passes and both share the CHAT_AVAILABLE /
# openai-client plumbing above.
#
# Uses its own env var (LLM_API_KEY) rather than VALIDATE_KEY above, since
# these two checks may reasonably run against different keys/quotas.
def verify_inferred_skills_with_llm(
graph_result: dict,
only_flagged: bool = True,
) -> list:
inferred_skills = graph_result.get("inferred_skills", [])
known_skills = graph_result.get("known_skills", [])
candidate_name = graph_result.get("candidate", "Candidate")
to_verify = [s for s in inferred_skills if s.get("low_data", True)] if only_flagged else list(inferred_skills)
to_verify_names = {s["skill"] for s in to_verify if "skill" in s}
passthrough = [s for s in inferred_skills if s.get("skill") not in to_verify_names]
if not to_verify:
return inferred_skills
if not CHAT_AVAILABLE:
logger.warning("[SkillVerifier] openai package not installed β€” skipping verification.")
return inferred_skills
if not API_KEY:
logger.warning("[SkillVerifier] API_KEY not set β€” skipping verification.")
return inferred_skills
inference_payload = {
"candidate": candidate_name,
"known_skills": known_skills,
"unmatched_skills": graph_result.get("unmatched_skills", []),
"inferred_skills": to_verify,
"domains": graph_result.get("domains", []),
"similar_skills": graph_result.get("similar_skills", []),
"skill_score": graph_result.get("skill_score"),
"total_competencies": graph_result.get("total_competencies"),
}
prompt = f"""You are reviewing skill inferences made by a graph neural network for a candidate evaluation system.
Candidate: {candidate_name}
Known skills (confirmed): {', '.join(known_skills)}
Here is the full inference object produced by the model:
{json.dumps(inference_payload, indent=2)}
The model inferred the candidate LIKELY also knows the skills listed in "inferred_skills", but each one is flagged as resting on THIN training data (the model saw very few real-world examples), so they need a sanity check.
Your task:
1. For each entry in "inferred_skills", judge whether it's a REALISTIC inference given the "known_skills" β€” does it make sense that someone with these known skills would also know this inferred skill? Remove any entry that doesn't realistically follow. Correct the "confidence" value where it looks wrong (0.0-1.0 scale). Be conservative β€” when genuinely unsure, keep the entry but lower its confidence rather than dropping it outright.
2. Check "unmatched_skills" for any skill that is just a case-insensitive or formatting duplicate of something already in "known_skills" (e.g. "typescript" vs "TypeScript", "Node.js" vs "nodejs"). Remove such duplicates from "unmatched_skills" β€” they are not genuinely unmatched.
3. Do not modify "candidate" or "total_competencies".
4. Include "domains": [{{"domain": <string>, "confidence": <0-1>}}] and "similar_skills": [{{"skill": <string>, "similar_to": <string>}}], if the provided skills are related to that domain.
5. Compute the skill score based on the information provided and return a rating between 0-10 with 1 decimal point.
Return ONLY the full corrected JSON object, in the exact same top-level shape as the input above (same keys: "candidate", "known_skills", "unmatched_skills", "inferred_skills", "domains", "similar_skills", "skill_score", "total_competencies"), with "inferred_skills" filtered/corrected per step 1 and "unmatched_skills" corrected per step 2. No markdown, no explanation outside the JSON."""
try:
client = OpenAI(api_key=API_KEY, timeout=TIMEOUT)
response = client.chat.completions.create(
model=MODEL,
messages=[{"role": "user", "content": prompt}],
temperature=0,
**({"response_format": {"type": "json_object"}} if "gpt-4" in MODEL or "gpt-5" in MODEL else {}),
)
raw = response.choices[0].message.content.strip()
raw = raw.removeprefix("```json").removeprefix("```").removesuffix("```").strip()
parsed = json.loads(raw)
if isinstance(parsed, dict) and "inferred_skills" in parsed:
corrected_entries = parsed["inferred_skills"]
elif isinstance(parsed, list):
corrected_entries = parsed
else:
corrected_entries = next((v for v in parsed.values() if isinstance(v, list)), [])
corrected_by_skill = {}
for e in corrected_entries:
if isinstance(e, dict) and "skill" in e:
corrected_by_skill[e["skill"]] = e # last one wins if the LLM ever duplicates a skill name
except Exception as e:
logger.error("[SkillVerifier] LLM call/parse failed (%s) β€” returning raw GNN output for the flagged entries.", e)
return inferred_skills
corrected = []
for entry in to_verify:
llm_entry = corrected_by_skill.get(entry.get("skill"))
if llm_entry is None:
continue # not present in the LLM's returned array = dropped
updated = dict(entry)
updated.update(llm_entry)
corrected.append(updated)
result = passthrough + corrected
result.sort(key=lambda x: x.get("confidence", 0), reverse=True)
return result
def verify_skill_graph_result(graph_result: dict, only_flagged: bool = True) -> dict:
"""
Takes the FULL dict gnn_skill_graph.py's SkillGraph.infer() produced
(candidate, known_skills, inferred_skills, domains, similar_skills,
skill_score, ...) and returns a new dict with inferred_skills
replaced by the LLM-corrected list. All other keys pass through
unchanged. skill_score is NOT recomputed here.
Fail-safe: if verification is skipped or fails for any reason, this
returns graph_result with inferred_skills unchanged β€” never raises.
"""
corrected_inferred = verify_inferred_skills_with_llm(
graph_result,
only_flagged=only_flagged,
)
updated = dict(graph_result)
updated["inferred_skills"] = corrected_inferred
updated["total_competencies"] = len(graph_result.get("known_skills", [])) + len(corrected_inferred)
return updated