Spaces:
Paused
Paused
Download verifier.py from DarkSting/Test_Space: direct link, hf CLI and curl.
- Browser
- Download file 14.3 kB
-
https://huggingface.co/spaces/DarkSting/Test_Space/resolve/main/verifier.py
- Command line
-
hf download hf://spaces/DarkSting/Test_Space/verifier.py
-
curl -L -o verifier.py https://huggingface.co/spaces/DarkSting/Test_Space/resolve/main/verifier.py
14.3 kB
| import base64 | |
| import json | |
| import logging | |
| import os | |
| logger = logging.getLogger(__name__) | |
| try: | |
| from openai import OpenAI | |
| CHAT_AVAILABLE = True | |
| except ImportError: | |
| CHAT_AVAILABLE = False | |
| API_KEY = os.getenv("VALIDATE_KEY", "") | |
| MODEL = os.getenv("MODEL", "") | |
| TIMEOUT = 60 | |
| # Schema mirrors app.py's SYSTEM_PROMPT exactly β this module must never | |
| # drift from what the local model outputs. | |
| _SYSTEM_PROMPT = """You are a resume-parsing quality checker. You will be given a resume \ | |
| (as a PDF or as extracted text) and a JSON extraction of it produced by another model. Carefully \ | |
| verify the JSON against the actual resume content and correct any fields that are wrong, missing, \ | |
| or incomplete. Return ONLY a single valid JSON object β no markdown, no commentary, no code fences \ | |
| β using exactly this schema (use null for missing scalar fields and [] for missing lists): | |
| { | |
| "name": "Full Name or null", | |
| "email": "email@example.com or null", | |
| "phone": "phone number or null", | |
| "skills": ["skill1", "skill2"], | |
| "programming_languages": ["Python", "JavaScript"], | |
| "experience": [ | |
| { | |
| "job_title": "Job Title", | |
| "company": "Company Name", | |
| "start_date": "MM/YYYY or MM/DD/YYYY or null", | |
| "end_date": "MM/YYYY or MM/DD/YYYY or Present or null", | |
| "responsibilities": ["responsibility1", "responsibility2"] | |
| } | |
| ], | |
| "total_experience_years": 0.0, | |
| "education": [ | |
| { | |
| "degree": "Degree Name", | |
| "field": "Field of Study or null", | |
| "institution": "Institution Name or null", | |
| "year": 2020 | |
| } | |
| ], | |
| "certifications": ["cert1", "cert2"], | |
| "projects": ["Project Name 1", "Project Name 2"], | |
| "publications": ["publication1"], | |
| "achievements": ["achievement1"], | |
| "summary": "Brief professional summary or null" | |
| } | |
| Be thorough and accurate. Infer programming languages and technical skills even if not stated under an \ | |
| explicit "skills" heading. Do not invent information that isn't supported by the resume. If the given \ | |
| JSON is already correct and complete, return it unchanged.""" | |
| # --------------------------------------------------------------------------- | |
| # Public entry point β resume-parsing verification | |
| # --------------------------------------------------------------------------- | |
| def verify(model_json: dict, resume_text: str = "", pdf_bytes: bytes = None) -> dict: | |
| """ | |
| Verify/correct the local model's extraction. | |
| Parameters | |
| ---------- | |
| model_json : dict Raw JSON produced by the local Gemma model | |
| resume_text : str Resume plain text (used when no PDF is available, | |
| or alongside the PDF for extra context) | |
| pdf_bytes : bytes Original PDF bytes, when available | |
| check against the real document rather than just | |
| the locally-extracted text | |
| Returns | |
| ------- | |
| dict Either the corrected JSON the original model_json unchanged | |
| (tagged parser_source="model_only") if verification was | |
| skipped or failed. Never raises. | |
| """ | |
| if not CHAT_AVAILABLE: | |
| logger.warning("[Verifier] openai package not installed β skipping verification.") | |
| return {**model_json, "parser_source": "model_only"} | |
| if not CHAT_AVAILABLE: | |
| logger.warning("[Verifier] API_KEY not set β skipping verification.") | |
| return {**model_json, "parser_source": "model_only"} | |
| try: | |
| corrected = _call_API(model_json, resume_text=resume_text, pdf_bytes=pdf_bytes) | |
| except Exception as e: | |
| logger.error("[Verifier] Verification call failed: %s", e) | |
| return {**model_json, "parser_source": "model_only"} | |
| if corrected is None: | |
| return {**model_json, "parser_source": "model_only"} | |
| # Safety net β never let a worse/emptier result overwrite a | |
| # good local-model result. | |
| if _quality_score(corrected) < _quality_score(model_json): | |
| logger.warning("[Verifier] Corrected result looked worse than model output β keeping model output.") | |
| return {**model_json, "parser_source": "model_only"} | |
| corrected["parser_source"] = "raw_model" | |
| return corrected | |
| # --------------------------------------------------------------------------- | |
| # Internal helpers β resume-parsing verification | |
| # --------------------------------------------------------------------------- | |
| def _call_API(model_json: dict, resume_text: str, pdf_bytes: bytes = None) -> dict: | |
| """Returns corrected dict, or None if the call/parse failed.""" | |
| model_json_str = json.dumps(_strip_internal_keys(model_json), indent=2) | |
| user_instruction = ( | |
| "Here is a resume and a JSON extraction of it produced by another model.\n\n" | |
| "Carefully compare the JSON against the resume. If every field is correct and " | |
| "complete, return the SAME JSON unchanged. If anything is wrong, missing, or " | |
| "incomplete (wrong name, missed skills, wrong dates, missing education/projects/" | |
| "certifications, etc.), return a CORRECTED version of the JSON with the accurate " | |
| "values instead. Always return the full JSON object in the exact schema described, " | |
| "never partial output.\n\n" | |
| f"Extracted JSON to verify:\n{model_json_str}" | |
| ) | |
| logger.info("[Verifier] Calling (%s) to verify/correct model extraction β¦", MODEL) | |
| client = OpenAI(api_key=API_KEY, timeout=TIMEOUT) | |
| if pdf_bytes: | |
| b64_pdf = base64.b64encode(pdf_bytes).decode("utf-8") | |
| response = client.responses.create( | |
| model=MODEL, | |
| temperature=0, | |
| input=[ | |
| {"role": "system", "content": _SYSTEM_PROMPT}, | |
| { | |
| "role": "user", | |
| "content": [ | |
| { | |
| "type": "input_file", | |
| "filename": "resume.pdf", | |
| "file_data": f"data:application/pdf;base64,{b64_pdf}", | |
| }, | |
| {"type": "input_text", "text": user_instruction}, | |
| ], | |
| }, | |
| ], | |
| text={"format": {"type": "json_object"}}, | |
| ) | |
| content = response.output_text | |
| else: | |
| completion = client.chat.completions.create( | |
| model=MODEL, | |
| response_format={"type": "json_object"}, | |
| temperature=0, | |
| messages=[ | |
| {"role": "system", "content": _SYSTEM_PROMPT}, | |
| {"role": "user", "content": f"Resume text:\n{resume_text}\n\n{user_instruction}"}, | |
| ], | |
| ) | |
| content = completion.choices[0].message.content | |
| try: | |
| raw = json.loads(content) | |
| except json.JSONDecodeError as e: | |
| logger.error("[Verifier] returned invalid JSON: %s", e) | |
| return None | |
| logger.info("[Verifier] Verification/correction pass complete.") | |
| return raw | |
| def _strip_internal_keys(data: dict) -> dict: | |
| """Drop any bookkeeping keys before showing the JSON.""" | |
| keys = ( | |
| "name", "email", "phone", "skills", "programming_languages", | |
| "experience", "total_experience_years", "education", | |
| "certifications", "projects", "publications", "achievements", | |
| "summary", | |
| ) | |
| return {k: data.get(k) for k in keys if k in data} | |
| def _quality_score(data: dict) -> int: | |
| """Rough heuristic: how many meaningful fields are populated.""" | |
| score = 0 | |
| for key in ("name", "skills", "programming_languages", "experience", | |
| "education", "certifications", "projects"): | |
| if data.get(key): | |
| score += 1 | |
| return score | |
| # --------------------------------------------------------------------------- | |
| # Public entry point β skill-graph inference verification | |
| # --------------------------------------------------------------------------- | |
| # | |
| # Separate from the resume-parsing verify() above: this checks a GNN's | |
| # skill-graph inferences (from gnn_skill_graph.py's SkillGraph.infer()) | |
| # rather than resume-extraction JSON. Kept in the same module since both | |
| # are "second-opinion LLM check" passes and both share the CHAT_AVAILABLE / | |
| # openai-client plumbing above. | |
| # | |
| # Uses its own env var (LLM_API_KEY) rather than VALIDATE_KEY above, since | |
| # these two checks may reasonably run against different keys/quotas. | |
| def verify_inferred_skills_with_llm( | |
| graph_result: dict, | |
| only_flagged: bool = True, | |
| ) -> list: | |
| inferred_skills = graph_result.get("inferred_skills", []) | |
| known_skills = graph_result.get("known_skills", []) | |
| candidate_name = graph_result.get("candidate", "Candidate") | |
| to_verify = [s for s in inferred_skills if s.get("low_data", True)] if only_flagged else list(inferred_skills) | |
| to_verify_names = {s["skill"] for s in to_verify if "skill" in s} | |
| passthrough = [s for s in inferred_skills if s.get("skill") not in to_verify_names] | |
| if not to_verify: | |
| return inferred_skills | |
| if not CHAT_AVAILABLE: | |
| logger.warning("[SkillVerifier] openai package not installed β skipping verification.") | |
| return inferred_skills | |
| if not API_KEY: | |
| logger.warning("[SkillVerifier] API_KEY not set β skipping verification.") | |
| return inferred_skills | |
| inference_payload = { | |
| "candidate": candidate_name, | |
| "known_skills": known_skills, | |
| "unmatched_skills": graph_result.get("unmatched_skills", []), | |
| "inferred_skills": to_verify, | |
| "domains": graph_result.get("domains", []), | |
| "similar_skills": graph_result.get("similar_skills", []), | |
| "skill_score": graph_result.get("skill_score"), | |
| "total_competencies": graph_result.get("total_competencies"), | |
| } | |
| prompt = f"""You are reviewing skill inferences made by a graph neural network for a candidate evaluation system. | |
| Candidate: {candidate_name} | |
| Known skills (confirmed): {', '.join(known_skills)} | |
| Here is the full inference object produced by the model: | |
| {json.dumps(inference_payload, indent=2)} | |
| The model inferred the candidate LIKELY also knows the skills listed in "inferred_skills", but each one is flagged as resting on THIN training data (the model saw very few real-world examples), so they need a sanity check. | |
| Your task: | |
| 1. For each entry in "inferred_skills", judge whether it's a REALISTIC inference given the "known_skills" β does it make sense that someone with these known skills would also know this inferred skill? Remove any entry that doesn't realistically follow. Correct the "confidence" value where it looks wrong (0.0-1.0 scale). Be conservative β when genuinely unsure, keep the entry but lower its confidence rather than dropping it outright. | |
| 2. Check "unmatched_skills" for any skill that is just a case-insensitive or formatting duplicate of something already in "known_skills" (e.g. "typescript" vs "TypeScript", "Node.js" vs "nodejs"). Remove such duplicates from "unmatched_skills" β they are not genuinely unmatched. | |
| 3. Do not modify "candidate" or "total_competencies". | |
| 4. Include "domains": [{{"domain": <string>, "confidence": <0-1>}}] and "similar_skills": [{{"skill": <string>, "similar_to": <string>}}], if the provided skills are related to that domain. | |
| 5. Compute the skill score based on the information provided and return a rating between 0-10 with 1 decimal point. | |
| Return ONLY the full corrected JSON object, in the exact same top-level shape as the input above (same keys: "candidate", "known_skills", "unmatched_skills", "inferred_skills", "domains", "similar_skills", "skill_score", "total_competencies"), with "inferred_skills" filtered/corrected per step 1 and "unmatched_skills" corrected per step 2. No markdown, no explanation outside the JSON.""" | |
| try: | |
| client = OpenAI(api_key=API_KEY, timeout=TIMEOUT) | |
| response = client.chat.completions.create( | |
| model=MODEL, | |
| messages=[{"role": "user", "content": prompt}], | |
| temperature=0, | |
| **({"response_format": {"type": "json_object"}} if "gpt-4" in MODEL or "gpt-5" in MODEL else {}), | |
| ) | |
| raw = response.choices[0].message.content.strip() | |
| raw = raw.removeprefix("```json").removeprefix("```").removesuffix("```").strip() | |
| parsed = json.loads(raw) | |
| if isinstance(parsed, dict) and "inferred_skills" in parsed: | |
| corrected_entries = parsed["inferred_skills"] | |
| elif isinstance(parsed, list): | |
| corrected_entries = parsed | |
| else: | |
| corrected_entries = next((v for v in parsed.values() if isinstance(v, list)), []) | |
| corrected_by_skill = {} | |
| for e in corrected_entries: | |
| if isinstance(e, dict) and "skill" in e: | |
| corrected_by_skill[e["skill"]] = e # last one wins if the LLM ever duplicates a skill name | |
| except Exception as e: | |
| logger.error("[SkillVerifier] LLM call/parse failed (%s) β returning raw GNN output for the flagged entries.", e) | |
| return inferred_skills | |
| corrected = [] | |
| for entry in to_verify: | |
| llm_entry = corrected_by_skill.get(entry.get("skill")) | |
| if llm_entry is None: | |
| continue # not present in the LLM's returned array = dropped | |
| updated = dict(entry) | |
| updated.update(llm_entry) | |
| corrected.append(updated) | |
| result = passthrough + corrected | |
| result.sort(key=lambda x: x.get("confidence", 0), reverse=True) | |
| return result | |
| def verify_skill_graph_result(graph_result: dict, only_flagged: bool = True) -> dict: | |
| """ | |
| Takes the FULL dict gnn_skill_graph.py's SkillGraph.infer() produced | |
| (candidate, known_skills, inferred_skills, domains, similar_skills, | |
| skill_score, ...) and returns a new dict with inferred_skills | |
| replaced by the LLM-corrected list. All other keys pass through | |
| unchanged. skill_score is NOT recomputed here. | |
| Fail-safe: if verification is skipped or fails for any reason, this | |
| returns graph_result with inferred_skills unchanged β never raises. | |
| """ | |
| corrected_inferred = verify_inferred_skills_with_llm( | |
| graph_result, | |
| only_flagged=only_flagged, | |
| ) | |
| updated = dict(graph_result) | |
| updated["inferred_skills"] = corrected_inferred | |
| updated["total_competencies"] = len(graph_result.get("known_skills", [])) + len(corrected_inferred) | |
| return updated |