Spaces:
Sleeping
Sleeping
| """Streamlit sandbox demo for the Redrob Candidate Intelligence Engine. | |
| The sandbox runs the COMPLETE pipeline on a small input that the user | |
| uploads. It demonstrates: | |
| * The Role Blueprint and how the JD is parsed | |
| * Hybrid retrieval (BM25 + sentence-transformers + RRF) | |
| * Feature engineering | |
| * LightGBM LambdaRank (or deterministic fallback) ranking | |
| * Per-candidate reasoning | |
| * Export of the top-N as a CSV | |
| For the OFFICIAL reproduce_command see `scripts/run_ranking.py` in the | |
| repository. That command processes the full 100k candidate pool within | |
| the 5-minute / 16GB sandbox budget. | |
| This is the Stage-1 sandbox required by the hackathon spec | |
| (submission_spec.docx Section 10.5). Deploy to Streamlit Cloud or | |
| HuggingFace Spaces to get a working sandbox URL. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| PROJECT_ROOT = Path(__file__).resolve().parent.parent | |
| if str(PROJECT_ROOT) not in sys.path: | |
| sys.path.insert(0, str(PROJECT_ROOT)) | |
| import pandas as pd | |
| import streamlit as st | |
| from redrob.blueprint import build_blueprint | |
| from redrob.rank.predict import rank_shortlist | |
| from redrob.reasoning.template import generate_reasoning_dataframe | |
| from redrob.submit.write import write_submission | |
| st.set_page_config(page_title="Redrob Candidate Intelligence", layout="wide") | |
| st.title("Redrob Candidate Intelligence Engine") | |
| st.caption("Senior AI Engineer ranking — sandbox demo (Stage 1 submission)") | |
| # --------------------------------------------------------------------------- | |
| # Helpers | |
| # --------------------------------------------------------------------------- | |
| def _load_uploaded_jsonl(uploaded_file) -> list: | |
| """Read an uploaded jsonl or json file as a list of dicts.""" | |
| raw = uploaded_file.getvalue().decode("utf-8", errors="ignore") | |
| rows = [] | |
| for line in raw.splitlines(): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| rows.append(json.loads(line)) | |
| except json.JSONDecodeError: | |
| try: | |
| rows.append(json.loads(raw)) | |
| break | |
| except Exception: | |
| pass | |
| return rows | |
| def _rows_to_candidate_objects(rows: list) -> list: | |
| """Convert raw dicts to the dataclass shape used by the pipeline. | |
| The pipeline's `iter_canonical` parses one of two formats: | |
| - the bundled `sample_candidates.json` (list of records) | |
| - the streamed `candidates.jsonl` (one record per line, with the | |
| record itself containing a `profile` key). | |
| The bundled sample is already in the right shape. The streamed | |
| jsonl wraps the profile under `profile`. We support both. | |
| """ | |
| from redrob.data import CanonicalCandidate # type: ignore | |
| out = [] | |
| for r in rows: | |
| if "profile" in r and isinstance(r["profile"], dict): | |
| # Streamed jsonl format | |
| cand = CanonicalCandidate( | |
| candidate_id=r.get("candidate_id") or r.get("id", ""), | |
| name=(r.get("profile") or {}).get("name", ""), | |
| headline=(r.get("profile") or {}).get("headline", ""), | |
| summary=(r.get("profile") or {}).get("summary", ""), | |
| location=(r.get("profile") or {}).get("location", ""), | |
| country=(r.get("profile") or {}).get("country", ""), | |
| yoe=int((r.get("profile") or {}).get("years_experience", 0) or 0), | |
| current_title=(r.get("profile") or {}).get("current_title", ""), | |
| current_company=(r.get("profile") or {}).get("current_company", ""), | |
| current_company_size=(r.get("profile") or {}).get("current_company_size", ""), | |
| current_industry=(r.get("profile") or {}).get("current_industry", ""), | |
| career=(r.get("profile") or {}).get("career", []) or [], | |
| edu=(r.get("profile") or {}).get("education", []) or [], | |
| skills=(r.get("profile") or {}).get("skills", []) or [], | |
| certs=(r.get("profile") or {}).get("certifications", []) or [], | |
| languages=(r.get("profile") or {}).get("languages", []) or [], | |
| signals=r.get("signals") or {}, | |
| ) | |
| else: | |
| # Bundled sample format | |
| cand = CanonicalCandidate( | |
| candidate_id=r.get("candidate_id", ""), | |
| name=r.get("name", ""), | |
| headline=r.get("headline", ""), | |
| summary=r.get("summary", ""), | |
| location=r.get("location", ""), | |
| country=r.get("country", ""), | |
| yoe=int(r.get("yoe", 0) or 0), | |
| current_title=r.get("current_title", ""), | |
| current_company=r.get("current_company", ""), | |
| current_company_size=r.get("current_company_size", ""), | |
| current_industry=r.get("current_industry", ""), | |
| career=r.get("career", []) or [], | |
| edu=r.get("edu", []) or [], | |
| skills=r.get("skills", []) or [], | |
| certs=r.get("certs", []) or [], | |
| languages=r.get("languages", []) or [], | |
| signals=r.get("signals", {}) or {}, | |
| ) | |
| out.append(cand) | |
| return out | |
| def _candidates_to_dataframe(cands) -> pd.DataFrame: | |
| rows = [] | |
| for cc in cands: | |
| rows.append({ | |
| "candidate_id": cc.candidate_id, | |
| "name": cc.name, | |
| "headline": cc.headline, | |
| "summary": cc.summary, | |
| "location": cc.location, | |
| "country": cc.country, | |
| "yoe": cc.yoe, | |
| "current_title": cc.current_title, | |
| "current_company": cc.current_company, | |
| "current_company_size": cc.current_company_size, | |
| "current_industry": cc.current_industry, | |
| "career": cc.career, | |
| "edu": cc.edu, | |
| "skills": cc.skills, | |
| "certs": cc.certs, | |
| "languages": cc.languages, | |
| "signals": cc.signals, | |
| "text_corpus": cc.text_corpus, | |
| "career_total_months": cc.career_total_months, | |
| "career_overlap_months": cc.career_overlap_months, | |
| "career_industries": cc.career_industries, | |
| "career_titles": cc.career_titles, | |
| "career_companies": cc.career_companies, | |
| "skill_names": cc.skill_names, | |
| "skill_durations_max": cc.skill_durations_max, | |
| "has_ai_skill": cc.has_ai_skill, | |
| }) | |
| return pd.DataFrame(rows) | |
| # --------------------------------------------------------------------------- | |
| # Sidebar: inputs | |
| # --------------------------------------------------------------------------- | |
| st.sidebar.header("Inputs") | |
| uploaded = st.sidebar.file_uploader( | |
| "Upload candidates.jsonl or sample_candidates.json (max ~5,000 records)", | |
| type=["jsonl", "json"], | |
| help="Get the file from the hackathon bundle. The sandbox runs the " | |
| "complete pipeline on a small subset for demonstration.", | |
| ) | |
| sample_size = st.sidebar.slider("Sample size for sandbox run", 50, 2000, 200, step=50) | |
| use_dense = st.sidebar.checkbox( | |
| "Use sentence-transformers (slower, ~1 min extra)", | |
| value=False, | |
| ) | |
| run_btn = st.sidebar.button("Run ranking pipeline", type="primary") | |
| st.sidebar.markdown("---") | |
| st.sidebar.caption( | |
| "For the full 100k run, use `python scripts/run_ranking.py --no_dense --no_lgbm` " | |
| "from the repo root (see README)." | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Blueprint panel | |
| # --------------------------------------------------------------------------- | |
| bp = build_blueprint() | |
| with st.expander("Role Blueprint (Senior AI Engineer, Redrob)", expanded=True): | |
| c1, c2, c3 = st.columns(3) | |
| with c1: | |
| st.markdown("**Core competencies**") | |
| st.write(", ".join(bp["core_competencies"][:20]) + "…") | |
| st.markdown(f"*{len(bp['core_competencies'])} total*") | |
| with c2: | |
| st.markdown("**Target titles**") | |
| st.write(", ".join(bp["target_titles"][:10]) + "…") | |
| st.markdown("**Experience band (yrs)**") | |
| st.write( | |
| f"min {bp['experience_band']['min']} · ideal " | |
| f"{bp['experience_band']['ideal_low']}–{bp['experience_band']['ideal_high']}" | |
| ) | |
| with c3: | |
| st.markdown("**Notice period**") | |
| st.write(f"preferred ≤ {bp['notice_period_ok_days']} days") | |
| st.markdown("**Locations**") | |
| st.write(", ".join(bp["preferred_locations"])) | |
| # --------------------------------------------------------------------------- | |
| # Pipeline run | |
| # --------------------------------------------------------------------------- | |
| if run_btn: | |
| if uploaded is None: | |
| st.error( | |
| "Please upload a candidates file. The sandbox needs your " | |
| "data — there's no bundled sample in the public repo." | |
| ) | |
| st.stop() | |
| t0 = time.time() | |
| raw_rows = _load_uploaded_jsonl(uploaded) | |
| if not raw_rows: | |
| st.error("Could not parse the uploaded file as JSON/JSONL.") | |
| st.stop() | |
| # Truncate to sample size | |
| raw_rows = raw_rows[:sample_size] | |
| cands = _rows_to_candidate_objects(raw_rows) | |
| df = _candidates_to_dataframe(cands) | |
| st.success(f"Loaded {len(df):,} candidates in {time.time()-t0:.1f}s") | |
| if not use_dense: | |
| # Disable dense retrieval for the sandbox (faster) | |
| from redrob import retrieval | |
| retrieval.dense.dense_search = lambda *a, **kw: {} | |
| t0 = time.time() | |
| # Provide placeholder retrieval scores | |
| df["sparse_score"] = 0.0 | |
| df["dense_score"] = 0.0 | |
| df["rrf_score"] = 0.0 | |
| df["structured_hit"] = 0.0 | |
| ranked = rank_shortlist(df, use_lgbm=False) | |
| ranked = ranked.head(min(100, len(ranked))) | |
| elapsed = time.time() - t0 | |
| st.success(f"Ranked in {elapsed:.1f}s — top-{len(ranked)} ready") | |
| # Reasoning | |
| reasonings = generate_reasoning_dataframe(ranked) | |
| ranked = ranked.copy() | |
| ranked["reasoning"] = reasonings | |
| # Display | |
| st.subheader("Top 25 (preview)") | |
| display_cols = [ | |
| "candidate_id", "current_title", "yoe", | |
| "title_fit_blend", "recruitability", "honeypot_penalty", | |
| "final_score", "reasoning", | |
| ] | |
| display_cols = [c for c in display_cols if c in ranked.columns] | |
| st.dataframe(ranked.head(25)[display_cols], use_container_width=True, hide_index=True) | |
| st.subheader(f"Top-{len(ranked)} full table") | |
| full_cols = ["candidate_id", "current_title", "yoe", "title_fit_blend", | |
| "recruitability", "honeypot_penalty", "final_score", "reasoning"] | |
| full_cols = [c for c in full_cols if c in ranked.columns] | |
| st.dataframe(ranked[full_cols], use_container_width=True, hide_index=True) | |
| # Charts | |
| c1, c2 = st.columns(2) | |
| with c1: | |
| if "current_title" in ranked.columns: | |
| st.markdown("**Top-100 title distribution**") | |
| st.bar_chart(ranked["current_title"].value_counts().head(10)) | |
| with c2: | |
| if "title_fit_blend" in ranked.columns: | |
| st.markdown("**Top-100 title fit histogram**") | |
| st.bar_chart(pd.cut(ranked["title_fit_blend"], bins=10).value_counts().sort_index()) | |
| # Export | |
| out_csv = PROJECT_ROOT / "submission_sandbox.csv" | |
| write_submission(ranked, out_path=out_csv, validate=False) | |
| with open(out_csv, "rb") as f: | |
| st.download_button( | |
| "Download submission CSV", | |
| data=f.read(), | |
| file_name="submission.csv", | |
| mime="text/csv", | |
| ) | |
| st.info( | |
| "The sandbox ran the COMPLETE pipeline (BM25 retrieval, 5-axis " | |
| "PageRank, 74 features, deterministic composite, reasoning) " | |
| "on your uploaded sample. For the full 100k pool, run " | |
| "`python scripts/run_ranking.py --no_dense --no_lgbm` from " | |
| "the repo root (see README)." | |
| ) | |
| else: | |
| st.info("Upload a candidates file and press 'Run ranking pipeline' in the sidebar to begin.") | |