Blame for app.py

"""Streamlit sandbox demo for the Redrob Candidate Intelligence Engine.



The sandbox runs the COMPLETE pipeline on a small input that the user

uploads. It demonstrates:



  * The Role Blueprint and how the JD is parsed

  * Hybrid retrieval (BM25 + sentence-transformers + RRF)

  * Feature engineering

  * LightGBM LambdaRank (or deterministic fallback) ranking

  * Per-candidate reasoning

  * Export of the top-N as a CSV



For the OFFICIAL reproduce_command see `scripts/run_ranking.py` in the

repository. That command processes the full 100k candidate pool within

the 5-minute / 16GB sandbox budget.



This is the Stage-1 sandbox required by the hackathon spec

(submission_spec.docx Section 10.5). Deploy to Streamlit Cloud or

HuggingFace Spaces to get a working sandbox URL.

"""
from __future__ import annotations

import json
import sys
import time
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

import pandas as pd
import streamlit as st

from redrob.blueprint import build_blueprint
from redrob.rank.predict import rank_shortlist
from redrob.reasoning.template import generate_reasoning_dataframe
from redrob.submit.write import write_submission


st.set_page_config(page_title="Redrob Candidate Intelligence", layout="wide")

st.title("Redrob Candidate Intelligence Engine")
st.caption("Senior AI Engineer ranking — sandbox demo (Stage 1 submission)")

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _load_uploaded_jsonl(uploaded_file) -> list:
    """Read an uploaded jsonl or json file as a list of dicts."""
    raw = uploaded_file.getvalue().decode("utf-8", errors="ignore")
    rows = []
    for line in raw.splitlines():
        line = line.strip()
        if not line:
            continue
        try:
            rows.append(json.loads(line))
        except json.JSONDecodeError:
            try:
                rows.append(json.loads(raw))
                break
            except Exception:
                pass
    return rows


def _rows_to_candidate_objects(rows: list) -> list:
    """Convert raw dicts to the dataclass shape used by the pipeline.



    The pipeline's `iter_canonical` parses one of two formats:

      - the bundled `sample_candidates.json` (list of records)

      - the streamed `candidates.jsonl` (one record per line, with the

        record itself containing a `profile` key).

    The bundled sample is already in the right shape. The streamed

    jsonl wraps the profile under `profile`. We support both.

    """
    from redrob.data import CanonicalCandidate  # type: ignore
    out = []
    for r in rows:
        if "profile" in r and isinstance(r["profile"], dict):
            # Streamed jsonl format
            cand = CanonicalCandidate(
                candidate_id=r.get("candidate_id") or r.get("id", ""),
                name=(r.get("profile") or {}).get("name", ""),
                headline=(r.get("profile") or {}).get("headline", ""),
                summary=(r.get("profile") or {}).get("summary", ""),
                location=(r.get("profile") or {}).get("location", ""),
                country=(r.get("profile") or {}).get("country", ""),
                yoe=int((r.get("profile") or {}).get("years_experience", 0) or 0),
                current_title=(r.get("profile") or {}).get("current_title", ""),
                current_company=(r.get("profile") or {}).get("current_company", ""),
                current_company_size=(r.get("profile") or {}).get("current_company_size", ""),
                current_industry=(r.get("profile") or {}).get("current_industry", ""),
                career=(r.get("profile") or {}).get("career", []) or [],
                edu=(r.get("profile") or {}).get("education", []) or [],
                skills=(r.get("profile") or {}).get("skills", []) or [],
                certs=(r.get("profile") or {}).get("certifications", []) or [],
                languages=(r.get("profile") or {}).get("languages", []) or [],
                signals=r.get("signals") or {},
            )
        else:
            # Bundled sample format
            cand = CanonicalCandidate(
                candidate_id=r.get("candidate_id", ""),
                name=r.get("name", ""),
                headline=r.get("headline", ""),
                summary=r.get("summary", ""),
                location=r.get("location", ""),
                country=r.get("country", ""),
                yoe=int(r.get("yoe", 0) or 0),
                current_title=r.get("current_title", ""),
                current_company=r.get("current_company", ""),
                current_company_size=r.get("current_company_size", ""),
                current_industry=r.get("current_industry", ""),
                career=r.get("career", []) or [],
                edu=r.get("edu", []) or [],
                skills=r.get("skills", []) or [],
                certs=r.get("certs", []) or [],
                languages=r.get("languages", []) or [],
                signals=r.get("signals", {}) or {},
            )
        out.append(cand)
    return out


def _candidates_to_dataframe(cands) -> pd.DataFrame:
    rows = []
    for cc in cands:
        rows.append({
            "candidate_id": cc.candidate_id,
            "name": cc.name,
            "headline": cc.headline,
            "summary": cc.summary,
            "location": cc.location,
            "country": cc.country,
            "yoe": cc.yoe,
            "current_title": cc.current_title,
            "current_company": cc.current_company,
            "current_company_size": cc.current_company_size,
            "current_industry": cc.current_industry,
            "career": cc.career,
            "edu": cc.edu,
            "skills": cc.skills,
            "certs": cc.certs,
            "languages": cc.languages,
            "signals": cc.signals,
            "text_corpus": cc.text_corpus,
            "career_total_months": cc.career_total_months,
            "career_overlap_months": cc.career_overlap_months,
            "career_industries": cc.career_industries,
            "career_titles": cc.career_titles,
            "career_companies": cc.career_companies,
            "skill_names": cc.skill_names,
            "skill_durations_max": cc.skill_durations_max,
            "has_ai_skill": cc.has_ai_skill,
        })
    return pd.DataFrame(rows)


# ---------------------------------------------------------------------------
# Sidebar: inputs
# ---------------------------------------------------------------------------
st.sidebar.header("Inputs")
uploaded = st.sidebar.file_uploader(
    "Upload candidates.jsonl or sample_candidates.json (max ~5,000 records)",
    type=["jsonl", "json"],
    help="Get the file from the hackathon bundle. The sandbox runs the "
         "complete pipeline on a small subset for demonstration.",
)
sample_size = st.sidebar.slider("Sample size for sandbox run", 50, 2000, 200, step=50)
use_dense = st.sidebar.checkbox(
    "Use sentence-transformers (slower, ~1 min extra)",
    value=False,
)
run_btn = st.sidebar.button("Run ranking pipeline", type="primary")

st.sidebar.markdown("---")
st.sidebar.caption(
    "For the full 100k run, use `python scripts/run_ranking.py --no_dense --no_lgbm` "
    "from the repo root (see README)."
)

# ---------------------------------------------------------------------------
# Blueprint panel
# ---------------------------------------------------------------------------
bp = build_blueprint()
with st.expander("Role Blueprint (Senior AI Engineer, Redrob)", expanded=True):
    c1, c2, c3 = st.columns(3)
    with c1:
        st.markdown("**Core competencies**")
        st.write(", ".join(bp["core_competencies"][:20]) + "…")
        st.markdown(f"*{len(bp['core_competencies'])} total*")
    with c2:
        st.markdown("**Target titles**")
        st.write(", ".join(bp["target_titles"][:10]) + "…")
        st.markdown("**Experience band (yrs)**")
        st.write(
            f"min {bp['experience_band']['min']} · ideal "
            f"{bp['experience_band']['ideal_low']}–{bp['experience_band']['ideal_high']}"
        )
    with c3:
        st.markdown("**Notice period**")
        st.write(f"preferred ≤ {bp['notice_period_ok_days']} days")
        st.markdown("**Locations**")
        st.write(", ".join(bp["preferred_locations"]))

# ---------------------------------------------------------------------------
# Pipeline run
# ---------------------------------------------------------------------------
if run_btn:
    if uploaded is None:
        st.error(
            "Please upload a candidates file. The sandbox needs your "
            "data — there's no bundled sample in the public repo."
        )
        st.stop()

    t0 = time.time()
    raw_rows = _load_uploaded_jsonl(uploaded)
    if not raw_rows:
        st.error("Could not parse the uploaded file as JSON/JSONL.")
        st.stop()

    # Truncate to sample size
    raw_rows = raw_rows[:sample_size]
    cands = _rows_to_candidate_objects(raw_rows)
    df = _candidates_to_dataframe(cands)
    st.success(f"Loaded {len(df):,} candidates in {time.time()-t0:.1f}s")

    if not use_dense:
        # Disable dense retrieval for the sandbox (faster)
        from redrob import retrieval
        retrieval.dense.dense_search = lambda *a, **kw: {}

    t0 = time.time()
    # Provide placeholder retrieval scores
    df["sparse_score"] = 0.0
    df["dense_score"] = 0.0
    df["rrf_score"] = 0.0
    df["structured_hit"] = 0.0

    ranked = rank_shortlist(df, use_lgbm=False)
    ranked = ranked.head(min(100, len(ranked)))
    elapsed = time.time() - t0
    st.success(f"Ranked in {elapsed:.1f}s — top-{len(ranked)} ready")

    # Reasoning
    reasonings = generate_reasoning_dataframe(ranked)
    ranked = ranked.copy()
    ranked["reasoning"] = reasonings

    # Display
    st.subheader("Top 25 (preview)")
    display_cols = [
        "candidate_id", "current_title", "yoe",
        "title_fit_blend", "recruitability", "honeypot_penalty",
        "final_score", "reasoning",
    ]
    display_cols = [c for c in display_cols if c in ranked.columns]
    st.dataframe(ranked.head(25)[display_cols], use_container_width=True, hide_index=True)

    st.subheader(f"Top-{len(ranked)} full table")
    full_cols = ["candidate_id", "current_title", "yoe", "title_fit_blend",
                 "recruitability", "honeypot_penalty", "final_score", "reasoning"]
    full_cols = [c for c in full_cols if c in ranked.columns]
    st.dataframe(ranked[full_cols], use_container_width=True, hide_index=True)

    # Charts
    c1, c2 = st.columns(2)
    with c1:
        if "current_title" in ranked.columns:
            st.markdown("**Top-100 title distribution**")
            st.bar_chart(ranked["current_title"].value_counts().head(10))
    with c2:
        if "title_fit_blend" in ranked.columns:
            st.markdown("**Top-100 title fit histogram**")
            st.bar_chart(pd.cut(ranked["title_fit_blend"], bins=10).value_counts().sort_index())

    # Export
    out_csv = PROJECT_ROOT / "submission_sandbox.csv"
    write_submission(ranked, out_path=out_csv, validate=False)
    with open(out_csv, "rb") as f:
        st.download_button(
            "Download submission CSV",
            data=f.read(),
            file_name="submission.csv",
            mime="text/csv",
        )

    st.info(
        "The sandbox ran the COMPLETE pipeline (BM25 retrieval, 5-axis "
        "PageRank, 74 features, deterministic composite, reasoning) "
        "on your uploaded sample. For the full 100k pool, run "
        "`python scripts/run_ranking.py --no_dense --no_lgbm` from "
        "the repo root (see README)."
    )
else:
    st.info("Upload a candidates file and press 'Run ranking pipeline' in the sidebar to begin.")