"""Graph-assisted features (the 'graphify' layer). The graph does NOT rank — it derives three supporting signals that nudge the hybrid scorer via a bounded `graph_boost` in [graph_boost.min, graph_boost.max]: 1. peer_quality — a FAISS kNN similarity graph over candidate embeddings. Are your nearest-neighbour candidates on-role? Genuine fits cluster with genuine fits; isolated keyword-stuffers and honeypots cluster with off-role/odd profiles. 2. skill_coherence — a networkx skill co-occurrence graph. Do a candidate's core-AI skills form a tight, frequently-co-occurring cluster (coherent expertise) or a scattered stuff list? 3. company_product — a networkx candidate↔company bipartite graph. What fraction of a company's people sit in product industries (vs IT-services/consulting)? Propagates a product signal. All of this is computed in the OFFLINE precompute step and cached to `graph_features.parquet`; the ranking step only loads that table (cheap). Built with networkx (in-memory, no external DB) + faiss-cpu for kNN — fully reproducible inside the Stage-3 sandbox, no network. """ from __future__ import annotations from pathlib import Path import numpy as np import pandas as pd from . import jd from .config import load_config from .features import title_fit from .ingest import stream_candidates def _lower(s) -> str: return s.lower() if isinstance(s, str) else "" # --------------------------------------------------------------------------- # component computations # --------------------------------------------------------------------------- def peer_quality(embeddings: np.ndarray, role_quality: np.ndarray, k: int, min_sim: float) -> np.ndarray: """For each candidate, the similarity-weighted mean role-quality of its k nearest neighbours (excluding itself). Uses a FAISS inner-product index on L2-normalised embeddings (so inner product == cosine). """ import faiss n, d = embeddings.shape index = faiss.IndexFlatIP(d) index.add(embeddings) k_eff = min(k + 1, n) # +1 for self; clamp for small samples (demo) sims, idx = index.search(embeddings, k_eff) out = np.empty(n, dtype=np.float32) for i in range(n): neigh_sims, neigh_idx = sims[i], idx[i] mask = (neigh_idx != i) & (neigh_idx >= 0) & (neigh_sims >= min_sim) if not mask.any(): out[i] = role_quality[i] # isolated node: fall back to its own quality continue w = neigh_sims[mask] out[i] = float(np.average(role_quality[neigh_idx[mask]], weights=w)) return out def skill_coherence(skill_lists: list[list[str]]) -> np.ndarray: """Build a networkx co-occurrence graph over core-AI skills, then score each candidate by how strongly their AI skills co-occur with each other across the pool (coherent expertise vs scattered keyword stuffing). """ import networkx as nx from itertools import combinations g = nx.Graph() ai_lists = [] for skills in skill_lists: ai = [s for s in skills if s in jd.CORE_AI_SKILLS] ai_lists.append(ai) for a, b in combinations(sorted(set(ai)), 2): if g.has_edge(a, b): g[a][b]["w"] += 1 else: g.add_edge(a, b, w=1) # Normalise edge weights by the global max so coherence is in ~[0,1]. max_w = max((data["w"] for _, _, data in g.edges(data=True)), default=1) out = np.zeros(len(ai_lists), dtype=np.float32) for i, ai in enumerate(ai_lists): uniq = sorted(set(ai)) if len(uniq) < 2: out[i] = 0.5 # neutral: nothing to (in)cohere; rely on other signals continue pairs = list(combinations(uniq, 2)) strength = sum(g[a][b]["w"] for a, b in pairs if g.has_edge(a, b)) out[i] = min(1.0, (strength / len(pairs)) / max_w * 8.0) return out def company_product_scores(companies: list[str], industries: list[str]) -> np.ndarray: """networkx candidate↔company bipartite graph: each company's product-ness is the fraction of its people in product industries; propagate back to candidates. """ import networkx as nx g = nx.Graph() for i, comp in enumerate(companies): if not comp: continue is_prod = industries[i] in jd.PRODUCT_INDUSTRIES cnode = ("company", comp) g.add_node(cnode) g.add_edge(("cand", i), cnode, product=int(is_prod)) comp_score: dict = {} for comp in {("company", c) for c in companies if c}: edges = g.edges(comp, data=True) vals = [e[2]["product"] for e in edges] comp_score[comp] = sum(vals) / len(vals) if vals else 0.5 out = np.full(len(companies), 0.5, dtype=np.float32) for i, comp in enumerate(companies): if comp: out[i] = comp_score.get(("company", comp), 0.5) return out # --------------------------------------------------------------------------- # core computation (shared by offline precompute and the in-memory demo) # --------------------------------------------------------------------------- def compute_boosts(embeddings, role_quality, skill_lists, companies, industries, cfg): """Return (peer, coherence, product, graph_boost) arrays for a set of candidates. Works on the full pool (offline) or a small sample (the Streamlit demo). """ gcfg = cfg["graph"] peer = peer_quality(embeddings, role_quality, gcfg["knn_neighbors"], gcfg["knn_min_similarity"]) coher = skill_coherence(skill_lists) prod = company_product_scores(companies, industries) w = gcfg["boost_weights"] raw = w["peer"] * peer + w["skill_coherence"] * coher + w["company_product"] * prod bmin = cfg["scoring"]["graph_boost"]["min"] bmax = cfg["scoring"]["graph_boost"]["max"] boost = bmin + (bmax - bmin) * raw return peer, coher, prod, boost.astype(np.float32) def _collect_inputs(records): """Extract role_quality / skill_lists / companies / industries from candidate dicts.""" role_quality = np.array([title_fit(c) for c in records], dtype=np.float32) skill_lists = [[_lower(s.get("name", "")) for s in c.get("skills", [])] for c in records] companies = [_lower(c.get("profile", {}).get("current_company", "")) for c in records] industries = [_lower(c.get("profile", {}).get("current_industry", "")) for c in records] return role_quality, skill_lists, companies, industries # --------------------------------------------------------------------------- # offline driver # --------------------------------------------------------------------------- def precompute_graph(cfg: dict) -> None: paths = cfg["paths"] ids = np.load(paths["candidate_ids"]) embeddings = np.load(paths["embeddings"]) records = [] for i, c in enumerate(stream_candidates(paths["candidates"])): if i >= len(ids): break assert c["candidate_id"] == ids[i], "candidate order must match cached embeddings" records.append(c) role_quality, skill_lists, companies, industries = _collect_inputs(records) print("Computing graph features (FAISS kNN + networkx) ...") peer, coher, prod, boost = compute_boosts( embeddings, role_quality, skill_lists, companies, industries, cfg) df = pd.DataFrame({ "candidate_id": ids, "peer_quality": peer, "skill_coherence": coher, "company_product": prod, "graph_boost": boost, }) Path(paths["artifacts_dir"]).mkdir(parents=True, exist_ok=True) df.to_parquet(paths["graph_features"], index=False) print(f"Saved graph features {df.shape} -> {paths['graph_features']}") print(f"graph_boost: min={boost.min():.3f} mean={boost.mean():.3f} max={boost.max():.3f}") def load_graph_boosts(cfg: dict) -> dict | None: path = Path(cfg["paths"]["graph_features"]) if not path.exists(): return None df = pd.read_parquet(path) return dict(zip(df["candidate_id"], df["graph_boost"].astype(float))) if __name__ == "__main__": precompute_graph(load_config())