Spaces:
Running on Zero
Running on Zero
Download scripts/mesurer_performances.py from Myz0/CV-Matching: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/spaces/Myz0/CV-Matching/resolve/main/scripts/mesurer_performances.py
- Command line
-
hf download hf://spaces/Myz0/CV-Matching/scripts/mesurer_performances.py
-
curl -L -o mesurer_performances.py https://huggingface.co/spaces/Myz0/CV-Matching/resolve/main/scripts/mesurer_performances.py
11.5 kB
| """Mesure la qualité du classement et la vitesse du service, et garde des traces. | |
| Écrit dans resultats/<date>/ : | |
| machine_<device>.json la machine et les versions (GPU, pilote, torch...) + pip_freeze.txt | |
| qualite.json mesures de classement de chaque méthode testée (GPU seulement) | |
| scores_cv_avp.csv score de chaque CV sur chaque AVP, méthode retenue (GPU seulement) | |
| repartition.json scores des bons CV et des hors-domaine, seuils des libellés | |
| temps_<device>.json durées : démarrage, /match, classement d'un vivier | |
| Lancer depuis la racine du projet (le dataset complet : $env:DATASET_REVISION=...) : | |
| python -m scripts.mesurer_performances --device gpu | |
| python -m scripts.mesurer_performances --device cpu # carte graphique masquée, ~20 min | |
| Puis : python -m scripts.graphiques resultats/<date> | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import platform | |
| import statistics | |
| import subprocess | |
| import sys | |
| import time | |
| from datetime import datetime | |
| from pathlib import Path | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--device", choices=["gpu", "cpu"], default="gpu") | |
| parser.add_argument("--sortie", default=None, help="dossier (défaut : resultats/<date du jour>)") | |
| args = parser.parse_args() | |
| if args.device == "cpu": | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # avant l'import de torch | |
| import numpy as np | |
| import torch | |
| from app import config, scoring | |
| from app.avp_store import AVPStore | |
| from app.classement import classer_candidats | |
| from app.cv_texte import phrases_du_cv | |
| from app.encodeur import Encodeur | |
| from app.matching import matcher | |
| from scripts.evaluer_classement import charger_attendus, mesurer | |
| RACINE = Path(__file__).resolve().parent.parent | |
| SORTIE = Path(args.sortie) if args.sortie else RACINE / "resultats" / datetime.now().strftime("%Y-%m-%d") | |
| SORTIE.mkdir(parents=True, exist_ok=True) | |
| CV_TEST = [0, 6, 16] # auditeur, chef d'agence, facteur | |
| AVP_VIVIER = "3134-26-1419/SR" # classement des 23 CV pour l'AVP Auditeur | |
| # (nom affiché, mode, SIM_BAS, SIM_HAUT) - mode : bge (rampe seule), rr (reranker seul), hyb | |
| VARIANTES = [ | |
| ("Avant : bge-m3, rampe 0,45/0,70", "bge", 0.45, 0.70), | |
| ("bge-m3, rampe 0,55/0,75", "bge", 0.55, 0.75), | |
| ("bge-m3, rampe 0,575/0,775", "bge", 0.575, 0.775), | |
| ("Reranker seul", "rr", None, None), | |
| ("Hybride, rampe 0,55/0,75", "hyb", 0.55, 0.75), | |
| ("Hybride, rampe 0,575/0,775 (retenu)", "hyb", 0.575, 0.775), | |
| ] | |
| def ecrire(nom: str, donnees) -> None: | |
| (SORTIE / nom).write_text(json.dumps(donnees, ensure_ascii=False, indent=2), encoding="utf-8") | |
| print(f" -> {SORTIE / nom}") | |
| def commande(cmd: list[str]) -> str: | |
| try: | |
| return subprocess.run(cmd, capture_output=True, text=True, timeout=60).stdout.strip() | |
| except Exception: | |
| return "" | |
| def fiche_machine() -> dict: | |
| import gradio | |
| import sentence_transformers | |
| import transformers | |
| fiche = { | |
| "date": datetime.now().isoformat(timespec="seconds"), | |
| "device_demande": args.device, | |
| "os": platform.platform(), | |
| "python": platform.python_version(), | |
| "coeurs_logiques": os.cpu_count(), | |
| "threads_torch": torch.get_num_threads(), | |
| "torch": torch.__version__, | |
| "cuda_disponible": torch.cuda.is_available(), | |
| "transformers": transformers.__version__, | |
| "sentence_transformers": sentence_transformers.__version__, | |
| "gradio": gradio.__version__, | |
| "modele": config.MODEL_NAME, | |
| "reranker": config.RERANKER_MODEL or None, | |
| "dataset": config.DATASET_REPO, | |
| "dataset_revision": config.DATASET_REVISION, | |
| "rampe": [scoring.SIM_BAS, scoring.SIM_HAUT], | |
| "seuils_libelle": config.SEUILS_LIBELLE, | |
| } | |
| if platform.system() == "Windows": | |
| fiche["processeur"] = commande(["powershell", "-NoProfile", "-Command", | |
| "(Get-CimInstance Win32_Processor).Name"]) | |
| ram = commande(["powershell", "-NoProfile", "-Command", | |
| "(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory"]) | |
| fiche["ram_go"] = round(int(ram) / 2**30, 1) if ram.isdigit() else None | |
| else: | |
| fiche["processeur"] = platform.processor() | |
| if torch.cuda.is_available(): | |
| fiche["gpu"] = torch.cuda.get_device_name(0) | |
| fiche["gpu_vram_go"] = round(torch.cuda.get_device_properties(0).total_memory / 2**30, 1) | |
| fiche["cuda_torch"] = torch.version.cuda | |
| fiche["pilote_nvidia"] = commande(["nvidia-smi", "--query-gpu=driver_version", | |
| "--format=csv,noheader"]) | |
| (SORTIE / "pip_freeze.txt").write_text( | |
| commande([sys.executable, "-m", "pip", "freeze"]), encoding="utf-8") | |
| return fiche | |
| def chrono(f, n: int = 3) -> dict: | |
| """Durée médiane de n appels (après un appel de chauffe fait par l'appelant).""" | |
| durees = [] | |
| for _ in range(n): | |
| t = time.perf_counter() | |
| f() | |
| if torch.cuda.is_available(): | |
| torch.cuda.synchronize() | |
| durees.append(time.perf_counter() - t) | |
| return {"mediane_s": round(statistics.median(durees), 3), "essais_s": [round(d, 3) for d in durees]} | |
| def matrice_variante(sims_cv, poids, mode, bas, haut) -> np.ndarray: | |
| m = np.zeros((len(sims_cv), len(poids))) | |
| for c, ligne in enumerate(sims_cv): | |
| for a, (x, w) in enumerate(zip(ligne, poids)): | |
| if x is None or w.sum() <= 0: | |
| continue | |
| sims, rr = x | |
| if mode == "rr": | |
| d = rr[0] | |
| else: | |
| d, _ = scoring.couvertures(sims, rr if mode == "hyb" else None, bas, haut) | |
| m[c, a] = float(np.sum(w * d) / w.sum()) | |
| return m | |
| def main(): | |
| print(f"Sortie : {SORTIE}") | |
| fiche = fiche_machine() | |
| ecrire(f"machine_{args.device}.json", fiche) | |
| print(f"Machine : {fiche.get('processeur')} | GPU : {fiche.get('gpu', 'aucun')} | torch {fiche['torch']}") | |
| temps = {"device": args.device} | |
| cvs = json.loads((RACINE / "exemples/cv_list.json").read_text(encoding="utf-8")) | |
| # 1. Démarrage du service : chargement des modèles, puis encodage des AVP | |
| t = time.perf_counter() | |
| from app.reranker import Reranker | |
| reranker = Reranker(config.RERANKER_MODEL) | |
| encodeur = Encodeur(config.MODEL_NAME) | |
| temps["chargement_modeles_s"] = round(time.perf_counter() - t, 1) | |
| t = time.perf_counter() | |
| store = AVPStore(encodeur) | |
| store.charger() | |
| temps["encodage_avps_s"] = round(time.perf_counter() - t, 1) | |
| temps["nb_avps"] = len(store.prepares) | |
| print(f"Démarrage : modèles {temps['chargement_modeles_s']} s, " | |
| f"{temps['nb_avps']} AVP {temps['encodage_avps_s']} s") | |
| # 2. /match (côté candidat), sans puis avec reranker | |
| temps["match"] = {} | |
| for nom, rr in (("sans_reranker", None), ("avec_reranker", reranker)): | |
| scoring.RERANKER = rr | |
| appel = lambda i: matcher(cvs[i], store, encodeur, config.SEUIL_COUVERTURE, | |
| top=5, inclure_fermes=True) | |
| appel(CV_TEST[0]) # chauffe | |
| n = 3 if (rr is None or args.device == "gpu") else 1 # CPU + reranker : ~4 min/appel | |
| temps["match"][nom] = {str(i): chrono(lambda: appel(i), n) for i in CV_TEST} | |
| meds = [v["mediane_s"] for v in temps["match"][nom].values()] | |
| print(f" /match {nom:14s} : {', '.join(f'{x:.2f} s' for x in meds)}") | |
| # 3. Classement d'un vivier de 23 CV pour un AVP (côté recruteur) | |
| temps["classement_23_cv"] = {} | |
| for nom, rr in (("sans_reranker", None), ("avec_reranker", reranker)): | |
| if rr is not None and args.device == "cpu": | |
| temps["classement_23_cv"][nom] = None # ~23 x 4 min : non mesuré | |
| continue | |
| scoring.RERANKER = rr | |
| temps["classement_23_cv"][nom] = chrono( | |
| lambda: classer_candidats(AVP_VIVIER, cvs, store, encodeur, | |
| seuil=config.SEUIL_COUVERTURE, top=23), n=1) | |
| print(f" classement 23 CV {nom:14s} : {temps['classement_23_cv'][nom]['mediane_s']:.1f} s") | |
| ecrire(f"temps_{args.device}.json", temps) | |
| if args.device == "cpu": | |
| return | |
| # 4. Qualité du classement de chaque méthode (mêmes mesures que evaluer_classement) | |
| scoring.RERANKER = reranker | |
| ids = list(store.prepares) | |
| attendus = charger_attendus(ids) | |
| poids = [store.prepares[i].poids for i in ids] | |
| sims_cv = [] | |
| for cv in cvs: | |
| textes = [t for t, _ in phrases_du_cv(cv)] | |
| vecs = encodeur.encoder(textes) if textes else None | |
| rr = scoring.probas_reranker_avps(store.prepares, textes, vecs) if vecs is not None else {} | |
| sims_cv.append([(store.prepares[i].vecs_attendus @ vecs.T, rr.get(i)) | |
| if vecs is not None and len(store.prepares[i].attendus) else None | |
| for i in ids]) | |
| qualite = [] | |
| for nom, mode, bas, haut in VARIANTES: | |
| m = matrice_variante(sims_cv, poids, mode, bas, haut) | |
| qualite.append({"methode": nom, "mode": mode, "sim_bas": bas, "sim_haut": haut, | |
| **mesurer(m, ids, attendus)}) | |
| print(f" {nom:38s} {qualite[-1]}") | |
| ecrire("qualite.json", {"nb_cv_etiquetes": sum(v is not None for v in attendus.values()), | |
| "nb_hors_domaine": sum(v is None for v in attendus.values()), | |
| "methodes": qualite}) | |
| # 5. Méthode retenue : matrice complète CV x AVP et répartition des scores | |
| m = matrice_variante(sims_cv, poids, "hyb", scoring.SIM_BAS, scoring.SIM_HAUT) | |
| # Le libellé se lit sur le pourcentage affiché : même calcul sans les savoir-être | |
| m_pct = matrice_variante(sims_cv, [store.prepares[i].poids * ~store.prepares[i].savoir_etre for i in ids], | |
| "hyb", scoring.SIM_BAS, scoring.SIM_HAUT) | |
| lignes = ["cv;profil;attendu;" + ";".join(ids)] | |
| for c, cv in enumerate(cvs): | |
| cible = attendus.get(c, "ignore") if c in attendus else "ignore" | |
| cible = "hors_domaine" if cible is None else ("ignore" if cible == "ignore" else "|".join(cible)) | |
| lignes.append(f"{c};{cv.get('_profil', '')};{cible};" + | |
| ";".join(f"{100 * x:.1f}" for x in m[c])) | |
| (SORTIE / "scores_cv_avp.csv").write_text("\n".join(lignes), encoding="utf-8-sig") | |
| print(f" -> {SORTIE / 'scores_cv_avp.csv'}") | |
| bons = [{"cv": c, "profil": cvs[c].get("_profil"), "avp": a, "score": round(100 * m[c, ids.index(a)], 1), | |
| "pourcentage": round(100 * m_pct[c, ids.index(a)], 1), "libelle": scoring.libelle(m_pct[c, ids.index(a)])} | |
| for c, cib in attendus.items() if cib for a in cib] | |
| hors = [{"cv": c, "profil": cvs[c].get("_profil"), "avp": ids[int(m[c].argmax())], | |
| "score": round(100 * m[c].max(), 1), "pourcentage": round(100 * m_pct[c, int(m[c].argmax())], 1), | |
| "libelle": scoring.libelle(m_pct[c, int(m[c].argmax())])} | |
| for c, cib in attendus.items() if cib is None] | |
| ecrire("repartition.json", {"seuils_libelle": config.SEUILS_LIBELLE, | |
| "bons_cv_sur_leur_avp": bons, "hors_domaine_meilleur_avp": hors}) | |
| if __name__ == "__main__": | |
| main() | |