Checker: gen_quality_metrics.py + watch_gen_quality_bg1.sh (scripts only, add-only)

v3_scale/gen/gen_quality_metrics.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Checker worker (Toronto, 24 oct. 2026, 12 h 30) — post-traitement LECTURE SEULE des sorties de gen_eval.py (ne relance aucun modèle).
+Entrée : samples/<tag>__<décodage>__<prompt>.txt (PROMPT / VRAI / GÉNÉRÉ, écrits par gen_eval.py) + results_gen_v1.jsonl (contrôle croisé).
+Par défaut : décodage fused, 5 prompts en1 en2 fr1 fr2 code (= gen_5p_report.py), tags = clés de report_5p.json ayant des samples.
+Métriques (sur le texte GÉNÉRÉ, 200 o) :
+  - valid_word_rate : mots (≥2 lettres, même regex que gen_eval) présents dans lexicon_v3.json (vocabulaire de corpus_v3/train.bin, fréq ≥3) ;
+    vérifié = valid_word_rate de results_gen_v1.jsonl (même texte, même lexique).
+  - répétition : distinct-1/2/3 sur les mots (types/occurrences), rep_word3 = part des trigrammes de mots déjà vus, rep_byte8 (= rep_rate8 gen_eval),
+    distinct_byte4.
+  - langue/code : part FR des mots-outils (FR/(FR+EN), listes de gen_eval), ratio de caractères de code []{}()=;_#<>, étiquettes lang/dom ;
+    DÉRIVE prompt→génération (étiquette changée ; |Δ part FR| ; Δ ratio code) ET génération vs vraie continuation (comme gen_eval).
+Sortie : agrégats seulement (aucun texte) -> gen_quality_metrics.json (+ .md) dans checker/gen/, ou OUTP=chemin_sans_extension.
+Usage : gen_quality_metrics.py [TAG ...]   (env : DEC=fused, SAMPLES=..., IN=..., OUTP=...)"""
+import os, sys, json, re
+D = os.path.dirname(os.path.abspath(__file__)); P5 = ["en1", "en2", "fr1", "fr2", "code"]
+DEC = os.environ.get("DEC", "fused"); SAM = os.environ.get("SAMPLES", f"{D}/samples"); IN = os.environ.get("IN", f"{D}/results_gen_v1.jsonl")
+OUTP = os.environ.get("OUTP", f"{D}/gen_quality_metrics")
+LEX = set(json.load(open(f"{D}/lexicon_v3.json")))
+FR = set("le la les de des du un une et est que qui dans pour pas sur au aux il elle ne se ce son sa ses avec par plus mais".split())
+EN = set("the of and to a in is that it was for on with as his her he she they not but at by be this had".split())
+WRE = re.compile(r"[a-zà-ÿœæç']+"); CODE = set("[]{}()=;_#<>")
+def words(t): return [w.strip("'") for w in WRE.findall(t.lower()) if len(w.strip("'")) >= 2]
+def fr_en(t): ws = words(t); return sum(w in FR for w in ws), sum(w in EN for w in ws)
+def lang(t): f, e = fr_en(t); return "fr" if f > e else ("en" if e > f else "?")
+def fr_share(t): f, e = fr_en(t); return f / (f + e) if f + e else None
+def code_ratio(t): return sum(c in CODE for c in t) / len(t) if t else 0.0
+def dom(t): return "code" if t and code_ratio(t) >= 0.04 else "prose"
+def distinct(seq, n):
+    g = [tuple(seq[i:i + n]) for i in range(len(seq) - n + 1)]; return (len(set(g)) / len(g)) if g else None
+def rep_rate(seq, n):
+    seen, r, tot = set(), 0, 0
+    for i in range(len(seq) - n + 1):
+        s = tuple(seq[i:i + n]) if isinstance(seq, list) else seq[i:i + n]; tot += 1; r += s in seen; seen.add(s)
+    return r / tot if tot else None
+def parse(path):
+    t = open(path, encoding="utf-8").read()
+    a = t.index("PROMPT:\n"); b = t.index("\n\nVRAI:\n"); c = t.index("\n\nGÉNÉRÉ:\n")
+    return t[a + 8:b], t[b + 8:c], t[c + 10:].removesuffix("\n")
+REF = {}
+if os.path.exists(IN):
+    for l in open(IN):
+        r = json.loads(l)
+        if r["decoding"] == DEC: REF[(r["tag"], r["prompt"])] = r
+tags = sys.argv[1:]
+if not tags:
+    rp = f"{D}/report_5p.json"; base = list(json.load(open(rp))) if os.path.exists(rp) else sorted({k[0] for k in REF})
+    tags = [t for t in base if all(os.path.exists(f"{SAM}/{t}__{DEC}__{p}.txt") for p in P5)]
+r3 = lambda x: None if x is None else round(x, 4)
+out = {"script": os.path.abspath(__file__), "decoding": DEC, "prompts": P5, "lexicon": "lexicon_v3.json (corpus_v3 train, fréq≥3, %d mots)" % len(LEX), "tags": {}}
+rows = ["| tag | mots valides | distinct-1 | distinct-2 | rep. trigr. mots | rep. 8-gr. octets | dérive langue prompt→gén | dérive domaine prompt→gén | dérive langue vs vrai | contrôle gen_eval |",
+        "|---|---|---|---|---|---|---|---|---|---|"]
+for t in tags:
+    per = {}
+    for p in P5:
+        pr, tr, g = parse(f"{SAM}/{t}__{DEC}__{p}.txt"); ws = words(g); gb = g.encode("utf-8")
+        fp, fg = fr_share(pr), fr_share(g)
+        m = {"n_words": len(ws), "valid_word_rate": r3(sum(w in LEX for w in ws) / max(len(ws), 1)),
+             "distinct_1": r3(distinct(ws, 1)), "distinct_2": r3(distinct(ws, 2)), "distinct_3": r3(distinct(ws, 3)),
+             "rep_word3": r3(rep_rate(ws, 3)), "rep_byte8": r3(rep_rate(gb, 8)), "distinct_byte4": r3(distinct(list(gb), 4)),
+             "lang_prompt": lang(pr), "lang_gen": lang(g), "lang_true": lang(tr), "dom_prompt": dom(pr), "dom_gen": dom(g), "dom_true": dom(tr),
+             "fr_share_prompt": r3(fp), "fr_share_gen": r3(fg), "fr_share_abs_delta": r3(abs(fp - fg)) if fp is not None and fg is not None else None,
+             "code_ratio_prompt": r3(code_ratio(pr)), "code_ratio_gen": r3(code_ratio(g)), "code_ratio_delta": r3(code_ratio(g) - code_ratio(pr)),
+             "lang_drift_prompt_gen": lang(g) != lang(pr), "dom_drift_prompt_gen": dom(g) != dom(pr),
+             "lang_drift_true_gen": lang(g) != lang(tr), "dom_drift_true_gen": dom(g) != dom(tr)}
+        ref = REF.get((t, p))
+        m["xcheck_gen_eval"] = None if ref is None else {k: ref[k] == m[k2] if isinstance(ref[k], str) else abs(ref[k] - (m[k2] or 0)) < 1e-3
+            for k, k2 in (("valid_word_rate", "valid_word_rate"), ("rep_rate8", "rep_byte8"), ("lang_gen", "lang_gen"), ("lang_true", "lang_true"), ("dom_gen", "dom_gen"))}
+        per[p] = m
+    mean = lambda k: r3(sum(per[p][k] for p in P5 if per[p][k] is not None) / max(1, sum(per[p][k] is not None for p in P5)))
+    cnt = lambda k: sum(per[p][k] for p in P5)
+    xc = [v for p in P5 if per[p]["xcheck_gen_eval"] for v in per[p]["xcheck_gen_eval"].values()]
+    o = {"valid_word_rate_mean": mean("valid_word_rate"), "distinct_1_mean": mean("distinct_1"), "distinct_2_mean": mean("distinct_2"),
+         "distinct_3_mean": mean("distinct_3"), "rep_word3_mean": mean("rep_word3"), "rep_byte8_mean": mean("rep_byte8"), "distinct_byte4_mean": mean("distinct_byte4"),
+         "lang_drift_prompt_gen_n": cnt("lang_drift_prompt_gen"), "dom_drift_prompt_gen_n": cnt("dom_drift_prompt_gen"),
+         "lang_drift_true_gen_n": cnt("lang_drift_true_gen"), "dom_drift_true_gen_n": cnt("dom_drift_true_gen"),
+         "fr_share_abs_delta_mean": mean("fr_share_abs_delta"), "xcheck_gen_eval_ok": f"{sum(xc)}/{len(xc)}", "per_prompt": per}
+    out["tags"][t] = o
+    rows.append(f"| {t} | {100*o['valid_word_rate_mean']:.1f} % | {o['distinct_1_mean']:.3f} | {o['distinct_2_mean']:.3f} | {100*o['rep_word3_mean']:.1f} % | {100*o['rep_byte8_mean']:.1f} % | {o['lang_drift_prompt_gen_n']}/5 | {o['dom_drift_prompt_gen_n']}/5 | {o['lang_drift_true_gen_n']}/5 | {o['xcheck_gen_eval_ok']} |")
+json.dump(out, open(OUTP + ".json", "w"), indent=1, ensure_ascii=False); open(OUTP + ".md", "w").write("\n".join(rows) + "\n"); print("\n".join(rows))

v3_scale/gen/watch_gen_quality_bg1.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Checker worker (Toronto, 24 oct. 2026, 12 h 47) — veilleur LÉGER, lecture seule de la chaîne (ne modifie aucun script de chaîne, ne lance aucun modèle).
+# Attend (sondage 120 s) : (1) les 5 samples/bg1_m8__fused__{en1,en2,fr1,fr2,code}.txt (écrits par gen_eval.py via watch_scale.sh),
+# (2) le rapport génération de la chaîne : bg1_m8 dans scale/logs/chain_post_m8_2245.state ET report_5p.json[bg1_m8] complet.
+# Puis UNE fois : nice 10, OMP_NUM_THREADS=1, ulimit -v 3,5 GiB : gen_quality_metrics.py bg1_m8 -> gen_quality_metrics_bg1_m8.{json,md}
+# (agrégats/métriques numériques seulement, aucun texte), et sort. Abandon après 48 h. Journal : logs/watch_gen_quality_bg1.log.
+G=/workspace/spira_shared/checker/gen; S=/workspace/spira_shared/checker/scale; PY=/workspace/SPIRA-venv/bin/python; TAG=bg1_m8
+LOG=$G/logs/watch_gen_quality_bg1.log; ST=$S/logs/chain_post_m8_2245.state; MAXW=$((48*3600)); T0=$(date +%s)
+export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1
+log(){ echo "$(TZ=America/Toronto date '+%F %H:%M:%S %Z') [watch_gen_quality_bg1 pid $$] $*" >> $LOG; }
+exec 9>$G/logs/watch_gen_quality_bg1.lock; flock -n 9 || { log "déjà actif, sortie"; exit 1; }
+log "démarré ; attend samples/${TAG}__fused__{en1,en2,fr1,fr2,code}.txt + rapport gén. (state + report_5p.json)"
+ready(){
+  for p in en1 en2 fr1 fr2 code; do [ -s $G/samples/${TAG}__fused__$p.txt ] || return 1; done
+  grep -qx "$TAG" $ST 2>/dev/null || return 1
+  $PY -c "import json,sys;d=json.load(open('$G/report_5p.json')).get('$TAG');sys.exit(0 if d and d.get('status')!='incomplet' and d.get('n_prompts')==5 else 1)" 2>/dev/null
+}
+while :; do
+  if ready; then
+    log "prêt -> gen_quality_metrics.py $TAG"
+    ( ulimit -v 3670016; cd $G && OUTP=$G/gen_quality_metrics_${TAG} nice -n 10 $PY $G/gen_quality_metrics.py $TAG ) >> $LOG 2>&1; rc=$?
+    log "gen_quality_metrics EXIT $rc -> gen_quality_metrics_${TAG}.json/.md"; exit $rc
+  fi
+  [ $(( $(date +%s) - T0 )) -ge $MAXW ] && { log "abandon après 48 h (non prêt)"; exit 2; }
+  sleep 120
+done