Upload folder using huggingface_hub

README.md

@@ -115,6 +115,24 @@ this card are measured, never estimated.
 The ruler was **never** used for training or fitting; all numbers at T=1,
 same split, same protocol, paired per-case predictions.
 
+**This release:**
+
+| metric | Trium (this release) | best single model (E1) |
+|---|---|---|
+| Overall accuracy (all questions) | **0.7118** | 0.6835 |
+| Accuracy on the human-adjudicated hard core | **0.6895** | 0.6566 |
+| Category accuracy | **0.7012** | 0.6773 |
+| Decision accuracy | **0.6892** | 0.6892 |
+| Urgency accuracy | **0.7450** | 0.6839 |
+| Category macro-F1 (classes present in the test split) | **0.6018** | 0.5714 |
+
+The hard core is the 497 human-adjudicated hard cases of the ruler
+(subset `nucleo`, 1,491 of the 2,259 test answers); macro-F1 follows the
+lineage's standard rule (mean F1 over every category appearing in the
+split's gold or predictions — 13 here). Category accuracy/macro-F1 of
+Trium come from its B2 head; decision from E1; urgency from the v4
+adapter.
+
 | system | category | decision | urgency | overall |
 |---|---|---|---|---|
 | JEV (frontier decision API, teacher/reference) | **0.7251** | **0.6892** | 0.7238 | **0.7127** |

REPRODUCAO.md

@@ -21,13 +21,13 @@ few hundredths of a pp between library versions):
 
 ```
 # 753 casos, 2259 respostas
-TRIUM  cat 0.7012 dec 0.6892 urg 0.7450 geral 0.7118
-CompA  cat 0.6826 dec 0.6494 urg 0.7450 geral 0.6923
-v6     cat 0.6826 dec 0.6494 urg 0.6521 geral 0.6614
-b2     cat 0.7012 dec 0.6680 urg 0.6707 geral 0.6799
-e1     cat 0.6773 dec 0.6892 urg 0.6839 geral 0.6835
-v4     cat 0.2470 dec 0.1633 urg 0.7450 geral 0.3851
-base   cat 0.5817 dec 0.2988 urg 0.5511 geral 0.4772
+TRIUM  cat 0.7012 dec 0.6892 urg 0.7450 geral 0.7118 hard 0.6895 macroF1 0.6018(13)
+CompA  cat 0.6826 dec 0.6494 urg 0.7450 geral 0.6923 hard 0.6693
+v6     cat 0.6826 dec 0.6494 urg 0.6521 geral 0.6614 hard 0.6398 macroF1 0.5898(13)
+b2     cat 0.7012 dec 0.6680 urg 0.6707 geral 0.6799 hard 0.6506 macroF1 0.6018(13)
+e1     cat 0.6773 dec 0.6892 urg 0.6839 geral 0.6835 hard 0.6566 macroF1 0.5714(13)
+v4     cat 0.2470 dec 0.1633 urg 0.7450 geral 0.3851 hard 0.3803
+base   cat 0.5817 dec 0.2988 urg 0.5511 geral 0.4772 hard 0.4681
 bootstrap Trium vs v6:    Δ +5.05 p.p. IC95 [+3.54; +6.60]
 bootstrap Trium vs CompA: Δ +1.95 p.p. IC95 [+1.15; +2.74]
 bootstrap Trium vs base (clef-flash zero-shot): Δ +23.46 p.p. IC95 [+21.16; +25.72]

manifest.json

@@ -1,9 +1,9 @@
 {
- "gerado": "2026-10-23T12:29:41-03:00",
+ "gerado": "2026-10-23T16:08:13-03:00",
  "release": "veredicto",
  "arquivos": {
-  "README.md": "c639bf03a5177218648525052dd3ff382efa92f5d4de52132562c5a1d99141b2",
-  "REPRODUCAO.md": "32fce7453198ce9c1599d7b0458e9ae99c70ca57f87058a2b18dd0924be5d33e",
+  "README.md": "9688d0912fd17c9917e4d37ac914049b39a898dff6f7d6aa774da2d897f22584",
+  "REPRODUCAO.md": "e56439e8311f274cb28c1b17e8e91d943b506adfdbf6784a88b99e37f6d8ff30",
   "heads/veredicto_e1_step-2576/joint_head.pt": "2665935d6b018483015d09739165cbf4248425922c60754f105f1cfb8011e92a",
   "heads/veredicto_e1_step-2576/joint_head_config.json": "71018e6a3d73723a5a8b4a151c5ceb82a1c2a5658f7d80168fe6c82afede0f36",
   "heads/veredicto_e1_step-2576/manifesto.json": "620e185e8d527bfa36415b89236b3510155a781436b1b531c37701a75a8a9d29",
@@ -20,7 +20,7 @@
   "heads/veredicto_b2_step-2208/processor/tokenizer_config.json": "7488550a894dcf741616126e322ea31d66cf9457b59ec5825f4c7efdced421f8",
   "adapters/veredicto-lora-v4/adapter_config.json": "719d9a473d3f9fd7d9fbe729b77bf58accae8d1d733cb8f875b5ecfe82e3b6b4",
   "adapters/veredicto-lora-v4/adapter_model.safetensors": "efe526ccca100a32b11d44a62fbf2ebaf659c499f8f14db1543c0abd0420a07f",
-  "scripts/auditoria_card.py": "ada7ac01aa253b10498142d2691ce78d11d003ca19856ed72c7ca204586f87e7",
+  "scripts/auditoria_card.py": "8b9799a54474d7ab6e37a701dabfa3ef29daa817c9dfc19539f61ee0ca4c7575",
   "scripts/avaliar_veredicto.py": "14fe1a5624977a9fb3004c0eebb5066a6560e7fa77a04c707e9df2922a2f8c66",
   "scripts/paridade_multi.py": "16d8eff0af7d3e045f3453420137ed6f13550d39dc61decc232f8448b5c4ba04",
   "scripts/servir_veredicto_multi.py": "485eeb2731e0d3c3a887a4226ff446683deb9fd8f88865d1a84fe07569a778f4",

scripts/auditoria_card.py

@@ -43,6 +43,16 @@ def load_raw(path):
     return d
 
 
+def load_pred_raw(path):
+    """(caso,q) -> índice predito, na ordem de option_ids do próprio arquivo."""
+    d = {}
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            r = json.loads(line)
+            d[(r["caso"], r["q"])] = int(r["pred"])
+    return d
+
+
 def load_ops(path):
     """(caso,q) -> lista de option_ids (ordem em que pred/rotulo indexam)."""
     d = {}
@@ -53,6 +63,16 @@ def load_ops(path):
     return d
 
 
+def load_subset(path):
+    """(caso,q) -> subset ('nucleo' = hard core arbitrado por humano)."""
+    d = {}
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            r = json.loads(line)
+            d[(r["caso"], r["q"])] = r.get("subset", "nao_informado")
+    return d
+
+
 def load_ouro(path, seed=42):
     """rótulos do teste (mesmo split da régua) como hit-flags de um oráculo."""
     here = Path(__file__).resolve().parent
@@ -111,6 +131,24 @@ def acc(casos, fonte):
     return out
 
 
+def macro_f1_categoria(raw_cat, pred_cat, ops):
+    """macro-F1 de categoria com a mesma regra da linhagem Decisum (avaliar_v4_bench):
+    média do F1 sobre as classes que aparecem no ouro OU nas predições do teste
+    (no teste atual: 12 classes; fraude_suspeita e reembolso têm n ínfimo no ouro)."""
+    cls = {}
+    def slot(lab):
+        return cls.setdefault(lab, [0, 0, 0])
+    for key, gold_idx in raw_cat.items():
+        real, pred = ops[key][gold_idx], ops[key][pred_cat[key]]
+        if real == pred:
+            slot(real)[0] += 1
+        else:
+            slot(real)[2] += 1
+            slot(pred)[1] += 1
+    f1s = [2 * tp / max(1, 2 * tp + fp + fn) for tp, fp, fn in cls.values()]
+    return sum(f1s) / len(f1s), len(f1s)
+
+
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--dir", required=True)
@@ -126,6 +164,24 @@ def main():
     casos = sorted({c for c, _ in keys})
     print(f"# {len(casos)} casos, {len(keys)} respostas")
 
+    subset = load_subset(d / "preds_v6.jsonl")
+    assert all(subset[k] in ("nucleo", "aleatorio") for k in keys), "subsets inesperados"
+    nucleo = [k for k in keys if subset[k] == "nucleo"]
+    ops = load_ops(d / "preds_v6.jsonl")
+
+    def hard(src):
+        """acc no hard core (subset 'nucleo'); src = roteado (tarefa->hits) ou mono."""
+        roteado = all(isinstance(k, str) for k in src)
+        return sum((src[k[1]] if roteado else src)[k] for k in nucleo) / len(nucleo)
+
+    def macro_f1(src_raw_cat):
+        """macro-F1 de categoria no critério da linhagem Decisum (ver macro_f1_
+        categoria); src_raw_cat: (caso,'categoria') -> índice predito."""
+        raw_cat = load_raw(d / "preds_v6.jsonl")
+        raw_cat = {k: v for k, v in raw_cat.items() if k[1] == "categoria"}
+        return macro_f1_categoria(raw_cat, src_raw_cat,
+                                  {k: ops[k] for k in raw_cat})
+
     base_path = d / "preds_base_clef.jsonl"
     base = load_preds(base_path) if base_path.exists() else None
 
@@ -138,8 +194,19 @@ def main():
         modelos.append(("base", base))
     for nome, fonte in modelos:
         a = acc(casos, fonte)
-        print(f"{nome:6s} cat {a['categoria']:.4f} dec {a['decisao']:.4f} "
-              f"urg {a['urgencia']:.4f} geral {a['geral']:.4f}")
+        line = (f"{nome:6s} cat {a['categoria']:.4f} dec {a['decisao']:.4f} "
+                f"urg {a['urgencia']:.4f} geral {a['geral']:.4f} "
+                f"hard {hard(fonte):.4f}")
+        # macro-F1 de categoria:TRIUM usa a cabeça de categoria (b2). A base
+        # zero-shot fica de fora: preds_base_clef.jsonl não traz option_ids,
+        # então seus índices não podem ser rotulados a partir do arquivo.
+        cat_file = {"TRIUM": "preds_b2.jsonl", "v6": "preds_v6.jsonl",
+                    "b2": "preds_b2.jsonl", "e1": "preds_e1.jsonl"}.get(nome)
+        if cat_file is not None:
+            src_pred = load_pred_raw(d / cat_file)
+            m, n_cls = macro_f1({k: v for k, v in src_pred.items() if k[1] == "categoria"})
+            line += f" macroF1 {m:.4f}({n_cls})"
+        print(line)
 
     def compose(fonte):
         if all(isinstance(k, str) for k in fonte):