thefinalboss commited on
Commit
cc4149a
·
verified ·
1 Parent(s): 9710a52

v3_scale: broad_gutenberg_v1 manifest + recomptes 64 o (delta vs broad_local_v1 = 0, valid_vs_base)

Browse files
v3_scale/corpora/broad_gutenberg_v1/manifest.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "variant": "broad_gutenberg_v1",
3
+ "construction": "train.bin = broad_local_v1/train.bin + b'\\n\\n' + corpus_gutenberg_v2/train.bin ; test.bin et oneshot.bin = copies de broad_local_v1 (test20k / fresh_en inchang\u00e9s)",
4
+ "sources": {
5
+ "broad_local_v1": {
6
+ "train_sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
7
+ "composition": {
8
+ "en": 28654965,
9
+ "fr": 214337,
10
+ "code": 6860045
11
+ }
12
+ },
13
+ "corpus_gutenberg_v2": {
14
+ "train_sha256": "75afd55a4e800715d6870ec88ed12e5c01a42b6caa53243ad267a35c55525fb3",
15
+ "n_books": 568,
16
+ "en_bytes": 186503118,
17
+ "fr_bytes": 69470991,
18
+ "source": "Project Gutenberg via aleph.pglaf.org (approuv\u00e9 23/10 16 h 04)"
19
+ }
20
+ },
21
+ "files": {
22
+ "train.bin": {
23
+ "bytes": 290810502,
24
+ "sha256": "5af6398fc853bce270e430f8336fc2b136318c2e762ead19e92f5d180de083f2"
25
+ },
26
+ "test.bin": {
27
+ "bytes": 894718,
28
+ "sha256": "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9"
29
+ },
30
+ "oneshot.bin": {
31
+ "bytes": 119763,
32
+ "sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a"
33
+ }
34
+ },
35
+ "composition_approx": {
36
+ "en": 215158083,
37
+ "fr": 69685328,
38
+ "code": 6860045
39
+ },
40
+ "ratio_en_fr": 3.088,
41
+ "share": {
42
+ "en": 0.7376,
43
+ "fr": 0.2389,
44
+ "code": 0.0235
45
+ },
46
+ "contamination_check": "checker/recount/recount_concat_v1.py (delta vs broad_local_v1) -> checker/recount/recount_broad_gutenberg_v1.json"
47
+ }
v3_scale/recount/delta_concat_v1.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Checker — delta chevauchement 64 o : concat (recount_concat_v1.py) moins base broad_local_v1 (même script). seen_* exclus, code à part."""
3
+ import json
4
+ R = "/workspace/spira_shared/checker/recount"; c = json.load(open(f"{R}/recount_broad_gutenberg_v1.json")); b = json.load(open(f"{R}/recount_broad_local_v1.json"))
5
+ EX = {"seen_old", "seen_broad", "seen_v3", "m7_seen"}; SEP = {"neither_code", "m7_neither_code", "m7_freshcode"}
6
+ d = {k: c["overlap64"][k]["confirmed"] - b["overlap64"][k]["confirmed"] for k in c["overlap64"]}
7
+ c.update(base_json=f"{R}/recount_broad_local_v1.json", base_train_sha256=b["train_sha256"], delta_vs_base=d, excluded_seen=sorted(EX),
8
+ reported_separately={k: {"concat": c["overlap64"][k]["confirmed"], "base": b["overlap64"][k]["confirmed"]} for k in SEP},
9
+ valid_vs_base=all(v == 0 for k, v in d.items() if k not in EX))
10
+ json.dump(c, open(f"{R}/recount_broad_gutenberg_v1.json", "w"), indent=1); print("valid_vs_base", c["valid_vs_base"], d)
v3_scale/recount/recount_broad_gutenberg_v1.json ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train_sha256": "5af6398fc853bce270e430f8336fc2b136318c2e762ead19e92f5d180de083f2",
3
+ "train_bytes": 290810502,
4
+ "overlap64": {
5
+ "fc2": {
6
+ "n_64grams": 199729,
7
+ "confirmed": 0
8
+ },
9
+ "fresh_en": {
10
+ "n_64grams": 119700,
11
+ "confirmed": 0
12
+ },
13
+ "fresh_fr": {
14
+ "n_64grams": 1088625,
15
+ "confirmed": 0
16
+ },
17
+ "neither_code": {
18
+ "n_64grams": 545973,
19
+ "confirmed": 545558
20
+ },
21
+ "test20k": {
22
+ "n_64grams": 894655,
23
+ "confirmed": 33766
24
+ },
25
+ "queries": {
26
+ "n_64grams": 27737,
27
+ "confirmed": 0
28
+ },
29
+ "test_v3": {
30
+ "n_64grams": 3486590,
31
+ "confirmed": 0
32
+ },
33
+ "seen_old": {
34
+ "n_64grams": 199937,
35
+ "confirmed": 199877
36
+ },
37
+ "seen_broad": {
38
+ "n_64grams": 199937,
39
+ "confirmed": 199937
40
+ },
41
+ "seen_v3": {
42
+ "n_64grams": 199937,
43
+ "confirmed": 199217
44
+ },
45
+ "m7_fresh_fr": {
46
+ "n_64grams": 1088625,
47
+ "confirmed": 0
48
+ },
49
+ "m7_freshcode": {
50
+ "n_64grams": 199937,
51
+ "confirmed": 163445
52
+ },
53
+ "m7_seen": {
54
+ "n_64grams": 199937,
55
+ "confirmed": 199877
56
+ },
57
+ "m7_neither_code": {
58
+ "n_64grams": 545973,
59
+ "confirmed": 545558
60
+ },
61
+ "m7_test20k": {
62
+ "n_64grams": 319877,
63
+ "confirmed": 1405
64
+ },
65
+ "m7_fresh_en": {
66
+ "n_64grams": 119700,
67
+ "confirmed": 0
68
+ },
69
+ "m7_queries": {
70
+ "n_64grams": 27737,
71
+ "confirmed": 0
72
+ },
73
+ "v3_test": {
74
+ "n_64grams": 3486590,
75
+ "confirmed": 0
76
+ },
77
+ "v3_oneshot": {
78
+ "n_64grams": 119700,
79
+ "confirmed": 0
80
+ }
81
+ },
82
+ "valid_zero_overlap": false,
83
+ "base_json": "/workspace/spira_shared/checker/recount/recount_broad_local_v1.json",
84
+ "base_train_sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
85
+ "delta_vs_base": {
86
+ "fc2": 0,
87
+ "fresh_en": 0,
88
+ "fresh_fr": 0,
89
+ "neither_code": 0,
90
+ "test20k": 0,
91
+ "queries": 0,
92
+ "test_v3": 0,
93
+ "seen_old": 0,
94
+ "seen_broad": 0,
95
+ "seen_v3": 0,
96
+ "m7_fresh_fr": 0,
97
+ "m7_freshcode": 0,
98
+ "m7_seen": 0,
99
+ "m7_neither_code": 0,
100
+ "m7_test20k": 0,
101
+ "m7_fresh_en": 0,
102
+ "m7_queries": 0,
103
+ "v3_test": 0,
104
+ "v3_oneshot": 0
105
+ },
106
+ "excluded_seen": [
107
+ "m7_seen",
108
+ "seen_broad",
109
+ "seen_old",
110
+ "seen_v3"
111
+ ],
112
+ "reported_separately": {
113
+ "m7_neither_code": {
114
+ "concat": 545558,
115
+ "base": 545558
116
+ },
117
+ "neither_code": {
118
+ "concat": 545558,
119
+ "base": 545558
120
+ },
121
+ "m7_freshcode": {
122
+ "concat": 163445,
123
+ "base": 163445
124
+ }
125
+ },
126
+ "valid_vs_base": true
127
+ }
v3_scale/recount/recount_broad_local_v1.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train_sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
3
+ "train_bytes": 34835257,
4
+ "overlap64": {
5
+ "fc2": {
6
+ "n_64grams": 199729,
7
+ "confirmed": 0
8
+ },
9
+ "fresh_en": {
10
+ "n_64grams": 119700,
11
+ "confirmed": 0
12
+ },
13
+ "fresh_fr": {
14
+ "n_64grams": 1088625,
15
+ "confirmed": 0
16
+ },
17
+ "neither_code": {
18
+ "n_64grams": 545973,
19
+ "confirmed": 545558
20
+ },
21
+ "test20k": {
22
+ "n_64grams": 894655,
23
+ "confirmed": 33766
24
+ },
25
+ "queries": {
26
+ "n_64grams": 27737,
27
+ "confirmed": 0
28
+ },
29
+ "test_v3": {
30
+ "n_64grams": 3486590,
31
+ "confirmed": 0
32
+ },
33
+ "seen_old": {
34
+ "n_64grams": 199937,
35
+ "confirmed": 199877
36
+ },
37
+ "seen_broad": {
38
+ "n_64grams": 199937,
39
+ "confirmed": 199937
40
+ },
41
+ "seen_v3": {
42
+ "n_64grams": 199937,
43
+ "confirmed": 199217
44
+ },
45
+ "m7_fresh_fr": {
46
+ "n_64grams": 1088625,
47
+ "confirmed": 0
48
+ },
49
+ "m7_freshcode": {
50
+ "n_64grams": 199937,
51
+ "confirmed": 163445
52
+ },
53
+ "m7_seen": {
54
+ "n_64grams": 199937,
55
+ "confirmed": 199877
56
+ },
57
+ "m7_neither_code": {
58
+ "n_64grams": 545973,
59
+ "confirmed": 545558
60
+ },
61
+ "m7_test20k": {
62
+ "n_64grams": 319877,
63
+ "confirmed": 1405
64
+ },
65
+ "m7_fresh_en": {
66
+ "n_64grams": 119700,
67
+ "confirmed": 0
68
+ },
69
+ "m7_queries": {
70
+ "n_64grams": 27737,
71
+ "confirmed": 0
72
+ },
73
+ "v3_test": {
74
+ "n_64grams": 3486590,
75
+ "confirmed": 0
76
+ },
77
+ "v3_oneshot": {
78
+ "n_64grams": 119700,
79
+ "confirmed": 0
80
+ }
81
+ },
82
+ "valid_zero_overlap": false
83
+ }
v3_scale/recount/recount_concat_v1.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Checker — recompte indépendant chevauchement 64 o corpus_gutenberg_v2/train.bin vs toutes les fenêtres d'éval (lecture seule).
3
+ Multiplicateur différent du builder (0xC2B2AE3D27D4EB4F), train haché par tranches de 8 Mo, confirmation octet à octet."""
4
+ import sys, os, glob, json, hashlib, numpy as np
5
+ sys.path.insert(0, "/workspace/spira_shared/checker"); sys.path.insert(0, "/workspace/spira_shared/checker/scale")
6
+ from bench_common import fixed_windows
7
+ K = 64; P = np.uint64(0xC2B2AE3D27D4EB4F); S = "/workspace/spira_shared"; C = sys.argv[1] if len(sys.argv) > 1 else f"{S}/corpus_gutenberg_v2"; rd = lambda p: open(p, "rb").read()
8
+ def H(b):
9
+ a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1; h = np.zeros(max(n, 0), np.uint64)
10
+ with np.errstate(over="ignore"):
11
+ for j in range(K): h = h * P + a[j:j + n] + np.uint64(13)
12
+ return h
13
+ W = {k: v[0] for k, v in fixed_windows().items()}
14
+ for c, p in {"old": f"{S}/broad/old_local_v1", "broad": f"{S}/broad/broad_local_v1", "v3": f"{S}/corpus_v3"}.items(): W[f"seen_{c}"] = rd(f"{p}/train.bin")[2_600_000:2_800_000]
15
+ for p in glob.glob(f"{S}/spira_m7/windows/*.bin"): W["m7_" + os.path.basename(p)[:-4]] = rd(p)
16
+ W["v3_test"] = rd(f"{S}/corpus_v3/test.bin"); W["v3_oneshot"] = rd(f"{S}/corpus_v3/oneshot.bin")
17
+ tr = rd(f"{C}/train.bin"); out = {"train_sha256": hashlib.sha256(tr).hexdigest(), "train_bytes": len(tr), "overlap64": {}}
18
+ # basse mémoire : on hache les fenêtres (petites) et on balaie train par tranches de 8 Mo
19
+ WH = {k: H(b) for k, b in W.items()}; allh = np.unique(np.concatenate(list(WH.values())))
20
+ found = set()
21
+ for s0 in range(0, len(tr), 8_000_000):
22
+ seg = tr[s0:s0 + 8_000_000 + K - 1]; h = H(seg); idx = np.nonzero(np.isin(h, allh))[0]
23
+ for i in idx: found.add(seg[i:i + K])
24
+ for k, b in W.items():
25
+ hw = WH[k]; cand = np.nonzero(np.isin(hw, allh))[0]
26
+ conf = sum(1 for p in range(len(hw)) if b[p:p + K] in found) if found else 0
27
+ out["overlap64"][k] = {"n_64grams": int(len(hw)), "confirmed": int(conf)}; print(k, out["overlap64"][k], flush=True)
28
+ out["valid_zero_overlap"] = all(v["confirmed"] == 0 for v in out["overlap64"].values())
29
+ BASE = sys.argv[2] if len(sys.argv) > 2 else None
30
+ if BASE:
31
+ bj = json.load(open(BASE)); EX = {"seen_old", "seen_broad", "seen_v3", "m7_seen"}; SEP = {"neither_code", "m7_neither_code", "m7_freshcode"}
32
+ out["base_json"] = BASE; out["delta_vs_base"] = {k: out["overlap64"][k]["confirmed"] - bj["overlap64"][k]["confirmed"] for k in out["overlap64"]}
33
+ out["excluded_seen"] = sorted(EX); out["reported_separately"] = {k: out["overlap64"][k]["confirmed"] for k in SEP}
34
+ out["valid_vs_base"] = all(v == 0 for k, v in out["delta_vs_base"].items() if k not in EX)
35
+ json.dump(out, open(f"{S}/checker/recount/recount_" + os.path.basename(C.rstrip("/")) + ".json", "w"), indent=1); print("VALID" if out["valid_zero_overlap"] else "INVALID", "valid_vs_base", out.get("valid_vs_base"), out.get("delta_vs_base"))
v3_scale/recount/recount_gutenberg_v2_2059.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train_bytes": 255975243,
3
+ "train_sha256": "75afd55a4e800715d6870ec88ed12e5c01a42b6caa53243ad267a35c55525fb3",
4
+ "train_sha_match": true,
5
+ "bytes_match": true,
6
+ "en_plus_fr": 255974109,
7
+ "n_en": 395,
8
+ "n_fr": 173,
9
+ "win_sha_mismatch": [],
10
+ "hash_candidates": 0,
11
+ "confirmed_64grams": {
12
+ "fc2": 0,
13
+ "fresh_en": 0,
14
+ "fresh_fr": 0,
15
+ "neither_code": 0,
16
+ "test20k": 0,
17
+ "queries": 0,
18
+ "test_v3": 0,
19
+ "seen_old": 0,
20
+ "seen_broad": 0,
21
+ "seen_v3": 0,
22
+ "m7_fresh_fr": 0,
23
+ "m7_freshcode": 0,
24
+ "m7_seen": 0,
25
+ "m7_neither_code": 0,
26
+ "m7_test20k": 0,
27
+ "m7_fresh_en": 0,
28
+ "m7_queries": 0,
29
+ "v3_test": 0,
30
+ "v3_oneshot": 0
31
+ },
32
+ "total_confirmed": 0
33
+ }
v3_scale/recount/recount_gutenberg_v2_2059.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Checker recount indépendant corpus_gutenberg_v2 : sha256/taille train.bin, sha256 fenêtres vs manifeste,
3
+ contamination 64 o avec un hash différent (P2, offset 13) + confirmation octet à octet. Lecture seule."""
4
+ import sys, json, hashlib, numpy as np
5
+ D = "/workspace/spira_shared/corpus_gutenberg_v2"; K = 64
6
+ m = json.load(open(f"{D}/manifest.json"))
7
+ tb = open(f"{D}/train.bin", "rb").read()
8
+ out = {"train_bytes": len(tb), "train_sha256": hashlib.sha256(tb).hexdigest()}
9
+ out["train_sha_match"] = out["train_sha256"] == m["train_sha256"]; out["bytes_match"] = len(tb) == m["train_bytes"]
10
+ out["en_plus_fr"] = m["en_bytes"] + m["fr_bytes"]; out["n_en"] = len(m["book_ids_en"]); out["n_fr"] = len(m["book_ids_fr"])
11
+ # Correctif 21 h 07 : ne plus importer build_corpus_v2 (sans garde __main__, l'import relançait le build et réécrivait train.bin/manifest).
12
+ import os, glob
13
+ sys.path.insert(0, "/workspace/spira_shared/checker"); sys.path.insert(0, "/workspace/spira_shared/checker/scale")
14
+ def windows():
15
+ from bench_common import fixed_windows
16
+ S = "/workspace/spira_shared"; rd = lambda p: open(p, "rb").read(); W = {k: v[0] for k, v in fixed_windows().items()}
17
+ for c, p in {"old": f"{S}/broad/old_local_v1", "broad": f"{S}/broad/broad_local_v1", "v3": f"{S}/corpus_v3"}.items():
18
+ W[f"seen_{c}"] = rd(f"{p}/train.bin")[2_600_000:2_800_000]
19
+ for p in glob.glob(f"{S}/spira_m7/windows/*.bin"): W["m7_" + os.path.basename(p)[:-4]] = rd(p)
20
+ W["v3_test"] = rd(f"{S}/corpus_v3/test.bin"); W["v3_oneshot"] = rd(f"{S}/corpus_v3/oneshot.bin")
21
+ return W
22
+ W = windows()
23
+ out["win_sha_mismatch"] = [k for k in W if hashlib.sha256(W[k]).hexdigest() != m["windows"][k]["sha256"]]
24
+ P = np.uint64(0xC2B2AE3D27D4EB4F)
25
+ def H(b):
26
+ a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1
27
+ if n <= 0: return np.zeros(0, np.uint64)
28
+ h = np.zeros(n, np.uint64)
29
+ with np.errstate(over="ignore"):
30
+ for j in range(K): h = h * P + a[j:j + n] + np.uint64(13)
31
+ return h
32
+ keys = list(W); hs, ks, js = [], [], []
33
+ for n, k in enumerate(keys):
34
+ h = H(W[k]); hs.append(h); ks.append(np.full(len(h), n, np.int16)); js.append(np.arange(len(h)))
35
+ hs = np.concatenate(hs); ks = np.concatenate(ks); js = np.concatenate(js); o = np.argsort(hs); hs, ks, js = hs[o], ks[o], js[o]
36
+ hit = set(); cand = 0; C = 8_000_000
37
+ for s in range(0, len(tb) - K + 1, C):
38
+ seg = tb[s:s + C + K - 1]; h = H(seg)
39
+ lo = np.searchsorted(hs, h, "left"); hi = np.searchsorted(hs, h, "right"); idx = np.nonzero(hi > lo)[0]; cand += len(idx)
40
+ for i in idx:
41
+ g = seg[i:i + K]
42
+ for q in range(lo[i], hi[i]):
43
+ k = keys[ks[q]]; j = int(js[q])
44
+ if W[k][j:j + K] == g: hit.add((k, j))
45
+ res = {k: 0 for k in keys}
46
+ for k, j in hit: res[k] += 1
47
+ out["hash_candidates"] = int(cand); out["confirmed_64grams"] = res; out["total_confirmed"] = sum(res.values())
48
+ print(json.dumps(out)); json.dump(out, open("/workspace/spira_shared/checker/recount/recount_gutenberg_v2_2059.json", "w"), indent=1)
v3_scale/scale/test20k_clean.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Checker — test20k sur positions PROPRES : position contaminée si un octet de son patch cible est couvert par un 64-gram présent
3
+ dans le train.bin du corpus du modèle (confirmé octet à octet). Lit hits_v4/<tag>__test20k.npz. Usage : test20k_clean.py TAG CORPUS"""
4
+ import sys, json, numpy as np
5
+ sys.path.insert(0, "/workspace/spira_shared/checker/scale"); from bench_common import CORPORA, fixed_windows
6
+ K = 64; P = np.uint64(0xC2B2AE3D27D4EB4F); tag, corp = sys.argv[1:3]; D = "/workspace/spira_shared/checker/scale"
7
+ def H(b):
8
+ a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1; h = np.zeros(max(n, 0), np.uint64)
9
+ with np.errstate(over="ignore"):
10
+ for j in range(K): h = h * P + a[j:j + n] + np.uint64(13)
11
+ return h
12
+ w = fixed_windows()["test20k"][0]; hw = H(w); uw = np.unique(hw); tr = open(f"{CORPORA[corp]}/train.bin", "rb").read(); found = set()
13
+ for s in range(0, len(tr), 8_000_000):
14
+ seg = tr[s:s + 8_000_000 + K - 1]; h = H(seg)
15
+ for i in np.nonzero(np.isin(h, uw))[0]: found.add(seg[i:i + K])
16
+ cov = np.zeros(len(w) + 1, np.int32)
17
+ for i in range(len(hw)):
18
+ if w[i:i + K] in found: cov[i] += 1; cov[i + K] -= 1
19
+ covered = np.cumsum(cov)[:len(w)] > 0
20
+ z = np.load(f"{D}/hits_v4/{tag}__test20k.npz"); st, ln, fh, iv = z["tgt_start"], z["tgt_len"], z["fused_hit"].astype(bool), z["invocab"].astype(bool)
21
+ n = min(len(st), len(fh)); cont = np.array([covered[st[t]:st[t] + ln[t]].any() for t in range(n)]); fh, iv = fh[:n], iv[:n]; cl = ~cont
22
+ r = {"tag": tag, "corpus": corp, "positions": int(n), "positions_contaminated": int(cont.sum()), "bytes_covered": int(covered.sum()),
23
+ "acc_invocab_all": 100 * float(fh[iv].mean()), "acc_invocab_clean": 100 * float(fh[iv & cl].mean()) if (iv & cl).any() else None}
24
+ json.dump(r, open(f"{D}/verdicts/{tag}__test20k_clean.json", "w"), indent=1); print(json.dumps(r))