v3_scale: broad_gutenberg_v1 manifest + recomptes 64 o (delta vs broad_local_v1 = 0, valid_vs_base)
Browse files- v3_scale/corpora/broad_gutenberg_v1/manifest.json +47 -0
- v3_scale/recount/delta_concat_v1.py +10 -0
- v3_scale/recount/recount_broad_gutenberg_v1.json +127 -0
- v3_scale/recount/recount_broad_local_v1.json +83 -0
- v3_scale/recount/recount_concat_v1.py +35 -0
- v3_scale/recount/recount_gutenberg_v2_2059.json +33 -0
- v3_scale/recount/recount_gutenberg_v2_2059.py +48 -0
- v3_scale/scale/test20k_clean.py +24 -0
v3_scale/corpora/broad_gutenberg_v1/manifest.json
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"variant": "broad_gutenberg_v1",
|
| 3 |
+
"construction": "train.bin = broad_local_v1/train.bin + b'\\n\\n' + corpus_gutenberg_v2/train.bin ; test.bin et oneshot.bin = copies de broad_local_v1 (test20k / fresh_en inchang\u00e9s)",
|
| 4 |
+
"sources": {
|
| 5 |
+
"broad_local_v1": {
|
| 6 |
+
"train_sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
|
| 7 |
+
"composition": {
|
| 8 |
+
"en": 28654965,
|
| 9 |
+
"fr": 214337,
|
| 10 |
+
"code": 6860045
|
| 11 |
+
}
|
| 12 |
+
},
|
| 13 |
+
"corpus_gutenberg_v2": {
|
| 14 |
+
"train_sha256": "75afd55a4e800715d6870ec88ed12e5c01a42b6caa53243ad267a35c55525fb3",
|
| 15 |
+
"n_books": 568,
|
| 16 |
+
"en_bytes": 186503118,
|
| 17 |
+
"fr_bytes": 69470991,
|
| 18 |
+
"source": "Project Gutenberg via aleph.pglaf.org (approuv\u00e9 23/10 16 h 04)"
|
| 19 |
+
}
|
| 20 |
+
},
|
| 21 |
+
"files": {
|
| 22 |
+
"train.bin": {
|
| 23 |
+
"bytes": 290810502,
|
| 24 |
+
"sha256": "5af6398fc853bce270e430f8336fc2b136318c2e762ead19e92f5d180de083f2"
|
| 25 |
+
},
|
| 26 |
+
"test.bin": {
|
| 27 |
+
"bytes": 894718,
|
| 28 |
+
"sha256": "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9"
|
| 29 |
+
},
|
| 30 |
+
"oneshot.bin": {
|
| 31 |
+
"bytes": 119763,
|
| 32 |
+
"sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a"
|
| 33 |
+
}
|
| 34 |
+
},
|
| 35 |
+
"composition_approx": {
|
| 36 |
+
"en": 215158083,
|
| 37 |
+
"fr": 69685328,
|
| 38 |
+
"code": 6860045
|
| 39 |
+
},
|
| 40 |
+
"ratio_en_fr": 3.088,
|
| 41 |
+
"share": {
|
| 42 |
+
"en": 0.7376,
|
| 43 |
+
"fr": 0.2389,
|
| 44 |
+
"code": 0.0235
|
| 45 |
+
},
|
| 46 |
+
"contamination_check": "checker/recount/recount_concat_v1.py (delta vs broad_local_v1) -> checker/recount/recount_broad_gutenberg_v1.json"
|
| 47 |
+
}
|
v3_scale/recount/delta_concat_v1.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Checker — delta chevauchement 64 o : concat (recount_concat_v1.py) moins base broad_local_v1 (même script). seen_* exclus, code à part."""
|
| 3 |
+
import json
|
| 4 |
+
R = "/workspace/spira_shared/checker/recount"; c = json.load(open(f"{R}/recount_broad_gutenberg_v1.json")); b = json.load(open(f"{R}/recount_broad_local_v1.json"))
|
| 5 |
+
EX = {"seen_old", "seen_broad", "seen_v3", "m7_seen"}; SEP = {"neither_code", "m7_neither_code", "m7_freshcode"}
|
| 6 |
+
d = {k: c["overlap64"][k]["confirmed"] - b["overlap64"][k]["confirmed"] for k in c["overlap64"]}
|
| 7 |
+
c.update(base_json=f"{R}/recount_broad_local_v1.json", base_train_sha256=b["train_sha256"], delta_vs_base=d, excluded_seen=sorted(EX),
|
| 8 |
+
reported_separately={k: {"concat": c["overlap64"][k]["confirmed"], "base": b["overlap64"][k]["confirmed"]} for k in SEP},
|
| 9 |
+
valid_vs_base=all(v == 0 for k, v in d.items() if k not in EX))
|
| 10 |
+
json.dump(c, open(f"{R}/recount_broad_gutenberg_v1.json", "w"), indent=1); print("valid_vs_base", c["valid_vs_base"], d)
|
v3_scale/recount/recount_broad_gutenberg_v1.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"train_sha256": "5af6398fc853bce270e430f8336fc2b136318c2e762ead19e92f5d180de083f2",
|
| 3 |
+
"train_bytes": 290810502,
|
| 4 |
+
"overlap64": {
|
| 5 |
+
"fc2": {
|
| 6 |
+
"n_64grams": 199729,
|
| 7 |
+
"confirmed": 0
|
| 8 |
+
},
|
| 9 |
+
"fresh_en": {
|
| 10 |
+
"n_64grams": 119700,
|
| 11 |
+
"confirmed": 0
|
| 12 |
+
},
|
| 13 |
+
"fresh_fr": {
|
| 14 |
+
"n_64grams": 1088625,
|
| 15 |
+
"confirmed": 0
|
| 16 |
+
},
|
| 17 |
+
"neither_code": {
|
| 18 |
+
"n_64grams": 545973,
|
| 19 |
+
"confirmed": 545558
|
| 20 |
+
},
|
| 21 |
+
"test20k": {
|
| 22 |
+
"n_64grams": 894655,
|
| 23 |
+
"confirmed": 33766
|
| 24 |
+
},
|
| 25 |
+
"queries": {
|
| 26 |
+
"n_64grams": 27737,
|
| 27 |
+
"confirmed": 0
|
| 28 |
+
},
|
| 29 |
+
"test_v3": {
|
| 30 |
+
"n_64grams": 3486590,
|
| 31 |
+
"confirmed": 0
|
| 32 |
+
},
|
| 33 |
+
"seen_old": {
|
| 34 |
+
"n_64grams": 199937,
|
| 35 |
+
"confirmed": 199877
|
| 36 |
+
},
|
| 37 |
+
"seen_broad": {
|
| 38 |
+
"n_64grams": 199937,
|
| 39 |
+
"confirmed": 199937
|
| 40 |
+
},
|
| 41 |
+
"seen_v3": {
|
| 42 |
+
"n_64grams": 199937,
|
| 43 |
+
"confirmed": 199217
|
| 44 |
+
},
|
| 45 |
+
"m7_fresh_fr": {
|
| 46 |
+
"n_64grams": 1088625,
|
| 47 |
+
"confirmed": 0
|
| 48 |
+
},
|
| 49 |
+
"m7_freshcode": {
|
| 50 |
+
"n_64grams": 199937,
|
| 51 |
+
"confirmed": 163445
|
| 52 |
+
},
|
| 53 |
+
"m7_seen": {
|
| 54 |
+
"n_64grams": 199937,
|
| 55 |
+
"confirmed": 199877
|
| 56 |
+
},
|
| 57 |
+
"m7_neither_code": {
|
| 58 |
+
"n_64grams": 545973,
|
| 59 |
+
"confirmed": 545558
|
| 60 |
+
},
|
| 61 |
+
"m7_test20k": {
|
| 62 |
+
"n_64grams": 319877,
|
| 63 |
+
"confirmed": 1405
|
| 64 |
+
},
|
| 65 |
+
"m7_fresh_en": {
|
| 66 |
+
"n_64grams": 119700,
|
| 67 |
+
"confirmed": 0
|
| 68 |
+
},
|
| 69 |
+
"m7_queries": {
|
| 70 |
+
"n_64grams": 27737,
|
| 71 |
+
"confirmed": 0
|
| 72 |
+
},
|
| 73 |
+
"v3_test": {
|
| 74 |
+
"n_64grams": 3486590,
|
| 75 |
+
"confirmed": 0
|
| 76 |
+
},
|
| 77 |
+
"v3_oneshot": {
|
| 78 |
+
"n_64grams": 119700,
|
| 79 |
+
"confirmed": 0
|
| 80 |
+
}
|
| 81 |
+
},
|
| 82 |
+
"valid_zero_overlap": false,
|
| 83 |
+
"base_json": "/workspace/spira_shared/checker/recount/recount_broad_local_v1.json",
|
| 84 |
+
"base_train_sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
|
| 85 |
+
"delta_vs_base": {
|
| 86 |
+
"fc2": 0,
|
| 87 |
+
"fresh_en": 0,
|
| 88 |
+
"fresh_fr": 0,
|
| 89 |
+
"neither_code": 0,
|
| 90 |
+
"test20k": 0,
|
| 91 |
+
"queries": 0,
|
| 92 |
+
"test_v3": 0,
|
| 93 |
+
"seen_old": 0,
|
| 94 |
+
"seen_broad": 0,
|
| 95 |
+
"seen_v3": 0,
|
| 96 |
+
"m7_fresh_fr": 0,
|
| 97 |
+
"m7_freshcode": 0,
|
| 98 |
+
"m7_seen": 0,
|
| 99 |
+
"m7_neither_code": 0,
|
| 100 |
+
"m7_test20k": 0,
|
| 101 |
+
"m7_fresh_en": 0,
|
| 102 |
+
"m7_queries": 0,
|
| 103 |
+
"v3_test": 0,
|
| 104 |
+
"v3_oneshot": 0
|
| 105 |
+
},
|
| 106 |
+
"excluded_seen": [
|
| 107 |
+
"m7_seen",
|
| 108 |
+
"seen_broad",
|
| 109 |
+
"seen_old",
|
| 110 |
+
"seen_v3"
|
| 111 |
+
],
|
| 112 |
+
"reported_separately": {
|
| 113 |
+
"m7_neither_code": {
|
| 114 |
+
"concat": 545558,
|
| 115 |
+
"base": 545558
|
| 116 |
+
},
|
| 117 |
+
"neither_code": {
|
| 118 |
+
"concat": 545558,
|
| 119 |
+
"base": 545558
|
| 120 |
+
},
|
| 121 |
+
"m7_freshcode": {
|
| 122 |
+
"concat": 163445,
|
| 123 |
+
"base": 163445
|
| 124 |
+
}
|
| 125 |
+
},
|
| 126 |
+
"valid_vs_base": true
|
| 127 |
+
}
|
v3_scale/recount/recount_broad_local_v1.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"train_sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
|
| 3 |
+
"train_bytes": 34835257,
|
| 4 |
+
"overlap64": {
|
| 5 |
+
"fc2": {
|
| 6 |
+
"n_64grams": 199729,
|
| 7 |
+
"confirmed": 0
|
| 8 |
+
},
|
| 9 |
+
"fresh_en": {
|
| 10 |
+
"n_64grams": 119700,
|
| 11 |
+
"confirmed": 0
|
| 12 |
+
},
|
| 13 |
+
"fresh_fr": {
|
| 14 |
+
"n_64grams": 1088625,
|
| 15 |
+
"confirmed": 0
|
| 16 |
+
},
|
| 17 |
+
"neither_code": {
|
| 18 |
+
"n_64grams": 545973,
|
| 19 |
+
"confirmed": 545558
|
| 20 |
+
},
|
| 21 |
+
"test20k": {
|
| 22 |
+
"n_64grams": 894655,
|
| 23 |
+
"confirmed": 33766
|
| 24 |
+
},
|
| 25 |
+
"queries": {
|
| 26 |
+
"n_64grams": 27737,
|
| 27 |
+
"confirmed": 0
|
| 28 |
+
},
|
| 29 |
+
"test_v3": {
|
| 30 |
+
"n_64grams": 3486590,
|
| 31 |
+
"confirmed": 0
|
| 32 |
+
},
|
| 33 |
+
"seen_old": {
|
| 34 |
+
"n_64grams": 199937,
|
| 35 |
+
"confirmed": 199877
|
| 36 |
+
},
|
| 37 |
+
"seen_broad": {
|
| 38 |
+
"n_64grams": 199937,
|
| 39 |
+
"confirmed": 199937
|
| 40 |
+
},
|
| 41 |
+
"seen_v3": {
|
| 42 |
+
"n_64grams": 199937,
|
| 43 |
+
"confirmed": 199217
|
| 44 |
+
},
|
| 45 |
+
"m7_fresh_fr": {
|
| 46 |
+
"n_64grams": 1088625,
|
| 47 |
+
"confirmed": 0
|
| 48 |
+
},
|
| 49 |
+
"m7_freshcode": {
|
| 50 |
+
"n_64grams": 199937,
|
| 51 |
+
"confirmed": 163445
|
| 52 |
+
},
|
| 53 |
+
"m7_seen": {
|
| 54 |
+
"n_64grams": 199937,
|
| 55 |
+
"confirmed": 199877
|
| 56 |
+
},
|
| 57 |
+
"m7_neither_code": {
|
| 58 |
+
"n_64grams": 545973,
|
| 59 |
+
"confirmed": 545558
|
| 60 |
+
},
|
| 61 |
+
"m7_test20k": {
|
| 62 |
+
"n_64grams": 319877,
|
| 63 |
+
"confirmed": 1405
|
| 64 |
+
},
|
| 65 |
+
"m7_fresh_en": {
|
| 66 |
+
"n_64grams": 119700,
|
| 67 |
+
"confirmed": 0
|
| 68 |
+
},
|
| 69 |
+
"m7_queries": {
|
| 70 |
+
"n_64grams": 27737,
|
| 71 |
+
"confirmed": 0
|
| 72 |
+
},
|
| 73 |
+
"v3_test": {
|
| 74 |
+
"n_64grams": 3486590,
|
| 75 |
+
"confirmed": 0
|
| 76 |
+
},
|
| 77 |
+
"v3_oneshot": {
|
| 78 |
+
"n_64grams": 119700,
|
| 79 |
+
"confirmed": 0
|
| 80 |
+
}
|
| 81 |
+
},
|
| 82 |
+
"valid_zero_overlap": false
|
| 83 |
+
}
|
v3_scale/recount/recount_concat_v1.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Checker — recompte indépendant chevauchement 64 o corpus_gutenberg_v2/train.bin vs toutes les fenêtres d'éval (lecture seule).
|
| 3 |
+
Multiplicateur différent du builder (0xC2B2AE3D27D4EB4F), train haché par tranches de 8 Mo, confirmation octet à octet."""
|
| 4 |
+
import sys, os, glob, json, hashlib, numpy as np
|
| 5 |
+
sys.path.insert(0, "/workspace/spira_shared/checker"); sys.path.insert(0, "/workspace/spira_shared/checker/scale")
|
| 6 |
+
from bench_common import fixed_windows
|
| 7 |
+
K = 64; P = np.uint64(0xC2B2AE3D27D4EB4F); S = "/workspace/spira_shared"; C = sys.argv[1] if len(sys.argv) > 1 else f"{S}/corpus_gutenberg_v2"; rd = lambda p: open(p, "rb").read()
|
| 8 |
+
def H(b):
|
| 9 |
+
a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1; h = np.zeros(max(n, 0), np.uint64)
|
| 10 |
+
with np.errstate(over="ignore"):
|
| 11 |
+
for j in range(K): h = h * P + a[j:j + n] + np.uint64(13)
|
| 12 |
+
return h
|
| 13 |
+
W = {k: v[0] for k, v in fixed_windows().items()}
|
| 14 |
+
for c, p in {"old": f"{S}/broad/old_local_v1", "broad": f"{S}/broad/broad_local_v1", "v3": f"{S}/corpus_v3"}.items(): W[f"seen_{c}"] = rd(f"{p}/train.bin")[2_600_000:2_800_000]
|
| 15 |
+
for p in glob.glob(f"{S}/spira_m7/windows/*.bin"): W["m7_" + os.path.basename(p)[:-4]] = rd(p)
|
| 16 |
+
W["v3_test"] = rd(f"{S}/corpus_v3/test.bin"); W["v3_oneshot"] = rd(f"{S}/corpus_v3/oneshot.bin")
|
| 17 |
+
tr = rd(f"{C}/train.bin"); out = {"train_sha256": hashlib.sha256(tr).hexdigest(), "train_bytes": len(tr), "overlap64": {}}
|
| 18 |
+
# basse mémoire : on hache les fenêtres (petites) et on balaie train par tranches de 8 Mo
|
| 19 |
+
WH = {k: H(b) for k, b in W.items()}; allh = np.unique(np.concatenate(list(WH.values())))
|
| 20 |
+
found = set()
|
| 21 |
+
for s0 in range(0, len(tr), 8_000_000):
|
| 22 |
+
seg = tr[s0:s0 + 8_000_000 + K - 1]; h = H(seg); idx = np.nonzero(np.isin(h, allh))[0]
|
| 23 |
+
for i in idx: found.add(seg[i:i + K])
|
| 24 |
+
for k, b in W.items():
|
| 25 |
+
hw = WH[k]; cand = np.nonzero(np.isin(hw, allh))[0]
|
| 26 |
+
conf = sum(1 for p in range(len(hw)) if b[p:p + K] in found) if found else 0
|
| 27 |
+
out["overlap64"][k] = {"n_64grams": int(len(hw)), "confirmed": int(conf)}; print(k, out["overlap64"][k], flush=True)
|
| 28 |
+
out["valid_zero_overlap"] = all(v["confirmed"] == 0 for v in out["overlap64"].values())
|
| 29 |
+
BASE = sys.argv[2] if len(sys.argv) > 2 else None
|
| 30 |
+
if BASE:
|
| 31 |
+
bj = json.load(open(BASE)); EX = {"seen_old", "seen_broad", "seen_v3", "m7_seen"}; SEP = {"neither_code", "m7_neither_code", "m7_freshcode"}
|
| 32 |
+
out["base_json"] = BASE; out["delta_vs_base"] = {k: out["overlap64"][k]["confirmed"] - bj["overlap64"][k]["confirmed"] for k in out["overlap64"]}
|
| 33 |
+
out["excluded_seen"] = sorted(EX); out["reported_separately"] = {k: out["overlap64"][k]["confirmed"] for k in SEP}
|
| 34 |
+
out["valid_vs_base"] = all(v == 0 for k, v in out["delta_vs_base"].items() if k not in EX)
|
| 35 |
+
json.dump(out, open(f"{S}/checker/recount/recount_" + os.path.basename(C.rstrip("/")) + ".json", "w"), indent=1); print("VALID" if out["valid_zero_overlap"] else "INVALID", "valid_vs_base", out.get("valid_vs_base"), out.get("delta_vs_base"))
|
v3_scale/recount/recount_gutenberg_v2_2059.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"train_bytes": 255975243,
|
| 3 |
+
"train_sha256": "75afd55a4e800715d6870ec88ed12e5c01a42b6caa53243ad267a35c55525fb3",
|
| 4 |
+
"train_sha_match": true,
|
| 5 |
+
"bytes_match": true,
|
| 6 |
+
"en_plus_fr": 255974109,
|
| 7 |
+
"n_en": 395,
|
| 8 |
+
"n_fr": 173,
|
| 9 |
+
"win_sha_mismatch": [],
|
| 10 |
+
"hash_candidates": 0,
|
| 11 |
+
"confirmed_64grams": {
|
| 12 |
+
"fc2": 0,
|
| 13 |
+
"fresh_en": 0,
|
| 14 |
+
"fresh_fr": 0,
|
| 15 |
+
"neither_code": 0,
|
| 16 |
+
"test20k": 0,
|
| 17 |
+
"queries": 0,
|
| 18 |
+
"test_v3": 0,
|
| 19 |
+
"seen_old": 0,
|
| 20 |
+
"seen_broad": 0,
|
| 21 |
+
"seen_v3": 0,
|
| 22 |
+
"m7_fresh_fr": 0,
|
| 23 |
+
"m7_freshcode": 0,
|
| 24 |
+
"m7_seen": 0,
|
| 25 |
+
"m7_neither_code": 0,
|
| 26 |
+
"m7_test20k": 0,
|
| 27 |
+
"m7_fresh_en": 0,
|
| 28 |
+
"m7_queries": 0,
|
| 29 |
+
"v3_test": 0,
|
| 30 |
+
"v3_oneshot": 0
|
| 31 |
+
},
|
| 32 |
+
"total_confirmed": 0
|
| 33 |
+
}
|
v3_scale/recount/recount_gutenberg_v2_2059.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Checker recount indépendant corpus_gutenberg_v2 : sha256/taille train.bin, sha256 fenêtres vs manifeste,
|
| 3 |
+
contamination 64 o avec un hash différent (P2, offset 13) + confirmation octet à octet. Lecture seule."""
|
| 4 |
+
import sys, json, hashlib, numpy as np
|
| 5 |
+
D = "/workspace/spira_shared/corpus_gutenberg_v2"; K = 64
|
| 6 |
+
m = json.load(open(f"{D}/manifest.json"))
|
| 7 |
+
tb = open(f"{D}/train.bin", "rb").read()
|
| 8 |
+
out = {"train_bytes": len(tb), "train_sha256": hashlib.sha256(tb).hexdigest()}
|
| 9 |
+
out["train_sha_match"] = out["train_sha256"] == m["train_sha256"]; out["bytes_match"] = len(tb) == m["train_bytes"]
|
| 10 |
+
out["en_plus_fr"] = m["en_bytes"] + m["fr_bytes"]; out["n_en"] = len(m["book_ids_en"]); out["n_fr"] = len(m["book_ids_fr"])
|
| 11 |
+
# Correctif 21 h 07 : ne plus importer build_corpus_v2 (sans garde __main__, l'import relançait le build et réécrivait train.bin/manifest).
|
| 12 |
+
import os, glob
|
| 13 |
+
sys.path.insert(0, "/workspace/spira_shared/checker"); sys.path.insert(0, "/workspace/spira_shared/checker/scale")
|
| 14 |
+
def windows():
|
| 15 |
+
from bench_common import fixed_windows
|
| 16 |
+
S = "/workspace/spira_shared"; rd = lambda p: open(p, "rb").read(); W = {k: v[0] for k, v in fixed_windows().items()}
|
| 17 |
+
for c, p in {"old": f"{S}/broad/old_local_v1", "broad": f"{S}/broad/broad_local_v1", "v3": f"{S}/corpus_v3"}.items():
|
| 18 |
+
W[f"seen_{c}"] = rd(f"{p}/train.bin")[2_600_000:2_800_000]
|
| 19 |
+
for p in glob.glob(f"{S}/spira_m7/windows/*.bin"): W["m7_" + os.path.basename(p)[:-4]] = rd(p)
|
| 20 |
+
W["v3_test"] = rd(f"{S}/corpus_v3/test.bin"); W["v3_oneshot"] = rd(f"{S}/corpus_v3/oneshot.bin")
|
| 21 |
+
return W
|
| 22 |
+
W = windows()
|
| 23 |
+
out["win_sha_mismatch"] = [k for k in W if hashlib.sha256(W[k]).hexdigest() != m["windows"][k]["sha256"]]
|
| 24 |
+
P = np.uint64(0xC2B2AE3D27D4EB4F)
|
| 25 |
+
def H(b):
|
| 26 |
+
a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1
|
| 27 |
+
if n <= 0: return np.zeros(0, np.uint64)
|
| 28 |
+
h = np.zeros(n, np.uint64)
|
| 29 |
+
with np.errstate(over="ignore"):
|
| 30 |
+
for j in range(K): h = h * P + a[j:j + n] + np.uint64(13)
|
| 31 |
+
return h
|
| 32 |
+
keys = list(W); hs, ks, js = [], [], []
|
| 33 |
+
for n, k in enumerate(keys):
|
| 34 |
+
h = H(W[k]); hs.append(h); ks.append(np.full(len(h), n, np.int16)); js.append(np.arange(len(h)))
|
| 35 |
+
hs = np.concatenate(hs); ks = np.concatenate(ks); js = np.concatenate(js); o = np.argsort(hs); hs, ks, js = hs[o], ks[o], js[o]
|
| 36 |
+
hit = set(); cand = 0; C = 8_000_000
|
| 37 |
+
for s in range(0, len(tb) - K + 1, C):
|
| 38 |
+
seg = tb[s:s + C + K - 1]; h = H(seg)
|
| 39 |
+
lo = np.searchsorted(hs, h, "left"); hi = np.searchsorted(hs, h, "right"); idx = np.nonzero(hi > lo)[0]; cand += len(idx)
|
| 40 |
+
for i in idx:
|
| 41 |
+
g = seg[i:i + K]
|
| 42 |
+
for q in range(lo[i], hi[i]):
|
| 43 |
+
k = keys[ks[q]]; j = int(js[q])
|
| 44 |
+
if W[k][j:j + K] == g: hit.add((k, j))
|
| 45 |
+
res = {k: 0 for k in keys}
|
| 46 |
+
for k, j in hit: res[k] += 1
|
| 47 |
+
out["hash_candidates"] = int(cand); out["confirmed_64grams"] = res; out["total_confirmed"] = sum(res.values())
|
| 48 |
+
print(json.dumps(out)); json.dump(out, open("/workspace/spira_shared/checker/recount/recount_gutenberg_v2_2059.json", "w"), indent=1)
|
v3_scale/scale/test20k_clean.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Checker — test20k sur positions PROPRES : position contaminée si un octet de son patch cible est couvert par un 64-gram présent
|
| 3 |
+
dans le train.bin du corpus du modèle (confirmé octet à octet). Lit hits_v4/<tag>__test20k.npz. Usage : test20k_clean.py TAG CORPUS"""
|
| 4 |
+
import sys, json, numpy as np
|
| 5 |
+
sys.path.insert(0, "/workspace/spira_shared/checker/scale"); from bench_common import CORPORA, fixed_windows
|
| 6 |
+
K = 64; P = np.uint64(0xC2B2AE3D27D4EB4F); tag, corp = sys.argv[1:3]; D = "/workspace/spira_shared/checker/scale"
|
| 7 |
+
def H(b):
|
| 8 |
+
a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1; h = np.zeros(max(n, 0), np.uint64)
|
| 9 |
+
with np.errstate(over="ignore"):
|
| 10 |
+
for j in range(K): h = h * P + a[j:j + n] + np.uint64(13)
|
| 11 |
+
return h
|
| 12 |
+
w = fixed_windows()["test20k"][0]; hw = H(w); uw = np.unique(hw); tr = open(f"{CORPORA[corp]}/train.bin", "rb").read(); found = set()
|
| 13 |
+
for s in range(0, len(tr), 8_000_000):
|
| 14 |
+
seg = tr[s:s + 8_000_000 + K - 1]; h = H(seg)
|
| 15 |
+
for i in np.nonzero(np.isin(h, uw))[0]: found.add(seg[i:i + K])
|
| 16 |
+
cov = np.zeros(len(w) + 1, np.int32)
|
| 17 |
+
for i in range(len(hw)):
|
| 18 |
+
if w[i:i + K] in found: cov[i] += 1; cov[i + K] -= 1
|
| 19 |
+
covered = np.cumsum(cov)[:len(w)] > 0
|
| 20 |
+
z = np.load(f"{D}/hits_v4/{tag}__test20k.npz"); st, ln, fh, iv = z["tgt_start"], z["tgt_len"], z["fused_hit"].astype(bool), z["invocab"].astype(bool)
|
| 21 |
+
n = min(len(st), len(fh)); cont = np.array([covered[st[t]:st[t] + ln[t]].any() for t in range(n)]); fh, iv = fh[:n], iv[:n]; cl = ~cont
|
| 22 |
+
r = {"tag": tag, "corpus": corp, "positions": int(n), "positions_contaminated": int(cont.sum()), "bytes_covered": int(covered.sum()),
|
| 23 |
+
"acc_invocab_all": 100 * float(fh[iv].mean()), "acc_invocab_clean": 100 * float(fh[iv & cl].mean()) if (iv & cl).any() else None}
|
| 24 |
+
json.dump(r, open(f"{D}/verdicts/{tag}__test20k_clean.json", "w"), indent=1); print(json.dumps(r))
|