v3_scale/checker: recompte dédup 64 o corpus_diverse_v1 (REPORT, agrégats assainis sans plages ni texte, dedup64_div1.py 5230e995, run.sh) — raw 186812/687/231506 ; norm 149315/454/221174

v3_scale/checker/recount/corpus_diverse_v1_dedup/REPORT.md

@@ -0,0 +1,26 @@
+# Checker — recompte dedup 64-grammes corpus_diverse_v1 (agrégats)
+
+Rapport reconstitué le 30 oct. 2026 (Toronto) à partir des seuls JSON d'agrégats (le REPORT.md d'origine est absent du disque après la restauration de la boîte). Aucune position, masque, extrait de texte ni donnée d'éval n'est publié.
+
+- train : corpus_diverse_v1/train.bin, 530,137,703 o, sha256 c813244c… (ok=True)
+- script dedup64_div1.py sha256 5230e9959b08e9c16972b8b5d3920a54ec6b17c8fd489b14b5585cc1dd314428 ; k=64 ; Horner mod 2^64 P=0x9E3779B97F4A7C15 (logique dedup64.py 4bee4293…), confirmation octet à octet
+
+| forme | positions train appariées | plages fusionnées | octets dans plages | faux positifs hash | positions seg multilang |
+|---|---|---|---|---|---|
+| brut | 186,812 | 687 | 231,506 | 0 | 0 |
+| normalisé | 149,315 | 454 | 221,174 | 0 | 117 |
+
+Normalisation : suppr. \r ; A-Z->a-z ASCII ; suites [ \t\n\v\f] -> 1 espace ; en flux
+
+| fenêtre | n64 brut | trouvés brut | octets éval couverts brut | n64 norm | trouvés norm | octets couverts norm |
+|---|---|---|---|---|---|---|
+| test20k_old | 319,877 | 1,405 | 4,263 | 298,660 | 1,915 | 3,944 |
+| fc2 | 199,729 | 0 | 0 | 157,783 | 22 | 113 |
+| fresh_en | 119,700 | 0 | 0 | 119,587 | 0 | 0 |
+| fresh_fr | 1,088,625 | 0 | 0 | 1,067,982 | 77 | 143 |
+| test_v3 | 3,486,590 | 0 | 0 | 3,390,446 | 17 | 149 |
+| queries | 27,737 | 0 | 0 | 27,688 | 0 | 0 |
+| bg1_test | 894,655 | 33,766 | 45,852 | 823,920 | 26,765 | 42,756 |
+
+Autotests (selftest.json) : ALL_PASS = True.
+Détails complets : dedup64_aggregates.json.

v3_scale/checker/recount/corpus_diverse_v1_dedup/dedup64_aggregates.json

@@ -0,0 +1,385 @@
+{
+ "what": "Checker — dedup 64-grammes corpus_diverse_v1/train.bin vs fenêtres d'éval — AGRÉGATS ASSAINIS (échantillons de plages train retirés, sans texte, sans masques)",
+ "source_json_sha256": {
+  "dedup64_raw.json": "ef939f41937bd18536fa0aff33b97a47fa3c8c0b16caec2daf8bfbb9302b3d52",
+  "dedup64_norm.json": "b73e5099ccad14ee4fb1c80c06904601ab50f5ced0160ca4aaade0d199189fa0"
+ },
+ "raw": {
+  "form": "raw",
+  "k": 64,
+  "hash": "Horner mod 2^64 P=0x9E3779B97F4A7C15 (logique dedup64.py 4bee4293…), confirmation octet à octet",
+  "normalisation": null,
+  "train": "/workspace/spira_shared/corpus_diverse_v1/train.bin",
+  "train_bytes": 530137703,
+  "train_sha256": "c813244c618eb20dbc34e2d7472ac29e10b0c22344e8117a0c25b1186124e6d9",
+  "train_sha_ok": true,
+  "train_64gram_positions_hashed": 530137640,
+  "table_unique_hashes": 5777106,
+  "table_internal_collisions": 0,
+  "candidates": 186812,
+  "hash_false_positives_rejected": 0,
+  "train_positions_matched_any": 186812,
+  "train_ranges_merged_any": 687,
+  "train_bytes_in_ranges_any": 231506,
+  "train_positions_seg_multilang_any": 0,
+  "windows": {
+   "test20k_old": {
+    "n64": 319877,
+    "n64_found": 1405,
+    "frac64_found": 0.004392313295422928,
+    "n64_distinct_found": 1386,
+    "eval_bytes": 319940,
+    "eval_bytes_covered": 4263,
+    "eval_bytes_excl_ctx64": 6596,
+    "window_sha256": "ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22",
+    "train_positions_matched": 13111,
+    "train_positions_seg_bg1": 13111,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 133,
+    "train_bytes_in_ranges": 22280,
+    "xcheck_mask_2134": {
+     "key": "m7_test20k",
+     "mask_covered": 4263,
+     "mask_excl": 6596,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   },
+   "fc2": {
+    "n64": 199729,
+    "n64_found": 0,
+    "frac64_found": 0.0,
+    "n64_distinct_found": 0,
+    "eval_bytes": 199792,
+    "eval_bytes_covered": 0,
+    "eval_bytes_excl_ctx64": 0,
+    "window_sha256": "b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3",
+    "train_positions_matched": 0,
+    "train_positions_seg_bg1": 0,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 0,
+    "train_bytes_in_ranges": 0,
+    "xcheck_mask_2134": {
+     "key": "bench_fc2",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   },
+   "fresh_en": {
+    "n64": 119700,
+    "n64_found": 0,
+    "frac64_found": 0.0,
+    "n64_distinct_found": 0,
+    "eval_bytes": 119763,
+    "eval_bytes_covered": 0,
+    "eval_bytes_excl_ctx64": 0,
+    "window_sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a",
+    "train_positions_matched": 0,
+    "train_positions_seg_bg1": 0,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 0,
+    "train_bytes_in_ranges": 0,
+    "xcheck_mask_2134": {
+     "key": "bench_fresh_en",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   },
+   "fresh_fr": {
+    "n64": 1088625,
+    "n64_found": 0,
+    "frac64_found": 0.0,
+    "n64_distinct_found": 0,
+    "eval_bytes": 1088688,
+    "eval_bytes_covered": 0,
+    "eval_bytes_excl_ctx64": 0,
+    "window_sha256": "875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715",
+    "train_positions_matched": 0,
+    "train_positions_seg_bg1": 0,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 0,
+    "train_bytes_in_ranges": 0,
+    "xcheck_mask_2134": {
+     "key": "bench_fresh_fr",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   },
+   "test_v3": {
+    "n64": 3486590,
+    "n64_found": 0,
+    "frac64_found": 0.0,
+    "n64_distinct_found": 0,
+    "eval_bytes": 3486653,
+    "eval_bytes_covered": 0,
+    "eval_bytes_excl_ctx64": 0,
+    "window_sha256": "a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84",
+    "train_positions_matched": 0,
+    "train_positions_seg_bg1": 0,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 0,
+    "train_bytes_in_ranges": 0,
+    "xcheck_mask_2134": {
+     "key": "bench_test_v3",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   },
+   "queries": {
+    "n64": 27737,
+    "n64_found": 0,
+    "frac64_found": 0.0,
+    "n64_distinct_found": 0,
+    "eval_bytes": 27800,
+    "eval_bytes_covered": 0,
+    "eval_bytes_excl_ctx64": 0,
+    "window_sha256": "5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706",
+    "train_positions_matched": 0,
+    "train_positions_seg_bg1": 0,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 0,
+    "train_bytes_in_ranges": 0,
+    "xcheck_mask_2134": {
+     "key": "bench_queries",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   },
+   "bg1_test": {
+    "n64": 894655,
+    "n64_found": 33766,
+    "frac64_found": 0.03774192286412081,
+    "n64_distinct_found": 32879,
+    "eval_bytes": 894718,
+    "eval_bytes_covered": 45852,
+    "eval_bytes_excl_ctx64": 54828,
+    "window_sha256": "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9",
+    "train_positions_matched": 186809,
+    "train_positions_seg_bg1": 186809,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 687,
+    "train_bytes_in_ranges": 231503,
+    "xcheck_mask_2134": {
+     "key": "broad_test_bin",
+     "mask_covered": 45852,
+     "mask_excl": 54828,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   }
+  },
+  "files": {
+   "masks_raw.npz": "8f56c0edf25cafb368c1a2c848ec926fe389c6695863b37d5c5895b93350b7b0",
+   "train_ranges_raw.npz": "31115e29fee3d0f794a5357bce1e942b846ad491e91fecc14e566839ced220bc"
+  },
+  "elapsed_s": 74.0,
+  "peak_rss_kB": 217304,
+  "script_sha256": "5230e9959b08e9c16972b8b5d3920a54ec6b17c8fd489b14b5585cc1dd314428"
+ },
+ "norm": {
+  "form": "norm",
+  "k": 64,
+  "hash": "Horner mod 2^64 P=0x9E3779B97F4A7C15 (logique dedup64.py 4bee4293…), confirmation octet à octet",
+  "normalisation": "suppr. \\r ; A-Z->a-z ASCII ; suites [ \\t\\n\\v\\f] -> 1 espace ; en flux",
+  "train": "/workspace/spira_shared/corpus_diverse_v1/train.bin",
+  "train_bytes": 530137703,
+  "train_sha256": "c813244c618eb20dbc34e2d7472ac29e10b0c22344e8117a0c25b1186124e6d9",
+  "train_sha_ok": true,
+  "train_64gram_positions_hashed": 489807340,
+  "table_unique_hashes": 5551296,
+  "table_internal_collisions": 0,
+  "candidates": 149315,
+  "hash_false_positives_rejected": 0,
+  "train_positions_matched_any": 149315,
+  "train_ranges_merged_any": 454,
+  "train_bytes_in_ranges_any": 221174,
+  "train_positions_seg_multilang_any": 117,
+  "windows": {
+   "test20k_old": {
+    "n64": 298660,
+    "n64_found": 1915,
+    "frac64_found": 0.006411973481550927,
+    "n64_distinct_found": 1907,
+    "eval_bytes": 319940,
+    "eval_bytes_covered": 3944,
+    "eval_bytes_excl_ctx64": 5448,
+    "window_sha256": "ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22",
+    "train_positions_matched": 12453,
+    "train_positions_seg_bg1": 12453,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 54,
+    "train_bytes_in_ranges": 17298,
+    "xcheck_mask_2134": {
+     "key": "m7_test20k",
+     "mask_covered": 4263,
+     "mask_excl": 6596,
+     "covered_identical_to_mask": false,
+     "covered_outside_mask_excl": 647,
+     "excl_identical_to_mask": false
+    }
+   },
+   "fc2": {
+    "n64": 157783,
+    "n64_found": 22,
+    "frac64_found": 0.0001394320047153369,
+    "n64_distinct_found": 22,
+    "eval_bytes": 199792,
+    "eval_bytes_covered": 113,
+    "eval_bytes_excl_ctx64": 177,
+    "window_sha256": "b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3",
+    "train_positions_matched": 122,
+    "train_positions_seg_bg1": 22,
+    "train_positions_seg_multilang": 100,
+    "train_ranges_merged": 6,
+    "train_bytes_in_ranges": 573,
+    "xcheck_mask_2134": {
+     "key": "bench_fc2",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": false,
+     "covered_outside_mask_excl": 113,
+     "excl_identical_to_mask": false
+    }
+   },
+   "fresh_en": {
+    "n64": 119587,
+    "n64_found": 0,
+    "frac64_found": 0.0,
+    "n64_distinct_found": 0,
+    "eval_bytes": 119763,
+    "eval_bytes_covered": 0,
+    "eval_bytes_excl_ctx64": 0,
+    "window_sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a",
+    "train_positions_matched": 0,
+    "train_positions_seg_bg1": 0,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 0,
+    "train_bytes_in_ranges": 0,
+    "xcheck_mask_2134": {
+     "key": "bench_fresh_en",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   },
+   "fresh_fr": {
+    "n64": 1067982,
+    "n64_found": 77,
+    "frac64_found": 7.209859342198652e-05,
+    "n64_distinct_found": 77,
+    "eval_bytes": 1088688,
+    "eval_bytes_covered": 143,
+    "eval_bytes_excl_ctx64": 207,
+    "window_sha256": "875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715",
+    "train_positions_matched": 1112,
+    "train_positions_seg_bg1": 1112,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 18,
+    "train_bytes_in_ranges": 2270,
+    "xcheck_mask_2134": {
+     "key": "bench_fresh_fr",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": false,
+     "covered_outside_mask_excl": 143,
+     "excl_identical_to_mask": false
+    }
+   },
+   "test_v3": {
+    "n64": 3390446,
+    "n64_found": 17,
+    "frac64_found": 5.014089591752825e-06,
+    "n64_distinct_found": 17,
+    "eval_bytes": 3486653,
+    "eval_bytes_covered": 149,
+    "eval_bytes_excl_ctx64": 277,
+    "window_sha256": "a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84",
+    "train_positions_matched": 17,
+    "train_positions_seg_bg1": 17,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 2,
+    "train_bytes_in_ranges": 144,
+    "xcheck_mask_2134": {
+     "key": "bench_test_v3",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": false,
+     "covered_outside_mask_excl": 149,
+     "excl_identical_to_mask": false
+    }
+   },
+   "queries": {
+    "n64": 27688,
+    "n64_found": 0,
+    "frac64_found": 0.0,
+    "n64_distinct_found": 0,
+    "eval_bytes": 27800,
+    "eval_bytes_covered": 0,
+    "eval_bytes_excl_ctx64": 0,
+    "window_sha256": "5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706",
+    "train_positions_matched": 0,
+    "train_positions_seg_bg1": 0,
+    "train_positions_seg_multilang": 0,
+    "train_ranges_merged": 0,
+    "train_bytes_in_ranges": 0,
+    "xcheck_mask_2134": {
+     "key": "bench_queries",
+     "mask_covered": 0,
+     "mask_excl": 0,
+     "covered_identical_to_mask": true,
+     "covered_outside_mask_excl": 0,
+     "excl_identical_to_mask": true
+    }
+   },
+   "bg1_test": {
+    "n64": 823920,
+    "n64_found": 26765,
+    "frac64_found": 0.03248494999514516,
+    "n64_distinct_found": 26046,
+    "eval_bytes": 894718,
+    "eval_bytes_covered": 42756,
+    "eval_bytes_excl_ctx64": 49727,
+    "window_sha256": "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9",
+    "train_positions_matched": 148061,
+    "train_positions_seg_bg1": 148044,
+    "train_positions_seg_multilang": 17,
+    "train_ranges_merged": 428,
+    "train_bytes_in_ranges": 218184,
+    "xcheck_mask_2134": {
+     "key": "broad_test_bin",
+     "mask_covered": 45852,
+     "mask_excl": 54828,
+     "covered_identical_to_mask": false,
+     "covered_outside_mask_excl": 1484,
+     "excl_identical_to_mask": false
+    }
+   }
+  },
+  "files": {
+   "masks_norm.npz": "7a3d76b5a20475a35dc0113efc0648d358df63f805eb05320affdf6f036df465",
+   "train_ranges_norm.npz": "e8cfddf83e1c2b0bace101d9a9a784f5e5ad1363c2aa4a1d365146b604cfb2a9"
+  },
+  "elapsed_s": 78.2,
+  "peak_rss_kB": 233784,
+  "script_sha256": "5230e9959b08e9c16972b8b5d3920a54ec6b17c8fd489b14b5585cc1dd314428"
+ }
+}

v3_scale/checker/recount/corpus_diverse_v1_dedup/dedup64_div1.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""SPIRA Checker — 25 oct. 2026 (Toronto) — recompte INDÉPENDANT de dédup 64 o : corpus_diverse_v1/train.bin vs fenêtres d'éval.
+Logique reprise de checker/recount/bg1_test_dedup64/dedup64.py (sha 4bee4293…) : Horner mod 2^64 (P = 0x9E3779B97F4A7C15) sur chaque
+64-gramme, table triée + np.searchsorted, train lu par blocs + 63 o de recouvrement, CONFIRMATION octet à octet de chaque candidat.
+Deux formes (un processus par forme, pour tenir < 300 Mo RSS) :
+  raw  : octets bruts ;
+  norm : \r supprimés, A-Z -> a-z (ASCII seulement), toute suite d'espaces (espace, \t, \n, \v, \f) -> un seul espace ; normalisation
+         en flux (état « dernier octet = espace » propagé entre blocs) ; chaque octet normalisé garde son offset d'origine.
+Tous les débuts de 64-grammes des fenêtres sont hachés (les positions notées par le banc sont tirées par bench_common, non séparables
+du fichier seul ; « tous » est un sur-ensemble). Ne lit que des fichiers ; écrit UNIQUEMENT dans son dossier. Aucun texte en sortie.
+Usage : dedup64_div1.py raw|norm   |   dedup64_div1.py selftest"""
+import sys, os, json, hashlib, time, resource
+import numpy as np
+S = "/workspace/spira_shared"; OD = os.path.dirname(os.path.abspath(__file__)); K = 64; P = np.uint64(0x9E3779B97F4A7C15)
+TRAIN = f"{S}/corpus_diverse_v1/train.bin"; TRAIN_SHA = "c813244c618eb20dbc34e2d7472ac29e10b0c22344e8117a0c25b1186124e6d9"
+SEG2 = 290810504  # début du segment multi_lang_v1 (manifest.segments) ; [290810502, 290810504) = séparateur b"\n\n"
+WIN = {  # nom : (chemin, sha256 attendu, clé du masque propre dans clean_masks/broad_gutenberg_v1_2134.npz)
+    "test20k_old": ("spira_m7/windows/test20k.bin", "ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22", "m7_test20k"),
+    "fc2": ("broad/freshcode_v2.bin", "b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3", "bench_fc2"),
+    "fresh_en": ("spira_m7/windows/fresh_en.bin", "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a", "bench_fresh_en"),
+    "fresh_fr": ("spira_m7/windows/fresh_fr.bin", "875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715", "bench_fresh_fr"),
+    "test_v3": ("corpus_v3/test.bin", "a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84", "bench_test_v3"),
+    "queries": ("spira_m7/windows/queries.bin", "5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706", "bench_queries"),
+    "bg1_test": ("broad/broad_gutenberg_v1/test.bin", "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9", "broad_test_bin"),
+}
+MASKS = f"{S}/checker/scale/clean_masks/broad_gutenberg_v1_2134.npz"
+WS = np.zeros(256, bool); WS[[9, 10, 11, 12, 32]] = True
+LOW = np.arange(256, dtype=np.uint8); LOW[65:91] += 32
+RSS_CAP_KB = 295 * 1024
+def rss(): return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+def guard():
+    if rss() > RSS_CAP_KB: raise SystemExit(f"ABANDON : pic RSS {rss()} kB > {RSS_CAP_KB} kB")
+
+def H(a):
+    """a : uint8 ; hash Horner des 64-grammes commençant en 0..len-64."""
+    n = len(a) - K + 1
+    if n <= 0: return np.zeros(0, np.uint64)
+    h = np.zeros(n, np.uint64)
+    with np.errstate(over="ignore"):
+        for k in range(K): h *= P; h += a[k:k + n]
+    return h
+
+class Norm:
+    """normalisation en flux ; feed(bytes, offset) -> (octets normalisés uint8, offsets d'origine int64)."""
+    def __init__(self): self.prev_ws = False
+    def feed(self, a, off):
+        a = np.frombuffer(a, np.uint8) if not isinstance(a, np.ndarray) else a
+        kcr = a != 13; idx = np.flatnonzero(kcr); b = LOW[a[idx]]; ws = WS[b]; b[ws] = 32
+        if len(b) == 0: return b, idx.astype(np.int64)
+        pw = np.empty_like(ws); pw[0] = self.prev_ws; pw[1:] = ws[:-1]; keep = ~(ws & pw); self.prev_ws = bool(ws[-1])
+        return b[keep], idx[keep].astype(np.int64) + off
+
+def form_bytes(raw, form):
+    a = np.frombuffer(raw, np.uint8)
+    if form == "raw": return a, None
+    b, m = Norm().feed(a, 0); return b, m.astype(np.int32)
+
+def build_table(wins, form):
+    """wins : {nom: bytes}. Renvoie U (hash uniques triés), REP (offset global d'un représentant dans E), BITS (fenêtres), E, infos."""
+    names = list(wins); parts, maps, base, starts = [], {}, 0, {}
+    for nm in names:
+        b, m = form_bytes(wins.pop(nm), form); parts.append(b.copy() if form == "raw" else b); maps[nm] = m; starts[nm] = (base, len(b)); base += len(b)
+    E = np.concatenate(parts); del parts
+    # version économe (tient sous prlimit --as 300 Mio) : table pré-allouée, G et B déduits de l'indice de tri
+    cnt = np.array([max(starts[nm][1] - K + 1, 0) for nm in names], np.int64); cum = np.concatenate([[0], np.cumsum(cnt)])
+    Hc = np.empty(int(cum[-1]), np.uint64)
+    for i, nm in enumerate(names):
+        b0, L = starts[nm]; Hc[cum[i]:cum[i + 1]] = H(E[b0:b0 + L])
+    o = np.argsort(Hc, kind="stable").astype(np.int32); guard(); Hc = Hc[o]; guard()
+    first = np.ones(len(Hc), bool); first[1:] = Hc[1:] != Hc[:-1]; U = Hc[first]; del Hc
+    wbase = np.array([starts[nm][0] for nm in names], np.int64)
+    def gofs(ix):  # indice dans Hc concaténé -> (offset global dans E, fenêtre)
+        w = np.searchsorted(cum, ix, "right") - 1; return (ix - cum[w] + wbase[w]), w
+    fidx = np.flatnonzero(first).astype(np.int32); REP = np.empty(len(fidx), np.int32); BITS = np.zeros(len(fidx), np.uint8)
+    W8 = np.empty(len(o), np.uint8)
+    for s in range(0, len(o), 1 << 20):
+        w = np.searchsorted(cum, o[s:s + (1 << 20)], "right") - 1; W8[s:s + (1 << 20)] = np.left_shift(1, w).astype(np.uint8); del w
+    for s in range(0, len(fidx), 1 << 20):
+        REP[s:s + (1 << 20)] = gofs(o[fidx[s:s + (1 << 20)]])[0].astype(np.int32)
+    BITS[:] = np.bitwise_or.reduceat(W8, fidx); del W8; guard()
+    # collisions internes : même hash, octets différents
+    first = ~first; dup = np.flatnonzero(first).astype(np.int32); del first; coll = 0; ar = np.arange(K)
+    for s in range(0, len(dup), 20000):
+        d = dup[s:s + 20000]; g = np.searchsorted(fidx, d, "right") - 1; gd = gofs(o[d])[0]
+        coll += int((E[gd[:, None] + ar] != E[REP[g].astype(np.int64)[:, None] + ar]).any(1).sum())
+    del o, dup, fidx; guard()
+    return U, REP, BITS, E, starts, maps, coll, names
+
+def scan(src, U, REP, BITS, E, form, chunk, nwin):
+    """src : chemin ou bytes. Renvoie FOUND (bool sur U), hits (offsets train début/fin, bits), compteurs."""
+    FOUND = np.zeros(len(U), bool); hs, he, hb = [], [], []; cand = fp = npos = 0; sha = hashlib.sha256(); ar = np.arange(K)
+    nz = Norm(); tail = np.zeros(0, np.uint8); tidx = np.zeros(0, np.int64); pos = 0
+    f = open(src, "rb") if isinstance(src, str) else None; mv = memoryview(src) if f is None else None
+    while True:
+        blk = f.read(chunk) if f else bytes(mv[pos:pos + chunk])
+        if not blk: break
+        sha.update(blk); a = np.frombuffer(blk, np.uint8)
+        if form == "raw": nb, ni = a, None
+        else: nb, ni = nz.feed(a, pos)
+        buf = np.concatenate([tail, nb])
+        if form == "raw": cst = pos - len(tail)
+        else: bidx = np.concatenate([tidx, ni])
+        h = H(buf); npos += len(h)
+        if len(h):
+            i = np.searchsorted(U, h); i[i >= len(U)] = 0; hit = np.flatnonzero(U[i] == h)
+            if len(hit):
+                cand += len(hit); ui = i[hit]; ok = np.ones(len(hit), bool)
+                for s in range(0, len(hit), 20000):
+                    j = hit[s:s + 20000]; r = REP[ui[s:s + 20000]]
+                    ok[s:s + 20000] = (buf[j[:, None] + ar] == E[r.astype(np.int64)[:, None] + ar]).all(1)
+                fp += int((~ok).sum()); hit = hit[ok]; ui = ui[ok]; FOUND[ui] = True
+                if form == "raw": st = hit.astype(np.int64) + cst; en = st + K
+                else: st = bidx[hit]; en = bidx[hit + K - 1] + 1
+                hs.append(st); he.append(en); hb.append(BITS[ui])
+        pos += len(blk); keep = min(K - 1, len(buf)); tail = buf[len(buf) - keep:].copy()
+        if form != "raw": tidx = bidx[len(bidx) - keep:].copy()
+        del h, buf; guard()
+    if f: f.close()
+    cat = lambda L, dt: np.concatenate(L) if L else np.zeros(0, dt)
+    return FOUND, cat(hs, np.int64), cat(he, np.int64), cat(hb, np.uint8), cand, fp, npos, pos, sha.hexdigest()
+
+def merge(st, en):
+    if len(st) == 0: return np.zeros((0, 2), np.int64)
+    o = np.argsort(st, kind="stable"); st, en = st[o], en[o]; em = np.maximum.accumulate(en)
+    new = np.ones(len(st), bool); new[1:] = st[1:] > em[:-1]; si = np.flatnonzero(new); ei = np.r_[si[1:], len(st)] - 1
+    return np.stack([st[si], em[ei]], 1)
+
+def per_window(U, FOUND, E, starts, maps, names, nwin_bytes):
+    out, cov_all = {}, {}
+    for nm in names:
+        b0, L = starts[nm]; h = H(E[b0:b0 + L]); fw = np.zeros(len(h), bool)
+        for s0 in range(0, len(h), 1 << 19): fw[s0:s0 + (1 << 19)] = FOUND[np.searchsorted(U, h[s0:s0 + (1 << 19)])]
+        nh = int(len(h)); ndf = int(len(np.unique(h[fw]))); n0 = nwin_bytes[nm]; d = np.zeros(n0 + 1, np.int32); s = np.flatnonzero(fw)
+        if maps[nm] is None: a0, a1 = s, s + K
+        else: m = maps[nm]; a0, a1 = m[s].astype(np.int64), m[s + K - 1].astype(np.int64) + 1
+        np.add.at(d, a0, 1); np.add.at(d, a1, -1); np.cumsum(d, dtype=np.int32, out=d); cov = d[:n0] > 0; del d, h
+        c = np.zeros(n0 + 1, np.int32); np.cumsum(cov, dtype=np.int32, out=c[1:]); lo = np.zeros(n0, np.int32); lo[K:] = c[:n0 - K]
+        np.subtract(c[1:], lo, out=lo); excl = lo > 0; del c, lo
+        out[nm] = {"n64": nh, "n64_found": int(fw.sum()), "frac64_found": float(fw.mean()) if nh else 0.0,
+                   "n64_distinct_found": ndf,
+                   "eval_bytes": n0, "eval_bytes_covered": int(cov.sum()), "eval_bytes_excl_ctx64": int(excl.sum())}
+        cov_all[nm] = (cov, excl)
+    return out, cov_all
+
+def run(form):
+    t0 = time.time(); wins = {}; wsha = {}
+    for nm, (rel, sh, _) in WIN.items():
+        d = open(f"{S}/{rel}", "rb").read(); s = hashlib.sha256(d).hexdigest(); assert s == sh, (nm, s); wins[nm] = d; wsha[nm] = s
+    nbytes = {k: len(v) for k, v in wins.items()}; U, REP, BITS, E, starts, maps, coll, names = build_table(wins, form); del wins
+    print(f"{form} table {len(U)} uniques, collisions internes {coll}, rss {rss()} kB, {time.time()-t0:.1f} s", flush=True)
+    FOUND, st, en, bits, cand, fp, npos, tb, tsha = scan(TRAIN, U, REP, BITS, E, form, 1 << 20, len(names))
+    print(f"{form} scan {tb} o, cand {cand}, fp {fp}, rss {rss()} kB, {time.time()-t0:.1f} s", flush=True)
+    del REP, BITS; guard(); pw, covs = per_window(U, FOUND, E, starts, maps, names, nbytes)
+    M = np.load(MASKS); tr = {}; rng_all = merge(st, en)
+    for i, nm in enumerate(names):
+        sel = (bits & np.uint8(1 << i)) > 0; s1, e1 = st[sel], en[sel]; R = merge(s1, e1)
+        cov, excl = covs[nm]; mk = WIN[nm][2]; mc, me = M[f"{mk}__covered"], M[f"{mk}__excl"]
+        pw[nm].update({"window_sha256": wsha[nm], "train_positions_matched": int(sel.sum()),
+                       "train_positions_seg_bg1": int((s1 < SEG2).sum()), "train_positions_seg_multilang": int((s1 >= SEG2).sum()),
+                       "train_ranges_merged": int(len(R)), "train_bytes_in_ranges": int((R[:, 1] - R[:, 0]).sum()) if len(R) else 0,
+                       "train_range_offsets_sample": [[int(a), int(b)] for a, b in R[:3]] + ([[int(a), int(b)] for a, b in R[-2:]] if len(R) > 3 else []),
+                       "xcheck_mask_2134": {"key": mk, "mask_covered": int(mc.sum()), "mask_excl": int(me.sum()),
+                                            "covered_identical_to_mask": bool(np.array_equal(cov, mc)), "covered_outside_mask_excl": int((cov & ~me).sum()),
+                                            "excl_identical_to_mask": bool(np.array_equal(excl, me))}})
+        tr[nm] = R
+    np.savez_compressed(f"{OD}/masks_{form}.npz", **{f"{nm}__covered": np.packbits(covs[nm][0]) for nm in names},
+                        **{f"{nm}__excl_ctx64": np.packbits(covs[nm][1]) for nm in names}, **{f"{nm}__len": np.int64(nbytes[nm]) for nm in names})
+    np.savez_compressed(f"{OD}/train_ranges_{form}.npz", all=rng_all, **{nm: tr[nm] for nm in names})
+    sha = lambda p: hashlib.sha256(open(p, "rb").read()).hexdigest()
+    out = {"form": form, "k": K, "hash": "Horner mod 2^64 P=0x9E3779B97F4A7C15 (logique dedup64.py 4bee4293…), confirmation octet à octet",
+           "normalisation": None if form == "raw" else "suppr. \\r ; A-Z->a-z ASCII ; suites [ \\t\\n\\v\\f] -> 1 espace ; en flux",
+           "train": TRAIN, "train_bytes": tb, "train_sha256": tsha, "train_sha_ok": tsha == TRAIN_SHA, "train_64gram_positions_hashed": npos,
+           "table_unique_hashes": int(len(U)), "table_internal_collisions": coll, "candidates": cand, "hash_false_positives_rejected": fp,
+           "train_positions_matched_any": int(len(st)), "train_ranges_merged_any": int(len(rng_all)),
+           "train_bytes_in_ranges_any": int((rng_all[:, 1] - rng_all[:, 0]).sum()) if len(rng_all) else 0,
+           "train_positions_seg_multilang_any": int((st >= SEG2).sum()), "windows": pw,
+           "files": {f"masks_{form}.npz": sha(f"{OD}/masks_{form}.npz"), f"train_ranges_{form}.npz": sha(f"{OD}/train_ranges_{form}.npz")},
+           "elapsed_s": round(time.time() - t0, 1), "peak_rss_kB": rss(), "script_sha256": sha(os.path.abspath(__file__))}
+    json.dump(out, open(f"{OD}/dedup64_{form}.json", "w"), indent=1)
+    print(json.dumps({k: v["n64_found"] for k, v in pw.items()}), out["elapsed_s"], "s", out["peak_rss_kB"], "kB", flush=True)
+
+def selftest():
+    rng = np.random.default_rng(7); res = {}; ok_all = True
+    def rb(n): return bytes(rng.integers(0, 256, n, dtype=np.uint8))
+    def tx(n): return bytes(rng.choice(np.frombuffer(b"abcdefghij KLMNOP\n\r\t.,", np.uint8), n))
+    # T1 hash : Horner vectoriel == boucle Python
+    a = np.frombuffer(rb(300), np.uint8); h = H(a); M64 = (1 << 64) - 1
+    def hp(x):
+        v = 0
+        for c in x: v = (v * 0x9E3779B97F4A7C15 + int(c)) & M64
+        return v
+    res["T1_hash_vs_python"] = all(int(h[i]) == hp(a[i:i + K]) for i in (0, 1, 100, 236)); 
+    # T2 normalisation en flux == normalisation globale, plusieurs tailles de bloc (frontières au milieu de suites d'espaces / \r\n)
+    t = tx(5000) + b"  \r\n\r\n  X" + tx(3000); ref, refi = Norm().feed(np.frombuffer(t, np.uint8), 0)
+    py = bytes(t).replace(b"\r", b"").lower(); import re; py = re.sub(rb"[ \t\n\x0b\x0c]+", b" ", py)
+    t2 = True
+    for cs in (1, 7, 63, 64, 100, 4096):
+        nz = Norm(); parts, idxs = [], []
+        for s in range(0, len(t), cs): b, i = nz.feed(np.frombuffer(t[s:s + cs], np.uint8), s); parts.append(b); idxs.append(i)
+        t2 &= np.array_equal(np.concatenate(parts), ref) and np.array_equal(np.concatenate(idxs), refi)
+    res["T2_norm_stream_eq_global_eq_regex"] = bool(t2 and ref.tobytes() == py)
+    # T3/T4 : doublons plantés à cheval sur une frontière de bloc (raw exact ; norm : casse/espaces/\r modifiés)
+    ev = {"w0": tx(2000), "w1": tx(1500)}; CH = 1000
+    g_raw = ev["w0"][300:300 + K]
+    g_n = ev["w1"][500:500 + 80]; gnorm = Norm().feed(np.frombuffer(g_n, np.uint8), 0)[0].tobytes()
+    variant = gnorm.upper().replace(b" ", b" \r\n\t ")  # même forme normalisée, octets bruts différents
+    train = bytearray(tx(10000))
+    train = train.replace(g_raw, b"").replace(g_raw[:32], b"")  # pas d'occurrence fortuite
+    p1 = 3 * CH - 30; train[p1:p1 + K] = g_raw           # raw à cheval sur 3000
+    p2 = 6 * CH - 40; train[p2:p2 + len(variant)] = variant  # variante à cheval sur 6000
+    p3 = 8 * CH - 10; train[p3:p3 + 63] = g_raw[:63]        # quasi-doublon 63 o : ne doit PAS compter en raw
+    train = bytes(train)
+    for form in ("raw", "norm"):
+        U, REP, BITS, E, starts, maps, coll, names = build_table(dict(ev), form)
+        FOUND, st, en, bits, cand, fp, npos, tb, tsha = scan(train, U, REP, BITS, E, form, CH, len(names))
+        # brute force : ensemble Python de tous les 64-grammes de la forme
+        def grams(x): return {x[i:i + K] for i in range(len(x) - K + 1)}
+        fx = (lambda x: x) if form == "raw" else (lambda x: Norm().feed(np.frombuffer(x, np.uint8), 0)[0].tobytes())
+        eg = grams(fx(ev["w0"])) | grams(fx(ev["w1"])); tn = fx(train); bf = sum(1 for i in range(len(tn) - K + 1) if tn[i:i + K] in eg)
+        nfull = 0
+        for c in (1 << 20,):
+            F2 = scan(train, U, REP, BITS, E, form, c, len(names)); nfull = len(F2[1])
+        r = {"hits": int(len(st)), "bruteforce": bf, "one_chunk_hits": nfull, "starts": sorted(set(int(x) for x in st))[:5], "fp": fp, "coll": coll, "tsha_ok": tsha == hashlib.sha256(train).hexdigest()}
+        if form == "raw": r["planted_raw_found_at_p1"] = p1 in set(st.tolist()); r["near_miss_63_not_found"] = p3 not in set(st.tolist())
+        else: r["planted_variant_found_in_p2_range"] = bool(((st >= p2) & (st < p2 + len(variant))).any()); r["info_raw64_plant_seen_in_norm"] = bool(((st >= p1 - 5) & (st <= p1 + 5)).any())
+        r["pass"] = (r["hits"] == bf == nfull and r["tsha_ok"] and fp == 0 and all(v for k, v in r.items() if k.startswith("planted") or k.startswith("near")))
+        if form == "raw": r["pass"] &= r["hits"] >= 1
+        res[f"T3_{form}_chunk_boundary"] = r
+    ok_all = res["T1_hash_vs_python"] and res["T2_norm_stream_eq_global_eq_regex"] and res["T3_raw_chunk_boundary"]["pass"] and res["T3_norm_chunk_boundary"]["pass"]
+    res["ALL_PASS"] = bool(ok_all); res["peak_rss_kB"] = rss()
+    json.dump(res, open(f"{OD}/selftest.json", "w"), indent=1); print(json.dumps(res), flush=True)
+
+if __name__ == "__main__":
+    selftest() if sys.argv[1] == "selftest" else run(sys.argv[1])

v3_scale/checker/recount/corpus_diverse_v1_dedup/run.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Checker — lance une forme : prlimit --as 2 Gio (filet), nice 19, OMP 1 ; CHIEN DE GARDE RSS : sonde VmRSS toutes les 1 s,
+# tue UNIQUEMENT son propre processus python si VmRSS > 300 Mo (307200 kB). Pic VmHWM noté dans run_<form>.mem.
+export TZ=America/Toronto OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 MALLOC_MMAP_THRESHOLD_=65536 MALLOC_TRIM_THRESHOLD_=0
+cd "$(dirname "$0")"; F=$1; CAP=307200
+echo "$(date '+%F %T %Z') START $F" > run_$F.mem
+prlimit --as=2147483648 nice -n 19 /workspace/SPIRA-venv/bin/python -u dedup64_div1.py $F > run_$F.log 2>&1 & P=$!
+pk=0; killed=0
+while kill -0 $P 2>/dev/null; do
+  r=$(awk '/VmRSS/{print $2}' /proc/$P/status 2>/dev/null); h=$(awk '/VmHWM/{print $2}' /proc/$P/status 2>/dev/null)
+  [ -n "$h" ] && [ "$h" -gt "$pk" ] && pk=$h
+  if [ -n "$r" ] && [ "$r" -gt $CAP ]; then kill $P; killed=1; echo "$(date '+%T') WATCHDOG kill own pid $P VmRSS=$r kB" >> run_$F.mem; fi
+  sleep 1
+done
+wait $P; rc=$?; echo "$(date '+%F %T %Z') END $F rc=$rc killed=$killed VmHWM_polled_kB=$pk" >> run_$F.mem