Spencer Norton commited on
Generalize catalog/rule descriptions and add Intended Use & Legal Compliance section
Browse files- README.md +16 -5
- config.json +6 -6
- test_dht_db_sync.py +510 -10
README.md
CHANGED
|
@@ -22,7 +22,7 @@ pipeline_tag: other
|
|
| 22 |
|
| 23 |
<p align="center">
|
| 24 |
<strong>Decentralized <code>BEP-44</code> / <code>BEP-46</code> database synchronization engine featuring multi-layer CSAM & abuse blocking, <code>Ed25519</code>-signed blinded <code>SHA-256</code> tombstone purging, modular fine-grained category & language subscriptions with isolated <code>xxx</code> containment, <code>X25519</code> + <code>ChaCha20-Poly1305</code> AEAD encryption, and $K$-of-$N$ distinct-ASN Sybil quorum verification.</strong><br>
|
| 25 |
-
Designed for decentralized indexers (<code>Bitmagnet</code>
|
| 26 |
</p>
|
| 27 |
|
| 28 |
<p align="center">
|
|
@@ -97,7 +97,7 @@ sync_engine = ArmigerDHTDatabaseSync(
|
|
| 97 |
# 2. Ingest candidate record with CSAM & safety checks
|
| 98 |
ok, msg, record = sync_engine.ingest_candidate_record(
|
| 99 |
infohash="0123456789abcdef0123456789abcdef01234567",
|
| 100 |
-
title="
|
| 101 |
category="anime",
|
| 102 |
language="ja",
|
| 103 |
quality_tier="hd_1080p",
|
|
@@ -109,7 +109,7 @@ ok, msg, record = sync_engine.ingest_candidate_record(
|
|
| 109 |
print("Ingestion result:", ok, msg)
|
| 110 |
|
| 111 |
# 3. Search local database
|
| 112 |
-
results = sync_engine.db.search(query="
|
| 113 |
print("Found torrents:", len(results))
|
| 114 |
|
| 115 |
# 4. Apply Ed25519-signed blinded tombstone purge
|
|
@@ -173,13 +173,24 @@ Ran 27 tests in 0.011s - OK
|
|
| 173 |
| :--- | :---: | :---: | :--- |
|
| 174 |
| **[`NorviTech/armiger-swarm-survival`](https://huggingface.co/NorviTech/armiger-swarm-survival)** | **Yes** (`swarm_adapter.pt`, `swarm_survival.py`) | `163 KB` | **SwarmSurvivalNet v2 & Honeypot Residual Adapter:** **`96.83%`** Swarm Liveliness AUROC (vs. `53.04%` raw seeders), **`100.0%`** fake-seeder malware trap recall, **`0%`** content-category flip guarantee. |
|
| 175 |
| **[`NorviTech/armiger-swarm-priors`](https://huggingface.co/NorviTech/armiger-swarm-priors)** | **Yes** (`swarm_priors.py`) | `29.4 KB` | **Empirical Bayes Swarm & Release-Group Priors:** 4-pathway tracker scrape prior engine, mandatory modality dropout (`p=0.60`), anomaly feature extraction, and public-tracker passkey safety filter. |
|
| 176 |
-
| **[`NorviTech/armiger-zero-cue-catalog`](https://huggingface.co/NorviTech/armiger-zero-cue-catalog)** | **Yes** (`zero_cue_plugin.py`, `validated_zero_cue_catalog.json`) | `36.0 MB` | **Zero-Cue Catalog & Structural Disambiguator:** Offline 929,400-key normalized
|
| 177 |
| **[`NorviTech/armiger-audio-guard`](https://huggingface.co/NorviTech/armiger-audio-guard)** | **Yes** (`audio_language_guard.py`) | `4.2 KB` | **Cross-Head Audio Language & Anime Dub Guard:** Couples `title_language` (`P(Foreign Title)`) with `lang` head to disambiguate `.en.` prepositions, `<Foreign>.ENG.Sub`, regional dubs, and `MULTi (FR EN)` tags; reaches **`98.72%`** Gold Standard and **`99.88%`** Hard Edge-Case audio accuracy. |
|
| 178 |
-
| **[`NorviTech/armiger-calibration-dht`](https://huggingface.co/NorviTech/armiger-calibration-dht)** | **Yes** (`calibration.json`, `calibration_and_dht.py`, `armiger_dht_rsi.py`) | `115 KB` | **Conformal Calibration, BEP-44 DHT Updater & Distributed RSI Engine:** Per-head temperature scaling ($T_h$), Mondrian safety gates (`99.95%` junk balanced accuracy), `Ed25519` threshold-signed `BEP-44`/`BEP-46` DHT hot-reload, universal
|
| 179 |
| **[`NorviTech/armiger-dht-db-sync`](https://huggingface.co/NorviTech/armiger-dht-db-sync)** | **Yes** (`csam_guard.py`, `armiger_dht_db_sync.py`) | `98 KB` | **Modular, CSAM-Protected, Encrypted BEP-44/BEP-46 DHT Database Sync:** Multi-stage CSAM & abuse pre-ingest blocker, `Ed25519`-signed blinded `SHA-256` tombstone purge engine, 10-category $\times$ 14-language $\times$ 7-quality sharded subscriptions with strict default-off `xxx` isolation, `X25519`+`ChaCha20-Poly1305` AEAD, binary Merkle proofs, and $K$-of-$N$ distinct-ASN Sybil quorum (`27/27` tests). |
|
| 180 |
|
| 181 |
---
|
| 182 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
<p align="center">
|
| 184 |
<a href="https://norvitech.com"><img alt="Part of the NorviTech Suite — open-source apps for the Linux desktop and the self-hosted stack" src="https://norvitech.com/assets/banner.svg" width="640"></a>
|
| 185 |
</p>
|
|
|
|
| 22 |
|
| 23 |
<p align="center">
|
| 24 |
<strong>Decentralized <code>BEP-44</code> / <code>BEP-46</code> database synchronization engine featuring multi-layer CSAM & abuse blocking, <code>Ed25519</code>-signed blinded <code>SHA-256</code> tombstone purging, modular fine-grained category & language subscriptions with isolated <code>xxx</code> containment, <code>X25519</code> + <code>ChaCha20-Poly1305</code> AEAD encryption, and $K$-of-$N$ distinct-ASN Sybil quorum verification.</strong><br>
|
| 25 |
+
Designed for decentralized indexers (<code>Bitmagnet</code> and self-hosted library fleets) powered by <a href="https://huggingface.co/NorviTech/armiger-ultra"><code>NorviTech/armiger-ultra</code></a>.
|
| 26 |
</p>
|
| 27 |
|
| 28 |
<p align="center">
|
|
|
|
| 97 |
# 2. Ingest candidate record with CSAM & safety checks
|
| 98 |
ok, msg, record = sync_engine.ingest_candidate_record(
|
| 99 |
infohash="0123456789abcdef0123456789abcdef01234567",
|
| 100 |
+
title="Spring.Complete.1080p.BluRay.x265",
|
| 101 |
category="anime",
|
| 102 |
language="ja",
|
| 103 |
quality_tier="hd_1080p",
|
|
|
|
| 109 |
print("Ingestion result:", ok, msg)
|
| 110 |
|
| 111 |
# 3. Search local database
|
| 112 |
+
results = sync_engine.db.search(query="Spring", category="anime")
|
| 113 |
print("Found torrents:", len(results))
|
| 114 |
|
| 115 |
# 4. Apply Ed25519-signed blinded tombstone purge
|
|
|
|
| 173 |
| :--- | :---: | :---: | :--- |
|
| 174 |
| **[`NorviTech/armiger-swarm-survival`](https://huggingface.co/NorviTech/armiger-swarm-survival)** | **Yes** (`swarm_adapter.pt`, `swarm_survival.py`) | `163 KB` | **SwarmSurvivalNet v2 & Honeypot Residual Adapter:** **`96.83%`** Swarm Liveliness AUROC (vs. `53.04%` raw seeders), **`100.0%`** fake-seeder malware trap recall, **`0%`** content-category flip guarantee. |
|
| 175 |
| **[`NorviTech/armiger-swarm-priors`](https://huggingface.co/NorviTech/armiger-swarm-priors)** | **Yes** (`swarm_priors.py`) | `29.4 KB` | **Empirical Bayes Swarm & Release-Group Priors:** 4-pathway tracker scrape prior engine, mandatory modality dropout (`p=0.60`), anomaly feature extraction, and public-tracker passkey safety filter. |
|
| 176 |
+
| **[`NorviTech/armiger-zero-cue-catalog`](https://huggingface.co/NorviTech/armiger-zero-cue-catalog)** | **Yes** (`zero_cue_plugin.py`, `validated_zero_cue_catalog.json`) | `36.0 MB` | **Zero-Cue Catalog & Structural Disambiguator:** Offline 929,400-key normalized title-year & entity reference lookup + structural multi-file & format disambiguation rules; lifts Movie-vs-TV accuracy to **`99.46%`** and 11-class accuracy to **`98.48%`** (`98.93%` Hard Edge-Cases). |
|
| 177 |
| **[`NorviTech/armiger-audio-guard`](https://huggingface.co/NorviTech/armiger-audio-guard)** | **Yes** (`audio_language_guard.py`) | `4.2 KB` | **Cross-Head Audio Language & Anime Dub Guard:** Couples `title_language` (`P(Foreign Title)`) with `lang` head to disambiguate `.en.` prepositions, `<Foreign>.ENG.Sub`, regional dubs, and `MULTi (FR EN)` tags; reaches **`98.72%`** Gold Standard and **`99.88%`** Hard Edge-Case audio accuracy. |
|
| 178 |
+
| **[`NorviTech/armiger-calibration-dht`](https://huggingface.co/NorviTech/armiger-calibration-dht)** | **Yes** (`calibration.json`, `calibration_and_dht.py`, `armiger_dht_rsi.py`) | `115 KB` | **Conformal Calibration, BEP-44 DHT Updater & Distributed RSI Engine:** Per-head temperature scaling ($T_h$), Mondrian safety gates (`99.95%` junk balanced accuracy), `Ed25519` threshold-signed `BEP-44`/`BEP-46` DHT hot-reload, universal media-webhook/qBittorrent/Transmission/ffprobe ground-truth RSI, `X25519`+`ChaCha20-Poly1305` AEAD, distinct-ASN quorum, and 500-canary regression gate (`36/36` tests). |
|
| 179 |
| **[`NorviTech/armiger-dht-db-sync`](https://huggingface.co/NorviTech/armiger-dht-db-sync)** | **Yes** (`csam_guard.py`, `armiger_dht_db_sync.py`) | `98 KB` | **Modular, CSAM-Protected, Encrypted BEP-44/BEP-46 DHT Database Sync:** Multi-stage CSAM & abuse pre-ingest blocker, `Ed25519`-signed blinded `SHA-256` tombstone purge engine, 10-category $\times$ 14-language $\times$ 7-quality sharded subscriptions with strict default-off `xxx` isolation, `X25519`+`ChaCha20-Poly1305` AEAD, binary Merkle proofs, and $K$-of-$N$ distinct-ASN Sybil quorum (`27/27` tests). |
|
| 180 |
|
| 181 |
---
|
| 182 |
|
| 183 |
+
## Intended Use & Legal Compliance
|
| 184 |
+
|
| 185 |
+
`Armiger`, `Armiger Ultra`, and all bundled plugins are open-source machine learning and cryptographic metadata-classification utilities designed strictly for **lawful use cases**, including:
|
| 186 |
+
- Organizing and indexing licensed, public-domain, Creative Commons, and open-source digital media, software distributions (e.g., Linux ISOs, open-source games), and educational archives (e.g., OpenCourseWare, LibriVox, Project Gutenberg).
|
| 187 |
+
- Automated cybersecurity screening, fake-seeder honeypot detection, and disguised executable/malware filtering (`.mkv.exe`, `.scr`, `.vbs`) in peer-to-peer and decentralized storage research.
|
| 188 |
+
- Child safety (CSAM/CSAE) pre-ingest blocking, cryptographic tombstone revocation, and adult/NSFW namespace isolation for self-hosted and decentralized indexers.
|
| 189 |
+
|
| 190 |
+
**NorviTech does not condone, encourage, or support copyright infringement or the unauthorized distribution of copyrighted works.** Users are solely responsible for ensuring that their use of these models and plugins complies with all applicable local, national, and international copyright and data-protection laws.
|
| 191 |
+
|
| 192 |
+
---
|
| 193 |
+
|
| 194 |
<p align="center">
|
| 195 |
<a href="https://norvitech.com"><img alt="Part of the NorviTech Suite — open-source apps for the Linux desktop and the self-hosted stack" src="https://norvitech.com/assets/banner.svg" width="640"></a>
|
| 196 |
</p>
|
config.json
CHANGED
|
@@ -62,16 +62,16 @@
|
|
| 62 |
},
|
| 63 |
"artifacts": {
|
| 64 |
"csam_guard.py": {
|
| 65 |
-
"size_bytes":
|
| 66 |
-
"sha256": "
|
| 67 |
},
|
| 68 |
"armiger_dht_db_sync.py": {
|
| 69 |
-
"size_bytes":
|
| 70 |
-
"sha256": "
|
| 71 |
},
|
| 72 |
"test_dht_db_sync.py": {
|
| 73 |
-
"size_bytes":
|
| 74 |
-
"sha256": "
|
| 75 |
}
|
| 76 |
}
|
| 77 |
}
|
|
|
|
| 62 |
},
|
| 63 |
"artifacts": {
|
| 64 |
"csam_guard.py": {
|
| 65 |
+
"size_bytes": 23295,
|
| 66 |
+
"sha256": "3749e36abf4e121de9eea655a5ad9055d0946a1dc8b9753b656bbadaabfde36b"
|
| 67 |
},
|
| 68 |
"armiger_dht_db_sync.py": {
|
| 69 |
+
"size_bytes": 44070,
|
| 70 |
+
"sha256": "fd10adfc4ac9e82b093bcbb1f94b7f00c92184b45e0ad18640ed375509ae9415"
|
| 71 |
},
|
| 72 |
"test_dht_db_sync.py": {
|
| 73 |
+
"size_bytes": 59514,
|
| 74 |
+
"sha256": "8986cf54165e6e70c8260bc0c4304f3bd36fef3b9c98403794d3fe4bf07ed681"
|
| 75 |
}
|
| 76 |
}
|
| 77 |
}
|
test_dht_db_sync.py
CHANGED
|
@@ -94,11 +94,11 @@ class TestCSAMGuardAndSafetyGates(unittest.TestCase):
|
|
| 94 |
self.assertEqual(normalize_text(raw3), "teen porn")
|
| 95 |
|
| 96 |
def test_private_tracker_passkey_sanitization(self) -> None:
|
| 97 |
-
raw_title = "
|
| 98 |
sanitized = sanitize_title(raw_title)
|
| 99 |
self.assertNotIn("passkey=", sanitized)
|
| 100 |
self.assertNotIn("abc123secretkey", sanitized)
|
| 101 |
-
self.assertEqual(sanitized, "
|
| 102 |
|
| 103 |
def test_direct_csam_detection(self) -> None:
|
| 104 |
forbidden_samples = [
|
|
@@ -131,10 +131,10 @@ class TestCSAMGuardAndSafetyGates(unittest.TestCase):
|
|
| 131 |
|
| 132 |
def test_disguised_malware_dropper_in_media_torrents(self) -> None:
|
| 133 |
# Dangerous executable disguised as media in a movie torrent
|
| 134 |
-
malicious_files = ["
|
| 135 |
res = self.guard.inspect_torrent(
|
| 136 |
infohash="d" * 40,
|
| 137 |
-
title="
|
| 138 |
file_list=malicious_files,
|
| 139 |
category="movie",
|
| 140 |
)
|
|
@@ -253,7 +253,7 @@ class TestModularShardedSubscription(unittest.TestCase):
|
|
| 253 |
# Ingest mainstream record
|
| 254 |
rec_main = TorrentRecord(
|
| 255 |
infohash="1111222233334444555566667777888899990000",
|
| 256 |
-
title="
|
| 257 |
category="movie",
|
| 258 |
language="en",
|
| 259 |
quality_tier="hd_1080p",
|
|
@@ -263,7 +263,7 @@ class TestModularShardedSubscription(unittest.TestCase):
|
|
| 263 |
seeders=150,
|
| 264 |
leechers=10,
|
| 265 |
content_sha256="abcd" * 16,
|
| 266 |
-
file_list=["
|
| 267 |
safety_gate_passed=True,
|
| 268 |
created_at=100,
|
| 269 |
)
|
|
@@ -289,7 +289,7 @@ class TestModularShardedSubscription(unittest.TestCase):
|
|
| 289 |
self.assertFalse(sync_engine.db.insert_record(rec_adult))
|
| 290 |
|
| 291 |
# Verify search only sees mainstream
|
| 292 |
-
results = sync_engine.db.search("
|
| 293 |
self.assertEqual(len(results), 1)
|
| 294 |
self.assertEqual(results[0].category, "movie")
|
| 295 |
|
|
@@ -567,7 +567,7 @@ class TestEndToEndDHTSyncEngine(unittest.TestCase):
|
|
| 567 |
for i in range(1, 4):
|
| 568 |
ok, msg, rec = self.publisher.ingest_candidate_record(
|
| 569 |
infohash=f"abcdef{i:034x}",
|
| 570 |
-
title=f"
|
| 571 |
category="anime",
|
| 572 |
language="ja",
|
| 573 |
quality_tier="hd_1080p",
|
|
@@ -604,7 +604,7 @@ class TestEndToEndDHTSyncEngine(unittest.TestCase):
|
|
| 604 |
self.assertEqual(count, 3)
|
| 605 |
|
| 606 |
# Verify records exist in subscriber database
|
| 607 |
-
sub_records = self.subscriber.db.search("
|
| 608 |
self.assertEqual(len(sub_records), 3)
|
| 609 |
|
| 610 |
# Test tombstone purge on subscriber
|
|
@@ -619,7 +619,7 @@ class TestEndToEndDHTSyncEngine(unittest.TestCase):
|
|
| 619 |
self.assertEqual(purged, 1)
|
| 620 |
|
| 621 |
# Database should now have only 2 records
|
| 622 |
-
remaining = self.subscriber.db.search("
|
| 623 |
self.assertEqual(len(remaining), 2)
|
| 624 |
self.assertNotIn(records_to_shard[0].infohash, [r.infohash for r in remaining])
|
| 625 |
|
|
@@ -846,5 +846,505 @@ class TestEndToEndDHTSyncEngine(unittest.TestCase):
|
|
| 846 |
self.assertEqual(target, manifest_dummy.bep44_target())
|
| 847 |
|
| 848 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 849 |
if __name__ == "__main__":
|
| 850 |
unittest.main()
|
|
|
|
| 94 |
self.assertEqual(normalize_text(raw3), "teen porn")
|
| 95 |
|
| 96 |
def test_private_tracker_passkey_sanitization(self) -> None:
|
| 97 |
+
raw_title = "Big.Buck.Bunny.2008.1080p.BluRay[passkey=abc123secretkey]?passkey=xyz789"
|
| 98 |
sanitized = sanitize_title(raw_title)
|
| 99 |
self.assertNotIn("passkey=", sanitized)
|
| 100 |
self.assertNotIn("abc123secretkey", sanitized)
|
| 101 |
+
self.assertEqual(sanitized, "Big.Buck.Bunny.2008.1080p.BluRay")
|
| 102 |
|
| 103 |
def test_direct_csam_detection(self) -> None:
|
| 104 |
forbidden_samples = [
|
|
|
|
| 131 |
|
| 132 |
def test_disguised_malware_dropper_in_media_torrents(self) -> None:
|
| 133 |
# Dangerous executable disguised as media in a movie torrent
|
| 134 |
+
malicious_files = ["Big.Buck.Bunny.2008.1080p.mkv", "codec_update.mp4.exe", "instructions.txt"]
|
| 135 |
res = self.guard.inspect_torrent(
|
| 136 |
infohash="d" * 40,
|
| 137 |
+
title="Big.Buck.Bunny.2008.1080p.BluRay.x264",
|
| 138 |
file_list=malicious_files,
|
| 139 |
category="movie",
|
| 140 |
)
|
|
|
|
| 253 |
# Ingest mainstream record
|
| 254 |
rec_main = TorrentRecord(
|
| 255 |
infohash="1111222233334444555566667777888899990000",
|
| 256 |
+
title="Tears.of.Steel.2012.1080p",
|
| 257 |
category="movie",
|
| 258 |
language="en",
|
| 259 |
quality_tier="hd_1080p",
|
|
|
|
| 263 |
seeders=150,
|
| 264 |
leechers=10,
|
| 265 |
content_sha256="abcd" * 16,
|
| 266 |
+
file_list=["Tears.of.Steel.2012.1080p.mkv"],
|
| 267 |
safety_gate_passed=True,
|
| 268 |
created_at=100,
|
| 269 |
)
|
|
|
|
| 289 |
self.assertFalse(sync_engine.db.insert_record(rec_adult))
|
| 290 |
|
| 291 |
# Verify search only sees mainstream
|
| 292 |
+
results = sync_engine.db.search("Tears.of.Steel")
|
| 293 |
self.assertEqual(len(results), 1)
|
| 294 |
self.assertEqual(results[0].category, "movie")
|
| 295 |
|
|
|
|
| 567 |
for i in range(1, 4):
|
| 568 |
ok, msg, rec = self.publisher.ingest_candidate_record(
|
| 569 |
infohash=f"abcdef{i:034x}",
|
| 570 |
+
title=f"Wing It Open Series S04E{i:02d} 1080p",
|
| 571 |
category="anime",
|
| 572 |
language="ja",
|
| 573 |
quality_tier="hd_1080p",
|
|
|
|
| 604 |
self.assertEqual(count, 3)
|
| 605 |
|
| 606 |
# Verify records exist in subscriber database
|
| 607 |
+
sub_records = self.subscriber.db.search("Wing It Open Series")
|
| 608 |
self.assertEqual(len(sub_records), 3)
|
| 609 |
|
| 610 |
# Test tombstone purge on subscriber
|
|
|
|
| 619 |
self.assertEqual(purged, 1)
|
| 620 |
|
| 621 |
# Database should now have only 2 records
|
| 622 |
+
remaining = self.subscriber.db.search("Wing It Open Series")
|
| 623 |
self.assertEqual(len(remaining), 2)
|
| 624 |
self.assertNotIn(records_to_shard[0].infohash, [r.infohash for r in remaining])
|
| 625 |
|
|
|
|
| 846 |
self.assertEqual(target, manifest_dummy.bep44_target())
|
| 847 |
|
| 848 |
|
| 849 |
+
class TestPhase2CatalogRedTeamAndBenchmarks(unittest.TestCase):
|
| 850 |
+
"""Test Suite 6 (Phase 2): 10,000-Item Catalog Sharding, 120+ Adversarial Red-Team Traps & Benchmarks."""
|
| 851 |
+
|
| 852 |
+
def setUp(self) -> None:
|
| 853 |
+
self.root_priv, self.root_pub = generate_ed25519_keypair()
|
| 854 |
+
self.master_secret = os.urandom(32)
|
| 855 |
+
self.tombstone_store = TombstoneStore(trusted_root_pubkeys_hex=[self.root_pub.hex()])
|
| 856 |
+
self.guard = CSAMGuard(tombstone_store=self.tombstone_store)
|
| 857 |
+
self.safety_gate = ArmigerSafetyGate()
|
| 858 |
+
|
| 859 |
+
def test_10000_item_catalog_sharding_selective_sync_and_merkle_proofs(self) -> None:
|
| 860 |
+
"""Verify 10,000-item multi-category/multi-language sharding, selective sync, 0% xxx leakage, and 10k Merkle proofs."""
|
| 861 |
+
categories = list(ALL_CATEGORIES) # 10 categories (9 mainstream + 'xxx')
|
| 862 |
+
languages = ["en", "ja", "es", "fr", "de"] # 5 languages
|
| 863 |
+
qualities = ["uhd_4k", "hd_1080p"] # 2 quality tiers
|
| 864 |
+
epoch = "2026_q4"
|
| 865 |
+
|
| 866 |
+
publisher = ArmigerDHTDatabaseSync(
|
| 867 |
+
subscription=SubscriptionConfig(
|
| 868 |
+
categories=set(categories),
|
| 869 |
+
languages=set(languages),
|
| 870 |
+
quality_tiers=set(qualities),
|
| 871 |
+
epochs={epoch},
|
| 872 |
+
enable_xxx=True,
|
| 873 |
+
sync_mode=SyncMode.PUBLISHER,
|
| 874 |
+
),
|
| 875 |
+
master_secret=self.master_secret,
|
| 876 |
+
trusted_root_pubkeys_hex=[self.root_pub.hex()],
|
| 877 |
+
)
|
| 878 |
+
|
| 879 |
+
subscriber = ArmigerDHTDatabaseSync(
|
| 880 |
+
subscription=SubscriptionConfig(
|
| 881 |
+
categories={"movie", "tv_show", "anime"},
|
| 882 |
+
languages={"en", "ja"},
|
| 883 |
+
quality_tiers={"uhd_4k", "hd_1080p"},
|
| 884 |
+
epochs={epoch},
|
| 885 |
+
enable_xxx=False,
|
| 886 |
+
sync_mode=SyncMode.PULL_ONLY,
|
| 887 |
+
min_quorum=5,
|
| 888 |
+
min_distinct_asns=5,
|
| 889 |
+
),
|
| 890 |
+
master_secret=self.master_secret,
|
| 891 |
+
trusted_root_pubkeys_hex=[self.root_pub.hex()],
|
| 892 |
+
)
|
| 893 |
+
|
| 894 |
+
try:
|
| 895 |
+
# 1. Generate 10,000 catalog items partitioned into 100 shards (100 items per shard)
|
| 896 |
+
shards_map: dict[str, list[TorrentRecord]] = {}
|
| 897 |
+
total_generated = 0
|
| 898 |
+
|
| 899 |
+
for c_idx, cat in enumerate(categories):
|
| 900 |
+
for l_idx, lang in enumerate(languages):
|
| 901 |
+
for q_idx, qual in enumerate(qualities):
|
| 902 |
+
shard_id = SubscriptionConfig.format_shard_id(cat, lang, qual, epoch)
|
| 903 |
+
shard_records: list[TorrentRecord] = []
|
| 904 |
+
for item_idx in range(100):
|
| 905 |
+
global_id = (c_idx * 1000) + (l_idx * 200) + (q_idx * 100) + item_idx + 1
|
| 906 |
+
infohash = f"{global_id:040x}"
|
| 907 |
+
rec = TorrentRecord(
|
| 908 |
+
infohash=infohash,
|
| 909 |
+
title=f"Catalog_{cat}_{lang}_{qual}_Item_{global_id:05d}",
|
| 910 |
+
category=cat,
|
| 911 |
+
language=lang,
|
| 912 |
+
quality_tier=qual,
|
| 913 |
+
epoch=epoch,
|
| 914 |
+
size_bytes=1_000_000 * (item_idx + 1),
|
| 915 |
+
file_count=1,
|
| 916 |
+
seeders=50 + item_idx,
|
| 917 |
+
leechers=5,
|
| 918 |
+
content_sha256=f"{global_id:064x}",
|
| 919 |
+
file_list=[f"item_{global_id:05d}.mkv"],
|
| 920 |
+
safety_gate_passed=True,
|
| 921 |
+
created_at=1700000000 + global_id,
|
| 922 |
+
)
|
| 923 |
+
shard_records.append(rec)
|
| 924 |
+
total_generated += 1
|
| 925 |
+
shards_map[shard_id] = shard_records
|
| 926 |
+
|
| 927 |
+
self.assertEqual(total_generated, 10_000)
|
| 928 |
+
self.assertEqual(len(shards_map), 100)
|
| 929 |
+
|
| 930 |
+
# 2. Build & sign all 100 shards, verify 10,000 Merkle inclusion proofs, and feed to subscriber
|
| 931 |
+
distinct_asns = [15169, 13335, 16509, 7922, 3356]
|
| 932 |
+
reusable_node_keys = [generate_ed25519_keypair() for _ in distinct_asns]
|
| 933 |
+
|
| 934 |
+
merkle_proofs_verified = 0
|
| 935 |
+
subscribed_shards_applied = 0
|
| 936 |
+
unsubscribed_shards_skipped = 0
|
| 937 |
+
records_ingested = 0
|
| 938 |
+
|
| 939 |
+
for shard_id, recs in shards_map.items():
|
| 940 |
+
# Verify binary Merkle tree inclusion proof for every record in the shard
|
| 941 |
+
sorted_recs = sorted(recs, key=lambda r: r.infohash.lower())
|
| 942 |
+
leaves = [r.leaf_hash() for r in sorted_recs]
|
| 943 |
+
tree = MerkleTree(leaves)
|
| 944 |
+
for idx, r in enumerate(sorted_recs):
|
| 945 |
+
proof = tree.get_proof(idx)
|
| 946 |
+
self.assertTrue(MerkleTree.verify_proof(r.leaf_hash(), proof, tree.root_hex))
|
| 947 |
+
merkle_proofs_verified += 1
|
| 948 |
+
|
| 949 |
+
manifest, bundle = publisher.build_and_sign_shard(
|
| 950 |
+
shard_id=shard_id,
|
| 951 |
+
records=recs,
|
| 952 |
+
root_privkey=self.root_priv,
|
| 953 |
+
seq=1,
|
| 954 |
+
)
|
| 955 |
+
digest = manifest.canonical_digest()
|
| 956 |
+
attestations = []
|
| 957 |
+
for asn, (n_priv, n_pub) in zip(distinct_asns, reusable_node_keys):
|
| 958 |
+
sk = Ed25519PrivateKey.from_private_bytes(n_priv)
|
| 959 |
+
sig = sk.sign(digest)
|
| 960 |
+
attestations.append(
|
| 961 |
+
QuorumAttestation(
|
| 962 |
+
node_id=hashlib.sha1(n_pub + f"asn{asn}".encode()).hexdigest(),
|
| 963 |
+
asn=asn,
|
| 964 |
+
ip_digest=hashlib.sha256(f"198.51.100.{asn % 250}".encode()).hexdigest(),
|
| 965 |
+
pubkey_hex=n_pub.hex(),
|
| 966 |
+
signature_hex=sig.hex(),
|
| 967 |
+
timestamp=manifest.timestamp,
|
| 968 |
+
)
|
| 969 |
+
)
|
| 970 |
+
manifest_quorum = manifest.with_attestations(attestations)
|
| 971 |
+
|
| 972 |
+
ok, msg, count = subscriber.receive_and_apply_shard(manifest_quorum, bundle)
|
| 973 |
+
cat, lang, qual, _ = SubscriptionConfig.parse_shard_id(shard_id)
|
| 974 |
+
expected_match = (cat in {"movie", "tv_show", "anime"}) and (lang in {"en", "ja"})
|
| 975 |
+
if expected_match:
|
| 976 |
+
self.assertTrue(ok, f"Expected shard {shard_id} to be applied, got: {msg}")
|
| 977 |
+
self.assertEqual(count, 100)
|
| 978 |
+
subscribed_shards_applied += 1
|
| 979 |
+
records_ingested += count
|
| 980 |
+
else:
|
| 981 |
+
self.assertFalse(ok)
|
| 982 |
+
self.assertIn("unsubscribed_shard_filtered", msg)
|
| 983 |
+
self.assertEqual(count, 0)
|
| 984 |
+
unsubscribed_shards_skipped += 1
|
| 985 |
+
|
| 986 |
+
self.assertEqual(merkle_proofs_verified, 10_000)
|
| 987 |
+
self.assertEqual(subscribed_shards_applied, 12) # 3 categories * 2 languages * 2 qualities
|
| 988 |
+
self.assertEqual(unsubscribed_shards_skipped, 88)
|
| 989 |
+
self.assertEqual(records_ingested, 1_200)
|
| 990 |
+
|
| 991 |
+
# 3. Verify SQLite state & 0% 'xxx' leakage
|
| 992 |
+
self.assertEqual(subscriber.db.count_records(), 1_200)
|
| 993 |
+
self.assertEqual(subscriber.db.count_records(category="movie"), 400)
|
| 994 |
+
self.assertEqual(subscriber.db.count_records(category="tv_show"), 400)
|
| 995 |
+
self.assertEqual(subscriber.db.count_records(category="anime"), 400)
|
| 996 |
+
self.assertEqual(subscriber.db.count_records(category="xxx"), 0)
|
| 997 |
+
|
| 998 |
+
cur = subscriber.db.conn.cursor()
|
| 999 |
+
cur.execute("SELECT COUNT(*) FROM mainstream_torrents WHERE category = 'xxx'")
|
| 1000 |
+
self.assertEqual(cur.fetchone()[0], 0)
|
| 1001 |
+
cur.execute("SELECT DISTINCT category, language FROM mainstream_torrents")
|
| 1002 |
+
pairs = {(r[0], r[1]) for r in cur.fetchall()}
|
| 1003 |
+
self.assertEqual(
|
| 1004 |
+
pairs,
|
| 1005 |
+
{
|
| 1006 |
+
("movie", "en"), ("movie", "ja"),
|
| 1007 |
+
("tv_show", "en"), ("tv_show", "ja"),
|
| 1008 |
+
("anime", "en"), ("anime", "ja"),
|
| 1009 |
+
},
|
| 1010 |
+
)
|
| 1011 |
+
finally:
|
| 1012 |
+
publisher.close()
|
| 1013 |
+
subscriber.close()
|
| 1014 |
+
|
| 1015 |
+
def test_adversarial_csam_nonconsensual_malware_redteam_100_plus_traps(self) -> None:
|
| 1016 |
+
"""Red-team 120+ obfuscated CSAM, non-consensual, extreme violence, malware, conformal, and tombstone traps."""
|
| 1017 |
+
total_traps_verified = 0
|
| 1018 |
+
|
| 1019 |
+
# Surface A: 25 Unicode NFKD, Fullwidth, Mathematical & Cyrillic/Greek Homoglyph Traps
|
| 1020 |
+
homoglyph_traps = [
|
| 1021 |
+
"\uff23\uff33\uff21\uff2d Archive 2026", # Fullwidth CSAM
|
| 1022 |
+
"\uff23\uff33\uff21\uff25 Collection", # Fullwidth CSAE
|
| 1023 |
+
"\uff30\uff45\uff44\uff4f\uff50\uff48\uff49\uff4c\uff45", # Fullwidth Pedophile
|
| 1024 |
+
"\U0001d429\U0001d41e\U0001d41d\U0001d428\U0001d429\U0001d421\U0001d422\U0001d425\U0001d41e", # Math Bold
|
| 1025 |
+
"\U0001d450\U0001d455\U0001d456\U0001d459\U0001d451 \U0001d45d\U0001d45c\U0001d45f\U0001d45b", # Math Italic child porn
|
| 1026 |
+
"\U0001d52d\U0001d52f\U0001d522\U0001d531\U0001d522\U0001d522\U0001d52b \U0001d530\U0001d522\U0001d535", # Math Fraktur preteen sex
|
| 1027 |
+
"p\u00e9d\u00f6ph\u00efl\u00eb hidden pack", # Diacritic storm
|
| 1028 |
+
"pr\u00ebt\u00e9\u00e8n s\u00eax leak", # Diacritics preteen sex
|
| 1029 |
+
"l\u00f6l\u00eft\u00e4 h\u00e4rdc\u00f6r\u00eb", # Diacritics lolita hardcore
|
| 1030 |
+
"\u0440\u0435d\u043e\u0440hil\u0435 underground", # Cyrillic р, е, о
|
| 1031 |
+
"\u0441hild \u0440\u043ern video", # Cyrillic с, р, о
|
| 1032 |
+
"l\u043elit\u0430 h\u0430rd\u0441\u043er\u0435", # Cyrillic о, а, с, е
|
| 1033 |
+
"\u0455h\u043et\u0430\u0441\u043en collection", # Cyrillic ѕ, о, а, с
|
| 1034 |
+
"t\u043eddler \u0440\u043ern archive", # Cyrillic о, р
|
| 1035 |
+
"kinderg\u0430rten \u0455\u0435\u0445", # Cyrillic а, ѕ, е, х
|
| 1036 |
+
"inf\u0430nt \u0435r\u043eti\u0441", # Cyrillic а, е, о, с
|
| 1037 |
+
"und\u0435r\u0430ge \u0440\u043ern", # Cyrillic е, а, р, о
|
| 1038 |
+
"j\u0430ilb\u0430it pack", # Cyrillic а
|
| 1039 |
+
"in\u0441\u0435\u0455t \u0441hild", # Cyrillic с, е, ѕ
|
| 1040 |
+
"r\u0430\u0440\u0435 vid\u0435\u043e", # Cyrillic а, р, е, о
|
| 1041 |
+
"r\u0435v\u0435ng\u0435 \u0440\u043ern", # Cyrillic е, р, о
|
| 1042 |
+
"\u0455\u0440\u0443\u0441\u0430m u\u0440\u0455kirt", # Cyrillic ѕ, р, у, с, а
|
| 1043 |
+
"\u0455nuff film uncensored", # Cyrillic ѕ
|
| 1044 |
+
"r\u0435\u0430l b\u0435h\u0435\u0430ding", # Cyrillic е, а
|
| 1045 |
+
"\u03c1\u03b5d\u03bf\u03c1hil\u03b5 archive", # Greek ρ, ε, ο
|
| 1046 |
+
]
|
| 1047 |
+
self.assertEqual(len(homoglyph_traps), 25)
|
| 1048 |
+
for sample in homoglyph_traps:
|
| 1049 |
+
res = self.guard.inspect_torrent("a" * 40, sample, category="movie")
|
| 1050 |
+
self.assertFalse(res.is_safe, f"Homoglyph trap bypassed guard: {sample!r}")
|
| 1051 |
+
total_traps_verified += 1
|
| 1052 |
+
|
| 1053 |
+
# Surface B: 25 Zero-Width Joiner, Soft Hyphen & Bidirectional Override Traps
|
| 1054 |
+
zw_bidi_traps = [
|
| 1055 |
+
"c\u200bs\u200ca\u200dm leak",
|
| 1056 |
+
"c\u2060s\u2066a\u2069e dump",
|
| 1057 |
+
"p\u200be\u200cd\u200do\u200ep\u200fh\u202ai\u202el\u202ce",
|
| 1058 |
+
"ch\u200bild p\u200dorn",
|
| 1059 |
+
"cp\u200b v\u200cids",
|
| 1060 |
+
"cp\u2060 l\u2060inks",
|
| 1061 |
+
"cp\ufeff m\ufeffegas",
|
| 1062 |
+
"ch\u00adild er\u00adotic",
|
| 1063 |
+
"under\u200bage p\u200dorn",
|
| 1064 |
+
"pre\u200bteen s\u200dex",
|
| 1065 |
+
"pre\u202eteen n\u202cude",
|
| 1066 |
+
"loli\u200bta hard\u200ccore",
|
| 1067 |
+
"loli\u200dta hen\u200btai",
|
| 1068 |
+
"sho\u200btacon archive",
|
| 1069 |
+
"todd\u200bler p\u200dorn",
|
| 1070 |
+
"kinder\u200bgarten s\u200cex",
|
| 1071 |
+
"in\u200bfant n\u200dude",
|
| 1072 |
+
"elemen\u200btary er\u200cotic",
|
| 1073 |
+
"jail\u200bbait collection",
|
| 1074 |
+
"in\u200bcest mi\u200dnor",
|
| 1075 |
+
"non\u200bconsensual leak",
|
| 1076 |
+
"rape\u200b v\u200dideo",
|
| 1077 |
+
"revenge\u200b p\u200corn",
|
| 1078 |
+
"snuff\u200b f\u200dilm",
|
| 1079 |
+
"real\u200b ex\u202eecution",
|
| 1080 |
+
]
|
| 1081 |
+
self.assertEqual(len(zw_bidi_traps), 25)
|
| 1082 |
+
for sample in zw_bidi_traps:
|
| 1083 |
+
res = self.guard.inspect_torrent("b" * 40, sample, category="movie")
|
| 1084 |
+
self.assertFalse(res.is_safe, f"Zero-width/Bidi trap bypassed guard: {sample!r}")
|
| 1085 |
+
total_traps_verified += 1
|
| 1086 |
+
|
| 1087 |
+
# Surface C: 25 Heavy Leetspeak, Character Repetition & Delimiter Spacing Traps
|
| 1088 |
+
leet_traps = [
|
| 1089 |
+
"P3d0ph1l14_v1ds",
|
| 1090 |
+
"c_h_1_l_d__p_0_r_n",
|
| 1091 |
+
"pr3333t333n s333x",
|
| 1092 |
+
"l0l1t@ h@rdc0r3",
|
| 1093 |
+
"j@1lb@1t_2026",
|
| 1094 |
+
"r3v3ng3 p0rn",
|
| 1095 |
+
"5nuff f1lm",
|
| 1096 |
+
"b1@ckm@1l s3x",
|
| 1097 |
+
"5pyc@m uP5k1rt",
|
| 1098 |
+
"g@ng r@p3",
|
| 1099 |
+
"f0rc3d s3x",
|
| 1100 |
+
"c.s.a.m.pack",
|
| 1101 |
+
"c-s-a-e-collection",
|
| 1102 |
+
"k1nd3rg@rt3n p0rn",
|
| 1103 |
+
"t0ddl3r s3x",
|
| 1104 |
+
"1nf@nt p0rn",
|
| 1105 |
+
"3l3m3nt@ry 3r0t1c",
|
| 1106 |
+
"53xt0rt10n_leaks",
|
| 1107 |
+
"r3@l b3h3@d1ng",
|
| 1108 |
+
"r3@l 3x3cut10n",
|
| 1109 |
+
"l1v3 5u1c1d3 v1d30",
|
| 1110 |
+
"m@55@cr3 l1v35tr3@m",
|
| 1111 |
+
"r3@l t0rtur3 f1lm",
|
| 1112 |
+
"cru5h f3t15h k1ll",
|
| 1113 |
+
"1nc35t ch1ld",
|
| 1114 |
+
]
|
| 1115 |
+
self.assertEqual(len(leet_traps), 25)
|
| 1116 |
+
for sample in leet_traps:
|
| 1117 |
+
res = self.guard.inspect_torrent("c" * 40, sample, category="movie")
|
| 1118 |
+
self.assertFalse(res.is_safe, f"Leetspeak trap bypassed guard: {sample!r}")
|
| 1119 |
+
total_traps_verified += 1
|
| 1120 |
+
|
| 1121 |
+
# Surface D: 25 Nested Archive & Double-Extension Disguised Executable Malware Traps
|
| 1122 |
+
malware_file_traps = [
|
| 1123 |
+
("movie", ["Movie.2026.1080p.mp4.exe"]),
|
| 1124 |
+
("tv_show", ["Series.S01E01.mkv.scr"]),
|
| 1125 |
+
("anime", ["Episode_01.avi.bat"]),
|
| 1126 |
+
("book", ["Complete_Guide.pdf.vbs"]),
|
| 1127 |
+
("music", ["01_Lossless_Track.flac.ps1"]),
|
| 1128 |
+
("music", ["Album_Track_02.mp3.cmd"]),
|
| 1129 |
+
("book", ["Novel_Edition.epub.hta"]),
|
| 1130 |
+
("comic", ["Issue_001.cbz.jar"]),
|
| 1131 |
+
("movie", ["Trailer_4K.mov.msi"]),
|
| 1132 |
+
("comic", ["Cover_Art.jpg.pif"]),
|
| 1133 |
+
("tv_show", ["Subtitles_EN.srt.wsf"]),
|
| 1134 |
+
("music", ["Studio_Master.wav.vbe"]),
|
| 1135 |
+
("music", ["Podcast_Ep1.m4a.jse"]),
|
| 1136 |
+
("movie", ["WebRip_Stream.webm.com"]),
|
| 1137 |
+
("tv_show", ["Remux_2160p.mkv.cpl"]),
|
| 1138 |
+
("anime", ["OVA_Special.avi.wsh"]),
|
| 1139 |
+
("movie", ["Feature_Film.mp4.psm1"]),
|
| 1140 |
+
("book", ["Reference_Manual.pdf.application"]),
|
| 1141 |
+
("music", ["Bonus_Track.flac.gadget"]),
|
| 1142 |
+
("tv_show", ["Finale_1080p.mkv.msp"]),
|
| 1143 |
+
("Archive.zip/Nested/Video.mp4.exe", ["Archive.zip/Nested/Video.mp4.exe"]),
|
| 1144 |
+
("Season1.rar/S01E02.mkv.scr", ["Season1.rar/S01E02.mkv.scr"]),
|
| 1145 |
+
("Pack.7z/Clip.avi.bat", ["Pack.7z/Clip.avi.bat"]),
|
| 1146 |
+
("Library.tar.gz/Book.pdf.vbs", ["Library.tar.gz/Book.pdf.vbs"]),
|
| 1147 |
+
("Disc.iso/Audio/01.flac.ps1", ["Disc.iso/Audio/01.flac.ps1"]),
|
| 1148 |
+
]
|
| 1149 |
+
self.assertEqual(len(malware_file_traps), 25)
|
| 1150 |
+
for cat_or_path, flist in malware_file_traps:
|
| 1151 |
+
cat = cat_or_path if cat_or_path in ALL_CATEGORIES else None
|
| 1152 |
+
res = self.guard.inspect_torrent(
|
| 1153 |
+
infohash="d" * 40,
|
| 1154 |
+
title="Clean_Release_Title_2026",
|
| 1155 |
+
file_list=flist,
|
| 1156 |
+
category=cat,
|
| 1157 |
+
)
|
| 1158 |
+
self.assertFalse(res.is_safe, f"Malware dropper bypassed guard: {flist}")
|
| 1159 |
+
self.assertEqual(res.violation_type, CSAMViolationType.MALWARE_PAYLOAD)
|
| 1160 |
+
total_traps_verified += 1
|
| 1161 |
+
|
| 1162 |
+
# Surface E: 20 Mondrian Conformal Safety Gate Boundary Traps
|
| 1163 |
+
# E1: Safe just below thresholds
|
| 1164 |
+
ok_safe, _, _ = self.safety_gate.evaluate(p_adult=0.5450, p_junk=0.1009, p_excluded_scope=0.4769, target_category="movie")
|
| 1165 |
+
self.assertTrue(ok_safe)
|
| 1166 |
+
|
| 1167 |
+
conformal_traps = [
|
| 1168 |
+
# 5 junk boundary & high-risk cases (threshold = 0.1010)
|
| 1169 |
+
(0.01, 0.1010, 0.01, "movie", CSAMViolationType.SAFETY_GATE_JUNK),
|
| 1170 |
+
(0.01, 0.1011, 0.01, "tv_show", CSAMViolationType.SAFETY_GATE_JUNK),
|
| 1171 |
+
(0.01, 0.1500, 0.01, "anime", CSAMViolationType.SAFETY_GATE_JUNK),
|
| 1172 |
+
(0.01, 0.5000, 0.01, "software", CSAMViolationType.SAFETY_GATE_JUNK),
|
| 1173 |
+
(0.01, 0.9900, 0.01, "game", CSAMViolationType.SAFETY_GATE_JUNK),
|
| 1174 |
+
# 6 scope boundary & high-risk cases (threshold = 0.4770)
|
| 1175 |
+
(0.01, 0.01, 0.4770, "movie", CSAMViolationType.SAFETY_GATE_OUT_OF_SCOPE),
|
| 1176 |
+
(0.01, 0.01, 0.4771, "tv_show", CSAMViolationType.SAFETY_GATE_OUT_OF_SCOPE),
|
| 1177 |
+
(0.01, 0.01, 0.5500, "music", CSAMViolationType.SAFETY_GATE_OUT_OF_SCOPE),
|
| 1178 |
+
(0.01, 0.01, 0.7000, "book", CSAMViolationType.SAFETY_GATE_OUT_OF_SCOPE),
|
| 1179 |
+
(0.01, 0.01, 0.8500, "comic", CSAMViolationType.SAFETY_GATE_OUT_OF_SCOPE),
|
| 1180 |
+
(0.01, 0.01, 0.9900, "course", CSAMViolationType.SAFETY_GATE_OUT_OF_SCOPE),
|
| 1181 |
+
# 9 unisolated adult boundary cases across all 9 mainstream categories (threshold = 0.5451)
|
| 1182 |
+
(0.5451, 0.01, 0.01, "movie", CSAMViolationType.UNISOLATED_ADULT),
|
| 1183 |
+
(0.5451, 0.01, 0.01, "tv_show", CSAMViolationType.UNISOLATED_ADULT),
|
| 1184 |
+
(0.5451, 0.01, 0.01, "music", CSAMViolationType.UNISOLATED_ADULT),
|
| 1185 |
+
(0.5451, 0.01, 0.01, "game", CSAMViolationType.UNISOLATED_ADULT),
|
| 1186 |
+
(0.5451, 0.01, 0.01, "software", CSAMViolationType.UNISOLATED_ADULT),
|
| 1187 |
+
(0.5451, 0.01, 0.01, "anime", CSAMViolationType.UNISOLATED_ADULT),
|
| 1188 |
+
(0.5451, 0.01, 0.01, "book", CSAMViolationType.UNISOLATED_ADULT),
|
| 1189 |
+
(0.5451, 0.01, 0.01, "comic", CSAMViolationType.UNISOLATED_ADULT),
|
| 1190 |
+
(0.5451, 0.01, 0.01, "course", CSAMViolationType.UNISOLATED_ADULT),
|
| 1191 |
+
]
|
| 1192 |
+
self.assertEqual(len(conformal_traps), 20)
|
| 1193 |
+
for p_a, p_j, p_s, cat, expected_vtype in conformal_traps:
|
| 1194 |
+
ok, reason, vtype = self.safety_gate.evaluate(p_a, p_j, p_s, cat)
|
| 1195 |
+
self.assertFalse(ok, f"Conformal trap bypassed gate: {(p_a, p_j, p_s, cat)}")
|
| 1196 |
+
self.assertEqual(vtype, expected_vtype)
|
| 1197 |
+
total_traps_verified += 1
|
| 1198 |
+
|
| 1199 |
+
# Verify total adversarial traps >= 120
|
| 1200 |
+
self.assertGreaterEqual(total_traps_verified, 120)
|
| 1201 |
+
|
| 1202 |
+
# Surface F: Live Ed25519-Signed Blinded SHA-256 Tombstone Purge Propagation & Permanent Block
|
| 1203 |
+
engine = ArmigerDHTDatabaseSync(
|
| 1204 |
+
subscription=SubscriptionConfig(
|
| 1205 |
+
categories={"movie"},
|
| 1206 |
+
languages={"en"},
|
| 1207 |
+
quality_tiers={"hd_1080p"},
|
| 1208 |
+
epochs={"2026_q4"},
|
| 1209 |
+
enable_xxx=False,
|
| 1210 |
+
sync_mode=SyncMode.PUBLISHER,
|
| 1211 |
+
),
|
| 1212 |
+
master_secret=self.master_secret,
|
| 1213 |
+
trusted_root_pubkeys_hex=[self.root_pub.hex()],
|
| 1214 |
+
)
|
| 1215 |
+
try:
|
| 1216 |
+
all_hashes = [f"{i:040x}" for i in range(500, 550)]
|
| 1217 |
+
for idx, h in enumerate(all_hashes):
|
| 1218 |
+
ok, _, _ = engine.ingest_candidate_record(
|
| 1219 |
+
infohash=h,
|
| 1220 |
+
title=f"Verified_Mainstream_Film_{idx}",
|
| 1221 |
+
category="movie",
|
| 1222 |
+
language="en",
|
| 1223 |
+
quality_tier="hd_1080p",
|
| 1224 |
+
epoch="2026_q4",
|
| 1225 |
+
)
|
| 1226 |
+
self.assertTrue(ok)
|
| 1227 |
+
self.assertEqual(engine.db.count_records(), 50)
|
| 1228 |
+
|
| 1229 |
+
revoked_hashes = all_hashes[:15]
|
| 1230 |
+
tomb_manifest = sign_tombstone_list(
|
| 1231 |
+
seq=10,
|
| 1232 |
+
infohashes_or_blinded=revoked_hashes,
|
| 1233 |
+
private_key_bytes=self.root_priv,
|
| 1234 |
+
)
|
| 1235 |
+
ok_t, msg_t, new_t, purged_t = engine.apply_signed_tombstones(tomb_manifest)
|
| 1236 |
+
self.assertTrue(ok_t)
|
| 1237 |
+
self.assertEqual(new_t, 15)
|
| 1238 |
+
self.assertEqual(purged_t, 15)
|
| 1239 |
+
self.assertEqual(engine.db.count_records(), 35)
|
| 1240 |
+
|
| 1241 |
+
# Confirm permanent re-ingest block on all 15 tombstoned infohashes
|
| 1242 |
+
for rh in revoked_hashes:
|
| 1243 |
+
ok_re, msg_re, _ = engine.ingest_candidate_record(
|
| 1244 |
+
infohash=rh,
|
| 1245 |
+
title="Innocent_Renamed_Film",
|
| 1246 |
+
category="movie",
|
| 1247 |
+
language="en",
|
| 1248 |
+
quality_tier="hd_1080p",
|
| 1249 |
+
epoch="2026_q4",
|
| 1250 |
+
)
|
| 1251 |
+
self.assertFalse(ok_re)
|
| 1252 |
+
self.assertIn("tombstoned_hash", msg_re)
|
| 1253 |
+
finally:
|
| 1254 |
+
engine.close()
|
| 1255 |
+
|
| 1256 |
+
def test_throughput_and_latency_benchmarks(self) -> None:
|
| 1257 |
+
"""Measure CSAM Guard items/sec, AEAD encryption/decryption MB/s, and SQLite batch ingest speed."""
|
| 1258 |
+
import time
|
| 1259 |
+
|
| 1260 |
+
# 1. CSAM Guard inspection throughput (5,000 items)
|
| 1261 |
+
n_inspect = 5_000
|
| 1262 |
+
sample_titles = [
|
| 1263 |
+
("Clean.Movie.2026.2160p.WEB-DL.DDP5.1.Atmos.H.265", ["Movie.2026.2160p.mkv", "subs.srt"], True),
|
| 1264 |
+
("Ch1ld_p0rn_hidden_archive_2026", ["archive.zip"], False),
|
| 1265 |
+
("Disguised.Release.1080p", ["Release.1080p.mp4.exe"], False),
|
| 1266 |
+
("Anime.Series.S02E12.1080p.HEVC", ["Anime.S02E12.mkv"], True),
|
| 1267 |
+
("Secret_Bathroom_Spycam_Upskirt", ["clip.mp4"], False),
|
| 1268 |
+
]
|
| 1269 |
+
t0 = time.perf_counter()
|
| 1270 |
+
for i in range(n_inspect):
|
| 1271 |
+
title, flist, expected_safe = sample_titles[i % len(sample_titles)]
|
| 1272 |
+
res = self.guard.inspect_torrent(f"{i + 10000:040x}", title, file_list=flist, category="movie")
|
| 1273 |
+
self.assertEqual(res.is_safe, expected_safe)
|
| 1274 |
+
t_inspect = time.perf_counter() - t0
|
| 1275 |
+
inspect_rate = n_inspect / max(t_inspect, 1e-9)
|
| 1276 |
+
|
| 1277 |
+
# 2. AEAD Encryption & Decryption Throughput (4 MB payload x 5 iterations = 20 MB)
|
| 1278 |
+
shard_id = "movie/en/uhd_4k/2026_q4"
|
| 1279 |
+
epoch = "2026_q4"
|
| 1280 |
+
merkle_root = "a" * 64
|
| 1281 |
+
key = ShardCryptoEngine.derive_shard_key(self.master_secret, shard_id, epoch)
|
| 1282 |
+
payload_4mb = os.urandom(4 * 1024 * 1024)
|
| 1283 |
+
iters = 5
|
| 1284 |
+
total_mb = (len(payload_4mb) * iters) / (1024 * 1024)
|
| 1285 |
+
|
| 1286 |
+
t_enc0 = time.perf_counter()
|
| 1287 |
+
bundles = [
|
| 1288 |
+
ShardCryptoEngine.encrypt_shard(payload_4mb, key, shard_id, epoch, seq=i + 1, merkle_root=merkle_root)
|
| 1289 |
+
for i in range(iters)
|
| 1290 |
+
]
|
| 1291 |
+
t_enc = time.perf_counter() - t_enc0
|
| 1292 |
+
enc_mbps = total_mb / max(t_enc, 1e-9)
|
| 1293 |
+
|
| 1294 |
+
t_dec0 = time.perf_counter()
|
| 1295 |
+
for i, b in enumerate(bundles):
|
| 1296 |
+
pt = ShardCryptoEngine.decrypt_shard(b, key, shard_id, epoch, seq=i + 1, merkle_root=merkle_root)
|
| 1297 |
+
self.assertEqual(len(pt), len(payload_4mb))
|
| 1298 |
+
t_dec = time.perf_counter() - t_dec0
|
| 1299 |
+
dec_mbps = total_mb / max(t_dec, 1e-9)
|
| 1300 |
+
|
| 1301 |
+
# 3. SQLite Batch Ingest Throughput (10,000 records)
|
| 1302 |
+
n_sqlite = 10_000
|
| 1303 |
+
batch_records = [
|
| 1304 |
+
TorrentRecord(
|
| 1305 |
+
infohash=f"{i + 100000:040x}",
|
| 1306 |
+
title=f"Benchmark_Record_{i:05d}",
|
| 1307 |
+
category="movie",
|
| 1308 |
+
language="en",
|
| 1309 |
+
quality_tier="hd_1080p",
|
| 1310 |
+
epoch="2026_q4",
|
| 1311 |
+
size_bytes=2_000_000_000,
|
| 1312 |
+
file_count=2,
|
| 1313 |
+
seeders=100,
|
| 1314 |
+
leechers=10,
|
| 1315 |
+
content_sha256=f"{i + 100000:064x}",
|
| 1316 |
+
file_list=[f"movie_{i:05d}.mkv", f"movie_{i:05d}.srt"],
|
| 1317 |
+
safety_gate_passed=True,
|
| 1318 |
+
created_at=1700000000 + i,
|
| 1319 |
+
)
|
| 1320 |
+
for i in range(n_sqlite)
|
| 1321 |
+
]
|
| 1322 |
+
engine = ArmigerDHTDatabaseSync(
|
| 1323 |
+
subscription=SubscriptionConfig(enable_xxx=False),
|
| 1324 |
+
master_secret=self.master_secret,
|
| 1325 |
+
trusted_root_pubkeys_hex=[self.root_pub.hex()],
|
| 1326 |
+
)
|
| 1327 |
+
try:
|
| 1328 |
+
t_db0 = time.perf_counter()
|
| 1329 |
+
inserted = engine.db.insert_records_batch(batch_records)
|
| 1330 |
+
t_db = time.perf_counter() - t_db0
|
| 1331 |
+
sqlite_rate = inserted / max(t_db, 1e-9)
|
| 1332 |
+
self.assertEqual(inserted, n_sqlite)
|
| 1333 |
+
self.assertEqual(engine.db.count_records(), n_sqlite)
|
| 1334 |
+
finally:
|
| 1335 |
+
engine.close()
|
| 1336 |
+
|
| 1337 |
+
# Assert conservative floors to catch performance regressions across hosts
|
| 1338 |
+
self.assertGreater(inspect_rate, 5_000.0)
|
| 1339 |
+
self.assertGreater(enc_mbps, 50.0)
|
| 1340 |
+
self.assertGreater(dec_mbps, 50.0)
|
| 1341 |
+
self.assertGreater(sqlite_rate, 20_000.0)
|
| 1342 |
+
print(
|
| 1343 |
+
f"\n[BENCHMARK] CSAMGuard={inspect_rate:,.0f} items/s ({1e6/inspect_rate:.1f} us/item) | "
|
| 1344 |
+
f"AEAD_Enc={enc_mbps:,.1f} MB/s | AEAD_Dec={dec_mbps:,.1f} MB/s | "
|
| 1345 |
+
f"SQLite_Batch={sqlite_rate:,.0f} records/s"
|
| 1346 |
+
)
|
| 1347 |
+
|
| 1348 |
+
|
| 1349 |
if __name__ == "__main__":
|
| 1350 |
unittest.main()
|