Spaces:
Sleeping
Sleeping
Download tests/test_corpus_loop.py from Quantum927/tinymetatron-slm: direct link, hf CLI and curl.
- Browser
- Download file 8.66 kB
-
https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tests/test_corpus_loop.py
- Command line
-
hf download hf://spaces/Quantum927/tinymetatron-slm/tests/test_corpus_loop.py
-
curl -L -o test_corpus_loop.py https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tests/test_corpus_loop.py
8.66 kB
| """ | |
| tests/test_corpus_loop.py | |
| ======================== | |
| Automated smoke test for the corpus loop (Phase 3 completion). | |
| Verifies the invariant the manual smoke run got wrong: a *source-disjoint* | |
| split, where every source_id appears in exactly one of {train, val, hard_dev} | |
| β not merely text-disjoint within a source. | |
| Covers: | |
| * Full pipeline on the smoke fixture produces text- AND source-disjoint splits. | |
| * MANIFEST.json records split_policy / split_seed / source_counts / | |
| source_overlap / text_overlap, with every overlap count zero. | |
| * At least 3 independent source groups; experiment reaches FROZEN_CORPUS. | |
| * Deterministic: two clean runs produce the same corpus_hash. | |
| * Split rejects a corpus with too few source groups (< 3). | |
| * Corpus gate worker flags a nonzero source overlap and passes a clean one. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import shutil | |
| import sys | |
| import tempfile | |
| from pathlib import Path | |
| _REPO_ROOT = Path(__file__).resolve().parents[1] | |
| # Point the registry at a throwaway temp DB BEFORE importing the corpus loop so | |
| # the module-level init_db() in corpus_loop never touches state/registry.db. | |
| _TMP_REGISTRY = Path(tempfile.mkdtemp(prefix="corpus_test_registry_")) / "registry.db" | |
| os.environ["TINYMETATRON_DB"] = str(_TMP_REGISTRY) | |
| if str(_REPO_ROOT) not in sys.path: | |
| sys.path.insert(0, str(_REPO_ROOT)) | |
| import db # noqa: E402 | |
| db.set_db_path(str(_TMP_REGISTRY)) | |
| db.init_db(str(_TMP_REGISTRY)) | |
| from loops import corpus_loop # noqa: E402 | |
| import pytest # noqa: E402 | |
| _FIXTURE = _REPO_ROOT / "tests" / "fixtures" / "corpus_smoke.jsonl" | |
| # ββ fixtures ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def clean_registry(tmp_path) -> str: | |
| """A fresh temp registry DB per test (clean test registry).""" | |
| p = str(tmp_path / "registry.db") | |
| db.set_db_path(p) | |
| db.init_db(p) | |
| return p | |
| def corpus_dir(tmp_path) -> Path: | |
| """Temp raw-corpus dir containing a copy of the smoke fixture.""" | |
| d = tmp_path / "raw" | |
| d.mkdir() | |
| shutil.copy(_FIXTURE, d / "corpus_smoke.jsonl") | |
| return d | |
| def _load_jsonl(path: Path) -> list[dict]: | |
| return [json.loads(l) for l in open(path, encoding="utf-8") if l.strip()] | |
| def _texts(rows: list[dict]) -> set: | |
| return {r["text"] for r in rows} | |
| def _sources(rows: list[dict]) -> set: | |
| return {r.get("source", r.get("domain", "unknown")) for r in rows} | |
| # ββ source-disjoint smoke βββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def test_corpus_smoke_source_disjoint(clean_registry, corpus_dir, tmp_path) -> None: | |
| out = tmp_path / "out" | |
| summary = corpus_loop.run_corpus_pipeline({ | |
| "exp_id": "exp-smoke-001", | |
| "corpus_dir": str(corpus_dir), | |
| "output_dir": str(out), | |
| "seed": 42, | |
| "train_pct": 0.80, | |
| "val_pct": 0.10, | |
| }) | |
| corpus = out / "corpus" | |
| train = _load_jsonl(corpus / "train.jsonl") | |
| val = _load_jsonl(corpus / "val.jsonl") | |
| hard = _load_jsonl(corpus / "hard_dev.jsonl") | |
| tr_t, va_t, ha_t = _texts(train), _texts(val), _texts(hard) | |
| tr_s, va_s, ha_s = _sources(train), _sources(val), _sources(hard) | |
| # Every partition non-empty. | |
| assert train and val and hard, "all three splits must be non-empty" | |
| # Text-disjoint. | |
| assert tr_t.isdisjoint(va_t) | |
| assert tr_t.isdisjoint(ha_t) | |
| assert va_t.isdisjoint(ha_t) | |
| # Source-disjoint β the invariant the manual smoke run got wrong. | |
| assert tr_s.isdisjoint(va_s) | |
| assert tr_s.isdisjoint(ha_s) | |
| assert va_s.isdisjoint(ha_s) | |
| # At least 3 independent source groups overall. | |
| assert len(tr_s | va_s | ha_s) >= 3 | |
| # MANIFEST.json records the split policy invariants. | |
| man = json.loads((out / "MANIFEST.json").read_text(encoding="utf-8")) | |
| assert man["split_policy"] == "source_disjoint_capped_v1" | |
| assert man["split_seed"] == 42 | |
| assert man["source_counts"] == { | |
| "train": len(tr_s), "val": len(va_s), "hard_dev": len(ha_s), | |
| } | |
| for key in ("train_val", "train_hard_dev", "val_hard_dev"): | |
| assert man["source_overlap"][key] == 0, f"source_overlap[{key}] != 0" | |
| assert man["text_overlap"][key] == 0, f"text_overlap[{key}] != 0" | |
| assert sum(man["source_overlap"].values()) == 0 | |
| assert sum(man["text_overlap"].values()) == 0 | |
| # Gate passed β experiment frozen. | |
| assert summary["corpus_hash"] | |
| exp = db.get_loop_experiment("exp-smoke-001") | |
| assert exp["state"] == "FROZEN_CORPUS" | |
| # ββ determinism βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def test_corpus_smoke_deterministic(clean_registry, corpus_dir, tmp_path) -> None: | |
| out1 = tmp_path / "out1" | |
| s1 = corpus_loop.run_corpus_pipeline({ | |
| "exp_id": "exp-smoke-det-a", | |
| "corpus_dir": str(corpus_dir), | |
| "output_dir": str(out1), | |
| "seed": 42, | |
| }) | |
| out2 = tmp_path / "out2" | |
| s2 = corpus_loop.run_corpus_pipeline({ | |
| "exp_id": "exp-smoke-det-b", | |
| "corpus_dir": str(corpus_dir), | |
| "output_dir": str(out2), | |
| "seed": 42, | |
| }) | |
| assert s1["corpus_hash"] == s2["corpus_hash"], ( | |
| "deterministic split must produce identical corpus_hash across runs") | |
| # ββ too-few source groups rejected ββββββββββββββββββββββββββββββββββββββββββββ | |
| def test_split_rejects_too_few_sources(clean_registry, tmp_path) -> None: | |
| raw = tmp_path / "raw" | |
| raw.mkdir() | |
| rows = [ | |
| {"text": "Alpha source sentence one about network security fundamentals.", | |
| "source": "alpha"}, | |
| {"text": "Alpha source sentence two about cryptographic hash functions.", | |
| "source": "alpha"}, | |
| {"text": "Beta source sentence one about machine learning neural nets.", | |
| "source": "beta"}, | |
| {"text": "Beta source sentence two about gradient descent optimization.", | |
| "source": "beta"}, | |
| ] | |
| with open(raw / "in.jsonl", "w", encoding="utf-8") as f: | |
| for r in rows: | |
| f.write(json.dumps(r) + "\n") | |
| out = tmp_path / "out" | |
| with pytest.raises(RuntimeError, match="Worker error"): | |
| corpus_loop.run_corpus_pipeline({ | |
| "exp_id": "exp-smoke-few", | |
| "corpus_dir": str(raw), | |
| "output_dir": str(out), | |
| }) | |
| # Must not have been frozen. | |
| assert db.get_loop_experiment("exp-smoke-few")["state"] != "FROZEN_CORPUS" | |
| # ββ corpus gate worker: detects overlap, passes clean βββββββββββββββββββββββββ | |
| def test_corpus_gate_detects_overlap(tmp_path) -> None: | |
| man = tmp_path / "MANIFEST.json" | |
| payload = { | |
| "split_policy": "source_disjoint_v1", | |
| "split_seed": 42, | |
| "source_counts": {"train": 2, "val": 2, "hard_dev": 1}, | |
| "source_overlap": {"train_val": 1, "train_hard_dev": 0, "val_hard_dev": 0}, | |
| "text_overlap": {"train_val": 0, "train_hard_dev": 0, "val_hard_dev": 0}, | |
| } | |
| man.write_text(json.dumps(payload), encoding="utf-8") | |
| result = corpus_loop._run_worker( | |
| [sys.executable, "-m", "workers.corpus.corpus_gate", "--manifest", str(man)], | |
| tmp_path / "gate_art", 60, | |
| ) | |
| assert result["metrics"]["source_overlap_total"] == 1 | |
| # Gate pass_condition (source_overlap_total == 0) must evaluate to False. | |
| assert corpus_loop._eval_condition( | |
| result["metrics"]["source_overlap_total"], "==", 0) is False | |
| def test_corpus_gate_passes_clean(tmp_path) -> None: | |
| man = tmp_path / "MANIFEST.json" | |
| payload = { | |
| "split_policy": "source_disjoint_v1", | |
| "split_seed": 42, | |
| "source_counts": {"train": 2, "val": 1, "hard_dev": 1}, | |
| "source_overlap": {"train_val": 0, "train_hard_dev": 0, "val_hard_dev": 0}, | |
| "text_overlap": {"train_val": 0, "train_hard_dev": 0, "val_hard_dev": 0}, | |
| } | |
| man.write_text(json.dumps(payload), encoding="utf-8") | |
| result = corpus_loop._run_worker( | |
| [sys.executable, "-m", "workers.corpus.corpus_gate", "--manifest", str(man)], | |
| tmp_path / "gate_art", 60, | |
| ) | |
| assert result["metrics"]["source_overlap_total"] == 0 | |
| assert corpus_loop._eval_condition( | |
| result["metrics"]["source_overlap_total"], "==", 0) is True |