sponge-compsearch / classify_demo.py
EugeneBoondock's picture
Sponge online learners and the compression-scored search that found them
b999890 verified
Raw History Blame Contribute Delete
5.4 kB
"""Check the decision mode on real labelled text already on this machine.
Two tasks, each judged on snippets from files the classifier never saw:
language - which language is this snippet? (Python, C header, Tcl)
purpose - is this Python snippet test code or library code? (harder: same language)
python classify_demo.py
python classify_demo.py --learner runs/first/cands/00003/learner.py
"""
from __future__ import annotations
import argparse
import math
import random
import sysconfig
import time
import zlib
from pathlib import Path
from compsearch.classify import Classifier, load_learner_factory, log_loss
SNIPPET_BYTES = 300
TRAIN, CALIBRATE, TEST = 60, 25, 80 # snippets per label
MIN_FILE_BYTES = 2000
def snippets(files: list[Path], wanted: int, rng: random.Random) -> list[bytes]:
"""One snippet per file, so no file contributes to more than one split."""
found = []
for path in files:
try:
raw = path.read_bytes()
except OSError:
continue
if len(raw) < MIN_FILE_BYTES:
continue
start = rng.randrange(len(raw) - SNIPPET_BYTES)
found.append(raw[start:start + SNIPPET_BYTES])
if len(found) == wanted:
break
return found
def build_task(sources: dict[str, list[Path]], rng: random.Random) -> dict[str, dict[str, list[bytes]]]:
task = {"train": {}, "calibrate": {}, "test": {}}
for label, files in sources.items():
files = sorted(files)
rng.shuffle(files)
got = snippets(files, TRAIN + CALIBRATE + TEST, rng)
if len(got) < TRAIN + CALIBRATE + TEST:
raise SystemExit(f"not enough '{label}' files on this machine ({len(got)} usable)")
task["train"][label] = got[:TRAIN]
task["calibrate"][label] = got[TRAIN:TRAIN + CALIBRATE]
task["test"][label] = got[TRAIN + CALIBRATE:]
return task
def zlib_guess(train: dict[str, list[bytes]], text: bytes) -> str:
"""Baseline: classify by how much zlib output grows when the text is appended."""
def growth(label: str) -> int:
corpus = b"\n".join(train[label])[-30000:]
return len(zlib.compress(corpus + text, 9)) - len(zlib.compress(corpus, 9))
return min(train, key=growth)
def run_task(name: str, task: dict, factory) -> bool:
labels = list(task["train"])
classifier = Classifier(factory)
started = time.monotonic()
for round_index in range(TRAIN): # interleave labels: order should not matter
for label in labels:
classifier.learn(label, task["train"][label][round_index])
learn_seconds = time.monotonic() - started
temperature = classifier.calibrate(
[(label, text) for label in labels for text in task["calibrate"][label]])
tests = [(label, text) for label in labels for text in task["test"][label]]
started = time.monotonic()
scored = [(label, classifier.costs(text)) for label, text in tests]
judge_ms = 1000 * (time.monotonic() - started) / len(tests)
correct = sum(min(costs, key=costs.get) == label for label, costs in scored)
baseline = sum(zlib_guess(task["train"], text) == label for label, text in tests)
accuracy, base_accuracy, chance = correct / len(tests), baseline / len(tests), 1 / len(labels)
surprise = log_loss(scored, temperature)
print(f"\n{name}: {len(labels)} labels, {TRAIN} examples each, {len(tests)} unseen test snippets")
print(f" accuracy {accuracy:6.1%} (zlib baseline {base_accuracy:.1%}, chance {chance:.1%})")
print(f" calibration {surprise:.3f} bits of surprise per answer at temperature {temperature} "
f"(a coin-flip guesser scores {-math.log2(chance):.3f})")
print(f" speed learned {TRAIN * len(labels)} examples in {learn_seconds:.1f}s, "
f"{judge_ms:.0f} ms per judgment")
return accuracy > max(base_accuracy - 0.02, chance + 0.15)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--learner", default="seeds/sponge.py")
args = parser.parse_args()
factory = load_learner_factory(args.learner)
rng = random.Random(0)
stdlib = Path(sysconfig.get_paths()["stdlib"])
root = stdlib.parent
python_files = [p for p in stdlib.rglob("*.py") if "site-packages" not in p.parts]
is_test = lambda p: "test" in p.parts or "tests" in p.parts or p.name.startswith("test_") # noqa: E731
language = build_task({
"python": [p for p in python_files if not is_test(p)],
"c-header": list(root.rglob("*.h")),
"tcl": list(root.rglob("*.tcl")),
}, rng)
purpose = build_task({
"test-code": [p for p in python_files if is_test(p)],
"library-code": [p for p in python_files if not is_test(p)],
}, rng)
print(f"learner: {args.learner}")
passed = [run_task("language", language, factory), run_task("purpose", purpose, factory)]
print("\nresult:", "both tasks pass" if all(passed) else
"language passes, purpose does not" if passed[0] else "language task FAILED")
return 0 if passed[0] else 1
if __name__ == "__main__":
raise SystemExit(main())