"""Check the decision mode on real labelled text already on this machine. Two tasks, each judged on snippets from files the classifier never saw: language - which language is this snippet? (Python, C header, Tcl) purpose - is this Python snippet test code or library code? (harder: same language) python classify_demo.py python classify_demo.py --learner runs/first/cands/00003/learner.py """ from __future__ import annotations import argparse import math import random import sysconfig import time import zlib from pathlib import Path from compsearch.classify import Classifier, load_learner_factory, log_loss SNIPPET_BYTES = 300 TRAIN, CALIBRATE, TEST = 60, 25, 80 # snippets per label MIN_FILE_BYTES = 2000 def snippets(files: list[Path], wanted: int, rng: random.Random) -> list[bytes]: """One snippet per file, so no file contributes to more than one split.""" found = [] for path in files: try: raw = path.read_bytes() except OSError: continue if len(raw) < MIN_FILE_BYTES: continue start = rng.randrange(len(raw) - SNIPPET_BYTES) found.append(raw[start:start + SNIPPET_BYTES]) if len(found) == wanted: break return found def build_task(sources: dict[str, list[Path]], rng: random.Random) -> dict[str, dict[str, list[bytes]]]: task = {"train": {}, "calibrate": {}, "test": {}} for label, files in sources.items(): files = sorted(files) rng.shuffle(files) got = snippets(files, TRAIN + CALIBRATE + TEST, rng) if len(got) < TRAIN + CALIBRATE + TEST: raise SystemExit(f"not enough '{label}' files on this machine ({len(got)} usable)") task["train"][label] = got[:TRAIN] task["calibrate"][label] = got[TRAIN:TRAIN + CALIBRATE] task["test"][label] = got[TRAIN + CALIBRATE:] return task def zlib_guess(train: dict[str, list[bytes]], text: bytes) -> str: """Baseline: classify by how much zlib output grows when the text is appended.""" def growth(label: str) -> int: corpus = b"\n".join(train[label])[-30000:] return len(zlib.compress(corpus + text, 9)) - len(zlib.compress(corpus, 9)) return min(train, key=growth) def run_task(name: str, task: dict, factory) -> bool: labels = list(task["train"]) classifier = Classifier(factory) started = time.monotonic() for round_index in range(TRAIN): # interleave labels: order should not matter for label in labels: classifier.learn(label, task["train"][label][round_index]) learn_seconds = time.monotonic() - started temperature = classifier.calibrate( [(label, text) for label in labels for text in task["calibrate"][label]]) tests = [(label, text) for label in labels for text in task["test"][label]] started = time.monotonic() scored = [(label, classifier.costs(text)) for label, text in tests] judge_ms = 1000 * (time.monotonic() - started) / len(tests) correct = sum(min(costs, key=costs.get) == label for label, costs in scored) baseline = sum(zlib_guess(task["train"], text) == label for label, text in tests) accuracy, base_accuracy, chance = correct / len(tests), baseline / len(tests), 1 / len(labels) surprise = log_loss(scored, temperature) print(f"\n{name}: {len(labels)} labels, {TRAIN} examples each, {len(tests)} unseen test snippets") print(f" accuracy {accuracy:6.1%} (zlib baseline {base_accuracy:.1%}, chance {chance:.1%})") print(f" calibration {surprise:.3f} bits of surprise per answer at temperature {temperature} " f"(a coin-flip guesser scores {-math.log2(chance):.3f})") print(f" speed learned {TRAIN * len(labels)} examples in {learn_seconds:.1f}s, " f"{judge_ms:.0f} ms per judgment") return accuracy > max(base_accuracy - 0.02, chance + 0.15) def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--learner", default="seeds/sponge.py") args = parser.parse_args() factory = load_learner_factory(args.learner) rng = random.Random(0) stdlib = Path(sysconfig.get_paths()["stdlib"]) root = stdlib.parent python_files = [p for p in stdlib.rglob("*.py") if "site-packages" not in p.parts] is_test = lambda p: "test" in p.parts or "tests" in p.parts or p.name.startswith("test_") # noqa: E731 language = build_task({ "python": [p for p in python_files if not is_test(p)], "c-header": list(root.rglob("*.h")), "tcl": list(root.rglob("*.tcl")), }, rng) purpose = build_task({ "test-code": [p for p in python_files if is_test(p)], "library-code": [p for p in python_files if not is_test(p)], }, rng) print(f"learner: {args.learner}") passed = [run_task("language", language, factory), run_task("purpose", purpose, factory)] print("\nresult:", "both tasks pass" if all(passed) else "language passes, purpose does not" if passed[0] else "language task FAILED") return 0 if passed[0] else 1 if __name__ == "__main__": raise SystemExit(main())