Download classify_demo.py from EugeneBoondock/sponge-compsearch: direct link, hf CLI and curl.
- Browser
- Download file 5.4 kB
-
https://huggingface.co/EugeneBoondock/sponge-compsearch/resolve/main/classify_demo.py
- Command line
-
hf download hf://EugeneBoondock/sponge-compsearch/classify_demo.py
-
curl -L -o classify_demo.py https://huggingface.co/EugeneBoondock/sponge-compsearch/resolve/main/classify_demo.py
5.4 kB
| """Check the decision mode on real labelled text already on this machine. | |
| Two tasks, each judged on snippets from files the classifier never saw: | |
| language - which language is this snippet? (Python, C header, Tcl) | |
| purpose - is this Python snippet test code or library code? (harder: same language) | |
| python classify_demo.py | |
| python classify_demo.py --learner runs/first/cands/00003/learner.py | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import math | |
| import random | |
| import sysconfig | |
| import time | |
| import zlib | |
| from pathlib import Path | |
| from compsearch.classify import Classifier, load_learner_factory, log_loss | |
| SNIPPET_BYTES = 300 | |
| TRAIN, CALIBRATE, TEST = 60, 25, 80 # snippets per label | |
| MIN_FILE_BYTES = 2000 | |
| def snippets(files: list[Path], wanted: int, rng: random.Random) -> list[bytes]: | |
| """One snippet per file, so no file contributes to more than one split.""" | |
| found = [] | |
| for path in files: | |
| try: | |
| raw = path.read_bytes() | |
| except OSError: | |
| continue | |
| if len(raw) < MIN_FILE_BYTES: | |
| continue | |
| start = rng.randrange(len(raw) - SNIPPET_BYTES) | |
| found.append(raw[start:start + SNIPPET_BYTES]) | |
| if len(found) == wanted: | |
| break | |
| return found | |
| def build_task(sources: dict[str, list[Path]], rng: random.Random) -> dict[str, dict[str, list[bytes]]]: | |
| task = {"train": {}, "calibrate": {}, "test": {}} | |
| for label, files in sources.items(): | |
| files = sorted(files) | |
| rng.shuffle(files) | |
| got = snippets(files, TRAIN + CALIBRATE + TEST, rng) | |
| if len(got) < TRAIN + CALIBRATE + TEST: | |
| raise SystemExit(f"not enough '{label}' files on this machine ({len(got)} usable)") | |
| task["train"][label] = got[:TRAIN] | |
| task["calibrate"][label] = got[TRAIN:TRAIN + CALIBRATE] | |
| task["test"][label] = got[TRAIN + CALIBRATE:] | |
| return task | |
| def zlib_guess(train: dict[str, list[bytes]], text: bytes) -> str: | |
| """Baseline: classify by how much zlib output grows when the text is appended.""" | |
| def growth(label: str) -> int: | |
| corpus = b"\n".join(train[label])[-30000:] | |
| return len(zlib.compress(corpus + text, 9)) - len(zlib.compress(corpus, 9)) | |
| return min(train, key=growth) | |
| def run_task(name: str, task: dict, factory) -> bool: | |
| labels = list(task["train"]) | |
| classifier = Classifier(factory) | |
| started = time.monotonic() | |
| for round_index in range(TRAIN): # interleave labels: order should not matter | |
| for label in labels: | |
| classifier.learn(label, task["train"][label][round_index]) | |
| learn_seconds = time.monotonic() - started | |
| temperature = classifier.calibrate( | |
| [(label, text) for label in labels for text in task["calibrate"][label]]) | |
| tests = [(label, text) for label in labels for text in task["test"][label]] | |
| started = time.monotonic() | |
| scored = [(label, classifier.costs(text)) for label, text in tests] | |
| judge_ms = 1000 * (time.monotonic() - started) / len(tests) | |
| correct = sum(min(costs, key=costs.get) == label for label, costs in scored) | |
| baseline = sum(zlib_guess(task["train"], text) == label for label, text in tests) | |
| accuracy, base_accuracy, chance = correct / len(tests), baseline / len(tests), 1 / len(labels) | |
| surprise = log_loss(scored, temperature) | |
| print(f"\n{name}: {len(labels)} labels, {TRAIN} examples each, {len(tests)} unseen test snippets") | |
| print(f" accuracy {accuracy:6.1%} (zlib baseline {base_accuracy:.1%}, chance {chance:.1%})") | |
| print(f" calibration {surprise:.3f} bits of surprise per answer at temperature {temperature} " | |
| f"(a coin-flip guesser scores {-math.log2(chance):.3f})") | |
| print(f" speed learned {TRAIN * len(labels)} examples in {learn_seconds:.1f}s, " | |
| f"{judge_ms:.0f} ms per judgment") | |
| return accuracy > max(base_accuracy - 0.02, chance + 0.15) | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__, | |
| formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--learner", default="seeds/sponge.py") | |
| args = parser.parse_args() | |
| factory = load_learner_factory(args.learner) | |
| rng = random.Random(0) | |
| stdlib = Path(sysconfig.get_paths()["stdlib"]) | |
| root = stdlib.parent | |
| python_files = [p for p in stdlib.rglob("*.py") if "site-packages" not in p.parts] | |
| is_test = lambda p: "test" in p.parts or "tests" in p.parts or p.name.startswith("test_") # noqa: E731 | |
| language = build_task({ | |
| "python": [p for p in python_files if not is_test(p)], | |
| "c-header": list(root.rglob("*.h")), | |
| "tcl": list(root.rglob("*.tcl")), | |
| }, rng) | |
| purpose = build_task({ | |
| "test-code": [p for p in python_files if is_test(p)], | |
| "library-code": [p for p in python_files if not is_test(p)], | |
| }, rng) | |
| print(f"learner: {args.learner}") | |
| passed = [run_task("language", language, factory), run_task("purpose", purpose, factory)] | |
| print("\nresult:", "both tasks pass" if all(passed) else | |
| "language passes, purpose does not" if passed[0] else "language task FAILED") | |
| return 0 if passed[0] else 1 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |