Download neurv/prep_codenet.py from Prahaladha/learned-computer: direct link, hf CLI and curl.
- Browser
- Download file 3.96 kB
-
https://huggingface.co/Prahaladha/learned-computer/resolve/main/neurv/prep_codenet.py
- Command line
-
hf download hf://Prahaladha/learned-computer/neurv/prep_codenet.py
-
curl -L -o prep_codenet.py https://huggingface.co/Prahaladha/learned-computer/resolve/main/neurv/prep_codenet.py
3.96 kB
| """CodeNet C submissions -> programs that compile for our 8 KiB RV32IM machine and run on it. | |
| For each submission: drop its #include lines, add our io.h, compile with zig, run on a random input on the real chip (Unicorn). | |
| Keeps: image + input + instruction count. Logs why the others failed (link error symbol, too big, crash). | |
| Usage: python prep_codenet.py <parquet ...> --out DIR [--limit N] [--procs P] [--max_steps S] | |
| """ | |
| import argparse | |
| import collections | |
| import json | |
| import multiprocessing as mp | |
| import os | |
| import random | |
| import re | |
| import subprocess | |
| import sys | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| INC = re.compile(r"^\s*#\s*include\s*[<\"][^>\"]*[>\"]\s*$", re.M) | |
| UNDEF = re.compile(r"undefined symbol: (\S+)") | |
| def rand_input(rng): | |
| """A plausible stdin: a few lines of small integers (most CodeNet programs read numbers).""" | |
| k = rng.choice([1, 2, 3, 5, 8]) | |
| lines = [str(k)] + [" ".join(str(rng.randint(0, rng.choice([9, 99, 999]))) for _ in range(rng.randint(1, 6))) for _ in range(k)] | |
| return ("\n".join(lines) + "\n").encode() | |
| def work(job): | |
| import rv | |
| sid, code, seed = job | |
| rng = random.Random(seed) | |
| src = '#include "io.h"\n' + INC.sub("", code) | |
| try: | |
| img = rv.compile_c(src, "p") | |
| except subprocess.CalledProcessError as e: | |
| err = (e.stderr or b"").decode(errors="replace") | |
| u = UNDEF.findall(err) | |
| if u: | |
| return sid, "undef", sorted(set(u))[:4], None | |
| if "overflow" in err or "will not fit" in err or "exceeds" in err: | |
| return sid, "too_big", [], None | |
| return sid, "compile", [err.strip().splitlines()[-1][:120] if err.strip() else "?"], None | |
| except ValueError: | |
| return sid, "too_big", [], None | |
| except Exception as e: # timeout etc | |
| return sid, "compile", [type(e).__name__], None | |
| inp = rand_input(rng) | |
| steps, out, halted = rv.run(img, inp, max_steps=MAX_STEPS) | |
| if steps < 50: | |
| return sid, "tiny", [], None | |
| return sid, "ok" if halted else "no_halt", [], {"id": sid, "img": img.hex(), "inp": inp.hex(), "steps": steps, | |
| "halted": halted, "out": out[:120].decode("latin-1")} | |
| def init(ms): | |
| global MAX_STEPS | |
| MAX_STEPS = ms | |
| os.environ["RV_TMP"] = "/dev/shm/neurv/tmp" | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("parquet", nargs="+") | |
| ap.add_argument("--out", required=True) | |
| ap.add_argument("--limit", type=int, default=0) | |
| ap.add_argument("--procs", type=int, default=64) | |
| ap.add_argument("--max_steps", type=int, default=300000) | |
| a = ap.parse_args() | |
| import pyarrow.parquet as pq | |
| os.makedirs(a.out, exist_ok=True) | |
| os.makedirs("/dev/shm/neurv/tmp", exist_ok=True) | |
| jobs = [] | |
| for f in a.parquet: | |
| t = pq.read_table(f, columns=["s_id", "code", "status"]) | |
| for sid, code, st in zip(t["s_id"].to_pylist(), t["code"].to_pylist(), t["status"].to_pylist()): | |
| if code and len(code) < 6000: | |
| jobs.append((sid, code, hash(sid) & 0xFFFFFFFF)) | |
| random.Random(0).shuffle(jobs) | |
| if a.limit: | |
| jobs = jobs[:a.limit] | |
| print(f"jobs {len(jobs)}", flush=True) | |
| cnt, undef = collections.Counter(), collections.Counter() | |
| with open(f"{a.out}/programs.jsonl", "w") as fo, mp.get_context("fork").Pool(a.procs, init, (a.max_steps,)) as pool: | |
| for i, (sid, kind, info, rec) in enumerate(pool.imap_unordered(work, jobs, chunksize=8)): | |
| cnt[kind] += 1 | |
| if kind == "undef": | |
| undef.update(info) | |
| if rec: | |
| fo.write(json.dumps(rec) + "\n") | |
| if (i + 1) % 2000 == 0: | |
| print(f"{i + 1} {dict(cnt)}", flush=True) | |
| json.dump({"counts": cnt, "undef_top": undef.most_common(60)}, open(f"{a.out}/stats.json", "w"), indent=1) | |
| print("PREP_DONE", dict(cnt), "\nundef top:", undef.most_common(30), flush=True) | |