"""CodeNet C submissions -> programs that compile for our 8 KiB RV32IM machine and run on it. For each submission: drop its #include lines, add our io.h, compile with zig, run on a random input on the real chip (Unicorn). Keeps: image + input + instruction count. Logs why the others failed (link error symbol, too big, crash). Usage: python prep_codenet.py --out DIR [--limit N] [--procs P] [--max_steps S] """ import argparse import collections import json import multiprocessing as mp import os import random import re import subprocess import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) INC = re.compile(r"^\s*#\s*include\s*[<\"][^>\"]*[>\"]\s*$", re.M) UNDEF = re.compile(r"undefined symbol: (\S+)") def rand_input(rng): """A plausible stdin: a few lines of small integers (most CodeNet programs read numbers).""" k = rng.choice([1, 2, 3, 5, 8]) lines = [str(k)] + [" ".join(str(rng.randint(0, rng.choice([9, 99, 999]))) for _ in range(rng.randint(1, 6))) for _ in range(k)] return ("\n".join(lines) + "\n").encode() def work(job): import rv sid, code, seed = job rng = random.Random(seed) src = '#include "io.h"\n' + INC.sub("", code) try: img = rv.compile_c(src, "p") except subprocess.CalledProcessError as e: err = (e.stderr or b"").decode(errors="replace") u = UNDEF.findall(err) if u: return sid, "undef", sorted(set(u))[:4], None if "overflow" in err or "will not fit" in err or "exceeds" in err: return sid, "too_big", [], None return sid, "compile", [err.strip().splitlines()[-1][:120] if err.strip() else "?"], None except ValueError: return sid, "too_big", [], None except Exception as e: # timeout etc return sid, "compile", [type(e).__name__], None inp = rand_input(rng) steps, out, halted = rv.run(img, inp, max_steps=MAX_STEPS) if steps < 50: return sid, "tiny", [], None return sid, "ok" if halted else "no_halt", [], {"id": sid, "img": img.hex(), "inp": inp.hex(), "steps": steps, "halted": halted, "out": out[:120].decode("latin-1")} def init(ms): global MAX_STEPS MAX_STEPS = ms os.environ["RV_TMP"] = "/dev/shm/neurv/tmp" if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("parquet", nargs="+") ap.add_argument("--out", required=True) ap.add_argument("--limit", type=int, default=0) ap.add_argument("--procs", type=int, default=64) ap.add_argument("--max_steps", type=int, default=300000) a = ap.parse_args() import pyarrow.parquet as pq os.makedirs(a.out, exist_ok=True) os.makedirs("/dev/shm/neurv/tmp", exist_ok=True) jobs = [] for f in a.parquet: t = pq.read_table(f, columns=["s_id", "code", "status"]) for sid, code, st in zip(t["s_id"].to_pylist(), t["code"].to_pylist(), t["status"].to_pylist()): if code and len(code) < 6000: jobs.append((sid, code, hash(sid) & 0xFFFFFFFF)) random.Random(0).shuffle(jobs) if a.limit: jobs = jobs[:a.limit] print(f"jobs {len(jobs)}", flush=True) cnt, undef = collections.Counter(), collections.Counter() with open(f"{a.out}/programs.jsonl", "w") as fo, mp.get_context("fork").Pool(a.procs, init, (a.max_steps,)) as pool: for i, (sid, kind, info, rec) in enumerate(pool.imap_unordered(work, jobs, chunksize=8)): cnt[kind] += 1 if kind == "undef": undef.update(info) if rec: fo.write(json.dumps(rec) + "\n") if (i + 1) % 2000 == 0: print(f"{i + 1} {dict(cnt)}", flush=True) json.dump({"counts": cnt, "undef_top": undef.most_common(60)}, open(f"{a.out}/stats.json", "w"), indent=1) print("PREP_DONE", dict(cnt), "\nundef top:", undef.most_common(30), flush=True)