File size: 6,294 Bytes
48d68c4 0712b35 48d68c4 0712b35 48d68c4 5897fab 48d68c4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | """protocol.py — single source of truth for the slow-ring protocol.
Every battery in the lab must run the SAME sampling, and every result file
must be able to prove it did. Before this module the numbers lived in four
places: two runners (with DIFFERENT defaults — one shipped
presence_penalty=1.5, which the README itself records as "breaks thinking
templates"), verify.py (correct, via setdefault), and the markdown ledgers.
Result files carried {task_id, completion} and nothing else, so a battery
run at presence 1.5 was indistinguishable from one run at 0.0.
Rule: import from here, never hardcode. A drift between the code default
and the documented protocol is now impossible to introduce silently.
"""
import json
import os
import platform
import subprocess
import sys
import time
SERVER_BIN = os.environ.get(
"LLAMA_SERVER",
os.path.expanduser("~/llama.cpp/build/bin/llama-server"))
# The verified protocol (README.md §4). Env overrides still win, so a
# deliberate A/B (temp 0.0 vs 1.0 — SRIQ duel) is possible and recorded.
PROTOCOL = {
"temperature": 1.0,
"top_p": 0.95,
"top_k": 20,
"min_p": 0.0,
"presence_penalty": 0.0, # 1.5 is the vendor recipe; it breaks thinking
"repetition_penalty": 1.0,
"max_tokens": 2048, # 1024 truncates thinking models (MiMo: 11k+)
}
# Verdict-ring server settings. -c 8192 does NOT fit 8 GB VRAM with 6.4 GB of
# weights (proven: create_context fail) — the documented mitigation.
SERVER_ARGS = ["-ngl", "99", "-c", "8192", "--jinja", "--log-disable"]
def sample_params(environ=None):
"""Protocol dict with env overrides applied."""
env = os.environ if environ is None else environ
p = dict(PROTOCOL)
def _f(name, default):
try:
return float(env.get(name, default))
except ValueError:
return float(default)
p["temperature"] = _f("HE_TEMP", p["temperature"])
p["top_p"] = _f("HE_TOP_P", p["top_p"])
p["top_k"] = int(env.get("HE_TOP_K", p["top_k"]))
p["min_p"] = _f("HE_MIN_P", p["min_p"])
p["presence_penalty"] = _f("HE_PRESENCE", p["presence_penalty"])
p["repetition_penalty"] = _f("HE_REPEAT", p["repetition_penalty"])
p["max_tokens"] = int(env.get("HE_MAX_TOKENS", p["max_tokens"]))
return p
def _file_sha256(path, _bytes=1 << 26):
"""Hex sha256 of a model file (chunked; ~30-60 s for 5-10 GB).
The identity anchor: twins are byte-identical files, not same names."""
import hashlib
try:
h = hashlib.sha256()
with open(path, "rb") as f:
while True:
b = f.read(_bytes)
if not b:
break
h.update(b)
return h.hexdigest()
except Exception:
return None
def _git_sha(repo_root):
try:
out = subprocess.run(["git", "-C", repo_root, "rev-parse", "--short", "HEAD"],
capture_output=True, text=True, timeout=5)
return out.stdout.strip() or None
except Exception:
return None
def battery_meta(server_url, serve_model, environ=None, extra=None):
"""Self-describing record of how a battery was run.
Written to <results>.meta.json next to the sample file: a sidecar, not an
extra key inside the sample jsonl — the human_eval checker owns that
schema and must not be disturbed.
"""
env = dict(os.environ if environ is None else environ)
repo_root = os.path.dirname(os.path.abspath(__file__))
return {
"started": time.strftime("%Y-%m-%dT%H:%M:%S"),
"server_url": server_url,
"serve_model": serve_model,
"serve_sha256": _file_sha256(serve_model),
"sample": sample_params(env),
"server_bin": SERVER_BIN,
"server_args": SERVER_ARGS + (extra or []),
"git_sha": _git_sha(repo_root),
"dirty": bool(subprocess.run(
["git", "-C", repo_root, "diff", "--quiet"],
capture_output=True).returncode) if _git_sha(repo_root) else None,
"python": sys.version.split()[0],
"platform": platform.platform(),
"env_overrides": {k: v for k, v in env.items() if k.startswith("HE_")},
}
def banner(meta):
"""One-line protocol stamp for the battery log (goes to stdout)."""
s = meta["sample"]
return (f"protocol: temp={s['temperature']} top_p={s['top_p']} "
f"top_k={s['top_k']} min_p={s['min_p']} "
f"presence={s['presence_penalty']} rep={s['repetition_penalty']} "
f"max_tokens={s['max_tokens']} | "
f"git={meta.get('git_sha')} dirty={meta.get('dirty')}")
def write_meta(out_path, meta):
side = os.path.splitext(out_path)[0] + ".meta.json"
os.makedirs(os.path.dirname(side), exist_ok=True)
with open(side, "w") as f:
json.dump(meta, f, indent=2, sort_keys=True)
_append_ledger(out_path, meta)
return side
# Append-only battery ledger, inside the repo, written by the runner itself.
# The chain logs were the only build→path record that ever existed and they
# lived in /tmp, so they died with the boot (2026-09-28: /tmp/opencode/chain.log
# gone, 48 of 59 batteries unidentifiable). The sidecar lives next to the
# results and can be lost with them; this line cannot, because it is written
# at battery time into a file the runner owns.
LEDGER = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"provenance", "batteries.jsonl")
def _append_ledger(out_path, meta):
try:
os.makedirs(os.path.dirname(LEDGER), exist_ok=True)
rec = {
"out": os.path.basename(out_path),
"serve_model": meta.get("serve_model"),
"started": meta.get("started"),
"finished": meta.get("finished"),
"n_tasks": meta.get("n_tasks"),
"empties": meta.get("empties"),
"sample": meta.get("sample"),
"server_args": meta.get("server_args"),
"git_sha": meta.get("git_sha"),
"dirty": meta.get("dirty"),
}
with open(LEDGER, "a") as f:
f.write(json.dumps(rec, sort_keys=True) + "\n")
except Exception as e: # never lose a battery to a bookkeeping error
print(f"protocol: WARNING could not append to {LEDGER}: {e}")
|