decoder-bundle-minimization/work/e2e_bundle.sh

#!/usr/bin/env bash
# End-to-end proof of a packed B6 bundle, from a verifier's point of view:
#   unpack decompressor.zip in a fresh directory -> run its launcher on a real paq8px
#   archive of a real project blob -> output must be the known 17-byte original.
# Also exercises the paq8px stage alone on the 32 KiB real-blob-slice archive.
#
# usage: e2e_bundle.sh <label> <bundle-dir>
set -uo pipefail
B6=/root/hutter67/swarm67/B6-bundle
label=$1; bdir=$2
e=$B6/test/e2e_$label
rm -rf "$e"; mkdir -p "$e"
res=$B6/test/e2e_$label.json

# --- fresh-directory unpack, exactly as an external verifier would do ---
cp "$bdir/decompressor.zip" "$e/decompressor.zip"
python3 - <<PY
import zipfile, pathlib, hashlib, json
z = pathlib.Path("$e/decompressor.zip")
with zipfile.ZipFile(z) as zf:
    print("members:", [(i.filename, i.file_size, i.compress_size, i.compress_type) for i in zf.infolist()])
    zf.extractall("$e")
    bad = zf.testzip()
    print("zip integrity:", "OK" if bad is None else f"BAD {bad}")
PY
chmod +x "$e/d"

# --- real archives of real project blobs ---
cp /root/hutter67/swarm67/P5-bundles/fixtures/blob-paq.paq "$e/input archive.bin"    # P5-certified -0L chain fixture (41 B)
cp "$B6/test/level1/blob35_m1.paq8px216" "$e/blob35 archive.bin" 2>/dev/null || true
cp "$B6/test/level1/slice32k.paq8px216" "$e/slice32k archive.bin"

sha() { sha256sum "$1" | cut -d' ' -f1; }
EXPECT17=4ab9cb28f180436abbd3a76866c13eb3dc0797888908947f298eec7fb546bcd6
EXPECT32=252c561ddcdefb261e605f8e20f726e43de6a31b1b4cfb240563f5f7d4fd28e8

run() { # name, cmd..., captured separately below
  local name=$1; shift
  /usr/bin/time -v -o "$e/$name.time" "$@" >"$e/$name.log" 2>&1
  echo $?
}

rc1=$(run chain_p5fixture "$e/d" "$e/input archive.bin" "$e/out 17.bin")
rc2=$(run chain_blob35_m1 "$e/d" "$e/blob35 archive.bin" "$e/out 17b.bin")
python3 -c "import lzma,pathlib;p=pathlib.Path('$e');(p/'b').write_bytes(lzma.open(p/'b.xz').read())" && chmod +x "$e/b"
rc3=$(run paq8px_stage_slice32k "$e/b" -d "$e/slice32k archive.bin" "$e/out 32768.bin")

python3 - "$res" "$label" "$bdir" "$rc1" "$rc2" "$rc3" "$EXPECT17" "$EXPECT32" <<'PY'
import json, pathlib, sys, hashlib
res, label, bdir, rc1, rc2, rc3, exp17, exp32 = sys.argv[1:]
e = pathlib.Path(res).parent
e = pathlib.Path(str(pathlib.Path(bdir).parent.parent)) if False else pathlib.Path(f"/root/hutter67/swarm67/B6-bundle/test/e2e_{label}")

def sha(p):
    p = pathlib.Path(p)
    return hashlib.sha256(p.read_bytes()).hexdigest() if p.exists() else None

def rss(name):
    t = e / f"{name}.time"
    if not t.exists():
        return None
    for line in t.read_text().splitlines():
        if "Maximum resident set size" in line:
            return int(line.split()[-1])
    return None

def wall(name):
    t = e / f"{name}.time"
    if not t.exists():
        return None
    for line in t.read_text().splitlines():
        if "Elapsed (wall clock)" in line:
            return line.split()[-1]
    return None

members = {}
import zipfile
with zipfile.ZipFile(e / "decompressor.zip") as zf:
    for i in zf.infolist():
        members[i.filename] = {"file_size": i.file_size, "compress_size": i.compress_size,
                               "sha256": sha(e / i.filename)}

cases = [
    dict(name="chain_p5fixture", rc=int(rc1), out=str(e / "out 17.bin"), expected_sha=exp17, expected_bytes=17,
         what="P5-certified 41 B paq8px -0L archive of blob35.bin -> full chain -> 17 B original"),
    dict(name="chain_blob35_m1", rc=int(rc2), out=str(e / "out 17b.bin"), expected_sha=exp17, expected_bytes=17,
         what="fresh -1 archive of blob35.bin -> full chain -> 17 B original"),
    dict(name="paq8px_stage_slice32k", rc=int(rc3), out=str(e / "out 32768.bin"), expected_sha=exp32, expected_bytes=32768,
         what="real 32 KiB slice of blob67.bin, -1 archive -> shipped paq8px stage only"),
]
for c in cases:
    o = pathlib.Path(c["out"])
    c["out_bytes"] = o.stat().st_size if o.exists() else None
    c["out_sha256"] = sha(o)
    c["match"] = (c["out_sha256"] == c["expected_sha"])
    c["peak_rss_kb"] = rss(c["name"])
    c["wall"] = wall(c["name"])

bundle = e / "decompressor.zip"
doc = {
    "label": label,
    "bundle_dir": str(bdir),
    "bundle_bytes": bundle.stat().st_size,
    "bundle_sha256": sha(bundle),
    "extracted_payload_sha256": sha(e / "b"),
    "members": members,
    "cases": cases,
    "all_match": all(c["match"] and c["rc"] == 0 for c in cases),
}
pathlib.Path(res).write_text(json.dumps(doc, indent=1))
print(json.dumps(doc, indent=1))
PY