decoder-bundle-minimization/work/codec_sweep.sh

#!/usr/bin/env bash
# Codec sweep on the *original, unmodified* paq8px binary.
# Purpose: beat the P5 tuned-XZ payload (760,856 B) with zero behavioural risk
# (the bytes that execute are identical; only the container changes).
set -uo pipefail
B6=/root/hutter67/swarm67/B6-bundle
w=$B6/work
o=$B6/out/codec
mkdir -p "$o"
src=$w/paq8px.orig
res=$o/sweep.tsv
: >"$res"
echo -e "codec_bytes\trss_kb\tvariant\tnote" >"$o/sweep.sorted"

measure() { # $1 name, $2 output path, rest: command with OUT placeholder
  local name=$1 out=$2; shift 2
  local t=$o/$name.time
  local cmd=${*//OUT/$out}
  /usr/bin/time -v -o "$t" bash -c "$cmd" >"$o/$name.stdout" 2>&1
  local rc=$?
  local bytes=-1
  [[ -f $out ]] && bytes=$(stat -c%s "$out")
  local rss
  rss=$(awk '/Maximum resident set size/ {print $NF}' "$t" 2>/dev/null)
  printf '%s\t%s\trc=%s\trss_kb=%s\n' "$name" "$bytes" "$rc" "${rss:-?}" >>"$res"
  printf '%s\t%s\t%s\t%s\n' "$bytes" "${rss:-?}" "$name" "rc=$rc" >>"$o/sweep.sorted"
}

# --- baseline reproductions ---
measure xz_9e_plain  "$o/plain.xz"   'xz -9e -T1 --check=none -c '"$src"' > OUT'
measure xz_p5_tuned "$o/p5.xz"      'xz -c -T1 --memlimit-compress=190MiB --no-adjust --check=none --x86 --lzma2=preset=9e,dict=2097152 '"$src"' > OUT'

# --- tuned LZMA/x86 variants ---
for dict in 4194304 8388608 16777216 33554432 67108864; do
  measure "xz_x86_dict$((dict/1048576))m" "$o/x86_d$((dict/1048576))m.xz" \
    'xz -c -T1 --memlimit-compress=400MiB --no-adjust --check=none --x86 --lzma2=preset=9e,dict='"$dict"' '"$src"' > OUT'
done
for pb in 0 1 3 4; do
  measure "xz_x86_d16m_pb$pb" "$o/x86_d16m_pb$pb.xz" \
    'xz -c -T1 --memlimit-compress=400MiB --no-adjust --check=none --x86 --lzma2=preset=9e,dict=16777216,pb='"$pb"' '"$src"' > OUT'
done
for lc in 0 1 2 4; do
  measure "xz_x86_d16m_lc$lc" "$o/x86_d16m_lc$lc.xz" \
    'xz -c -T1 --memlimit-compress=400MiB --no-adjust --check=none --x86 --lzma2=preset=9e,dict=16777216,lc='"$lc"' '"$src"' > OUT'
done
measure xz_x86_bt4_nice273 "$o/x86_bt4.xz" \
  'xz -c -T1 --memlimit-compress=1024MiB --no-adjust --check=none --x86 "--lzma2=dict=268435456,lc=3,lp=0,pb=2,mode=normal,nice=273,mf=bt4,depth=0" '"$src"' > OUT'

# --- other codecs ---
measure zstd_22_long "$o/z22.zst"  'zstd -q --ultra -22 --long=27 -T1 -f -o OUT '"$src"
measure zstd_19_long "$o/z19.zst"  'zstd -q --ultra -19 --long=27 -T1 -f -o OUT '"$src"
measure brotli_q11_w24 "$o/br11.br" 'brotli -q 11 -w 24 -f -o OUT '"$src"
measure bzip2_9 "$o/bz2.bz2"       'bzip2 -9 -c '"$src"' > OUT'
measure 7z_ppmd "$o/ppmd.7z"       'rm -f OUT; 7z a -t7z -m0=PPMd:mem=1024m -mx=9 OUT '"$src"
measure 7z_lzma2_64m "$o/lzma2.7z" 'rm -f OUT; 7z a -t7z -m0=lzma2:preset=9e:dict=64m -mx=9 OUT '"$src"

# --- zpaq (project vendor binary) ---
if [[ -x /root/hutter67/vendor/zpaq715/zpaq ]]; then
  for m in 5 9; do
    measure "zpaq_m$m" "$o/zpaq_m$m.zpaq" 'rm -f OUT; /root/hutter67/vendor/zpaq715/zpaq a OUT '"$src"' -m'"$m"
  done
fi

sort -t$'\t' -k1 -n "$o/sweep.sorted" | column -t -s$'\t'