Buckets:

soixantesept's picture
download
raw
19.1 kB
{
"task": "minimize shipped paq8px decoder bundle without changing what it decodes",
"baseline": {
"bundle_bytes": 766308,
"bundle_sha256": "debed42c9ec425e5461dd886148814e55c71c52ea43b2675d80b563030074dce",
"source": "P5-bundles/paq8px-blob/decompressor.zip",
"members": {
"b.xz": 760856,
"v.xz": 4948,
"d_raw": 409,
"d_deflated": 236,
"framing": 268
},
"reproduced_byte_for_byte_by": "work/make_bundle.py --verify-baseline"
},
"winner": {
"label": "dyn_Os",
"bundle_bytes": 204244,
"bundle_sha256": "7a3899220efe888bd0d0e3affed875094afaea382ef1eda67eb1268a15b0071a",
"b_xz_bytes": 198792,
"b_xz_sha256": "35d1f4ff1886a9d6a9714cede09cb7c0d74ce00ca3e3b652d48ab311fc9f6960",
"binary_bytes": 476864,
"binary_sha256": "94cc29dbfb2dd1e9ac2ae96fd6699789935cc16585b145ed02ca83361e9764ce",
"raw_binary_bytes": 476864,
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Os/decompressor.zip"
},
"saving_vs_baseline_bytes": 562064,
"decode_chain": {
"steps": [
"unpack decompressor.zip",
"launcher d: b.xz -> paq8px, v.xz -> inverse",
"paq8px -d archive blob",
"inverse blob -> original"
],
"unchanged_vs_p5": [
"v.xz (4,948 B, sha 55e5f625\u2026)",
"launcher d (409 B, sha 9a9132bb\u2026)"
],
"changed": [
"b.xz payload (760,856 -> 198,792 B)"
]
},
"build": {
"source_commit": "29237fb44cb1995690e3eb72c6c3b1e4aede5791",
"source_tree": "P1-paq8px-cal/src/paq8px-29237fb\u2026(copied to work/paq8px-src, unmodified)",
"cxx_flags": "-DNDEBUG -fno-rtti -std=gnu++17 -ffp-contract=off -fno-fast-math -Os -flto=2 -ffunction-sections -fdata-sections -Wl,--gc-sections -fno-unwind-tables -fno-asynchronous-unwind-tables -Wl,-s",
"link": "dynamic (no -static), then strip --strip-all",
"march_native": false,
"march_native_reason": "AVX-512 portability risk on the verifier's host"
},
"packing": {
"payload_codec": "xz FORMAT_XZ CHECK_NONE, filters [FILTER_X86, FILTER_LZMA2]",
"lzma2": {
"dict_size": 524288,
"lc": 3,
"lp": 0,
"pb": 0,
"mode": "NORMAL",
"mf": "BT2",
"nice_len": 96,
"depth": 0
},
"zip": "b.xz STORED, v.xz STORED, d DEFLATED(9) mode 0755, DOS epoch timestamps"
},
"variants": {
"orig_static_p1": {
"description": "P1's shipped static binary, unmodified (baseline payload)",
"raw_binary_bytes": 1895080,
"binary_sha256": "659a4e2327c6bdc4fa5dea8df348ff3e9eba6c350be1308f7a9e47a95a0ffcff",
"b_xz_bytes": 761640,
"b_xz_sha256": "93428afa148db3aba68855c7ec9fd10367cd4b71f0646c4fbcadc5796cd21f8a",
"bundle_bytes": 767092,
"bundle_sha256": "df55287589ac25f059d7d5d2427084328e99a3f21d548a3efa66cda0bbc8d97a",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_orig_static_p1/decompressor.zip",
"binary_level_proof": {
"status": "PASS",
"same_binary_as": "orig",
"archives": [
"slice32k.paq8px216",
"small.paq8px216"
]
},
"bundle_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
}
},
"dyn_Os": {
"description": "same source, -Os -flto=2, dynamic, gc-sections, stripped",
"raw_binary_bytes": 476864,
"binary_sha256": "94cc29dbfb2dd1e9ac2ae96fd6699789935cc16585b145ed02ca83361e9764ce",
"b_xz_bytes": 198792,
"b_xz_sha256": "35d1f4ff1886a9d6a9714cede09cb7c0d74ce00ca3e3b652d48ab311fc9f6960",
"bundle_bytes": 204244,
"bundle_sha256": "7a3899220efe888bd0d0e3affed875094afaea382ef1eda67eb1268a15b0071a",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Os/decompressor.zip",
"binary_level_proof": {
"status": "PASS",
"same_binary_as": "dyn_Os",
"archives": [
"slice32k.paq8px216",
"small.paq8px216"
]
},
"bundle_level_proof": {
"status": "PASS"
},
"decode_proof": true
},
"dyn_Os_hard": {
"description": "-Os -flto=2 dynamic + hard size flags (no-plt/stack-protector off)",
"raw_binary_bytes": 473552,
"binary_sha256": "47605f744225600aebfa98b18fccebf414162e9e55049ece976e385bc7830637",
"b_xz_bytes": 201004,
"b_xz_sha256": "45ddbf9c0537011af92a83a6225206e3892bb6aa6e44a306be95f7b1aee28de7",
"bundle_bytes": 206456,
"bundle_sha256": "0fe90be2b6518b8e55031a6e67c3971046568f2c7af51672c455f8de8ed51213",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Os_hard/decompressor.zip",
"binary_level_proof": {
"status": "PASS",
"same_binary_as": "dyn_Os_hard",
"archives": [
"slice32k.paq8px216",
"small.paq8px216"
]
},
"bundle_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
}
},
"dyn_Os_flto1": {
"description": "-Os -flto=1 dynamic",
"raw_binary_bytes": 476864,
"binary_sha256": "94cc29dbfb2dd1e9ac2ae96fd6699789935cc16585b145ed02ca83361e9764ce",
"b_xz_bytes": 198792,
"b_xz_sha256": "35d1f4ff1886a9d6a9714cede09cb7c0d74ce00ca3e3b652d48ab311fc9f6960",
"bundle_bytes": 204244,
"bundle_sha256": "7a3899220efe888bd0d0e3affed875094afaea382ef1eda67eb1268a15b0071a",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Os_flto1/decompressor.zip",
"binary_level_proof": {
"status": "PASS",
"same_binary_as": "dyn_Os",
"archives": [
"slice32k.paq8px216",
"small.paq8px216"
]
},
"bundle_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
}
},
"dyn_Os_noflto": {
"description": "-Os dynamic, no LTO",
"raw_binary_bytes": 509632,
"binary_sha256": "8642bfbeb8ea6a2e0fd12b506db20ab9ce2d1c99b5b3ba141e5f08e74d86540a",
"b_xz_bytes": 204672,
"b_xz_sha256": "9a555edd11d00a5a31140cb8913e8ae83bb9e5f975941cbe568a6ed31c6cae63",
"bundle_bytes": 210124,
"bundle_sha256": "0cd663827adf08957c835e1ea4df30faa081295d5999d2d0dc2191eb75f75fac",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Os_noflto/decompressor.zip",
"binary_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
},
"bundle_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
}
},
"dyn_O3": {
"description": "P1 flags verbatim (-O3 -flto) except dynamic link instead of -static",
"raw_binary_bytes": 1012192,
"binary_sha256": "0d2524dae0dad033a7fb9a4e8574fc09e38b08176c45a9175474090398364dc4",
"b_xz_bytes": 437704,
"b_xz_sha256": "6af0068debfd1d8b297e93431b1841762ef8f802f2cca940794395ff20645445",
"bundle_bytes": 443156,
"bundle_sha256": "411bd72a8deb791e304f7d2ee56b86d35c6990166d90515b6d8f3de4be83b570",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_O3/decompressor.zip",
"binary_level_proof": {
"status": "PASS",
"same_binary_as": "dyn_O3",
"archives": [
"slice32k.paq8px216",
"small.paq8px216"
]
},
"bundle_level_proof": {
"status": "PASS"
},
"decode_proof": true
},
"dyn_O2": {
"description": "-O2 -flto=2 dynamic",
"raw_binary_bytes": 647648,
"binary_sha256": "db5958989eb0ab6d7d01970a28488dd9a3e4c76b9303a594e7d31c874ccfaffa",
"b_xz_bytes": 264480,
"b_xz_sha256": "7371c3d4ae369cf38ca45d6745ef90a2fdaa3c9619a4700f9b8c371b98bc3d71",
"bundle_bytes": 269932,
"bundle_sha256": "37c10e004f267e3e65f997b2409f6116540ca478252706aaedcf434506fa8d3b",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_O2/decompressor.zip",
"binary_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
},
"bundle_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
}
},
"dyn_Oz_clang": {
"description": "clang++-17 -Oz dynamic",
"raw_binary_bytes": 523208,
"binary_sha256": "88c848f3f310f6170753b6e37b92c68fb983e546c3710523356dc693f72e1e92",
"b_xz_bytes": 211248,
"b_xz_sha256": "b4f2667f050d7f966957fcee1cfd2030765e853a791ba573681a911a21a5364f",
"bundle_bytes": 216700,
"bundle_sha256": "47300c83eea0212bd90e1f5700b26170148409d18cd7164874635f7b1a4648b5",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Oz_clang/decompressor.zip",
"binary_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
},
"bundle_level_proof": {
"status": "not run",
"reason": "larger than the winner; not a shipping candidate"
}
},
"static_Os": {
"description": "-Os static (self-contained: no host libstdc++/libc needed)",
"raw_binary_bytes": 1277800,
"binary_sha256": "dc35a833502845853edf0e25346e6c93e82d27e504c0d12bb87f861194fc7a1d",
"b_xz_bytes": 480876,
"b_xz_sha256": "5c548a9e670d8aec1802602269cd005e4a55ab0165a59385dcd2c8610a704c64",
"bundle_bytes": 486328,
"bundle_sha256": "5f2c887528df97a9963b4e1f145b29ba0b21cacfd27dc38007dd7d5bc565a9d9",
"bundle_path": "/root/hutter67/swarm67/B6-bundle/out/bundle_static_Os/decompressor.zip",
"binary_level_proof": {
"status": "PASS",
"same_binary_as": "static_Os",
"archives": [
"slice32k.paq8px216",
"small.paq8px216"
]
},
"bundle_level_proof": {
"status": "PASS"
},
"decode_proof": true
}
},
"codec_comparison_on_winner_binary": {
"binary": "out/build/dyn_Os (476,864 B raw, sha256 94cc29dbfb2dd1e9ac2ae96fd6699789935cc16585b145ed02ca83361e9764ce)",
"rows": [
{
"name": "xz + BCJ/x86, LZMA2 dict512K lc3 lp0 pb0 mf=bt2 nice96 (SHIPPED b.xz)",
"bytes": 198792,
"note": "decoder is free on the host (launcher already uses Python lzma); winner"
},
{
"name": "zpaq zc56 stream (libzpaq level 5, 64 MiB block)",
"bytes": 187197,
"note": "smallest raw payload, but +16,476 B shipped libzpaq extractor = 203,673 B effective; loses"
},
{
"name": "UPX 4.2.4 --best --lzma (upx -d restores byte-identical)",
"bytes": 202768,
"note": "self-extracting; no host decoder, no xz payload; loses by 3,912 B"
},
{
"name": "7z PPMd mem=512m o=32 -mx=9",
"bytes": 204453,
"note": "needs a 7z binary on the host"
},
{
"name": "brotli -q 11 -w 24",
"bytes": 213027,
"note": "needs brotli on the host"
},
{
"name": "zstd --ultra -22 --long=27",
"bytes": 222742,
"note": "needs zstd on the host"
},
{
"name": "bzip2 -9",
"bytes": 248621,
"note": "host-decodable via Python bz2, but far larger than xz"
}
],
"on_original_binary_1895080B": [
{
"name": "P5 tuned xz (--x86 --lzma2=preset=9e,dict=2097152)",
"bytes": 760856,
"note": "the baseline b.xz actually shipped in the 766,308 B bundle"
},
{
"name": "xz best sweep (--x86 lzma2 dict16M lc4)",
"bytes": 760120,
"note": "736 B better than P5's choice"
},
{
"name": "UPX --best --lzma",
"bytes": 757436,
"note": "upx -d restores the binary byte-identically (cmp verified)"
},
{
"name": "zpaq zc56 / journaling -m5",
"bytes": 680245,
"note": "method 5 stream"
},
{
"name": "7z PPMd mem=1024m",
"bytes": 766383,
"note": ""
},
{
"name": "brotli -q 11 -w 24",
"bytes": 810902,
"note": ""
},
{
"name": "zstd --ultra -22 --long=27",
"bytes": 843201,
"note": ""
},
{
"name": "bzip2 -9",
"bytes": 918182,
"note": ""
}
],
"xz_parameter_sweep_on_dyn_Os": {
"winner": "--x86 + lzma2 dict=524288 lc=3 lp=0 pb=0 mf=bt2 nice_len=96 depth=0 = 198,792",
"alternatives": {
"lc=4,pb=0": 198940,
"lc=4,pb=2": 199228,
"lc=2,pb=0": 199552,
"lp=1": 199812,
"dict=4M/16M": 198856,
"no_BCJ": 208760,
"BCJ start=0x64f0": 198988,
"BCJ start=0x100000": 199136,
"text/rodata split, BCJ on code only": 198404,
"mf=bt4 nice=273 dict=1M (first best)": 198856
},
"note": "splitting .text from the rest and BCJ-ing only code saves 452 B but needs launcher reassembly (~+150 B), not shipped"
},
"extractor_experiment": {
"px_dec": "minimal libzpaq one-shot decompress(), 51,608 B raw / 16,476 B xz (built -Dunix -DNOJIT -Os, gc-sections, stripped)",
"verified": "decoded the zc56 payload back to a byte-identical dyn_Os (sha256 match), 2.96 s / 88,952 kB",
"verdict": "zpaq payload + extractor = 203,673 B > 198,856 B xz; not shipped"
}
},
"proofs": {
"binary_level": [
{
"binary": "orig",
"archive": "slice32k.paq8px216",
"out_bytes": "32768",
"out_sha256": "252c561ddcdefb261e605f8e20f726e43de6a31b1b4cfb240563f5f7d4fd28e8",
"match": "YES",
"rss_kb": "561508",
"wall": "0:10.42"
},
{
"binary": "dyn_O3",
"archive": "slice32k.paq8px216",
"out_bytes": "32768",
"out_sha256": "252c561ddcdefb261e605f8e20f726e43de6a31b1b4cfb240563f5f7d4fd28e8",
"match": "YES",
"rss_kb": "564120",
"wall": "0:09.38"
},
{
"binary": "dyn_Os",
"archive": "slice32k.paq8px216",
"out_bytes": "32768",
"out_sha256": "252c561ddcdefb261e605f8e20f726e43de6a31b1b4cfb240563f5f7d4fd28e8",
"match": "YES",
"rss_kb": "563812",
"wall": "0:12.25"
},
{
"binary": "static_Os",
"archive": "slice32k.paq8px216",
"out_bytes": "32768",
"out_sha256": "252c561ddcdefb261e605f8e20f726e43de6a31b1b4cfb240563f5f7d4fd28e8",
"match": "YES",
"rss_kb": "562876",
"wall": "0:13.42"
},
{
"binary": "dyn_Os_hard",
"archive": "slice32k.paq8px216",
"out_bytes": "32768",
"out_sha256": "252c561ddcdefb261e605f8e20f726e43de6a31b1b4cfb240563f5f7d4fd28e8",
"match": "YES",
"rss_kb": "563776",
"wall": "0:12.15"
},
{
"binary": "orig",
"archive": "small.paq8px216",
"out_bytes": "53",
"out_sha256": "c5cf4213e9e9568b3b7b8bb42e219d30cc66ed3adce28aa5292201d96f928430",
"match": "YES",
"rss_kb": "347900",
"wall": "0:00.25"
},
{
"binary": "dyn_O3",
"archive": "small.paq8px216",
"out_bytes": "53",
"out_sha256": "c5cf4213e9e9568b3b7b8bb42e219d30cc66ed3adce28aa5292201d96f928430",
"match": "YES",
"rss_kb": "350696",
"wall": "0:00.22"
},
{
"binary": "dyn_Os",
"archive": "small.paq8px216",
"out_bytes": "53",
"out_sha256": "c5cf4213e9e9568b3b7b8bb42e219d30cc66ed3adce28aa5292201d96f928430",
"match": "YES",
"rss_kb": "350280",
"wall": "0:00.27"
},
{
"binary": "static_Os",
"archive": "small.paq8px216",
"out_bytes": "53",
"out_sha256": "c5cf4213e9e9568b3b7b8bb42e219d30cc66ed3adce28aa5292201d96f928430",
"match": "YES",
"rss_kb": "347376",
"wall": "0:00.29"
},
{
"binary": "dyn_Os_hard",
"archive": "small.paq8px216",
"out_bytes": "53",
"out_sha256": "c5cf4213e9e9568b3b7b8bb42e219d30cc66ed3adce28aa5292201d96f928430",
"match": "YES",
"rss_kb": "350380",
"wall": "0:00.33"
}
],
"end_to_end": {
"label": "dyn_Os",
"bundle_dir": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Os",
"bundle_bytes": 204244,
"bundle_sha256": "7a3899220efe888bd0d0e3affed875094afaea382ef1eda67eb1268a15b0071a",
"extracted_payload_sha256": "94cc29dbfb2dd1e9ac2ae96fd6699789935cc16585b145ed02ca83361e9764ce",
"members": {
"b.xz": {
"file_size": 198792,
"compress_size": 198792,
"sha256": "35d1f4ff1886a9d6a9714cede09cb7c0d74ce00ca3e3b652d48ab311fc9f6960"
},
"v.xz": {
"file_size": 4948,
"compress_size": 4948,
"sha256": "55e5f6255319235b24f25d9c73b65481359f3c4f48943e0cfd381d47b128a904"
},
"d": {
"file_size": 409,
"compress_size": 236,
"sha256": "9a9132bb31c01fafd6dd3b18176f57b668e718b9ea5bbe76568733fda501786b"
}
},
"cases": [
{
"name": "chain_p5fixture",
"rc": 0,
"out": "/root/hutter67/swarm67/B6-bundle/test/e2e_dyn_Os/out 17.bin",
"expected_sha": "4ab9cb28f180436abbd3a76866c13eb3dc0797888908947f298eec7fb546bcd6",
"expected_bytes": 17,
"what": "P5-certified 41 B paq8px -0L archive of blob35.bin -> full chain -> 17 B original",
"out_bytes": 17,
"out_sha256": "4ab9cb28f180436abbd3a76866c13eb3dc0797888908947f298eec7fb546bcd6",
"match": true,
"peak_rss_kb": 21732,
"wall": "0:00.12"
},
{
"name": "chain_blob35_m1",
"rc": 0,
"out": "/root/hutter67/swarm67/B6-bundle/test/e2e_dyn_Os/out 17b.bin",
"expected_sha": "4ab9cb28f180436abbd3a76866c13eb3dc0797888908947f298eec7fb546bcd6",
"expected_bytes": 17,
"what": "fresh -1 archive of blob35.bin -> full chain -> 17 B original",
"out_bytes": 17,
"out_sha256": "4ab9cb28f180436abbd3a76866c13eb3dc0797888908947f298eec7fb546bcd6",
"match": true,
"peak_rss_kb": 532868,
"wall": "0:00.93"
},
{
"name": "paq8px_stage_slice32k",
"rc": 0,
"out": "/root/hutter67/swarm67/B6-bundle/test/e2e_dyn_Os/out 32768.bin",
"expected_sha": "252c561ddcdefb261e605f8e20f726e43de6a31b1b4cfb240563f5f7d4fd28e8",
"expected_bytes": 32768,
"what": "real 32 KiB slice of blob67.bin, -1 archive -> shipped paq8px stage only",
"out_bytes": 32768,
"out_sha256": "252c561ddcdefb261e605f8e20f726e43de6a31b1b4cfb240563f5f7d4fd28e8",
"match": true,
"peak_rss_kb": 563896,
"wall": "0:12.46"
}
],
"all_match": true
},
"flagship_minus9": {
"status": "NOT RUN BY B6 - cancelled by director after V1's timing correction",
"reason": "a full -9 decode of the 60.6 MB blob costs ~7 h at ~4.3 GB RSS (P1 receipt: 1 MB blob arm at -9 = 7:36 min / 4,042,460 kB, linear in blob size), so it would duplicate the verifier's certification decode; the waiter B6 had armed under locked_run.sh 14400 was cancelled before it ever acquired full67 and the lock was never taken by B6",
"owner_of_the_full_archive_proof": "V1-verify-queue, single certification decode of exactly this artifact",
"artifact_to_certify": {
"path": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Os/decompressor.zip",
"bytes": 204244,
"sha256": "7a3899220efe888bd0d0e3affed875094afaea382ef1eda67eb1268a15b0071a"
},
"archive": "/root/hutter67/swarm67/P1-paq8px-cal/runs/full_blob_m9/blob_full.paq8px216",
"expected_blob_bytes": 60634501,
"expected_blob_sha256": "9c869f141bc91a9b164e2cd4faecfea5bb648bd996c8c0f5e865b4e4010be425",
"expected_enwik8_bytes": 100000000,
"expected_enwik8_sha256": "2b49720ec4d78c3c9fabaee6e4179a5e997302b3a70029f30f2d582218c024a8",
"performance_envelope": {
"level1_decode_peak_rss_kb": 563896,
"original_binary_minus9_1mb_arm_peak_rss_kb": 4042460,
"estimated_full_decode": "~7-9 h wall, ~4.3 GB RSS"
}
}
},
"artifacts": {
"bundle_zip": "/root/hutter67/swarm67/B6-bundle/out/bundle_dyn_Os/decompressor.zip",
"binary": "/root/hutter67/swarm67/B6-bundle/out/build/dyn_Os",
"packer": "work/make_bundle.py",
"build_script": "work/build_variant.sh",
"e2e_proof": "work/e2e_bundle.sh -> test/e2e_dyn_Os.json",
"decode_proof": "work/decode_proof.sh -> test/decode_proof.tsv",
"independent_verify": "test/final_verify.json (system unzip, fresh directory)"
}
}

Xet Storage Details

Size:
19.1 kB
·
Xet hash:
456e1201c75501dbdd5945d269924c032a60d89fd59e9de15707816eb79f78f9

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.