"""Compare paired inference runs; preserve raw outputs and derive honest metrics."""
import argparse
import csv
import html
import json
import math
import statistics
from pathlib import Path
import numpy as np
from PIL import Image
from scripts.integrity import sha256, validate_pair
from scripts.provenance import validate_roles
def load_records(root):
rows = [json.loads(line) for line in (root / 'records.jsonl').read_text(encoding='utf-8').splitlines() if line]
records = {}
for row in rows:
key = (row['case_id'], row['seed'])
if key in records:
raise ValueError(f'Duplicate record: {key}')
if sha256(root / row['image']) != row['image_sha256']:
raise ValueError(f'Image hash mismatch: {key}')
records[key] = row
if not records:
raise ValueError('Empty benchmark run')
return records
def compare_pixels(a_path, b_path):
with Image.open(a_path) as a, Image.open(b_path) as b:
if a.size != b.size:
raise ValueError('Image dimensions differ')
a, b = a.convert('RGBA'), b.convert('RGBA')
aa, bb = np.asarray(a, dtype=np.float32) / 255, np.asarray(b, dtype=np.float32) / 255
# White compositing avoids invisible RGB values dominating a transparency comparison.
ac = aa[..., :3] * aa[..., 3:] + 1 - aa[..., 3:]
bc = bb[..., :3] * bb[..., 3:] + 1 - bb[..., 3:]
mse = float(np.mean((ac - bc) ** 2))
return {'rgb_mae_white': float(np.mean(np.abs(ac - bc))),
'rgb_psnr_white_db': None if mse == 0 else -10 * math.log10(mse),
'alpha_mae': float(np.mean(np.abs(aa[..., 3] - bb[..., 3])))}
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--baseline', default='artifacts/eval/bf16')
ap.add_argument('--quantized', default='artifacts/eval/int8')
ap.add_argument('--output', default='artifacts/eval')
args = ap.parse_args()
base, quant, out = Path(args.baseline), Path(args.quantized), Path(args.output)
a, b = load_records(base), load_records(quant)
if a.keys() != b.keys():
raise ValueError('Benchmark cases/seeds are incomplete or mismatched')
ea = json.loads((base / 'environment.json').read_text())
eb = json.loads((quant / 'environment.json').read_text())
validate_roles(ea['model_identity'], eb['model_identity'])
for key in ('offload_aux_fix', 'benchmark_sha256', 'runtime_helper_sha256'):
if ea.get(key) != eb.get(key):
raise ValueError(f'Benchmark implementation mismatch: {key}; report diagnostic runs separately')
for key in ('gpu', 'cuda', 'packages', 'offload', 'warmup', 'generator_device', 'cases_sha256'):
if ea[key] != eb[key]:
raise ValueError(f'Runtime mismatch: {key}')
rows = []
for key in a:
x, y = a[key], b[key]
validate_pair(x, y)
rows.append({'case_id': key[0], 'seed': key[1],
'bf16_seconds': x['seconds'], 'int8_seconds': y['seconds'],
'speed_ratio_bf16_over_int8': x['seconds'] / y['seconds'],
'bf16_peak_allocated_gib': x['peak_allocated_bytes'] / 2**30,
'int8_peak_allocated_gib': y['peak_allocated_bytes'] / 2**30,
**compare_pixels(base / x['image'], quant / y['image'])})
summary = {'pairs': len(rows), 'cases': len({r['case_id'] for r in rows}),
'bf16_mean_seconds': statistics.mean(r['bf16_seconds'] for r in rows),
'int8_mean_seconds': statistics.mean(r['int8_seconds'] for r in rows),
'bf16_max_allocated_gib': max(r['bf16_peak_allocated_gib'] for r in rows),
'int8_max_allocated_gib': max(r['int8_peak_allocated_gib'] for r in rows),
'width': next(iter(a.values()))['width'], 'height': next(iter(a.values()))['height'],
'steps': next(iter(a.values()))['steps'], 'warmup': ea['warmup'],
'bf16_weight_bytes': sum(f['size'] for f in ea['model_identity']['files'] if f['path'].endswith('.safetensors')),
'int8_weight_bytes': sum(f['size'] for f in eb['model_identity']['files'] if f['path'].endswith('.safetensors'))}
out.mkdir(parents=True, exist_ok=True)
(out / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8')
with (out / 'comparison.csv').open('w', encoding='utf-8', newline='') as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
lines = ['# Informal release evaluation: BF16 / INT8 comparison', '',
'This community evaluation is provided for reference only and does not represent any official evaluation.', '',
f'{len(rows)} paired outputs on {ea["gpu"]}; {summary["width"]}×{summary["height"]}, '
f'{summary["steps"]} steps, offload={ea["offload"]}, CFG=1, KV cache enabled.', '',
('Warmup excluded. ' if ea['warmup'] else 'No warmup performed. ') +
'Pixel metrics measure drift, not semantic quality. ' +
('The same BF16 portrait is used as input for every editing pair. ' if any(k[0] == 'edit' for k in a) else '') +
'The suite is small and does not establish a general quality ranking.', '',
'| Case | Seed | BF16 s | INT8 s | BF16 peak GiB | INT8 peak GiB | RGB MAE (white) |',
'|---|---:|---:|---:|---:|---:|---:|']
gallery = ['BF16 / INT8 comparison',
'',
'BF16 / INT8 comparison
Left: BF16. Right: saved INT8. '
'Identical settings and seeds. Alpha displayed over checkerboard.
']
if (out / 'young_woman/summary.json').exists():
lines[2:2] = ['Additional adult Chinese woman portrait pairs are reported separately: '
'[supplement](young_woman/report.md).', '']
gallery.append('Additional adult Chinese woman portrait pairs
')
for r in rows:
lines.append(f'| {r["case_id"]} | {r["seed"]} | {r["bf16_seconds"]:.2f} | '
f'{r["int8_seconds"]:.2f} | {r["bf16_peak_allocated_gib"]:.2f} | '
f'{r["int8_peak_allocated_gib"]:.2f} | {r["rgb_mae_white"]:.4f} |')
key = (r['case_id'], r['seed'])
import os
left = Path(os.path.relpath(base / a[key]['image'], out)).as_posix()
right = Path(os.path.relpath(quant / b[key]['image'], out)).as_posix()
gallery.append(f'{html.escape(r["case_id"])} — seed {r["seed"]}
'
f'{html.escape(a[key]["prompt"])}'
f'
})
'
f'
})