File size: 8,655 Bytes
6f8c80a ba48d54 6f8c80a 1435032 6f8c80a ba48d54 6f8c80a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | """Compare paired inference runs; preserve raw outputs and derive honest metrics."""
import argparse
import csv
import html
import json
import math
import statistics
from pathlib import Path
import numpy as np
from PIL import Image
from scripts.integrity import sha256, validate_pair
from scripts.provenance import validate_roles
def load_records(root):
rows = [json.loads(line) for line in (root / 'records.jsonl').read_text(encoding='utf-8').splitlines() if line]
records = {}
for row in rows:
key = (row['case_id'], row['seed'])
if key in records:
raise ValueError(f'Duplicate record: {key}')
if sha256(root / row['image']) != row['image_sha256']:
raise ValueError(f'Image hash mismatch: {key}')
records[key] = row
if not records:
raise ValueError('Empty benchmark run')
return records
def compare_pixels(a_path, b_path):
with Image.open(a_path) as a, Image.open(b_path) as b:
if a.size != b.size:
raise ValueError('Image dimensions differ')
a, b = a.convert('RGBA'), b.convert('RGBA')
aa, bb = np.asarray(a, dtype=np.float32) / 255, np.asarray(b, dtype=np.float32) / 255
# White compositing avoids invisible RGB values dominating a transparency comparison.
ac = aa[..., :3] * aa[..., 3:] + 1 - aa[..., 3:]
bc = bb[..., :3] * bb[..., 3:] + 1 - bb[..., 3:]
mse = float(np.mean((ac - bc) ** 2))
return {'rgb_mae_white': float(np.mean(np.abs(ac - bc))),
'rgb_psnr_white_db': None if mse == 0 else -10 * math.log10(mse),
'alpha_mae': float(np.mean(np.abs(aa[..., 3] - bb[..., 3])))}
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--baseline', default='artifacts/eval/bf16')
ap.add_argument('--quantized', default='artifacts/eval/int8')
ap.add_argument('--output', default='artifacts/eval')
args = ap.parse_args()
base, quant, out = Path(args.baseline), Path(args.quantized), Path(args.output)
a, b = load_records(base), load_records(quant)
if a.keys() != b.keys():
raise ValueError('Benchmark cases/seeds are incomplete or mismatched')
ea = json.loads((base / 'environment.json').read_text())
eb = json.loads((quant / 'environment.json').read_text())
validate_roles(ea['model_identity'], eb['model_identity'])
for key in ('offload_aux_fix', 'benchmark_sha256', 'runtime_helper_sha256'):
if ea.get(key) != eb.get(key):
raise ValueError(f'Benchmark implementation mismatch: {key}; report diagnostic runs separately')
for key in ('gpu', 'cuda', 'packages', 'offload', 'warmup', 'generator_device', 'cases_sha256'):
if ea[key] != eb[key]:
raise ValueError(f'Runtime mismatch: {key}')
rows = []
for key in a:
x, y = a[key], b[key]
validate_pair(x, y)
rows.append({'case_id': key[0], 'seed': key[1],
'bf16_seconds': x['seconds'], 'int8_seconds': y['seconds'],
'speed_ratio_bf16_over_int8': x['seconds'] / y['seconds'],
'bf16_peak_allocated_gib': x['peak_allocated_bytes'] / 2**30,
'int8_peak_allocated_gib': y['peak_allocated_bytes'] / 2**30,
**compare_pixels(base / x['image'], quant / y['image'])})
summary = {'pairs': len(rows), 'cases': len({r['case_id'] for r in rows}),
'bf16_mean_seconds': statistics.mean(r['bf16_seconds'] for r in rows),
'int8_mean_seconds': statistics.mean(r['int8_seconds'] for r in rows),
'bf16_max_allocated_gib': max(r['bf16_peak_allocated_gib'] for r in rows),
'int8_max_allocated_gib': max(r['int8_peak_allocated_gib'] for r in rows),
'width': next(iter(a.values()))['width'], 'height': next(iter(a.values()))['height'],
'steps': next(iter(a.values()))['steps'], 'warmup': ea['warmup'],
'bf16_weight_bytes': sum(f['size'] for f in ea['model_identity']['files'] if f['path'].endswith('.safetensors')),
'int8_weight_bytes': sum(f['size'] for f in eb['model_identity']['files'] if f['path'].endswith('.safetensors'))}
out.mkdir(parents=True, exist_ok=True)
(out / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8')
with (out / 'comparison.csv').open('w', encoding='utf-8', newline='') as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
lines = ['# Informal release evaluation: BF16 / INT8 comparison', '',
'This community evaluation is provided for reference only and does not represent any official evaluation.', '',
f'{len(rows)} paired outputs on {ea["gpu"]}; {summary["width"]}×{summary["height"]}, '
f'{summary["steps"]} steps, offload={ea["offload"]}, CFG=1, KV cache enabled.', '',
('Warmup excluded. ' if ea['warmup'] else 'No warmup performed. ') +
'Pixel metrics measure drift, not semantic quality. ' +
('The same BF16 portrait is used as input for every editing pair. ' if any(k[0] == 'edit' for k in a) else '') +
'The suite is small and does not establish a general quality ranking.', '',
'| Case | Seed | BF16 s | INT8 s | BF16 peak GiB | INT8 peak GiB | RGB MAE (white) |',
'|---|---:|---:|---:|---:|---:|---:|']
gallery = ['<!doctype html><meta charset="utf-8"><title>BF16 / INT8 comparison</title>',
'<style>body{font:16px system-ui;margin:32px;max-width:1500px}section{margin:40px 0}'
'.pair{display:grid;grid-template-columns:1fr 1fr;gap:20px}img{width:100%;background:'
'repeating-conic-gradient(#ddd 0 25%,#fff 0 50%) 0/24px 24px}pre{white-space:pre-wrap}</style>',
'<h1>BF16 / INT8 comparison</h1><p>Left: BF16. Right: saved INT8. '
'Identical settings and seeds. Alpha displayed over checkerboard.</p>']
if (out / 'young_woman/summary.json').exists():
lines[2:2] = ['Additional adult Chinese woman portrait pairs are reported separately: '
'[supplement](young_woman/report.md).', '']
gallery.append('<p><a href="young_woman/comparison.html">Additional adult Chinese woman portrait pairs</a></p>')
for r in rows:
lines.append(f'| {r["case_id"]} | {r["seed"]} | {r["bf16_seconds"]:.2f} | '
f'{r["int8_seconds"]:.2f} | {r["bf16_peak_allocated_gib"]:.2f} | '
f'{r["int8_peak_allocated_gib"]:.2f} | {r["rgb_mae_white"]:.4f} |')
key = (r['case_id'], r['seed'])
import os
left = Path(os.path.relpath(base / a[key]['image'], out)).as_posix()
right = Path(os.path.relpath(quant / b[key]['image'], out)).as_posix()
gallery.append(f'<section><h2>{html.escape(r["case_id"])} — seed {r["seed"]}</h2>'
f'<pre>{html.escape(a[key]["prompt"])}</pre><div class="pair">'
f'<a href="{html.escape(left, quote=True)}"><img src="{html.escape(left, quote=True)}" alt="BF16"></a>'
f'<a href="{html.escape(right, quote=True)}"><img src="{html.escape(right, quote=True)}" alt="INT8"></a></div></section>')
lines.extend(['', '## Summary', '',
f'Mean latency: BF16 {summary["bf16_mean_seconds"]:.2f}s; '
f'INT8 {summary["int8_mean_seconds"]:.2f}s.', '',
'[Interactive-sized side-by-side gallery](comparison.html). '
'Raw data: comparison.csv, bf16/records.jsonl, int8/records.jsonl. '
'Environment records include package versions and loading overhead.', '',
'[Qualitative observations](qualitative.md).', '',
'## Interpretation limits', '',
'- No CLIP, OCR, human preference, FID or benchmark leaderboard score is claimed.',
'- Peak CUDA allocated/reserved memory excludes other processes and display usage.',
'- Measured call latency includes transfers; disk writing and model loading are excluded.',
'- Paired images may diverge with quantization even when both remain plausible.',
'- Editing and transparency should be inspected in the gallery, including preserved details.'])
(out / 'report.md').write_text('\n'.join(lines) + '\n', encoding='utf-8')
(out / 'comparison.html').write_text('\n'.join(gallery), encoding='utf-8')
print(json.dumps(summary, indent=2))
if __name__ == '__main__':
main()
|