scripts/report.py

"""Compare paired inference runs; preserve raw outputs and derive honest metrics."""
import argparse
import csv
import html
import json
import math
import statistics
from pathlib import Path

import numpy as np
from PIL import Image
from scripts.integrity import sha256, validate_pair
from scripts.provenance import validate_roles


def load_records(root):
    rows = [json.loads(line) for line in (root / 'records.jsonl').read_text(encoding='utf-8').splitlines() if line]
    records = {}
    for row in rows:
        key = (row['case_id'], row['seed'])
        if key in records:
            raise ValueError(f'Duplicate record: {key}')
        if sha256(root / row['image']) != row['image_sha256']:
            raise ValueError(f'Image hash mismatch: {key}')
        records[key] = row
    if not records:
        raise ValueError('Empty benchmark run')
    return records


def compare_pixels(a_path, b_path):
    with Image.open(a_path) as a, Image.open(b_path) as b:
        if a.size != b.size:
            raise ValueError('Image dimensions differ')
        a, b = a.convert('RGBA'), b.convert('RGBA')
        aa, bb = np.asarray(a, dtype=np.float32) / 255, np.asarray(b, dtype=np.float32) / 255
        # White compositing avoids invisible RGB values dominating a transparency comparison.
        ac = aa[..., :3] * aa[..., 3:] + 1 - aa[..., 3:]
        bc = bb[..., :3] * bb[..., 3:] + 1 - bb[..., 3:]
        mse = float(np.mean((ac - bc) ** 2))
        return {'rgb_mae_white': float(np.mean(np.abs(ac - bc))),
                'rgb_psnr_white_db': None if mse == 0 else -10 * math.log10(mse),
                'alpha_mae': float(np.mean(np.abs(aa[..., 3] - bb[..., 3])))}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--baseline', default='artifacts/eval/bf16')
    ap.add_argument('--quantized', default='artifacts/eval/int8')
    ap.add_argument('--output', default='artifacts/eval')
    args = ap.parse_args()
    base, quant, out = Path(args.baseline), Path(args.quantized), Path(args.output)
    a, b = load_records(base), load_records(quant)
    if a.keys() != b.keys():
        raise ValueError('Benchmark cases/seeds are incomplete or mismatched')
    ea = json.loads((base / 'environment.json').read_text())
    eb = json.loads((quant / 'environment.json').read_text())
    validate_roles(ea['model_identity'], eb['model_identity'])
    for key in ('offload_aux_fix', 'benchmark_sha256', 'runtime_helper_sha256'):
        if ea.get(key) != eb.get(key):
            raise ValueError(f'Benchmark implementation mismatch: {key}; report diagnostic runs separately')
    for key in ('gpu', 'cuda', 'packages', 'offload', 'warmup', 'generator_device', 'cases_sha256'):
        if ea[key] != eb[key]:
            raise ValueError(f'Runtime mismatch: {key}')
    rows = []
    for key in a:
        x, y = a[key], b[key]
        validate_pair(x, y)
        rows.append({'case_id': key[0], 'seed': key[1],
                     'bf16_seconds': x['seconds'], 'int8_seconds': y['seconds'],
                     'speed_ratio_bf16_over_int8': x['seconds'] / y['seconds'],
                     'bf16_peak_allocated_gib': x['peak_allocated_bytes'] / 2**30,
                     'int8_peak_allocated_gib': y['peak_allocated_bytes'] / 2**30,
                     **compare_pixels(base / x['image'], quant / y['image'])})
    summary = {'pairs': len(rows), 'cases': len({r['case_id'] for r in rows}),
               'bf16_mean_seconds': statistics.mean(r['bf16_seconds'] for r in rows),
               'int8_mean_seconds': statistics.mean(r['int8_seconds'] for r in rows),
               'bf16_max_allocated_gib': max(r['bf16_peak_allocated_gib'] for r in rows),
               'int8_max_allocated_gib': max(r['int8_peak_allocated_gib'] for r in rows),
               'width': next(iter(a.values()))['width'], 'height': next(iter(a.values()))['height'],
               'steps': next(iter(a.values()))['steps'], 'warmup': ea['warmup'],
               'bf16_weight_bytes': sum(f['size'] for f in ea['model_identity']['files'] if f['path'].endswith('.safetensors')),
               'int8_weight_bytes': sum(f['size'] for f in eb['model_identity']['files'] if f['path'].endswith('.safetensors'))}
    out.mkdir(parents=True, exist_ok=True)
    (out / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8')
    with (out / 'comparison.csv').open('w', encoding='utf-8', newline='') as f:
        writer = csv.DictWriter(f, fieldnames=list(rows[0]))
        writer.writeheader()
        writer.writerows(rows)
    lines = ['# Informal release evaluation: BF16 / INT8 comparison', '',
             'This community evaluation is provided for reference only and does not represent any official evaluation.', '',
             f'{len(rows)} paired outputs on {ea["gpu"]}; {summary["width"]}×{summary["height"]}, '
             f'{summary["steps"]} steps, offload={ea["offload"]}, CFG=1, KV cache enabled.', '',
             ('Warmup excluded. ' if ea['warmup'] else 'No warmup performed. ') +
             'Pixel metrics measure drift, not semantic quality. ' +
             ('The same BF16 portrait is used as input for every editing pair. ' if any(k[0] == 'edit' for k in a) else '') +
             'The suite is small and does not establish a general quality ranking.', '',
             '| Case | Seed | BF16 s | INT8 s | BF16 peak GiB | INT8 peak GiB | RGB MAE (white) |',
             '|---|---:|---:|---:|---:|---:|---:|']
    gallery = ['<!doctype html><meta charset="utf-8"><title>BF16 / INT8 comparison</title>',
               '<style>body{font:16px system-ui;margin:32px;max-width:1500px}section{margin:40px 0}'
               '.pair{display:grid;grid-template-columns:1fr 1fr;gap:20px}img{width:100%;background:'
               'repeating-conic-gradient(#ddd 0 25%,#fff 0 50%) 0/24px 24px}pre{white-space:pre-wrap}</style>',
               '<h1>BF16 / INT8 comparison</h1><p>Left: BF16. Right: saved INT8. '
               'Identical settings and seeds. Alpha displayed over checkerboard.</p>']
    if (out / 'young_woman/summary.json').exists():
        lines[2:2] = ['Additional adult Chinese woman portrait pairs are reported separately: '
                      '[supplement](young_woman/report.md).', '']
        gallery.append('<p><a href="young_woman/comparison.html">Additional adult Chinese woman portrait pairs</a></p>')
    for r in rows:
        lines.append(f'| {r["case_id"]} | {r["seed"]} | {r["bf16_seconds"]:.2f} | '
                     f'{r["int8_seconds"]:.2f} | {r["bf16_peak_allocated_gib"]:.2f} | '
                     f'{r["int8_peak_allocated_gib"]:.2f} | {r["rgb_mae_white"]:.4f} |')
        key = (r['case_id'], r['seed'])
        import os
        left = Path(os.path.relpath(base / a[key]['image'], out)).as_posix()
        right = Path(os.path.relpath(quant / b[key]['image'], out)).as_posix()
        gallery.append(f'<section><h2>{html.escape(r["case_id"])} — seed {r["seed"]}</h2>'
                       f'<pre>{html.escape(a[key]["prompt"])}</pre><div class="pair">'
                       f'<a href="{html.escape(left, quote=True)}"><img src="{html.escape(left, quote=True)}" alt="BF16"></a>'
                       f'<a href="{html.escape(right, quote=True)}"><img src="{html.escape(right, quote=True)}" alt="INT8"></a></div></section>')
    lines.extend(['', '## Summary', '',
                  f'Mean latency: BF16 {summary["bf16_mean_seconds"]:.2f}s; '
                  f'INT8 {summary["int8_mean_seconds"]:.2f}s.', '',
                  '[Interactive-sized side-by-side gallery](comparison.html). '
                  'Raw data: comparison.csv, bf16/records.jsonl, int8/records.jsonl. '
                  'Environment records include package versions and loading overhead.', '',
                  '[Qualitative observations](qualitative.md).', '',
                  '## Interpretation limits', '',
                  '- No CLIP, OCR, human preference, FID or benchmark leaderboard score is claimed.',
                  '- Peak CUDA allocated/reserved memory excludes other processes and display usage.',
                  '- Measured call latency includes transfers; disk writing and model loading are excluded.',
                  '- Paired images may diverge with quantization even when both remain plausible.',
                  '- Editing and transparency should be inspected in the gallery, including preserved details.'])
    (out / 'report.md').write_text('\n'.join(lines) + '\n', encoding='utf-8')
    (out / 'comparison.html').write_text('\n'.join(gallery), encoding='utf-8')
    print(json.dumps(summary, indent=2))


if __name__ == '__main__':
    main()