scripts/model_card.py

"""Render complete bilingual evaluation sections from the measured evidence."""
import csv
import json
from pathlib import Path
from urllib.parse import quote

from scripts.report import load_records


LABELS = {
    'young_chinese_woman': ('Young Chinese adult woman portrait (supplement)', '中国年轻成年女性肖像（补充）'),
    'portrait': ('Older adult portrait', '老年女性肖像'),
    'english_text': ('English lettering', '英文文字生成'),
    'chinese_text': ('Chinese lettering', '中文文字生成'),
    'composition': ('Counting and spatial arrangement', '物体数量与空间关系'),
    'texture': ('Feather and branch texture', '羽毛与枝干纹理'),
    'rgba': ('RGBA transparency', 'RGBA 透明图'),
    'edit': ('Image editing: sweater recoloring', '图像编辑：毛衣改色'),
}
OBSERVATIONS = {
    'young_chinese_woman': (
        'Both seeds retain the broad framing, clothing and window lighting. Expression, facial contours, skin texture and individual hair strands change. No obvious large facial distortion was seen in these four outputs; this is not an identity-preservation score.',
        '两个种子的大体构图、衣着和窗边光线相近，表情、面部轮廓、皮肤纹理和发丝有变化。检查的四张图未见明显面部结构畸变；这不是身份保持评分。'),
    'portrait': (
        'Both depict the requested older woman, blue sweater and window. Face shape, wrinkles, framing and fabric details shift.',
        '两种精度均保留老年女性、蓝色毛衣与窗边构图；脸部形状、皱纹、取景和织物细节存在偏移。'),
    'english_text': (
        'Both inspected outputs visibly render CREATE WITH LIGHT and September 2026. Layout and desk objects differ.',
        '检查样例中的 CREATE WITH LIGHT 和 September 2026 均可读，版式与桌面物件有变化。'),
    'chinese_text': (
        'Both inspected outputs render the requested main title and subtitle, but also add unwanted, partly nonsensical footer text despite the no-extra-text instruction.',
        '检查样例的主标题与副标题均可读，但两种精度都添加了多余、部分不通顺的页脚文字，没有完全遵守“不添加其他文字”。'),
    'composition': (
        'Both show three cups in red/blue/yellow order and one green apple in front of the blue cup. Object sizes and spacing differ.',
        '两种精度均呈现红、蓝、黄三杯的顺序及蓝杯前的一个绿苹果；物体大小和间距存在变化。'),
    'texture': (
        'Both depict a plausible kingfisher; head angle, branch orientation and feather texture differ. Pixel drift alone does not establish worse semantic quality.',
        '两种精度都生成了合理的翠鸟形象；头部角度、枝干朝向与羽毛纹理明显变化。像素差异本身不能证明语义质量更差。'),
    'rgba': (
        'Both have a real varying alpha channel and a white sticker outline. Sampled background alpha is 0–1/255 and the subject is opaque. Page background colors are not part of the PNG; click through to inspect the original RGBA file.',
        '两种精度均有真实变化的 alpha 通道和白色贴纸轮廓。抽查背景 alpha 为 0–1/255，主体不透明。网页底色不属于 PNG 内容；可点击原图检查透明通道。'),
    'edit': (
        'Both shift the sweater toward red/burgundy, but both also show conspicuous oversharpening and contrast/texture changes across the image. Faithful preservation of lighting and skin texture is not achieved. The cause is not identified or attributed specifically to INT8.',
        '两种精度都把毛衣改为红色／酒红色，但整幅图像均出现明显过度锐化和对比度、纹理变化，未忠实保持原光线和皮肤质感。尚未确定原因，也不能将问题单独归因于 INT8。'),
}


def original_url(platform, path):
    if platform == 'modelscope':
        return ('https://modelscope.cn/api/v1/models/iximbox/Image21-INT8/repo'
                '?Revision=master&FilePath=' + quote(path, safe=''))
    return 'https://huggingface.co/ixim/Image21-INT8/resolve/main/' + quote(path, safe='/')


def _render_all(evaluation, platform):
    root = Path(evaluation)
    zh = platform == 'modelscope'
    def tr(en, cn):
        return cn if zh else en
    def label(case):
        return LABELS[case][int(zh)]
    groups = []
    for relative in ('', 'young_woman'):
        folder = root / relative
        records = {p: load_records(folder / p) for p in ('bf16', 'int8')}
        env = {p: json.loads((folder / p / 'environment.json').read_text()) for p in records}
        with (folder / 'comparison.csv').open(encoding='utf-8', newline='') as f:
            metrics = list(csv.DictReader(f))
        groups.append((relative, records, env, metrics))
    core, extra = groups
    rows = [(g, row) for g in groups for row in g[3]]
    count = len(rows)
    hardware_path = root / 'hardware.json'
    hardware = json.loads(hardware_path.read_text()) if hardware_path.exists() else {}
    gpu_detail = hardware.get('gpu_driver_memory', '').split(', ', 1)
    driver_memory = gpu_detail[1].replace(', ', ' / ') + ' MiB' if len(gpu_detail) == 2 else tr('Not recorded', '未记录')
    ram = f'{hardware["system_ram_bytes"]/2**30:.1f} GiB' if hardware.get('system_ram_bytes') else tr('Not recorded', '未记录')
    extra_summary = json.loads((root / 'young_woman/summary.json').read_text())
    lines = [tr('### Evaluation scope and runtime', '### 评测范围与运行环境'), '',
             tr(f'The main suite has {len(core[3])} pairs; the separate portrait supplement has {len(extra[3])} pairs. '
                f'All {count} pairs / {count * 2} original PNG images from these suites are shown below. Supplementary measurements are kept separate from the main summary average.',
                f'主评测七类用例共 {len(core[3])} 对，肖像专项另有 {len(extra[3])} 对。下文完整展示这两套用例的 {count} 对、{count * 2} 张原始 PNG。专项数据单独报告，不混入七类均值。'), '',
             tr('All previews link to their full-resolution original PNG. Click an image to open it; no crop, retouching or regenerated substitute is used.',
                '**点击任意样图即可打开原尺寸 PNG 大图。** 所有图片均为实际推理输出，没有裁剪、修图或使用其他模型替代。'), '',
             tr('| Setting | Recorded value |', '| 设置 | 实测记录 |'), '|---|---|',
             f'| GPU | {core[2]["bf16"]["gpu"]} |',
             tr('| Driver / board memory | ', '| 驱动 / 板载显存 | ') + driver_memory + ' |',
             tr('| System RAM | ', '| 系统内存 | ') + ram + ' |',
             f'| OS | {core[2]["bf16"]["platform"]} |',
             f'| Python / CUDA | {core[2]["bf16"]["python"]} / {core[2]["bf16"]["cuda"]} |',
             tr('| Generation | 1024×1024; 40 steps; CFG 1.0; KV cache enabled |', '| 生成设置 | 1024×1024；40 步；CFG 1.0；启用 KV 缓存 |'),
             tr('| RNG / seeds | CPU torch.Generator; 42, 123 |', '| 随机数 / 种子 | CPU torch.Generator；42、123 |'),
             tr('| Offload / attention | model CPU offload / default SDPA |', '| 卸载 / 注意力 | model CPU offload / 默认 SDPA |'),
             tr('| Warmup / order | One excluded full-settings warmup per precision per suite; BF16 then INT8 |', '| 预热 / 顺序 | 每个精度、每套用例各一次完整设置预热，不计入结果；先 BF16 后 INT8 |'), '',
             tr('| Package | Version |', '| 软件包 | 版本 |'), '|---|---|']
    lines.extend(f'| {name} | {version} |' for name, version in core[2]['bf16']['packages'].items())
    lines.extend(['', tr('### Supplementary portrait summary', '### 肖像专项汇总'), '',
                  tr('| Metric | BF16 | INT8 |', '| 指标 | BF16 | INT8 |'), '|---|---:|---:|',
                  tr('| Mean call latency (s) | ', '| 平均调用耗时（秒） | ') + f'{extra_summary["bf16_mean_seconds"]:.2f} | {extra_summary["int8_mean_seconds"]:.2f} |',
                  tr('| Maximum CUDA allocated (GiB) | ', '| 已分配显存最高值（GiB） | ') + f'{extra_summary["bf16_max_allocated_gib"]:.2f} | {extra_summary["int8_max_allocated_gib"]:.2f} |', '',
                  tr('### Per-image timing and memory', '### 每张图片的耗时与显存'), '',
                  tr('| Case | Seed | BF16 s | INT8 s | BF16 allocated GiB | INT8 allocated GiB |', '| 用例 | 种子 | BF16 秒 | INT8 秒 | BF16 已分配 GiB | INT8 已分配 GiB |'),
                  '|---|---:|---:|---:|---:|---:|'])
    for _, r in rows:
        lines.append(f'| {label(r["case_id"])} | {r["seed"]} | {float(r["bf16_seconds"]):.2f} | {float(r["int8_seconds"]):.2f} | {float(r["bf16_peak_allocated_gib"]):.2f} | {float(r["int8_peak_allocated_gib"]):.2f} |')
    lines.extend(['', tr('### Pixel drift (not a semantic quality score)', '### 像素差异（不能等同于语义质量评分）'), '',
                  tr('RGB MAE and PSNR use a white composite so invisible RGB values in transparent pixels do not dominate the comparison. Alpha MAE is measured separately.',
                     'RGB MAE 与 PSNR 在白底合成图上计算，避免透明像素中不可见的 RGB 值主导结果；alpha MAE 单独计算。'), '',
                  tr('| Case | Seed | RGB MAE | RGB PSNR dB | Alpha MAE |', '| 用例 | 种子 | RGB MAE | RGB PSNR dB | Alpha MAE |'), '|---|---:|---:|---:|---:|'])
    for _, r in rows:
        psnr = f'{float(r["rgb_psnr_white_db"]):.2f}' if r['rgb_psnr_white_db'] else '∞'
        lines.append(f'| {label(r["case_id"])} | {r["seed"]} | {float(r["rgb_mae_white"]):.5f} | {psnr} | {float(r["alpha_mae"]):.6f} |')
    lines.extend(['', tr('### Additional memory telemetry', '### 补充内存记录'), '',
                  tr('Loading is measured separately: BF16 peaks at '
                     f'{core[2]["bf16"]["load_peak_allocated_bytes"]/2**30:.2f} GiB CUDA allocated (loaded to CPU), '
                     f'INT8 at {core[2]["int8"]["load_peak_allocated_bytes"]/2**30:.2f} GiB with the bundled sequential loader. '
                     'Neither value is a minimum supported GPU capacity.',
                     '加载阶段单独记录：BF16 加载到 CPU，CUDA 分配峰值为 '
                     f'{core[2]["bf16"]["load_peak_allocated_bytes"]/2**30:.2f} GiB；INT8 使用配套顺序加载函数时为 '
                     f'{core[2]["int8"]["load_peak_allocated_bytes"]/2**30:.2f} GiB。两者均不等于最低支持显卡容量。'), '',
                  tr('Reserved memory is the PyTorch CUDA allocator peak. CPU RSS is the process snapshot after inference, not a peak.',
                     '保留显存为 PyTorch CUDA 分配器峰值。CPU RSS 为推理后的进程内存快照，不是峰值。'), '',
                  tr('| Case | Seed | BF16 reserved GiB | INT8 reserved GiB | BF16 RSS GiB | INT8 RSS GiB |', '| 用例 | 种子 | BF16 保留 GiB | INT8 保留 GiB | BF16 RSS GiB | INT8 RSS GiB |'),
                  '|---|---:|---:|---:|---:|---:|'])
    for g, r in rows:
        key = (r['case_id'], int(r['seed']))
        a, b = g[1]['bf16'][key], g[1]['int8'][key]
        values = [a['peak_reserved_bytes'], b['peak_reserved_bytes'], a['process_rss_after_bytes'], b['process_rss_after_bytes']]
        lines.append(f'| {label(r["case_id"])} | {r["seed"]} | ' + ' | '.join(f'{v/2**30:.2f}' for v in values) + ' |')
    lines.extend(['', tr('### All paired samples, prompts and observations', '### 全部对照样图、提示词与观察'), '',
                  tr('Left: BF16; right: saved and reloaded INT8. Same prompt and seed within each pair. The main seven-case observations primarily inspect seed 42; both supplementary seeds were inspected. This is not a blinded preference study or an OCR/CLIP/FID score.',
                     '每组左侧为 BF16，右侧为保存后重新加载的 INT8；同一组使用相同提示词与种子。七类主评测的定性观察主要基于种子 42；补充肖像的两个种子均已检查。这不是盲测偏好研究或 OCR／CLIP／FID 评分。'), ''])
    for g in (extra, core):
        case_ids = list(dict.fromkeys(r['case_id'] for r in g[3]))
        for case in case_ids:
            first = g[1]['bf16'][(case,42)]
            lines.extend([f'#### {label(case)}', '', tr('Exact measured prompt:', '实测原始提示词：'), '', '```text', first['prompt'], '```', '', OBSERVATIONS[case][int(zh)], ''])
            if case == 'edit':
                url = original_url(platform, 'evaluation/bf16/portrait-s42.png')
                lines.extend([tr(f'Both variants use [this identical BF16 portrait input]({url}).', f'两种精度均使用[同一张 BF16 肖像输入]({url})。'), ''])
            for seed in (42,123):
                a, b = g[1]['bf16'][(case,seed)], g[1]['int8'][(case,seed)]
                urls = [original_url(platform, '/'.join(x for x in ('evaluation',g[0],p,r['image']) if x)) for p,r in (('bf16',a),('int8',b))]
                caption = tr('Click to open original PNG', '点击查看原尺寸 PNG')
                alts = [f'{label(case)} · {p} · seed {seed}' for p in ('BF16','INT8')]
                lines.extend([f'**{tr("Seed", "种子")} {seed}**', '', f'| BF16 · {caption} | INT8 · {caption} |', '|---|---|',
                              '| ' + ' | '.join(f'[![{alt}]({url})]({url})' for alt,url in zip(alts,urls)) + ' |', '',
                              tr('SHA256 prefixes', 'SHA256 前 12 位') + f': BF16 `{a["image_sha256"][:12]}` / INT8 `{b["image_sha256"][:12]}`.', ''])
    lines.extend([tr('### Machine-readable evidence', '### 机器可读的原始证据'), '',
                  tr('The measured data and all images above are embedded in this card. Original files remain available for independent checking:',
                     '上文已在本卡片内展示评测数据与全部样图；以下原始文件同时保留，便于独立核验：'), '',
                  '- [Core CSV](evaluation/comparison.csv) · [Portrait CSV](evaluation/young_woman/comparison.csv)',
                  '- [Core BF16 JSONL](evaluation/bf16/records.jsonl) · [Core INT8 JSONL](evaluation/int8/records.jsonl)',
                  '- [Portrait BF16 JSONL](evaluation/young_woman/bf16/records.jsonl) · [Portrait INT8 JSONL](evaluation/young_woman/int8/records.jsonl)',
                  '- [Core environment](evaluation/bf16/environment.json) · [INT8 environment](evaluation/int8/environment.json)',
                  '- [File checksums](MANIFEST.json) · [Quantization provenance](conversion.json)', ''])
    return '\n'.join(lines)


def _sections(evaluation, platform):
    text = _render_all(evaluation, platform)
    zh = platform == 'modelscope'
    metrics = text.index('### 肖像专项汇总' if zh else '### Supplementary portrait summary')
    gallery = text.index('### 全部对照样图、提示词与观察' if zh else '### All paired samples, prompts and observations')
    evidence = text.index('### 机器可读的原始证据' if zh else '### Machine-readable evidence')
    return text[:metrics], text[metrics:gallery], text[gallery:evidence], text[evidence:]


def render_samples(evaluation, platform):
    return _sections(evaluation, platform)[2]


def render_details(evaluation, platform):
    scope, metrics, _, evidence = _sections(evaluation, platform)
    return '\n'.join((metrics, scope, evidence))