| """Validate supplementary editing pairs against the release's model identities.""" |
| import json |
| import hashlib |
| import zipfile |
| from pathlib import Path |
| from PIL import Image |
|
|
| from scripts.integrity import sha256, validate_pair |
| from scripts.report import load_records |
|
|
|
|
| def validate_editing(evaluation): |
| evaluation = Path(evaluation) |
| root = evaluation/'editing' |
| summary = json.loads((root/'summary.json').read_text(encoding='utf-8')) |
| cases = json.loads((root/'cases.json').read_text(encoding='utf-8')) |
| |
| with zipfile.ZipFile(root/'measured-cases.zip') as archive: |
| measured_cases = archive.read('editing.json') |
| if json.loads(measured_cases) != cases: |
| raise ValueError('Published case definitions differ from the measured case file') |
| expected = {(case['id'], 42) for case in cases} |
| if len(expected) != 3: |
| raise ValueError('Expected three declared editing cases') |
| records = {p: load_records(root/p) for p in ('bf16', 'int8')} |
| env = {p: json.loads((root/p/'environment.json').read_text()) for p in records} |
| for p in records: |
| core = json.loads((evaluation/p/'environment.json').read_text()) |
| if env[p]['model_identity']['fingerprint'] != core['model_identity']['fingerprint']: |
| raise ValueError('Editing model differs from main suite') |
| if records[p].keys() != expected or env[p]['before_load_memory']['allocated_bytes'] != 0: |
| raise ValueError('Incomplete editing set or nonzero startup allocation') |
| if not env[p]['warmup'] or not env[p]['vae_tiling'] or not env[p]['offload_aux_fix']: |
| raise ValueError('Missing warmup, VAE tiling or auxiliary offload') |
| if env[p]['cases_sha256'] != hashlib.sha256(measured_cases).hexdigest(): |
| raise ValueError('Declared editing cases changed') |
| for key in ('gpu', 'cuda', 'packages', 'offload', 'generator_device', 'source_sha256', 'vae_tiles'): |
| if env['bf16'][key] != env['int8'][key]: |
| raise ValueError('Editing runtime mismatch: '+key) |
| source_root = evaluation.parent/'scripts' |
| if source_root.exists(): |
| for name, digest in env['bf16']['source_sha256'].items(): |
| if sha256(source_root/name) != digest: |
| raise ValueError('Published editing implementation differs from measured code') |
| measures = [] |
| for case in cases: |
| a, b = (records[p][(case['id'], 42)] for p in ('bf16', 'int8')) |
| validate_pair(a, b) |
| input_hash = sha256(root/'inputs'/case['input_file']) |
| source = Path(case['source']) |
| if source.parts[0] != 'evaluation' or '..' in source.parts: |
| raise ValueError('Invalid source-image provenance path') |
| if sha256(evaluation/Path(*source.parts[1:])) != input_hash: |
| raise ValueError('Input differs from its documented BF16 source image') |
| for row in (a, b): |
| if (row['prompt'] != case['prompt'] or row['input_sha256'] != input_hash |
| or row['input_sha256'] != case['input_sha256']): |
| raise ValueError('Editing prompt/input mismatch') |
| if (row['width'], row['height'], row['output_resolution'], row['steps'], |
| row['cfg'], row['kv_cache'], row['vae_tiling'], row['offload']) != ( |
| 2048, 2048, 2048, 40, 1.0, True, True, 'model'): |
| raise ValueError('Unexpected editing settings') |
| for p in records: |
| row = records[p][(case['id'], 42)] |
| with Image.open(root/p/row['image']) as image: |
| if list(image.size) != row['actual_size'] or image.size != (2048, 2048): |
| raise ValueError('Editing PNG dimensions differ from recorded dimensions') |
| if image.mode != row['image_mode']: |
| raise ValueError('Editing PNG mode differs from recorded mode') |
| measures.append(dict(case_id=case['id'], bf16_seconds=a['seconds'], int8_seconds=b['seconds'], |
| bf16_peak_gib=a['peak_allocated_bytes']/2**30, |
| int8_peak_gib=b['peak_allocated_bytes']/2**30)) |
| if measures != summary['measurements']: |
| raise ValueError('Editing summary differs from measured records') |
| if [c['id'] for c in summary['cases']] != ['sweater_red'] + [c['id'] for c in cases]: |
| raise ValueError('Editing gallery does not include every declared case') |
| for item, case in zip(summary['cases'][1:], cases): |
| if item['prompt'] != case['prompt'] or item['input_sha256'] != case['input_sha256']: |
| raise ValueError('Gallery prompt/input differs from declared case') |
| for p in records: |
| record = records[p][(case['id'], 42)] |
| if item[p+'_sha256'] != record['image_sha256']: |
| raise ValueError('Gallery output differs from measured record') |
| native = json.loads((evaluation/'audit/summary.json').read_text())['native'] |
| if summary['cases'][0]['input_sha256'] != sha256(evaluation/'bf16/portrait-s42.png'): |
| raise ValueError('Existing sweater input differs from the BF16 portrait') |
| for p in records: |
| if summary['cases'][0][p+'_sha256'] != native[p]['image_sha256']: |
| raise ValueError('Existing sweater image differs from retained evidence') |
| for item in summary['cases']: |
| for key in ('input', 'bf16', 'int8'): |
| path = Path(item[key]) |
| if path.parts[0] != 'evaluation' or '..' in path.parts: |
| raise ValueError('Invalid editing gallery path') |
| if sha256(evaluation/Path(*path.parts[1:])) != item[key+'_sha256']: |
| raise ValueError('Editing gallery hash mismatch') |
| isolation = [json.loads(line) for line in (root/'process-isolation.jsonl').read_text().splitlines()] |
| if [row['precision'] for row in isolation] != ['bf16', 'int8']: |
| raise ValueError('Unexpected editing process order') |
| for row in isolation: |
| if row['exit_code'] != 0 or row['inference_pid'] != env[row['precision']]['pid']: |
| raise ValueError('Editing process did not finish successfully') |
| after_pids = {line.split(',')[0].strip() for line in row['gpu_processes_after'].splitlines()[1:]} |
| if str(row['inference_pid']) in after_pids: |
| raise ValueError('Editing process remains resident after exit') |
| return summary |
|
|
