Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/visualize_v2.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 31 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/visualize_v2.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/visualize_v2.py
-
curl -L -o visualize_v2.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/visualize_v2.py
31 kB
| """Export predeclared v2 comparisons from saved outputs, without model inference. | |
| Usage: python -m suite.visualize_v2 --run /outputs/v2-test --output /outputs/v2-showcase | |
| Selection is fixed: first two task IDs per model, seed 3, requested effects | |
| 0.2/0.5/0.8, raw and development-mapped controls, and all seven method IDs. | |
| Missing or unreachable cells stay visible; no replacement examples are chosen. | |
| """ | |
| import argparse | |
| from collections import Counter, defaultdict | |
| import hashlib | |
| import html | |
| import json | |
| import math | |
| from pathlib import Path | |
| import shutil | |
| import numpy as np | |
| from PIL import Image, ImageDraw | |
| METHODS = ( | |
| 'confidence_duration', 'random_duration', 'target_first_duration', | |
| 'target_last_duration', 'balanced_duration', 'control_aware_duration', | |
| 'confidence_amplitude', | |
| ) | |
| LABELS = { | |
| 'confidence_duration': 'Confidence / duration', | |
| 'random_duration': 'Random / duration', | |
| 'target_first_duration': 'Target first / duration', | |
| 'target_last_duration': 'Target last / duration', | |
| 'balanced_duration': 'Balanced / duration', | |
| 'control_aware_duration': 'Control aware / duration', | |
| 'confidence_amplitude': 'Confidence / amplitude', | |
| } | |
| TARGETS = (.2, .5, .8) | |
| MODES = ('raw', 'mapped') | |
| SEED = 3 | |
| def _json(path): | |
| return json.loads(Path(path).read_text(encoding='utf-8-sig')) | |
| def _write_json(path, data): | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text(json.dumps(data, ensure_ascii=False, indent=2, allow_nan=False), encoding='utf-8') | |
| def _lines(path): | |
| return [json.loads(line) for line in Path(path).read_text(encoding='utf-8-sig').splitlines() if line.strip()] | |
| def _finite(value): | |
| return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) | |
| def _task(job): | |
| return job['task']['id'] if isinstance(job['task'], dict) else str(job['task']) | |
| def _anchor_key(job): | |
| return job['model'], _task(job), job['seed'], job.get('steps') | |
| def _safe_name(text): | |
| readable = ''.join(c if c.isalnum() or c in '-_' else '_' for c in str(text))[:90] | |
| return readable + '_' + hashlib.sha256(str(text).encode()).hexdigest()[:8] | |
| def load_run(run): | |
| """Prefer locked expected slots, retaining planned-but-unreachable jobs.""" | |
| run = Path(run) | |
| expected, source = None, None | |
| for path in (run / 'expected_slots.json', run / 'plan/expected_slots.json'): | |
| if path.exists(): | |
| expected = _json(path)['expected_slots'] | |
| source = path | |
| break | |
| if expected is None: | |
| for path in (run / 'plan.jsonl', run / 'jobs.jsonl', run / 'plan/jobs.jsonl'): | |
| if path.exists(): | |
| expected, source = _lines(path), path | |
| break | |
| records = [] | |
| for path in sorted((run / 'jobs').glob('*/result.json')): | |
| row = _json(path) | |
| if not isinstance(row.get('job'), dict): | |
| raise ValueError(f'Result is missing job metadata: {path}') | |
| row = dict(row, _source_result=str(path), _source_folder=str(path.parent)) | |
| records.append(row) | |
| if expected is None: | |
| raise ValueError('A plan.jsonl, jobs.jsonl, or expected_slots.json is required; do not select examples from successful results alone') | |
| if len({j['id'] for j in expected}) != len(expected): | |
| raise ValueError('Duplicate job IDs in the plan') | |
| if len({r['job']['id'] for r in records}) != len(records): | |
| raise ValueError('Duplicate result job IDs') | |
| return expected, records, source | |
| def audit_dose(row): | |
| """Prefer saved counts; fall back to trace sums with explicit denominators.""" | |
| audit = row.get('metrics', {}).get('control_audit') or {} | |
| direct = audit.get('direct_copy_fraction', audit.get('target_direct_copy_fraction')) | |
| exposure = audit.get('exposure_per_target_step') | |
| if _finite(direct) and _finite(exposure): | |
| return dict(direct_copy_fraction=float(direct), exposure_per_target_step=float(exposure), | |
| source='metrics.control_audit', audit=audit) | |
| trace_path = Path(row['_source_folder']) / 'trace.jsonl' | |
| if not trace_path.exists(): | |
| return None | |
| trace = _lines(trace_path) | |
| if not trace or not all('target_count' in t for t in trace): | |
| return None | |
| target = trace[0]['target_count'] | |
| if not _finite(target) or target <= 0 or any(t['target_count'] != target for t in trace): | |
| return None | |
| required = ('target_direct_copy_count', 'target_temporary_count') | |
| if any(not all(_finite(t.get(k)) for k in required) for t in trace): | |
| return None | |
| direct_count = sum(t['target_direct_copy_count'] for t in trace) | |
| exposure_count = sum(t['target_temporary_count'] for t in trace) | |
| return dict(direct_copy_fraction=direct_count / target, | |
| exposure_per_target_step=exposure_count / (target * len(trace)), | |
| source='trace.jsonl', audit=dict(target_count=target, | |
| target_direct_copy_count=direct_count, | |
| reference_exposure_count=exposure_count, denoiser_calls=len(trace))) | |
| def collect_dose(records): | |
| """Compute common-anchor L1 improvements; alpha points are not IID samples.""" | |
| complete = [r for r in records if r.get('status') == 'complete'] | |
| candidates = defaultdict(list) | |
| for row in complete: | |
| j = row['job'] | |
| if (j.get('method_id') == 'confidence_duration' and j.get('alpha') == 0 | |
| and j.get('reference_copy', True) and j.get('control_schedule', 'duration') == 'duration'): | |
| value = row.get('metrics', {}).get('target_reference_l1') | |
| if _finite(value): | |
| candidates[_anchor_key(j)].append(row) | |
| anchors, ambiguous = {}, [] | |
| for key, rows in candidates.items(): | |
| preferred = [r for r in rows if r['job'].get('calibration_mode') == 'anchor'] or rows | |
| values = [r['metrics']['target_reference_l1'] for r in preferred] | |
| if max(values) - min(values) > 1e-6: | |
| ambiguous.append(dict(key=list(key), source_job_ids=[r['job']['id'] for r in preferred])) | |
| else: | |
| anchors[key] = preferred[0] | |
| points, skipped = [], Counter() | |
| for row in complete: | |
| j = row['job'] | |
| if j.get('study') != 'control_v2' and not j.get('experiment', '').startswith('v2_'): | |
| skipped['not_v2'] += 1 | |
| continue | |
| target_l1 = row.get('metrics', {}).get('target_reference_l1') | |
| anchor = anchors.get(_anchor_key(j)) | |
| anchor_value = row.get('metrics', {}).get('common_anchor_target_reference_l1') | |
| anchor_source = 'metrics.common_anchor_target_reference_l1' | |
| if not _finite(anchor_value) and anchor is not None: | |
| anchor_value = anchor['metrics']['target_reference_l1'] | |
| anchor_source = 'confidence_duration_alpha0_result' | |
| dose = audit_dose(row) | |
| if not _finite(target_l1): | |
| skipped['invalid_target_metric'] += 1 | |
| continue | |
| if not _finite(anchor_value): | |
| skipped['missing_or_ambiguous_confidence_alpha0_anchor'] += 1 | |
| continue | |
| if anchor is not None and abs(anchor_value - anchor['metrics']['target_reference_l1']) > 1e-6: | |
| skipped['stored_anchor_differs_from_anchor_result'] += 1 | |
| continue | |
| if dose is None: | |
| skipped['missing_dose_audit'] += 1 | |
| continue | |
| points.append(dict(model=j['model'], task=_task(j), seed=j['seed'], | |
| method_id=j.get('method_id', j.get('arm')), experiment=j.get('experiment'), | |
| calibration_mode=j.get('calibration_mode'), requested_effect=j.get('requested_effect'), | |
| alpha=j.get('alpha'), reference_copy=j.get('reference_copy', True), | |
| control_schedule=j.get('control_schedule', 'duration'), | |
| source_job_id=j['id'], anchor_job_id=anchor['job']['id'] if anchor else None, | |
| anchor_source=anchor_source, | |
| target_reference_l1=target_l1, | |
| improvement_from_common_confidence_alpha0=anchor_value - target_l1, | |
| **dose)) | |
| # The primary scatter only uses alpha=1 endpoints, then averages seeds per | |
| # task/method/experiment. Mapped/raw duplicates are not additional samples. | |
| groups = defaultdict(list) | |
| for p in points: | |
| if p['alpha'] == 1 and p['calibration_mode'] in (None, 'anchor', 'endpoint'): | |
| groups[(p['model'], p['task'], p['method_id'], p['experiment'], | |
| p['reference_copy'], p['control_schedule'])].append(p) | |
| means = [] | |
| for key, values in sorted(groups.items()): | |
| seeds = [p['seed'] for p in values] | |
| if len(set(seeds)) != len(seeds): | |
| raise ValueError(f'Duplicate endpoint per task/seed: {key}') | |
| means.append(dict(model=key[0], task=key[1], method_id=key[2], experiment=key[3], | |
| reference_copy=key[4], control_schedule=key[5], seeds=sorted(seeds), | |
| source_job_ids=[p['source_job_id'] for p in values], | |
| direct_copy_fraction=float(np.mean([p['direct_copy_fraction'] for p in values])), | |
| exposure_per_target_step=float(np.mean([p['exposure_per_target_step'] for p in values])), | |
| improvement_from_common_confidence_alpha0=float(np.mean([p['improvement_from_common_confidence_alpha0'] for p in values])))) | |
| summary = dict(complete_records=len(complete), dose_records=len(points), | |
| endpoint_task_method_points=len(means), skipped=dict(skipped), ambiguous_anchors=ambiguous, | |
| direct_copy_definition='sum of target positions directly copied at commitment / target token count; coincident samples count as copies', | |
| exposure_definition='sum of temporary target reference positions / (target token count * denoiser steps)', | |
| improvement_definition='target L1 of common confidence_duration alpha=0 minus target L1 of this output; positive is closer to the reference', | |
| scatter_unit='one task/method/experiment at alpha=1, averaging its seeds; no IID assumption across methods or tasks and no significance test', | |
| endpoint_copy_degeneracy='For a binary target mask, duration alpha=1, and reference_copy=True, target candidates are directly overridden at every commitment, so endpoint direct_copy_fraction is normally exactly 1. The endpoint direct-copy x axis may therefore be degenerate; inspect the full-alpha dose plot as well.', | |
| caveat='Dose association is descriptive and does not establish context coupling or a successful method. Native-coordinate response width is not a capacity metric.') | |
| return points, means, summary | |
| def aggregate_dose_sweeps(points): | |
| """Average seeds within task, then tasks equally at each method/alpha slot.""" | |
| fields = ('direct_copy_fraction', 'exposure_per_target_step', | |
| 'improvement_from_common_confidence_alpha0') | |
| groups = defaultdict(list) | |
| for p in points: | |
| if not _finite(p['alpha']): | |
| continue | |
| key = (p['model'], p['task'], p['method_id'], p['experiment'] or '', | |
| p['calibration_mode'] or 'native', p['alpha'], | |
| p['reference_copy'], p['control_schedule']) | |
| groups[key].append(p) | |
| task_means = [] | |
| for key, rows in sorted(groups.items()): | |
| # Different requested levels may map to exactly the same alpha. They | |
| # are kept in the job manifest, but do not multiply the physical dose. | |
| per_seed = defaultdict(list) | |
| for row in rows: | |
| per_seed[row['seed']].append(row) | |
| seed_means = [{field: float(np.mean([r[field] for r in values])) for field in fields} | |
| for values in per_seed.values()] | |
| task_means.append(dict(model=key[0], task=key[1], method_id=key[2], experiment=key[3], | |
| calibration_mode=key[4], alpha=key[5], reference_copy=key[6], control_schedule=key[7], | |
| seeds=sorted(per_seed), source_job_ids=[p['source_job_id'] for p in rows], | |
| **{field: float(np.mean([r[field] for r in seed_means])) for field in fields})) | |
| model_groups = defaultdict(list) | |
| for p in task_means: | |
| model_groups[(p['model'], p['method_id'], p['experiment'], p['calibration_mode'], | |
| p['alpha'], p['reference_copy'], p['control_schedule'])].append(p) | |
| summaries = [] | |
| for key, rows in sorted(model_groups.items()): | |
| summaries.append(dict(model=key[0], method_id=key[1], experiment=key[2], | |
| calibration_mode=key[3], alpha=key[4], reference_copy=key[5], control_schedule=key[6], | |
| tasks=[r['task'] for r in rows], task_count=len(rows), | |
| **{field: float(np.mean([r[field] for r in rows])) for field in fields})) | |
| return task_means, summaries | |
| def _copy(source, output, name): | |
| source, output = Path(source), Path(output) | |
| dest = output / 'media' / name | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| if source.resolve() != dest.resolve(): | |
| shutil.copy2(source, dest) | |
| return dest.relative_to(output).as_posix() | |
| def prepare_media(row, output): | |
| """Reuse saved PNG/MP4 or render saved XYZ; never call a generator.""" | |
| folder, output = Path(row['_source_folder']), Path(output) | |
| j = row['job'] | |
| stem = _safe_name(j['id']) | |
| media = dict(source_job_id=j['id'], source_result=row['_source_result']) | |
| png, mp4, joints = folder / 'output.png', folder / 'output.mp4', folder / 'output.joints.npy' | |
| if png.exists(): | |
| with Image.open(png) as check: | |
| check.verify() | |
| media['preview'] = _copy(png, output, stem + '.png') | |
| if mp4.exists(): | |
| media['video'] = _copy(mp4, output, stem + '.mp4') | |
| if j['model'] == 'momask' and joints.exists(): | |
| from .rendering import motion_frame | |
| xyz = np.load(joints, allow_pickle=False) | |
| if xyz.ndim == 4 and xyz.shape[0] == 1: | |
| xyz = xyz[0] | |
| if xyz.ndim != 3 or xyz.shape[1:] != (22, 3) or not len(xyz) or not np.isfinite(xyz).all(): | |
| raise ValueError(f'Invalid saved joints: {joints}') | |
| media['source_joints'] = str(joints) | |
| if 'preview' not in media: | |
| indices = np.linspace(0, len(xyz) - 1, 4).round().astype(int) | |
| strip = Image.new('RGB', (256 * len(indices), 256), 'white') | |
| for col, idx in enumerate(indices): | |
| strip.paste(motion_frame(xyz, int(idx), width=256), (col * 256, 0)) | |
| path = output / 'media' / (stem + '.png') | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| strip.save(path) | |
| media['preview'] = path.relative_to(output).as_posix() | |
| if 'video' not in media: | |
| stride = max(1, math.ceil(len(xyz) / 48)) | |
| frames = [motion_frame(xyz, idx, width=256) for idx in range(0, len(xyz), stride)] | |
| path = output / 'media' / (stem + '.gif') | |
| frames[0].save(path, save_all=True, append_images=frames[1:], | |
| duration=max(20, round(stride * 1000 / 20)), loop=0) | |
| media['animation'] = path.relative_to(output).as_posix() | |
| media['animation_note'] = f'Saved XYZ only, source 20 fps, every {stride}th frame, fixed renderer' | |
| media['result'] = _copy(folder / 'result.json', output, stem + '.result.json') | |
| if (folder / 'trace.jsonl').exists(): | |
| media['trace'] = _copy(folder / 'trace.jsonl', output, stem + '.trace.jsonl') | |
| return media | |
| def select_panels(expected, records): | |
| lookup = {r['job']['id']: r for r in records} | |
| slots = {} | |
| relevant = [j for j in expected if j.get('calibration_mode') in MODES] | |
| for job in relevant: | |
| beta = job.get('requested_effect') | |
| if not _finite(beta): | |
| continue | |
| key = (job['model'], _task(job), job['seed'], round(beta, 9), | |
| job['calibration_mode'], job.get('method_id')) | |
| if key in slots: | |
| raise ValueError(f'Duplicate qualitative slot: {key}') | |
| slots[key] = job | |
| tasks = defaultdict(set) | |
| for j in expected: | |
| tasks[j['model']].add(_task(j)) | |
| selections = {model: sorted(ids)[:2] for model, ids in sorted(tasks.items())} | |
| panels = [] | |
| anchor_one = {} | |
| for row in records: | |
| job = row['job'] | |
| if (row.get('status') == 'complete' and job.get('method_id') == 'confidence_duration' | |
| and job.get('alpha') == 1 and job.get('calibration_mode') == 'anchor'): | |
| value = row.get('metrics', {}).get('target_reference_l1') | |
| if _finite(value): | |
| anchor_one[_anchor_key(job)] = value | |
| for model, ids in selections.items(): | |
| for task in ids: | |
| for beta in TARGETS: | |
| cells = [] | |
| for mode in MODES: | |
| for method in METHODS: | |
| job = slots.get((model, task, SEED, beta, mode, method)) | |
| row = lookup.get(job['id']) if job else None | |
| status, reason = 'missing', 'No predeclared slot for this fixed selection' | |
| if job: | |
| status, reason = 'missing', 'Planned result is absent' | |
| if job.get('planning_status') == 'unreachable': | |
| status, reason = 'unreachable', job.get('unreachable_reason', 'Outside development mapping range') | |
| elif row: | |
| status = row.get('status', 'missing') | |
| reason = row.get('error', '') if status != 'complete' else '' | |
| if {k: v for k, v in row['job'].items()} != job: | |
| status, reason = 'plan_mismatch', 'Result metadata differs from the locked plan' | |
| metrics = row.get('metrics', {}) if row else {} | |
| cell = dict(method_id=method, calibration_mode=mode, | |
| status=status, reason=reason, job=job, _row=row, | |
| metrics={k: metrics[k] for k in ('target_reference_l1', 'common_collateral_l1', | |
| 'common_anchor_target_reference_l1') if _finite(metrics.get(k))}) | |
| l0 = metrics.get('common_anchor_target_reference_l1') | |
| l1 = anchor_one.get(_anchor_key(job)) if job else None | |
| current = metrics.get('target_reference_l1') | |
| if status == 'complete' and all(_finite(x) for x in (l0, l1, current)) and l0 - l1 > 1e-6: | |
| cell['achieved_effect'] = (l0 - current) / (l0 - l1) | |
| cell['absolute_effect_error'] = abs(cell['achieved_effect'] - beta) | |
| cells.append(cell) | |
| panels.append(dict(model=model, task=task, seed=SEED, | |
| requested_effect=beta, cells=cells)) | |
| return panels, selections | |
| def _placeholder(status): | |
| img = Image.new('RGB', (320, 240), '#f3f4f6') | |
| draw = ImageDraw.Draw(img) | |
| draw.text((20, 95), status.upper(), fill='#9b1c1c') | |
| draw.text((20, 120), 'No replacement example', fill='#444444') | |
| return img | |
| def save_panel(panel, output): | |
| import matplotlib | |
| matplotlib.use('Agg') | |
| import matplotlib.pyplot as plt | |
| fig, axes = plt.subplots(2, len(METHODS), figsize=(18, 6), squeeze=False) | |
| for i, cell in enumerate(panel['cells']): | |
| ax = axes[i // len(METHODS), i % len(METHODS)] | |
| preview = cell.get('media', {}).get('preview') | |
| img = Image.open(Path(output) / preview).convert('RGB') if preview else _placeholder(cell['status']) | |
| ax.imshow(img) | |
| ax.set_xticks([]) | |
| ax.set_yticks([]) | |
| job = cell.get('job') or {} | |
| alpha = job.get('alpha') | |
| value = f'{alpha:.4g}' if _finite(alpha) else 'N/A' | |
| ax.set_title(LABELS[cell['method_id']].replace(' / ', '\n'), fontsize=9) | |
| outcome = f"effect={cell['achieved_effect']:.2f}" if 'achieved_effect' in cell else cell['status'] | |
| ax.set_xlabel(f"{cell['calibration_mode']} | alpha={value}\n{outcome}", fontsize=8) | |
| if i % len(METHODS) == 0: | |
| ax.set_ylabel(cell['calibration_mode'].upper(), fontsize=11) | |
| fig.suptitle(f"{panel['model']} | {panel['task']} | seed {panel['seed']} | requested effect {panel['requested_effect']:.1f}\nFixed selection; raw and dev-mapped parameters; missing cells are retained", fontsize=12) | |
| fig.tight_layout(rect=(0, 0, 1, .91)) | |
| stem = _safe_name(f"{panel['model']}_{panel['task']}_beta{panel['requested_effect']:.1f}") | |
| files = {} | |
| for suffix in ('png', 'pdf'): | |
| dest = Path(output) / 'figures' / (stem + '.' + suffix) | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| fig.savefig(dest, dpi=160, bbox_inches='tight') | |
| files[suffix] = dest.relative_to(output).as_posix() | |
| plt.close(fig) | |
| return files | |
| def plot_dose(points, output, sweep=False): | |
| import matplotlib | |
| matplotlib.use('Agg') | |
| import matplotlib.pyplot as plt | |
| models = sorted({p['model'] for p in points}) | |
| if not models: | |
| return None | |
| all_methods = list(METHODS) + sorted({p['method_id'] for p in points} - set(METHODS)) | |
| colors = {method: plt.get_cmap('tab10')(i % 10) for i, method in enumerate(all_methods)} | |
| fig, axes = plt.subplots(2, len(models), figsize=(5 * len(models), 8), squeeze=False) | |
| for col, model in enumerate(models): | |
| for method in all_methods: | |
| rows = [p for p in points if p['model'] == model and p['method_id'] == method] | |
| if not rows: | |
| continue | |
| for ri, field in enumerate(('direct_copy_fraction', 'exposure_per_target_step')): | |
| axes[ri, col].scatter([p[field] for p in rows], | |
| [p['improvement_from_common_confidence_alpha0'] for p in rows], | |
| s=28, alpha=.7, color=colors[method], label=LABELS.get(method, method)) | |
| for ri, label in enumerate(('Direct copies / target tokens', 'Temporary reference exposure / target token / step')): | |
| axes[ri, col].axhline(0, color='#555555', lw=.8) | |
| axes[ri, col].set(xlabel=label, ylabel='Target L1 improvement from common alpha=0', title=model) | |
| axes[ri, col].grid(alpha=.15) | |
| handles, labels = [], [] | |
| for ax in axes[0]: | |
| for handle, label in zip(*ax.get_legend_handles_labels()): | |
| if label not in labels: | |
| handles.append(handle) | |
| labels.append(label) | |
| fig.legend(handles, labels, loc='lower center', ncol=min(4, len(labels)), fontsize=8) | |
| title = ('Response versus actual reference dose across native alpha settings\nEach point averages seeds within tasks, then tasks equally; alpha points are correlated, no significance test' | |
| if sweep else 'Endpoint effect versus actual reference dose\nEach point is one task/method at alpha=1, averaged over its seeds; descriptive association only') | |
| fig.suptitle(title, fontsize=12) | |
| fig.tight_layout(rect=(0, .12, 1, .91)) | |
| files = {} | |
| for suffix in ('png', 'pdf'): | |
| stem = 'sweep_reference_dose' if sweep else 'endpoint_reference_dose' | |
| path = Path(output) / 'figures' / (stem + '.' + suffix) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| fig.savefig(path, dpi=170, bbox_inches='tight') | |
| files[suffix] = path.relative_to(output).as_posix() | |
| plt.close(fig) | |
| return files | |
| def write_html(panels, output, dose_fig, sweep_fig): | |
| esc = html.escape | |
| sections = [] | |
| for panel in panels: | |
| cards = [] | |
| for cell in panel['cells']: | |
| media = cell.get('media', {}) | |
| if media.get('video'): | |
| visual = f'<video controls muted loop playsinline preload="none" src="{esc(media["video"])}"></video>' | |
| elif media.get('animation') or media.get('preview'): | |
| visual = f'<img loading="lazy" src="{esc(media.get("animation", media.get("preview")))}">' | |
| else: | |
| visual = f'<div class="missing">{esc(cell["status"].upper())}<br>No replacement example</div>' | |
| job = cell.get('job') or {} | |
| alpha = job.get('alpha') | |
| alpha_text = f'{alpha:.5g}' if _finite(alpha) else 'N/A' | |
| links = ' · '.join(f'<a href="{esc(media[k])}">{k}</a>' for k in ('result', 'trace', 'preview', 'video', 'animation') if k in media) | |
| effect = f'<p>Achieved effect: {cell["achieved_effect"]:.3f}<br>Absolute effect error: {cell["absolute_effect_error"]:.3f}</p>' if 'achieved_effect' in cell else '' | |
| metric_text = ', '.join(f'{esc(k)}: {v:.4f}' for k, v in cell.get('metrics', {}).items()) | |
| cards.append(f'<article><b>{esc(LABELS[cell["method_id"]])}</b><p>{esc(cell["calibration_mode"])} | alpha={alpha_text}</p>{visual}<p>{esc(cell["status"])} {esc(str(cell["reason"]))[:500]}</p>{effect}<small>{metric_text}</small><p>{links}</p><small>{esc(job.get("id", "No planned job"))}</small></article>') | |
| title = f'{panel["model"]} / {panel["task"]} / seed {panel["seed"]} / requested effect {panel["requested_effect"]:.1f}' | |
| sections.append(f'<section><h2>{esc(title)}</h2><p><a href="{panel["figures"]["png"]}">PNG grid</a> · <a href="{panel["figures"]["pdf"]}">PDF grid</a></p><div class="grid">'+''.join(cards)+'</div></section>') | |
| dose = f'<h2>Endpoint reference dose</h2><p>With a binary target mask, duration alpha=1 and direct reference copying, the copied target fraction is normally exactly 1. An endpoint x axis collapsed to one value is expected; inspect the all-alpha plot for dose variation.</p><img class="wide" src="{dose_fig["png"]}">' if dose_fig else '<p>No complete endpoint records with common anchors and dose audits; no endpoint dose plot or method-improvement conclusion is available.</p>' | |
| if sweep_fig: | |
| dose += f'<h2>Reference dose across alpha settings</h2><p>Seeds are averaged within each task, then tasks receive equal weight. Points at different alpha values are correlated; the plot is descriptive and carries no significance test.</p><img class="wide" src="{sweep_fig["png"]}">' | |
| content = '''<!doctype html><html lang="en"><meta charset="utf-8"><title>SNF v2 fixed-sample comparisons</title> | |
| <style>body{font-family:system-ui,sans-serif;margin:24px;color:#17212b;background:#fafbfc}h1{font-size:26px}h2{font-size:19px}section{margin:30px 0}.grid{display:grid;grid-template-columns:repeat(7,minmax(180px,1fr));gap:10px;overflow-x:auto}article{background:white;border:1px solid #d7dce0;border-radius:6px;padding:10px;overflow-wrap:anywhere}article img,video{width:100%;max-height:280px;object-fit:contain}small{color:#59616a}.missing{height:160px;background:#f2f2f2;color:#9b1c1c;padding:15px;display:flex;align-items:center}.wide{width:100%;max-width:1400px}p{line-height:1.45}.note{padding:16px;background:#edf3f8;border-left:4px solid #426f95}</style> | |
| <h1>Saved v2 outputs: fixed-sample comparisons</h1> | |
| <p class="note">Selection is locked to the first two task IDs per model in the plan, seed 3, requested effects 0.2 / 0.5 / 0.8, and all seven methods. Raw and development-mapped controls are shown together. Missing, failed, and unreachable slots are retained. These are actual saved outputs or renderings of saved XYZ; no model was run to create this page. Qualitative grids alone do not establish a method improvement.</p> | |
| <p><a href="manifest.json">Selection and source-job manifest</a> · <a href="dose_summary.json">Dose definitions and audit</a> · <a href="dose_task_means.json">Task-aggregated endpoint values</a></p>''' | |
| (Path(output) / 'index.html').write_text(content + dose + ''.join(sections) + '</html>', encoding='utf-8') | |
| def build_artifacts(run, output): | |
| run, output = Path(run), Path(output) | |
| output.mkdir(parents=True, exist_ok=True) | |
| expected, records, plan_source = load_run(run) | |
| points, means, dose_summary = collect_dose(records) | |
| _write_json(output / 'dose_records.json', points) | |
| _write_json(output / 'dose_task_means.json', means) | |
| _write_json(output / 'dose_summary.json', dose_summary) | |
| dose_fig = plot_dose(means, output) | |
| sweep_tasks, sweep_means = aggregate_dose_sweeps(points) | |
| _write_json(output / 'dose_sweep_task_means.json', sweep_tasks) | |
| _write_json(output / 'dose_sweep_summary_points.json', sweep_means) | |
| sweep_fig = plot_dose(sweep_means, output, sweep=True) | |
| panels, selected_tasks = select_panels(expected, records) | |
| prepared = {} | |
| for panel in panels: | |
| for cell in panel['cells']: | |
| row = cell.pop('_row') | |
| if cell['status'] == 'complete': | |
| job_id = row['job']['id'] | |
| try: | |
| if job_id not in prepared: | |
| prepared[job_id] = prepare_media(row, output) | |
| cell['media'] = prepared[job_id] | |
| if not any(k in cell['media'] for k in ('preview', 'video', 'animation')): | |
| cell['status'], cell['reason'] = 'missing_media', 'Completed result has no usable saved image, video, or joints' | |
| except (OSError, ValueError) as exc: | |
| cell['status'], cell['reason'] = 'invalid_media', str(exc) | |
| panel['figures'] = save_panel(panel, output) | |
| statuses = Counter(cell['status'] for panel in panels for cell in panel['cells']) | |
| manifest = dict(schema_version=1, source_run=str(run.resolve()), source_plan=str(plan_source.resolve()), | |
| source_plan_sha256=hashlib.sha256(plan_source.read_bytes()).hexdigest(), | |
| selection_rule=dict(task_order='lexicographic task ID from complete planned set, independent of result availability', | |
| tasks_per_model=2, selected_tasks=selected_tasks, seed=SEED, | |
| requested_effects=list(TARGETS), calibration_modes=list(MODES), method_ids=list(METHODS), | |
| substitution_allowed=False), | |
| scientific_scope='Diagnostic L1 comparisons and actual saved media; no method efficacy conclusion without held-out quantitative analysis. Native-coordinate response width is not a capacity metric.', | |
| selection_status_counts=dict(statuses), panels=panels, dose_figures=dose_fig, | |
| dose_sweep_figures=sweep_fig, | |
| dose_sweep_unit='For each model/method/alpha/experiment/mode, average seeds within task, then tasks equally. Duplicate mapped targets at identical alpha do not multiply task weight. Descriptive only.') | |
| _write_json(output / 'manifest.json', manifest) | |
| write_html(panels, output, dose_fig, sweep_fig) | |
| return dict(panels=len(panels), selected_media_jobs=len(prepared), statuses=dict(statuses), | |
| dose_endpoint_task_points=len(means), gallery=str(output / 'index.html'), | |
| manifest=str(output / 'manifest.json')) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--run', required=True) | |
| parser.add_argument('--output', required=True) | |
| args = parser.parse_args() | |
| print(json.dumps(build_artifacts(args.run, args.output), ensure_ascii=False, indent=2)) | |
| if __name__ == '__main__': | |
| main() | |