Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/design.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 8.01 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/design.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/design.py
-
curl -L -o design.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/design.py
8.01 kB
| """Pure-Python planning: no GPU or model imports.""" | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parents[1] | |
| MODELS = tuple(json.loads((ROOT / 'suite/models.json').read_text())) | |
| def read_tasks(path): | |
| tasks = [json.loads(line) for line in Path(path).read_text(encoding='utf-8').splitlines() if line.strip()] | |
| ids = [t['id'] for t in tasks] | |
| if len(ids) != len(set(ids)): | |
| raise ValueError('Task IDs must be unique') | |
| for t in tasks: | |
| if not t.get('prompt') or not (t.get('reference_path') or t.get('reference_prompt')): | |
| raise ValueError(f"Task {t['id']} needs prompt and reference_path/reference_prompt") | |
| if not set(t.get('models', MODELS)) <= set(MODELS): | |
| raise ValueError('Unknown model in task') | |
| for field, dims in [('target_time', 2), ('target_box', 4)]: | |
| if field in t: | |
| values = t[field] | |
| if len(values) != dims or not all(0 <= x <= 1 for x in values): | |
| raise ValueError(f'Invalid {field}') | |
| if (dims == 2 and values[0] >= values[1]) or (dims == 4 and (values[0] >= values[2] or values[1] >= values[3])): | |
| raise ValueError(f'Empty {field}') | |
| if t.get('split', 'test') not in ('dev', 'test'): | |
| raise ValueError('split must be dev or test') | |
| return tasks | |
| def job_id(job): | |
| return hashlib.sha256(json.dumps(job, sort_keys=True).encode()).hexdigest()[:24] | |
| def plan(tasks, profile, selected=MODELS, include_extras=None): | |
| cfg = json.loads((ROOT / 'suite/profiles.json').read_text())[profile] | |
| jobs = [] | |
| for model in selected: | |
| count = cfg['tasks'][model] if isinstance(cfg['tasks'], dict) else cfg['tasks'] | |
| subset = [t for t in tasks if t.get('split', 'test') == 'test' and model in t.get('models', MODELS)] | |
| if len(subset) < count: | |
| raise ValueError(f'{model}: profile needs {count} distinct tasks, found {len(subset)}. Use smoke or supply a real task manifest; tasks are never duplicated.') | |
| subset = subset[:count] | |
| n = cfg.get('video_alpha_points' if model in ('self_forcing', 'showo', 'sd35') else 'alpha_points', 9) | |
| alphas = cfg.get('alphas', [i/(n-1) for i in range(n)]) | |
| arms = cfg['video_arms'] if model == 'self_forcing' else ['local', 'guide'] if model == 'sd35' else cfg['orders'] | |
| for t in subset: | |
| for seed in cfg['seeds']: | |
| for arm in arms: | |
| for alpha in alphas: | |
| jobs.append(dict(model=model, task=t, seed=seed, alpha=alpha, arm=arm, experiment='main', | |
| guide_amplitude=.1, guide_duration=.05, soft_release=True)) | |
| extras = cfg['extras'] if include_extras is None else include_extras | |
| if extras: | |
| geometries=({'small_center':[.4,.4,.6,.6],'large_center':[.1,.1,.9,.9], | |
| 'edge':[0.,.25,.4,.75],'global':[0.,0.,1.,1.]} | |
| if model in ('meissonic','sd35','showo') else | |
| {'early':[.1,.3],'middle':[.4,.6],'late':[.7,.9],'global':[0.,1.]}) | |
| for t in subset[:min(4,len(subset))]: | |
| for name,region in geometries.items(): | |
| variant=dict(t,id=t['id']+'_'+name) | |
| variant['target_box' if model in ('meissonic','sd35','showo') else 'target_time']=region | |
| for alpha in [.0,.25,.5,.75,1.]: | |
| for arm in arms: | |
| jobs.append(dict(model=model,task=variant,seed=0,alpha=alpha,arm=arm,experiment='target_geometry')) | |
| # These subsets are predetermined, not chosen from successful outputs. | |
| for t in subset[:min(12, len(subset))]: | |
| for prior_seed in range(8): | |
| for alpha in [.25,.5,.75,1.]: | |
| jobs.append(dict(model=model,task=t,seed=0,prior_seed=prior_seed,alpha=alpha, | |
| arm=arms[0],experiment='reinjection_reroll')) | |
| for seed in range(8): | |
| for alpha in [.25, .5, .75, 1.]: | |
| jobs.append(dict(model=model, task=t, seed=seed, alpha=alpha, | |
| arm=arms[0], experiment='seed_robustness')) | |
| for alpha in [i/64 for i in range(65)]: | |
| for arm in arms: | |
| jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm=arm, experiment='dense')) | |
| dev = [t for t in tasks if t.get('split') == 'dev' and model in t.get('models', MODELS)] | |
| if not dev: | |
| raise ValueError(f'{model}: extras need independent split=dev tasks for recovery tuning/calibration') | |
| for t in dev: | |
| for tau in ([1.] if model=='sd35' else [.0, .02, .05, .1]): | |
| for amplitude in [.1, .3, 1.]: | |
| for alpha in [.0, .25, .5, .75, 1.]: | |
| jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, | |
| arm='guide', experiment='recovery_grid', | |
| guide_duration=tau, guide_amplitude=amplitude)) | |
| # A second control entry is logit bias for masked models, not token-ID blending. | |
| if model in ('meissonic', 'momask', 'showo'): | |
| for context in [0., .25, .5, .75, 1.]: | |
| for alpha in [.0, .25, .5, .75, 1.]: | |
| jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm='confidence', | |
| context_fraction=context, experiment='context_intervention')) | |
| for delay in [.0, .1, .25, .5]: | |
| for alpha in [.0, .25, .5, .75, 1.]: | |
| jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm='confidence', | |
| commitment_delay=delay, experiment='delayed_commitment')) | |
| for steps in [8, 18, 32, 64]: | |
| for arm in ['confidence', 'target_first', 'target_last']: | |
| for alpha in [.0, .25, .5, .75, 1.]: | |
| jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm=arm, | |
| steps=steps, experiment='granularity')) | |
| for alpha in alphas: | |
| jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm='logit_bias', experiment='control_entry')) | |
| for alpha in alphas: | |
| for soft in [False, True]: | |
| jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm=arms[0], | |
| soft_release=soft, experiment='release_ablation')) | |
| for t in subset[:min(12,len(subset))]: | |
| if model=='self_forcing': | |
| for rollback_blocks in [1,2,4]: | |
| for alpha in [.0,.25,.5,.75,1.]: | |
| jobs.append(dict(model=model,task=t,seed=0,alpha=alpha,arm='local', | |
| rollback_blocks=rollback_blocks,experiment='causal_cache_rollback')) | |
| for block_frames in [1,3,7]: | |
| for alpha in [.0,.25,.5,.75,1.]: | |
| jobs.append(dict(model=model,task=t,seed=0,alpha=alpha,arm='local', | |
| block_frames=block_frames,distribution_shift=block_frames!=3,experiment='causal_block_stress')) | |
| for attempts in [2,4,8]: | |
| for alpha in [.0,.25,.5,.75,1.]: | |
| jobs.append(dict(model=model,task=t,seed=0,alpha=alpha,arm=arms[0], | |
| resamples=attempts,experiment='reference_aware_resampling')) | |
| for j in jobs: | |
| j['id'] = job_id(j) | |
| return jobs | |