CharlesZhang-USTC's picture
Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified
Raw History Blame Contribute Delete
8.01 kB
"""Pure-Python planning: no GPU or model imports."""
import hashlib
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MODELS = tuple(json.loads((ROOT / 'suite/models.json').read_text()))
def read_tasks(path):
tasks = [json.loads(line) for line in Path(path).read_text(encoding='utf-8').splitlines() if line.strip()]
ids = [t['id'] for t in tasks]
if len(ids) != len(set(ids)):
raise ValueError('Task IDs must be unique')
for t in tasks:
if not t.get('prompt') or not (t.get('reference_path') or t.get('reference_prompt')):
raise ValueError(f"Task {t['id']} needs prompt and reference_path/reference_prompt")
if not set(t.get('models', MODELS)) <= set(MODELS):
raise ValueError('Unknown model in task')
for field, dims in [('target_time', 2), ('target_box', 4)]:
if field in t:
values = t[field]
if len(values) != dims or not all(0 <= x <= 1 for x in values):
raise ValueError(f'Invalid {field}')
if (dims == 2 and values[0] >= values[1]) or (dims == 4 and (values[0] >= values[2] or values[1] >= values[3])):
raise ValueError(f'Empty {field}')
if t.get('split', 'test') not in ('dev', 'test'):
raise ValueError('split must be dev or test')
return tasks
def job_id(job):
return hashlib.sha256(json.dumps(job, sort_keys=True).encode()).hexdigest()[:24]
def plan(tasks, profile, selected=MODELS, include_extras=None):
cfg = json.loads((ROOT / 'suite/profiles.json').read_text())[profile]
jobs = []
for model in selected:
count = cfg['tasks'][model] if isinstance(cfg['tasks'], dict) else cfg['tasks']
subset = [t for t in tasks if t.get('split', 'test') == 'test' and model in t.get('models', MODELS)]
if len(subset) < count:
raise ValueError(f'{model}: profile needs {count} distinct tasks, found {len(subset)}. Use smoke or supply a real task manifest; tasks are never duplicated.')
subset = subset[:count]
n = cfg.get('video_alpha_points' if model in ('self_forcing', 'showo', 'sd35') else 'alpha_points', 9)
alphas = cfg.get('alphas', [i/(n-1) for i in range(n)])
arms = cfg['video_arms'] if model == 'self_forcing' else ['local', 'guide'] if model == 'sd35' else cfg['orders']
for t in subset:
for seed in cfg['seeds']:
for arm in arms:
for alpha in alphas:
jobs.append(dict(model=model, task=t, seed=seed, alpha=alpha, arm=arm, experiment='main',
guide_amplitude=.1, guide_duration=.05, soft_release=True))
extras = cfg['extras'] if include_extras is None else include_extras
if extras:
geometries=({'small_center':[.4,.4,.6,.6],'large_center':[.1,.1,.9,.9],
'edge':[0.,.25,.4,.75],'global':[0.,0.,1.,1.]}
if model in ('meissonic','sd35','showo') else
{'early':[.1,.3],'middle':[.4,.6],'late':[.7,.9],'global':[0.,1.]})
for t in subset[:min(4,len(subset))]:
for name,region in geometries.items():
variant=dict(t,id=t['id']+'_'+name)
variant['target_box' if model in ('meissonic','sd35','showo') else 'target_time']=region
for alpha in [.0,.25,.5,.75,1.]:
for arm in arms:
jobs.append(dict(model=model,task=variant,seed=0,alpha=alpha,arm=arm,experiment='target_geometry'))
# These subsets are predetermined, not chosen from successful outputs.
for t in subset[:min(12, len(subset))]:
for prior_seed in range(8):
for alpha in [.25,.5,.75,1.]:
jobs.append(dict(model=model,task=t,seed=0,prior_seed=prior_seed,alpha=alpha,
arm=arms[0],experiment='reinjection_reroll'))
for seed in range(8):
for alpha in [.25, .5, .75, 1.]:
jobs.append(dict(model=model, task=t, seed=seed, alpha=alpha,
arm=arms[0], experiment='seed_robustness'))
for alpha in [i/64 for i in range(65)]:
for arm in arms:
jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm=arm, experiment='dense'))
dev = [t for t in tasks if t.get('split') == 'dev' and model in t.get('models', MODELS)]
if not dev:
raise ValueError(f'{model}: extras need independent split=dev tasks for recovery tuning/calibration')
for t in dev:
for tau in ([1.] if model=='sd35' else [.0, .02, .05, .1]):
for amplitude in [.1, .3, 1.]:
for alpha in [.0, .25, .5, .75, 1.]:
jobs.append(dict(model=model, task=t, seed=0, alpha=alpha,
arm='guide', experiment='recovery_grid',
guide_duration=tau, guide_amplitude=amplitude))
# A second control entry is logit bias for masked models, not token-ID blending.
if model in ('meissonic', 'momask', 'showo'):
for context in [0., .25, .5, .75, 1.]:
for alpha in [.0, .25, .5, .75, 1.]:
jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm='confidence',
context_fraction=context, experiment='context_intervention'))
for delay in [.0, .1, .25, .5]:
for alpha in [.0, .25, .5, .75, 1.]:
jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm='confidence',
commitment_delay=delay, experiment='delayed_commitment'))
for steps in [8, 18, 32, 64]:
for arm in ['confidence', 'target_first', 'target_last']:
for alpha in [.0, .25, .5, .75, 1.]:
jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm=arm,
steps=steps, experiment='granularity'))
for alpha in alphas:
jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm='logit_bias', experiment='control_entry'))
for alpha in alphas:
for soft in [False, True]:
jobs.append(dict(model=model, task=t, seed=0, alpha=alpha, arm=arms[0],
soft_release=soft, experiment='release_ablation'))
for t in subset[:min(12,len(subset))]:
if model=='self_forcing':
for rollback_blocks in [1,2,4]:
for alpha in [.0,.25,.5,.75,1.]:
jobs.append(dict(model=model,task=t,seed=0,alpha=alpha,arm='local',
rollback_blocks=rollback_blocks,experiment='causal_cache_rollback'))
for block_frames in [1,3,7]:
for alpha in [.0,.25,.5,.75,1.]:
jobs.append(dict(model=model,task=t,seed=0,alpha=alpha,arm='local',
block_frames=block_frames,distribution_shift=block_frames!=3,experiment='causal_block_stress'))
for attempts in [2,4,8]:
for alpha in [.0,.25,.5,.75,1.]:
jobs.append(dict(model=model,task=t,seed=0,alpha=alpha,arm=arms[0],
resamples=attempts,experiment='reference_aware_resampling'))
for j in jobs:
j['id'] = job_id(j)
return jobs