CharlesZhang-USTC's picture
Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified
Raw History Blame Contribute Delete
6.03 kB
"""Predeclared incremental study: calibration, direct-copy audit, and commit policies.
Planning imports no model code. Every recipe uses the same per-model denoiser
budget; control-aware scheduling is a candidate to test, not a claimed improvement.
"""
import argparse
import hashlib
import json
from collections import Counter
from pathlib import Path
from .design import ROOT, job_id, read_tasks
from .results import atomic_json
MODELS = ('meissonic', 'showo', 'momask')
VERSION = 'control_v2.1'
RECIPES = {
'confidence_duration': dict(arm='confidence'),
'random_duration': dict(arm='random'),
'target_first_duration': dict(arm='target_first'),
'target_last_duration': dict(arm='target_last'),
'balanced_duration': dict(arm='balanced'),
'control_aware_duration': dict(arm='control_aware', authority_weight=0.5),
'confidence_amplitude': dict(arm='confidence', control_schedule='amplitude'),
}
def recipe(method_id):
return dict(control_schedule='duration', reference_copy=True,
authority_weight=0.5, soft_release=True, **{
k: v for k, v in RECIPES[method_id].items()
if k not in ('control_schedule', 'authority_weight')
}) | {k: v for k, v in RECIPES[method_id].items()
if k in ('control_schedule', 'authority_weight')}
def make_job(model, task, seed, alpha, method_id, experiment, **overrides):
j = dict(study='control_v2', study_version=VERSION, model=model,
task=task, seed=seed, alpha=alpha, method_id=method_id,
experiment=experiment, **recipe(method_id))
j.update(overrides)
j['id'] = job_id(j)
return j
def write_plan(jobs, output, protocol):
ids = [j['id'] for j in jobs]
if len(ids) != len(set(ids)):
raise ValueError('Duplicate jobs in study')
out = Path(output); out.mkdir(parents=True, exist_ok=True)
text = ''.join(json.dumps(j, ensure_ascii=False, sort_keys=True) + '\n' for j in jobs)
destination = out / 'jobs.jsonl'
if destination.exists() and destination.read_text(encoding='utf8') != text:
raise ValueError('Plan changed; choose a new output directory')
destination.write_text(text, encoding='utf8')
summary = dict(protocol, version=VERSION, planned=len(jobs),
counts=dict(Counter(j['model'] for j in jobs)),
jobs_sha256=hashlib.sha256(text.encode()).hexdigest(),
methods=sorted({j['method_id'] for j in jobs}))
atomic_json(out / 'protocol.json', summary)
return summary
def plan_study(tasks_file, output, phase='dev', models=MODELS,
test_tasks=12, alpha_points=None):
tasks = read_tasks(tasks_file)
if not set(models) <= set(MODELS) or not models:
raise ValueError('This add-on supports the three masked generators only')
if phase not in ('smoke', 'dev', 'mechanism', 'test-sweep'):
raise ValueError('Unknown study phase')
count = alpha_points or (33 if phase == 'dev' else 17)
if count < 2 or test_tasks < 1:
raise ValueError('alpha_points >= 2 and test_tasks >= 1 required')
jobs = []
for model in models:
split = 'test' if phase == 'test-sweep' else 'dev'
subset = [t for t in tasks if t.get('split') == split and model in t.get('models', MODELS)]
if not subset:
raise ValueError(f'{model}: no {split} tasks')
if phase == 'smoke': subset = subset[:1]
if phase == 'test-sweep':
if len(subset) < test_tasks: raise ValueError(f'{model}: insufficient test tasks')
subset = subset[:test_tasks]
alphas = [0., .5, 1.] if phase in ('smoke', 'mechanism') else [i/(count-1) for i in range(count)]
seeds = [3, 4, 5] if phase == 'test-sweep' else [0]
for task in subset:
for seed in seeds:
for method_id in RECIPES:
if phase == 'mechanism' and method_id not in (
'confidence_duration', 'target_first_duration', 'target_last_duration'):
continue
for alpha in alphas:
jobs.append(make_job(model, task, seed, alpha, method_id,
'v2_' + phase.replace('-', '_')))
if phase == 'mechanism':
j = make_job(model, task, seed, alpha, method_id,
'v2_mechanism', reference_copy=False)
j['method_id'] = method_id.replace('_duration', '_context_only')
j['id'] = job_id({k: v for k, v in j.items() if k != 'id'})
jobs.append(j)
return write_plan(jobs, output, dict(
phase=phase, tasks_sha256=hashlib.sha256(Path(tasks_file).read_bytes()).hexdigest(),
split=split, seeds=seeds, alpha_points=len(alphas),
authority_weight_predeclared=0.5,
primary_outcome='held-out requested-effect accuracy, with failures retained',
secondary_outcomes=['common-baseline collateral', 'endpoint reach', 'seed variation',
'native alpha response concentration', 'actual reference dose', 'NFE'],
note='No dev/test task substitution or best-looking sample selection. Width is not a capacity metric.'))
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--tasks', default=str(ROOT / 'suite/tasks.method_v2.jsonl'))
p.add_argument('--output', required=True)
p.add_argument('--phase', choices=('smoke', 'dev', 'mechanism', 'test-sweep'), default='dev')
p.add_argument('--models', default=','.join(MODELS))
p.add_argument('--test-tasks', type=int, default=12)
p.add_argument('--alpha-points', type=int)
args = p.parse_args()
print(json.dumps(plan_study(args.tasks, args.output, args.phase,
tuple(args.models.split(',')), args.test_tasks, args.alpha_points), indent=2))
if __name__ == '__main__': main()