Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/method_v2.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 6.03 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/method_v2.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/method_v2.py
-
curl -L -o method_v2.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/method_v2.py
6.03 kB
| """Predeclared incremental study: calibration, direct-copy audit, and commit policies. | |
| Planning imports no model code. Every recipe uses the same per-model denoiser | |
| budget; control-aware scheduling is a candidate to test, not a claimed improvement. | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| from collections import Counter | |
| from pathlib import Path | |
| from .design import ROOT, job_id, read_tasks | |
| from .results import atomic_json | |
| MODELS = ('meissonic', 'showo', 'momask') | |
| VERSION = 'control_v2.1' | |
| RECIPES = { | |
| 'confidence_duration': dict(arm='confidence'), | |
| 'random_duration': dict(arm='random'), | |
| 'target_first_duration': dict(arm='target_first'), | |
| 'target_last_duration': dict(arm='target_last'), | |
| 'balanced_duration': dict(arm='balanced'), | |
| 'control_aware_duration': dict(arm='control_aware', authority_weight=0.5), | |
| 'confidence_amplitude': dict(arm='confidence', control_schedule='amplitude'), | |
| } | |
| def recipe(method_id): | |
| return dict(control_schedule='duration', reference_copy=True, | |
| authority_weight=0.5, soft_release=True, **{ | |
| k: v for k, v in RECIPES[method_id].items() | |
| if k not in ('control_schedule', 'authority_weight') | |
| }) | {k: v for k, v in RECIPES[method_id].items() | |
| if k in ('control_schedule', 'authority_weight')} | |
| def make_job(model, task, seed, alpha, method_id, experiment, **overrides): | |
| j = dict(study='control_v2', study_version=VERSION, model=model, | |
| task=task, seed=seed, alpha=alpha, method_id=method_id, | |
| experiment=experiment, **recipe(method_id)) | |
| j.update(overrides) | |
| j['id'] = job_id(j) | |
| return j | |
| def write_plan(jobs, output, protocol): | |
| ids = [j['id'] for j in jobs] | |
| if len(ids) != len(set(ids)): | |
| raise ValueError('Duplicate jobs in study') | |
| out = Path(output); out.mkdir(parents=True, exist_ok=True) | |
| text = ''.join(json.dumps(j, ensure_ascii=False, sort_keys=True) + '\n' for j in jobs) | |
| destination = out / 'jobs.jsonl' | |
| if destination.exists() and destination.read_text(encoding='utf8') != text: | |
| raise ValueError('Plan changed; choose a new output directory') | |
| destination.write_text(text, encoding='utf8') | |
| summary = dict(protocol, version=VERSION, planned=len(jobs), | |
| counts=dict(Counter(j['model'] for j in jobs)), | |
| jobs_sha256=hashlib.sha256(text.encode()).hexdigest(), | |
| methods=sorted({j['method_id'] for j in jobs})) | |
| atomic_json(out / 'protocol.json', summary) | |
| return summary | |
| def plan_study(tasks_file, output, phase='dev', models=MODELS, | |
| test_tasks=12, alpha_points=None): | |
| tasks = read_tasks(tasks_file) | |
| if not set(models) <= set(MODELS) or not models: | |
| raise ValueError('This add-on supports the three masked generators only') | |
| if phase not in ('smoke', 'dev', 'mechanism', 'test-sweep'): | |
| raise ValueError('Unknown study phase') | |
| count = alpha_points or (33 if phase == 'dev' else 17) | |
| if count < 2 or test_tasks < 1: | |
| raise ValueError('alpha_points >= 2 and test_tasks >= 1 required') | |
| jobs = [] | |
| for model in models: | |
| split = 'test' if phase == 'test-sweep' else 'dev' | |
| subset = [t for t in tasks if t.get('split') == split and model in t.get('models', MODELS)] | |
| if not subset: | |
| raise ValueError(f'{model}: no {split} tasks') | |
| if phase == 'smoke': subset = subset[:1] | |
| if phase == 'test-sweep': | |
| if len(subset) < test_tasks: raise ValueError(f'{model}: insufficient test tasks') | |
| subset = subset[:test_tasks] | |
| alphas = [0., .5, 1.] if phase in ('smoke', 'mechanism') else [i/(count-1) for i in range(count)] | |
| seeds = [3, 4, 5] if phase == 'test-sweep' else [0] | |
| for task in subset: | |
| for seed in seeds: | |
| for method_id in RECIPES: | |
| if phase == 'mechanism' and method_id not in ( | |
| 'confidence_duration', 'target_first_duration', 'target_last_duration'): | |
| continue | |
| for alpha in alphas: | |
| jobs.append(make_job(model, task, seed, alpha, method_id, | |
| 'v2_' + phase.replace('-', '_'))) | |
| if phase == 'mechanism': | |
| j = make_job(model, task, seed, alpha, method_id, | |
| 'v2_mechanism', reference_copy=False) | |
| j['method_id'] = method_id.replace('_duration', '_context_only') | |
| j['id'] = job_id({k: v for k, v in j.items() if k != 'id'}) | |
| jobs.append(j) | |
| return write_plan(jobs, output, dict( | |
| phase=phase, tasks_sha256=hashlib.sha256(Path(tasks_file).read_bytes()).hexdigest(), | |
| split=split, seeds=seeds, alpha_points=len(alphas), | |
| authority_weight_predeclared=0.5, | |
| primary_outcome='held-out requested-effect accuracy, with failures retained', | |
| secondary_outcomes=['common-baseline collateral', 'endpoint reach', 'seed variation', | |
| 'native alpha response concentration', 'actual reference dose', 'NFE'], | |
| note='No dev/test task substitution or best-looking sample selection. Width is not a capacity metric.')) | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--tasks', default=str(ROOT / 'suite/tasks.method_v2.jsonl')) | |
| p.add_argument('--output', required=True) | |
| p.add_argument('--phase', choices=('smoke', 'dev', 'mechanism', 'test-sweep'), default='dev') | |
| p.add_argument('--models', default=','.join(MODELS)) | |
| p.add_argument('--test-tasks', type=int, default=12) | |
| p.add_argument('--alpha-points', type=int) | |
| args = p.parse_args() | |
| print(json.dumps(plan_study(args.tasks, args.output, args.phase, | |
| tuple(args.models.split(',')), args.test_tasks, args.alpha_points), indent=2)) | |
| if __name__ == '__main__': main() | |