File size: 6,034 Bytes
a64600b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
"""Predeclared incremental study: calibration, direct-copy audit, and commit policies.

Planning imports no model code. Every recipe uses the same per-model denoiser
budget; control-aware scheduling is a candidate to test, not a claimed improvement.
"""
import argparse
import hashlib
import json
from collections import Counter
from pathlib import Path

from .design import ROOT, job_id, read_tasks
from .results import atomic_json

MODELS = ('meissonic', 'showo', 'momask')
VERSION = 'control_v2.1'
RECIPES = {
    'confidence_duration': dict(arm='confidence'),
    'random_duration': dict(arm='random'),
    'target_first_duration': dict(arm='target_first'),
    'target_last_duration': dict(arm='target_last'),
    'balanced_duration': dict(arm='balanced'),
    'control_aware_duration': dict(arm='control_aware', authority_weight=0.5),
    'confidence_amplitude': dict(arm='confidence', control_schedule='amplitude'),
}


def recipe(method_id):
    return dict(control_schedule='duration', reference_copy=True,
                authority_weight=0.5, soft_release=True, **{
                    k: v for k, v in RECIPES[method_id].items()
                    if k not in ('control_schedule', 'authority_weight')
                }) | {k: v for k, v in RECIPES[method_id].items()
                      if k in ('control_schedule', 'authority_weight')}


def make_job(model, task, seed, alpha, method_id, experiment, **overrides):
    j = dict(study='control_v2', study_version=VERSION, model=model,
             task=task, seed=seed, alpha=alpha, method_id=method_id,
             experiment=experiment, **recipe(method_id))
    j.update(overrides)
    j['id'] = job_id(j)
    return j


def write_plan(jobs, output, protocol):
    ids = [j['id'] for j in jobs]
    if len(ids) != len(set(ids)):
        raise ValueError('Duplicate jobs in study')
    out = Path(output); out.mkdir(parents=True, exist_ok=True)
    text = ''.join(json.dumps(j, ensure_ascii=False, sort_keys=True) + '\n' for j in jobs)
    destination = out / 'jobs.jsonl'
    if destination.exists() and destination.read_text(encoding='utf8') != text:
        raise ValueError('Plan changed; choose a new output directory')
    destination.write_text(text, encoding='utf8')
    summary = dict(protocol, version=VERSION, planned=len(jobs),
                   counts=dict(Counter(j['model'] for j in jobs)),
                   jobs_sha256=hashlib.sha256(text.encode()).hexdigest(),
                   methods=sorted({j['method_id'] for j in jobs}))
    atomic_json(out / 'protocol.json', summary)
    return summary


def plan_study(tasks_file, output, phase='dev', models=MODELS,
               test_tasks=12, alpha_points=None):
    tasks = read_tasks(tasks_file)
    if not set(models) <= set(MODELS) or not models:
        raise ValueError('This add-on supports the three masked generators only')
    if phase not in ('smoke', 'dev', 'mechanism', 'test-sweep'):
        raise ValueError('Unknown study phase')
    count = alpha_points or (33 if phase == 'dev' else 17)
    if count < 2 or test_tasks < 1:
        raise ValueError('alpha_points >= 2 and test_tasks >= 1 required')
    jobs = []
    for model in models:
        split = 'test' if phase == 'test-sweep' else 'dev'
        subset = [t for t in tasks if t.get('split') == split and model in t.get('models', MODELS)]
        if not subset:
            raise ValueError(f'{model}: no {split} tasks')
        if phase == 'smoke': subset = subset[:1]
        if phase == 'test-sweep':
            if len(subset) < test_tasks: raise ValueError(f'{model}: insufficient test tasks')
            subset = subset[:test_tasks]
        alphas = [0., .5, 1.] if phase in ('smoke', 'mechanism') else [i/(count-1) for i in range(count)]
        seeds = [3, 4, 5] if phase == 'test-sweep' else [0]
        for task in subset:
            for seed in seeds:
                for method_id in RECIPES:
                    if phase == 'mechanism' and method_id not in (
                            'confidence_duration', 'target_first_duration', 'target_last_duration'):
                        continue
                    for alpha in alphas:
                        jobs.append(make_job(model, task, seed, alpha, method_id,
                            'v2_' + phase.replace('-', '_')))
                        if phase == 'mechanism':
                            j = make_job(model, task, seed, alpha, method_id,
                                         'v2_mechanism', reference_copy=False)
                            j['method_id'] = method_id.replace('_duration', '_context_only')
                            j['id'] = job_id({k: v for k, v in j.items() if k != 'id'})
                            jobs.append(j)
    return write_plan(jobs, output, dict(
        phase=phase, tasks_sha256=hashlib.sha256(Path(tasks_file).read_bytes()).hexdigest(),
        split=split, seeds=seeds, alpha_points=len(alphas),
        authority_weight_predeclared=0.5,
        primary_outcome='held-out requested-effect accuracy, with failures retained',
        secondary_outcomes=['common-baseline collateral', 'endpoint reach', 'seed variation',
                            'native alpha response concentration', 'actual reference dose', 'NFE'],
        note='No dev/test task substitution or best-looking sample selection. Width is not a capacity metric.'))


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--tasks', default=str(ROOT / 'suite/tasks.method_v2.jsonl'))
    p.add_argument('--output', required=True)
    p.add_argument('--phase', choices=('smoke', 'dev', 'mechanism', 'test-sweep'), default='dev')
    p.add_argument('--models', default=','.join(MODELS))
    p.add_argument('--test-tasks', type=int, default=12)
    p.add_argument('--alpha-points', type=int)
    args = p.parse_args()
    print(json.dumps(plan_study(args.tasks, args.output, args.phase,
          tuple(args.models.split(',')), args.test_tasks, args.alpha_points), indent=2))


if __name__ == '__main__': main()