Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/create_method_tasks.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 4.96 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/create_method_tasks.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/create_method_tasks.py
-
curl -L -o create_method_tasks.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/create_method_tasks.py
4.96 kB
| """Fixed prospective author-curated tasks for the control-v2 add-on. | |
| These are diagnostic prompts, not a public benchmark. The full manifest is | |
| written before any v2 model inference and its hash is stored in every plan. | |
| """ | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| IMAGE_DEV = [ | |
| ('A yellow raincoat hanging on a coat rack in a hallway', 'A blue raincoat hanging on a coat rack in a hallway'), | |
| ('A green watering can on a stone garden bench', 'A red watering can on a stone garden bench'), | |
| ('A brown leather wallet on a gray desk', 'A black leather wallet on a gray desk'), | |
| ('A white ceramic teapot on a kitchen counter', 'A turquoise ceramic teapot on a kitchen counter'), | |
| ] | |
| IMAGE_TEST = [ | |
| ('A silver desk fan on a wooden side table', 'A red desk fan on a wooden side table'), | |
| ('A blue hard shell suitcase standing on a tiled floor', 'A yellow hard shell suitcase standing on a tiled floor'), | |
| ('A white mailbox mounted beside a garden gate', 'A green mailbox mounted beside a garden gate'), | |
| ('A beige knitted scarf folded on a park bench', 'A purple knitted scarf folded on a park bench'), | |
| ('A red fire hydrant beside a quiet sidewalk', 'A yellow fire hydrant beside a quiet sidewalk'), | |
| ('A black electric kettle on a white kitchen counter', 'A red electric kettle on a white kitchen counter'), | |
| ('A green garden hose coiled on a gray patio', 'A blue garden hose coiled on a gray patio'), | |
| ('A brown acoustic guitar leaning against a plain wall', 'A blue acoustic guitar leaning against a plain wall'), | |
| ('A gray sewing machine on a wooden workbench', 'A pink sewing machine on a wooden workbench'), | |
| ('A white football helmet on a locker room bench', 'An orange football helmet on a locker room bench'), | |
| ('A blue metal toolbox on a concrete garage floor', 'A red metal toolbox on a concrete garage floor'), | |
| ('A yellow rubber boot on a white display stand', 'A green rubber boot on a white display stand'), | |
| ] | |
| MOTION_DEV = [ | |
| ('A person takes small steps backward.', 'A person takes large steps backward.'), | |
| ('A person stretches one arm upward.', 'A person stretches both arms upward.'), | |
| ('A person turns slowly to the left.', 'A person turns quickly to the left.'), | |
| ('A person bends the knees slightly.', 'A person bends the knees deeply.'), | |
| ] | |
| MOTION_TEST = [ | |
| ('A person walks sideways with short steps.', 'A person walks sideways with wide steps.'), | |
| ('A person slowly raises both hands to shoulder height.', 'A person slowly raises both hands above the head.'), | |
| ('A person reaches forward with one hand.', 'A person reaches forward with both hands.'), | |
| ('A person makes a small bow.', 'A person makes a deep bow.'), | |
| ('A person steps over a low obstacle.', 'A person lifts the knees high while stepping over an obstacle.'), | |
| ('A person marches in place with low knees.', 'A person marches in place with high knees.'), | |
| ('A person swings both arms gently.', 'A person swings both arms widely.'), | |
| ('A person crouches briefly and stands up.', 'A person crouches deeply and stands up.'), | |
| ('A person moves forward with small hops.', 'A person moves forward with large hops.'), | |
| ('A person waves the right hand at chest height.', 'A person waves the right hand above the head.'), | |
| ('A person balances while holding the arms close to the body.', 'A person balances while stretching both arms sideways.'), | |
| ('A person takes a short step to the right and returns.', 'A person takes a wide step to the right and returns.'), | |
| ] | |
| def create(): | |
| rows = [] | |
| for modality, split, pairs in [('image', 'dev', IMAGE_DEV), ('motion', 'dev', MOTION_DEV), | |
| ('image', 'test', IMAGE_TEST), ('motion', 'test', MOTION_TEST)]: | |
| for i, (prompt, reference) in enumerate(pairs): | |
| row = dict(id=f'v2_{split}_{modality}_{i:03d}', split=split, | |
| prompt=prompt, reference_prompt=reference, | |
| reference_seed=9000 + len(rows), | |
| models=['meissonic', 'showo'] if modality == 'image' else ['momask'], | |
| reference_kind='model_generated', benchmark='author_curated_v2_prospective', | |
| confirmatory=split == 'test') | |
| if modality == 'image': row['target_box'] = [.2, .2, .8, .8] | |
| else: row['target_time'] = [[.25,.65],[.4,.8],[.1,.5]][i % 3] | |
| rows.append(row) | |
| return rows | |
| def main(): | |
| p = argparse.ArgumentParser(); p.add_argument('--output', required=True); args = p.parse_args() | |
| path = Path(args.output); path.parent.mkdir(parents=True, exist_ok=True) | |
| text = ''.join(json.dumps(t, ensure_ascii=False) + '\n' for t in create()) | |
| if path.exists() and path.read_text(encoding='utf8') != text: | |
| raise ValueError('Refusing to replace a different task manifest') | |
| path.write_text(text, encoding='utf8'); print(path) | |
| if __name__ == '__main__': main() | |