File size: 10,493 Bytes
840c3a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | #!/usr/bin/env python3
"""Rescore public ELF OWT outputs against the existing C64 baseline (no generation)."""
from __future__ import annotations
import argparse
import csv
from dataclasses import dataclass, asdict
import json
from pathlib import Path
import statistics
import sys
from elf_common import ROOT, asset, manifest, offline, read_jsonl, require_server, sha256, write_json
from run_elf_remaining import build_matrix, reusable_result
@dataclass
class C64Record:
"""Token-record interface consumed by the existing baseline evaluator."""
prompt_id: int
completion_id: int
prefix_token_ids: list[int]
continuation_token_ids: list[int]
reference_token_ids: list[int]
def c64_records(rows, prompts, tokenized, tokenizer, *, allow_short=False):
by_id = {p['prompt_id']: p for p in prompts}
records = []
for row, ids in zip(rows, tokenized, strict=True):
source = by_id[row['prompt_id']]
prefix, reference = source['prefix_token_ids'], source['reference_token_ids']
if len(prefix) != 64 or len(reference) != 64:
raise ValueError('Canonical prompt/reference must contain exactly 64 GPT-2 tokens')
# Prove the saved ELF condition/reference came from this exact prompt.
if row['input'] != tokenizer.decode(prefix, skip_special_tokens=False):
raise ValueError('Saved ELF input differs from canonical C64 prompt')
if row['reference'] != tokenizer.decode(reference, skip_special_tokens=False):
raise ValueError('Saved ELF reference differs from canonical C64 reference')
if not ids or (len(ids) < 64 and not allow_short):
raise ValueError('Short response cannot satisfy the 64-token scoring contract')
records.append(C64Record(row['prompt_id'], row['completion_id'], prefix, ids[:64], reference))
return records
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--root', type=Path, default=ROOT)
parser.add_argument('--task', choices=['owt', 'owt-prefix'], required=True)
parser.add_argument('--output', type=Path)
parser.add_argument('--batch-size', type=int, default=8)
parser.add_argument('--steps', nargs='+', type=int, choices=[1, 2, 4, 8, 16, 32, 1024])
parser.add_argument('--short-response-policy', choices=['strict', 'observed'], default='strict',
help='observed: separately report all nonempty responses up to 64 GPT-2 tokens')
parser.add_argument('--plan', action='store_true')
args = parser.parse_args()
if args.batch_size <= 0:
parser.error('--batch-size must be positive')
observed = args.short_response_policy == 'observed'
if observed and args.task != 'owt-prefix':
parser.error('observed short-response policy is only for owt-prefix')
if args.steps and len(args.steps) != len(set(args.steps)):
parser.error('duplicate steps')
root = args.root.resolve()
sys.path[:0] = [str(root), str(root / 'src')]
output = args.output or root / 'results' / (f'elf-owt-baseline-eval-{args.task}' + ('-observed' if observed else ''))
assets = manifest(root)['assets']
asset_hash = sha256(root / 'data/elf/assets.json')
cells, blocked = build_matrix(root, [args.task])
if blocked:
raise ValueError(f'Invalid inputs: {blocked}')
if args.steps:
cells = [cell for cell in cells if cell.steps in args.steps]
runs = []
for cell in cells:
found, rejected = reusable_result(root, cell, asset_hash, assets)
if not found:
raise ValueError(f'Missing verified generation: {cell.key}; first run run_elf_remaining.py; rejected={rejected}')
path = Path(found['run'])
runs.append((cell, path))
print(f'RESCORE {cell.key}: {path}', flush=True)
if args.plan:
print(f'{len(runs)} cells; no generation, no training; output={output}')
return
require_server()
offline()
from transformers import AutoTokenizer, AutoModelForCausalLM
from evaluation.generative_perplexity import compute_gen_ppl
from evaluation.conditional_perplexity import compute_conditional_gen_ppl, conditional_texts
from evaluation.conditional_evaluate import _gpt2_tokenized_self_bleu
from evaluation.unigram_entropy import unigram_entropy
tok = AutoTokenizer.from_pretrained(str(asset(root, 'gpt2')), local_files_only=True)
tok.pad_token = tok.eos_token
tok.padding_side = tok.truncation_side = 'right'
prompts = read_jsonl(root / 'data/conditional/owt-c64/prompts.jsonl') if args.task == 'owt-prefix' else []
dependencies = [Path(__file__), root / 'evaluation/generative_perplexity.py',
root / 'evaluation/conditional_perplexity.py', root / 'evaluation/conditional_evaluate.py',
root / 'evaluation/unigram_entropy.py', root / 'evaluation/self_bleu.py']
evaluator = {p.name: sha256(p) for p in dependencies}
model = None
summary = []
for cell, path in runs:
destination = output / f'steps_{cell.steps}' / 'metrics.json'
provenance = {'protocol': 'elf_owt_public_baseline_observed_v1' if observed else 'elf_owt_public_baseline_eval_v1', 'task': args.task,
'steps': cell.steps, 'source_run': str(path),
'samples_sha256': sha256(path / 'samples.jsonl'),
'generation_manifest_sha256': sha256(path / 'generation.json'),
'assets_manifest_sha256': asset_hash, 'evaluator': evaluator,
'batch_size': args.batch_size,
'prompts_sha256': sha256(root / 'data/conditional/owt-c64/prompts.jsonl') if prompts else None}
cached = json.loads(destination.read_text()) if destination.exists() else {}
if cached and cached.get('protocol') != provenance['protocol']:
raise ValueError(f'Different protocol already exists at {destination}; use a separate --output')
if all(cached.get(k) == v for k, v in provenance.items()) and type(cached.get('valid')) is bool:
result = cached
print(f'SKIP verified evaluation: {destination}', flush=True)
else:
rows = read_jsonl(path / 'samples.jsonl')
ids = [tok(row['generated'], add_special_tokens=False)['input_ids'] for row in rows]
lengths = list(map(len, ids))
limit = 64 if prompts else 1024
short = [row['id'] for row, tokens in zip(rows, ids) if len(tokens) < (64 if prompts else 2)]
invalid = [row['id'] for row, tokens in zip(rows, ids) if not tokens] if observed else short
result = {**provenance, 'sample_count': len(rows), 'prompt_count': 1024,
'metrics': {}, 'valid': not invalid,
'short_response_policy': args.short_response_policy,
'invalid_sample_ids': invalid,
'length_diagnostics': {'mean': statistics.fmean(lengths), 'min': min(lengths),
'max': max(lengths), 'below_64_count': sum(n < 64 for n in lengths),
'above_1024_count': sum(n > 1024 for n in lengths)},
'short_sample_ids': short,
'native_tokenizer': 't5-small', 'scoring_tokenizer': 'gpt2',
'scoring_limit': limit, 'completion_scope': 'all_saved_completions',
'output_policy': 'released ELF EOS-trimmed decoded text; GPT-2 retokenized; no fabricated padding'}
if invalid:
result['reason'] = ('Empty outputs' if observed else 'Short outputs') + ': cannot score the requested contract; none dropped, padded, or resampled'
else:
sliced = [tokens[:limit] for tokens in ids]
result['metrics']['entropy_gpt2_nats'] = statistics.fmean(unigram_entropy(tokens) for tokens in sliced)
records = c64_records(rows, prompts, ids, tok, allow_short=observed) if prompts else None
if model is None:
model = AutoModelForCausalLM.from_pretrained(str(asset(root, 'gpt2-large')), local_files_only=True).cuda().eval()
kwargs = dict(batch_size=args.batch_size, device='cuda',
model_revision=assets['gpt2-large']['revision'], tokenizer_revision=assets['gpt2']['revision'])
if prompts:
ppl = compute_conditional_gen_ppl(records, model, tok, tok, allow_short_continuations=observed, **kwargs)
result['metrics']['conditional_ppl'] = asdict(ppl)
texts = conditional_texts(records, tok, allow_short_continuations=observed)
result['metrics']['grouped_self_bleu'] = _gpt2_tokenized_self_bleu(records, [t.generated_suffix for t in texts], tok)
result['ppl_policy'] = 'existing compute_conditional_gen_ppl; canonical prefix IDs; decode joined IDs then retokenize; prompt loss excluded'
else:
ppl = compute_gen_ppl([row['generated'] for row in rows], model, tok, **kwargs)
result['metrics']['generative_ppl'] = asdict(ppl)
if ppl.sample_count != len(rows):
raise ValueError('PPL silently changed completion count')
write_json(destination, result)
metrics = result['metrics']
ppl = metrics.get('conditional_ppl', metrics.get('generative_ppl', {})).get('perplexity')
summary.append({'task': args.task, 'steps': cell.steps, 'protocol': result['protocol'], 'sample_count': result['sample_count'],
'valid': result['valid'], 'ppl': ppl, 'entropy': metrics.get('entropy_gpt2_nats'),
'short_count': len(result['short_sample_ids']), 'metrics_path': str(destination)})
print(json.dumps(summary[-1]), flush=True)
output.mkdir(parents=True, exist_ok=True)
summary_name = 'summary' + ('-steps-' + '-'.join(str(c.steps) for c, _ in runs) if args.steps else '') + '.csv'
with (output / summary_name).open('w') as handle:
writer = csv.DictWriter(handle, fieldnames=list(summary[0]))
writer.writeheader()
writer.writerows(summary)
print(f'SUMMARY: {output / summary_name}', flush=True)
if any(not row['valid'] for row in summary):
raise SystemExit(2)
if __name__ == '__main__':
main()
|