File size: 7,288 Bytes
87a0bf5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | #!/usr/bin/env python3
"""Summarize paired ADSD inference over the 263-question benchmark."""
import argparse
import hashlib
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def require(ok, message):
if not ok:
raise ValueError(message)
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def metrics(rows, native_correct):
seconds = sum(r['elapsed_seconds'] for r in rows)
tokens = sum(len(r['generated_ids']) for r in rows)
calls = sum(sum(r['counts']['total_step']) for r in rows)
drafted = sum(sum(r['counts']['draft_eval']) for r in rows)
accepted = sum(sum(r['counts']['accepted_draft_tokens']) for r in rows)
kept = kept_tokens = speed_numerator_tokens = 0
weighted_seconds = 0.0
for row in rows:
c = row['counts']
sizes = c['sample_length']
require(len(sizes) == len(c['draft_eval']), 'Inconsistent native block lists')
require(sum(sizes) == len(row['generated_ids']), 'Generated token accounting mismatch')
require(row['elapsed_seconds'] > 0, 'Nonpositive latency')
require(0 <= sum(c['accepted_draft_tokens']) <= sum(c['draft_eval']), 'Impossible acceptance')
valid = [s for d, s in zip(c['draft_eval'], sizes) if d == 10]
if valid:
kept += len(valid)
kept_tokens += sum(valid)
speed_numerator_tokens += sum(sizes)
weighted_seconds += row['elapsed_seconds'] * sum(valid) / sum(sizes)
require(seconds > 0 and calls > 0 and kept > 0 and weighted_seconds > 0, 'Insufficient counters')
return {
'samples': len(rows), 'native_accuracy': native_correct / len(rows),
'continuation_last_number_accuracy': sum(r['correct_last_number'] for r in rows) / len(rows),
'latency_seconds': seconds / len(rows),
'block_efficiency_gamma_matched': kept_tokens / kept,
'block_efficiency_all_blocks': tokens / calls,
'speed_gamma_scaled': speed_numerator_tokens / weighted_seconds * 10,
'throughput_real_tokens_per_second': tokens / seconds,
'acceptance_global': accepted / drafted,
'generated_tokens_mean': tokens / len(rows),
'capped_requests': sum(r['reached_cap'] for r in rows),
'gamma_matched_blocks': kept,
}
def collect(run_root, protocol, runtime_manifest=None):
expected_models = json.loads((ROOT / 'configs/model_hashes.json').read_text())
sample_manifest = json.loads((ROOT / 'configs/samples.json').read_text())
runtime_hash = protocol['runtime']['executed_manifest_sha256']
if runtime_manifest:
value = json.loads(runtime_manifest.read_text())
content = {k: value[k] for k in ('transformers_version', 'evaluator_sha256', 'overlay_hashes')}
fingerprint = hashlib.sha256(json.dumps(content, sort_keys=True).encode()).hexdigest()
require(fingerprint == protocol['runtime']['content_sha256'], 'Different runtime content')
runtime_hash = digest(runtime_manifest)
records, outputs = [], {}
for condition, parity in [('benign', 0), ('adsd', 1)]:
offset = protocol['evaluation']['offset']
count = protocol['evaluation']['samples']
seed = protocol['evaluation']['seed']
run = run_root / condition
status = json.loads((run / 'status.json').read_text())
require(status['status'] == 'complete' and status['exit_code'] == 0, f'Incomplete {run}')
manifest = json.loads((run / 'manifest.json').read_text())
require(manifest['runtime_manifest_sha256'] == runtime_hash, 'Unexpected runtime manifest')
args = manifest['args']
for key, value in {'samples': count, 'sample_offset': offset, 'seed': seed,
'cap': 512, 'mode': 'tokenwise'}.items():
require(args[key] == value, f'Unexpected {key} in {run}')
for split in ('train', 'test'):
require(manifest['dataset_sha256'][split] == protocol['dataset'][split + '_sha256'],
f'Wrong {split} dataset')
for role in ('target', 'draft'):
require(manifest['model_files'][role] == expected_models[role], f'Wrong {role} bytes')
require(bool(args['attack']) == bool(parity), 'Wrong attack condition')
if parity:
require(manifest['attack_sha256'] == digest(ROOT / 'configs/suffix.json'),
'Unexpected attack artifact')
paths = sorted(run.glob('row_*.json'))
chunk = [json.loads(p.read_text()) for p in paths]
require([r['index'] for r in chunk] == list(range(offset, offset + count)), 'Missing/duplicate rows')
for row in chunk:
require(hashlib.sha256(row['question'].encode()).hexdigest() == sample_manifest['question_sha256'][row['index']],
'Question differs from frozen sample list')
policy = json.loads((run / 'effective_policy.json').read_text())
for key, value in {'gamma': 10, 'temperature': 1.0, 'max_new_tokens': 512,
'num_samples': count, 'sample_offset': offset, 'seed': seed}.items():
require(policy['args'][key] == value, f'Wrong effective policy: {key}')
for role in ('model1', 'model2'):
require(policy['models'][role]['dtype'] == 'torch.float16', 'Wrong precision')
require(policy['models'][role]['attention'] == 'sdpa', 'Wrong attention kernel')
summary = json.loads((run / 'summary.json').read_text())
require(summary['native_parser']['samples'] == count, 'Native parser count mismatch')
native_correct = summary['native_parser']['correct']
rows = chunk
records.append({'run': str(run), 'manifest': manifest, 'status': status,
'summary': summary, 'effective_policy': policy,
'row_sha256': {p.name: digest(p) for p in paths}})
outputs[condition] = metrics(rows, native_correct)
return outputs, records
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--run-root', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--runtime-manifest', type=Path, help='Portable path remapping; content must match pinned runtime')
args = parser.parse_args()
protocol = json.loads((ROOT / 'configs/protocol.json').read_text())
try:
results, records = collect(args.run_root, protocol, args.runtime_manifest)
except (ValueError, FileNotFoundError, KeyError) as e:
raise SystemExit('Cannot summarize evaluation: ' + str(e))
benign, attack = results['benign'], results['adsd']
ratio = attack['latency_seconds'] / benign['latency_seconds']
result = {
'samples': protocol['evaluation']['samples'],
'seed': protocol['evaluation']['seed'],
'results': results,
'latency_ratio': ratio,
'slowdown_percent': (ratio - 1) * 100,
'records': records,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2, ensure_ascii=False) + '\n')
print(json.dumps({k: v for k, v in result.items() if k != 'records'}, indent=2))
if __name__ == '__main__':
main()
|