#!/usr/bin/env python3 """Independently audit full W4/Resurface receipts and summarize measured quality. This CPU-only audit recomputes PPL and recall from raw evaluation rows, verifies training schedule/provenance, and reads the actual serialized adapter. It does not replay GPU inference, prove the optimizer executed, or turn reported base identity checks into an independent post-training hash of all model tensors. """ from __future__ import annotations import argparse from collections import Counter import hashlib import json import math from pathlib import Path import re import struct import sys ROOT = Path(__file__).resolve().parents[1] SOURCE_SHA = '47c2766f6aad89d73beafbeaecb334aab902d7370906d081764a90bb7a8bbbcb' TOKENIZER_SHA = '5862e2f71caf762bc9845662be5fec2867deb58d874568235a02a36c5111cd09' PROSE_MANIFEST_SHA = 'facb2ca461615a4199781bd21784d642d6674f5b862641b3b9edac3fb499b89d' PROSE_CONTENT_SHA = '049dfb6847d778897fd19b67653c3189f29fbc19a36022e8a4801306d08228bc' ADAPTER_FORMAT = 'MAMBA2_POST_D_RESURFACE_FP16_V1' MATCH = re.compile(r'(?= 0 and expected_targets > 0, prefix + ': finite nonnegative NLL'): try: audit.close(row['ppl'], math.exp(row['nll'] / expected_targets), prefix + ': PPL from NLL') except OverflowError: audit.check(False, prefix + ': exponent overflow') signatures.append((row['start'], row['target_tokens'], row['token_sha256_int64le'])) targets = sum(row['target_tokens'] for row in rows) total_nll = math.fsum(row['nll'] for row in rows) audit.equal(targets, 264764, label + ': full target total') audit.equal(value['target_tokens'], targets, label + ': declared target total') audit.close(value['nll'], total_nll, label + ': summed raw NLL') ppl = math.exp(total_nll / targets) audit.close(value['ppl'], ppl, label + ': token-weighted PPL') return {'ppl': ppl, 'nll': total_nll, 'target_tokens': targets}, signatures def audit_mk(value, label, audit, tokenizer=None): rows = value['rows'] audit.equal(len(rows), 768, label + ': 768 raw rows') expected_order = [f'resurface-confirm-n{n}-t{t}-s{s}{suffix}' for n in (16, 64) for t in range(3) for s in range(64) for suffix in ('', '-removed')] audit.equal([r['id'] for r in rows], expected_order, label + ': exact CONFIRM identity/order') cells, correct, signatures = Counter(), Counter(), [] derived_correct = [] for i, row in enumerate(rows): prefix = f'{label}.row[{i}]' expected_condition = 'target_removed' if i % 2 else 'normal' audit.equal(row['condition'], expected_condition, prefix + ': paired condition') audit.equal(row['N'], 16 if i < 384 else 64, prefix + ': N cell') audit.equal(row['template'], (i % 384) // 128, prefix + ': template cell') audit.check(isinstance(row['expected'], str) and re.fullmatch(r'\d{6}', row['expected']) is not None, prefix + ': six-digit expected answer') audit.digest(row['prompt_token_sha256_int64le'], prefix + ': prompt hash') audit.check(type(row['prompt_tokens']) is int and 0 < row['prompt_tokens'] <= 4084, prefix + ': native context budget') audit.equal(row['cache_bytes'], 122028032, prefix + ': FP16 cache bytes') ids = row['generated_ids'] valid_ids = isinstance(ids, list) and 1 <= len(ids) <= 12 and all( type(t) is int and 0 <= t < 256000 for t in ids) audit.check(valid_ids, prefix + ': full-vocabulary generated IDs and length') if tokenizer is not None and valid_ids: audit.equal(tokenizer.decode(ids), row['output'], prefix + ': decoded generated IDs') eos = tokenizer.eos_id() if len(ids) < 12: audit.equal(ids[-1], eos, prefix + ': short generation ends with EOS') audit.check(eos not in ids[:-1], prefix + ': generation stops on first EOS') match = MATCH.search(row['output']) prediction = match.group(0) if match else None is_correct = prediction == row['expected'] audit.equal(row['prediction'], prediction, prefix + ': first standalone six-digit prediction') audit.check(type(row['correct']) is bool and row['correct'] == is_correct, prefix + ': correctness from decoded output') key = (row['condition'], row['N'], row['template']) cells[key] += 1 correct[key] += is_correct derived_correct.append(is_correct) signatures.append(tuple(row[k] for k in ('id', 'condition', 'N', 'template', 'expected', 'prompt_token_sha256_int64le', 'prompt_tokens'))) metrics = {} for condition in ('normal', 'target_removed'): for n in (16, 64): for t in range(3): audit.equal(cells[condition, n, t], 64, f'{label}: {condition}/N{n}/T{t} count') count = sum(v for k, v in cells.items() if k[0] == condition) wins = sum(v for k, v in correct.items() if k[0] == condition) summary = value['summary'][condition] audit.equal(summary['count'], count, f'{label}: {condition} summary count') audit.equal(summary['correct'], wins, f'{label}: {condition} summary correct') audit.close(summary['accuracy'], wins / count, f'{label}: {condition} summary accuracy') metrics[condition] = {'correct': wins, 'count': count, 'accuracy_percent': 100 * wins / count, 'by_N': {str(n): {'correct': sum(correct[condition, n, t] for t in range(3)), 'count': sum(cells[condition, n, t] for t in range(3))} for n in (16, 64)}} return metrics, signatures, derived_correct def audit_evaluation(report, label, audit, tokenizer=None, historical=False): expected_format = 'MAMBA2_SOURCE_RESURFACE_EVAL_V1' if historical else 'MAMBA2_W4_RESURFACE_EVAL_V1' audit.equal(report['format'], expected_format, label + ': format') audit.check(report['complete'] is True and report['smoke'] is False, label + ': completed full evaluation') audit.equal(report['split'], 'confirm', label + ': CONFIRM split') audit.equal(report['source_checkpoint_sha256'], SOURCE_SHA, label + ': official base hash') audit.equal(report['tokenizer_sha256'], TOKENIZER_SHA, label + ': tokenizer hash') audit.equal(report['ppl_windows'], 130, label + ': declared windows') audit.equal(report['mk_cases'], 768, label + ': declared MK count') audit.digest(report['protocol_sha256'], label + ': protocol hash') audit.digest(report['data_manifest_sha256'], label + ': data manifest hash') audit.equal(report['dataset']['total_tokens'], 264765, label + ': corpus token count') audit.equal(report['dataset']['split'], 'validation', label + ': prose split') audit.equal(report['dataset']['revision_argument'], 'b08601e04326c79dfdd32d625aee71d232d685c3', label + ': pinned WikiText revision') audit.equal(report['dataset']['tokenizer_sha256'], TOKENIZER_SHA, label + ': corpus tokenizer') metrics, window_sigs, prompt_sigs, derived = {}, {}, {}, {} for arm in ('baseline', 'adapter'): ppl, window_sigs[arm] = audit_ppl(report[arm + '_ppl'], f'{label}.{arm}_ppl', audit) mk, prompt_sigs[arm], derived[arm] = audit_mk(report[arm + '_mk'], f'{label}.{arm}_mk', audit, tokenizer) metrics[arm] = {'ppl': ppl['ppl'], 'nll': ppl['nll'], 'target_tokens': ppl['target_tokens'], 'mk': mk} audit.equal(window_sigs['baseline'], window_sigs['adapter'], label + ': paired prose identity') audit.equal(prompt_sigs['baseline'], prompt_sigs['adapter'], label + ': paired MK identity') comparisons = {} for parity, condition in enumerate(('normal', 'target_removed')): pairs = list(zip(derived['baseline'][parity::2], derived['adapter'][parity::2])) counts = Counter(pairs) comparison = {'both_correct': counts[True, True], 'both_wrong': counts[False, False], 'gained': counts[False, True], 'lost': counts[True, False], 'count': len(pairs), 'baseline_correct': sum(a for a, _ in pairs), 'adapter_correct': sum(b for _, b in pairs), 'delta_percentage_points': 100 * (counts[False, True] - counts[True, False]) / len(pairs)} for key, value in comparison.items(): if key == 'delta_percentage_points': audit.close(report['mk_comparison'][condition][key], value, f'{label}: paired {condition}/{key}') else: audit.equal(report['mk_comparison'][condition][key], value, f'{label}: paired {condition}/{key}') comparisons[condition] = comparison delta = 100 * (metrics['adapter']['ppl'] / metrics['baseline']['ppl'] - 1) audit.close(report['ppl_delta_percent'], delta, label + ': PPL delta') audit.equal(report['restored_probe']['rows'], report['baseline_mk']['rows'][:8], label + ': exact restored probe') frozen = report['frozen_base_check'] audit.check(frozen['identity_version_gradients_unchanged'] is True and frozen['tensors'] == 507 and frozen['parameters'] == 8236999680, label + ': reported frozen-base inventory') return {'arms': metrics, 'ppl_delta_percent': delta, 'mk_comparison': comparisons}, window_sigs['baseline'], prompt_sigs['baseline'] def audit_training(report, prose, audit): import torch audit.equal(report['format'], 'MAMBA2_W4_RESURFACE_TRAIN_V1', 'training: format') audit.check(report['complete'] is True and report['mode'] == 'formal', 'training: completed formal run') audit.check(report['zero_adapter_hidden_exact'] is True, 'training: reported exact initial teacher/student probe') audit.check(report['teacher_base_parameters_frozen'] is True, 'training: reported frozen teacher') audit.equal(report['successful_updates'], 1536, 'training: successful update count') binding = report['binding'] audit.equal(binding['successful_updates'], 1536, 'training binding: final candidate only') audit.equal(binding['source_checkpoint_sha256'], SOURCE_SHA, 'training binding: source hash') audit.equal(binding['tokenizer_sha256'], TOKENIZER_SHA, 'training binding: tokenizer hash') audit.equal(binding['adapter'], ADAPTER_FORMAT, 'training binding: adapter format') audit.equal(binding['prose_manifest_sha256'], PROSE_MANIFEST_SHA, 'training binding: prose manifest') audit.equal(binding['prose_tokens_int64le_sha256'], PROSE_CONTENT_SHA, 'training binding: prose content') for key in ('w4_manifest_sha256', 'train_manifest_sha256', 'prose_tokens_sha256', 'protocol_sha256'): audit.digest(binding[key], 'training binding: ' + key) schedule = torch.randperm(1536, generator=torch.Generator(device='cpu').manual_seed(2026092803)).tolist() prose_order = prose['schedule'] audit.equal(sorted(prose_order), list(range(448)), 'training: prose permutation') history = report['history'] audit.equal(report['attempts'], len(history), 'training: recorded attempt count') audit.check(1536 <= len(history) <= 1544, 'training: attempt budget') success, overflow, successful_ids = 0, 0, [] for i, row in enumerate(history): prefix = f'training.attempt[{i + 1}]' audit.equal(row['attempt'], i + 1, prefix + ': uninterrupted attempt ordinal') if not audit.check(success < 1536, prefix + ': no post-final attempt'): break entry = schedule[success] n, template, sample = (16 if entry < 768 else 64), (entry % 768) // 256, entry % 256 audit.equal(row['schedule_entry'], entry, prefix + ': frozen numeric permutation') audit.equal(row['case_id'], f'resurface-train-n{n}-t{template}-s{sample}', prefix + ': paired TRAIN identity') audit.equal(row['prose_window'], prose_order[success % 448], prefix + ': paired prose window') audit.equal(row['prose_start'], 512 * ((success // 448) % 4), prefix + ': paired prose segment') audit.check(type(row['answer_targets']) is int and 1 <= row['answer_targets'] <= 12, prefix + ': answer target count') audit.check(type(row['overflow']) is bool, prefix + ': boolean overflow') for key in ('mk_ce', 'prose_ce', 'prose_kl', 'prose_closure', 'seconds', 'loss_scale'): audit.check(finite(row[key]), prefix + ': finite ' + key) audit.check(row['seconds'] >= 0 and row['loss_scale'] > 0, prefix + ': time/scale range') if row['overflow']: overflow += 1 audit.check(row['gradient_norm_before_clip'] is None, prefix + ': no overflow optimizer step') else: success += 1 successful_ids.append(row['case_id']) audit.check(finite(row['gradient_norm_before_clip']) and row['gradient_norm_before_clip'] >= 0, prefix + ': finite accepted gradient norm') audit.equal(row['successful_updates'], success, prefix + ': uninterrupted successful counter') audit.equal(success, 1536, 'training: recomputed successful total') audit.check(overflow <= 8, 'training: overflow retry budget') audit.equal(len(set(successful_ids)), 1536, 'training: 1536 distinct successful examples') audit.equal([Path(c['path']).name for c in report['checkpoints']], [f'checkpoint_{n:04d}.pt' for n in (384, 768, 1152, 1536)], 'training: fixed checkpoint schedule') for checkpoint in report['checkpoints']: audit.digest(checkpoint['sha256'], 'training: checkpoint digest') audit.check(type(checkpoint['bytes']) is int and checkpoint['bytes'] > 0, 'training: checkpoint byte count') frozen = report['frozen_base_check'] audit.check(frozen['identity_version_gradients_unchanged'] is True and frozen['tensors'] == 507 and frozen['parameters'] == 8236999680, 'training: reported frozen-base inventory') return {'successful_updates': success, 'attempts': len(history), 'overflow_retries': overflow, 'distinct_successful_examples': len(set(successful_ids)), 'schedule_verified': True} def audit_adapter(path, training, evaluation, audit): import torch payload = torch.load(path, map_location='cpu', weights_only=True) audit.equal(payload['format'], ADAPTER_FORMAT, 'adapter: serialized format') audit.equal(payload['gate_mode'], 'soft', 'adapter: serialized soft gate') audit.equal(payload['variant'], 'post-D native norm-prehook; memoryless cross-head mixing', 'adapter: variant') audit.equal(payload['binding'], training['binding'], 'adapter: complete serialized/training binding') audit.equal(payload['geometry'], [{'width': 4096, 'heads': 128, 'head_dim': 64}] * 56, 'adapter: 56-layer geometry') expected = {f'layer{layer}.{field}': shape for layer in range(56) for field, shape in (('V_read', (128, 128)), ('g_read', (128,)), ('router_w', (4096,)), ('router_b', ()))} tensors = payload['tensors'] audit.equal(set(tensors), set(expected), 'adapter: complete tensor inventory') hashes, parameters, byte_count = {}, 0, 0 for key, value in tensors.items(): valid = isinstance(value, torch.Tensor) and value.dtype == torch.float16 audit.check(valid, 'adapter: FP16 tensor ' + key) if not valid: continue audit.equal(tuple(value.shape), expected.get(key), 'adapter: shape ' + key) audit.check(bool(torch.isfinite(value).all()), 'adapter: finite values ' + key) hashes[key] = hashlib.sha256(value.detach().contiguous().numpy().tobytes()).hexdigest() parameters += value.numel() byte_count += value.numel() * value.element_size() receipt = training['adapter'] digest = sha_file(path) audit.equal(digest, receipt['sha256'], 'adapter: actual file/training SHA') audit.equal(digest, evaluation['adapter_sha256'], 'adapter: actual file/evaluation SHA') audit.equal(Path(path).stat().st_size, receipt['bytes'], 'adapter: actual serialized bytes') audit.equal(hashes, receipt['tensor_sha256'], 'adapter: all serialized tensor hashes') audit.equal(parameters, 1154104, 'adapter: parameter count') audit.equal(receipt['parameters'], parameters, 'adapter: declared parameter count') audit.equal(receipt['payload_bytes'], byte_count, 'adapter: declared payload bytes') audit.equal(byte_count, 2308208, 'adapter: expected payload bytes') audit.equal(receipt['gate_mode'], 'soft', 'adapter: reported gate') audit.check(receipt['roundtrip_bitwise_equal'] is True, 'adapter: reported export roundtrip') return {'sha256': digest, 'file_bytes': Path(path).stat().st_size, 'payload_bytes': byte_count, 'parameters': parameters, 'tensors': len(tensors), 'serialized_binding_verified': True} def audit_package(manifest, manifest_path, training, evaluation, protocol_sha, audit, package_dir=None): audit.equal(manifest['format'], 'mamba2-independent-affine-w4-v1', 'package: independent W4 format') audit.equal(manifest['version'], 1, 'package: version') audit.check(manifest['complete'] is True, 'package: completed quantization') audit.equal(manifest['source_checkpoint_sha256'], SOURCE_SHA, 'package: official source hash') audit.equal(manifest['protocol_sha256'], protocol_sha, 'package: protocol hash') audit.equal(manifest['license'], 'Apache-2.0', 'package: base weight license') audit.equal(manifest['source_repo'], 'nvidia/mamba2-8b-3t-4k', 'package: source repository') audit.equal(manifest['source_revision'], 'b915550c63ba9359f88f44d1f6a600d85af27302', 'package: source revision') for key, value in (('parameter_count', 8236999680), ('tensor_count', 507), ('w4_tensor_count', 114), ('fp16_tensor_count', 393)): audit.equal(manifest[key], value, 'package: ' + key) expected_config = { 'd_model': 4096, 'd_intermediate': 0, 'n_layer': 56, 'vocab_size': 256000, 'ssm_cfg': {'layer': 'Mamba2', 'd_state': 128, 'd_conv': 4, 'expand': 2, 'headdim': 64, 'ngroups': 8, 'chunk_size': 128, 'rmsnorm': True, 'norm_before_gate': False, 'use_mem_eff_path': False}, 'rms_norm': True, 'residual_in_fp32': False, 'fused_add_norm': False, 'pad_vocab_size_multiple': 128, 'tie_embeddings': False} audit.equal(manifest['model_config'], expected_config, 'package: pure 56-layer Mamba2 architecture') q = manifest['quantization'] for key, expected in {'method': 'uniform_affine_centered_minmax_weight_mse', 'group_size': 128, 'clipping_factors': [1., .99, .98, .97, .96, .95, .94, .92, .90], 'scale_dtype': 'float16', 'offset_dtype': 'float16', 'code_bits': 4, 'code_order': 'low_nibble_first', 'comparison_dtype': 'float32', 'decoded_dtype': 'float16', 'source_reference_dtype': 'float16', 'training_data_used': False}.items(): audit.equal(q[key], expected, 'package: quantization ' + key) expected_w4 = {'backbone.embedding.weight': [256000, 4096], 'lm_head.weight': [256000, 4096]} for layer in range(56): expected_w4[f'backbone.layers.{layer}.mixer.in_proj.weight'] = [18560, 4096] expected_w4[f'backbone.layers.{layer}.mixer.out_proj.weight'] = [4096, 8192] entries = manifest['tensors'] audit.equal(len(entries), 507, 'package: tensor inventory count') names, kinds, total_parameters, total_bytes, w4_parameters = set(), Counter(), 0, 0, 0 for name, entry in entries.items(): prefix = 'package.' + name audit.check(isinstance(entry['file'], str) and re.fullmatch(r'[0-9]{4}\.w4bin', entry['file']) is not None and entry['file'] not in names, prefix + ': unique safe filename') names.add(entry['file']) shape = entry['shape'] audit.check(isinstance(shape, list) and shape and all(type(n) is int and n > 0 for n in shape), prefix + ': positive tensor shape') numel = math.prod(shape) audit.equal(entry['numel'], numel, prefix + ': element count') total_parameters += numel expected_kind = 'w4_affine_f16' if name in expected_w4 else 'fp16' audit.equal(entry['kind'], expected_kind, prefix + ': storage kind') kinds[entry['kind']] += 1 if expected_kind == 'w4_affine_f16': audit.equal(shape, expected_w4[name], prefix + ': projection/vocabulary shape') audit.equal(entry['group_size'], 128, prefix + ': group size') payload_bytes = shape[0] * (4 * ((shape[1] + 127) // 128) + (shape[1] + 1) // 2) w4_parameters += numel else: payload_bytes = 2 * numel audit.check(type(entry['file_bytes']) is int and entry['file_bytes'] > payload_bytes, prefix + ': packed payload plus header bytes') total_bytes += entry['file_bytes'] audit.digest(entry['sha256'], prefix + ': packed SHA') audit.digest(entry['decoded_sha256'], prefix + ': FP16 decoded SHA') if package_dir is not None: path = package_dir / entry['file'] audit.check(path.is_file() and not path.is_symlink() and path.resolve().parent == package_dir.resolve(), prefix + ': real local package file') audit.equal(path.stat().st_size, entry['file_bytes'], prefix + ': actual file bytes') audit.equal(sha_file(path), entry['sha256'], prefix + ': actual packed bytes hash') with path.open('rb') as stream: magic, version, header_length = struct.unpack('<8sII', stream.read(16)) audit.equal(magic, b'M2W4PK01', prefix + ': file magic') audit.equal(version, 1, prefix + ': file version') if not 0 < header_length <= 65536: raise ValueError(prefix + ': invalid header length') header = json.loads(stream.read(header_length)) audit.equal(header['kind'], entry['kind'], prefix + ': actual header storage kind') audit.equal(header['shape'], shape, prefix + ': actual header shape') audit.equal(16 + header_length + payload_bytes, entry['file_bytes'], prefix + ': complete payload length') audit.equal(set(name for name, entry in entries.items() if entry['kind'] == 'w4_affine_f16'), set(expected_w4), 'package: all 114 expected large matrices') audit.equal(kinds, Counter({'w4_affine_f16': 114, 'fp16': 393}), 'package: exact storage coverage') audit.equal(total_parameters, 8236999680, 'package: summed parameters') audit.equal(manifest['tensor_bytes'], total_bytes, 'package: summed packed file bytes') manifest_sha = sha_file(manifest_path) for label, report in (('training', training), ('evaluation', evaluation)): receipt = report['w4_package'] for key in ('format', 'source_checkpoint_sha256', 'parameter_count', 'tensor_count', 'w4_tensor_count', 'fp16_tensor_count', 'tensor_bytes'): audit.equal(receipt[key], manifest[key], label + ': loaded package receipt ' + key) audit.equal(receipt['manifest_sha256'], manifest_sha, label + ': loaded package manifest hash') audit.equal(receipt['resident_weight_dtype'], 'float16', label + ': decoded reference precision') audit.check(receipt['packed_resident_kernel'] is False, label + ': packed runtime non-claim') audit.check(receipt['file_hashes_verified'] is True and receipt['decoded_hashes_verified'] is True, label + ': loader reported packed and decoded hashes verified') return {'packed_tensor_file_bytes': total_bytes, 'manifest_bytes': Path(manifest_path).stat().st_size, 'weights_plus_manifest_bytes': total_bytes + Path(manifest_path).stat().st_size, 'effective_tensor_file_bits_per_all_parameters': 8 * total_bytes / total_parameters, 'parameters': total_parameters, 'w4_parameters': w4_parameters, 'fp16_parameters': total_parameters - w4_parameters, 'tensor_files': len(entries), 'packed_file_hashes_independently_rechecked': package_dir is not None, 'decoded_tensor_hashes_independently_rechecked': False, 'reference_resident_weight_bytes': 2 * total_parameters, 'note': 'Tensor files include headers. Added license/docs/tokenizer/adapter bytes are not included in base totals.'} def markdown(result): lines = ['# W4A16 Resurface result audit', '', f"Integrity audit: **{'PASS' if result['integrity_pass'] else 'FAIL'}** " f"({result['checks']} checks, {len(result['errors'])} errors).", ''] if 'evaluation' in result: lines += ['| Arm | PPL | Normal recall | Target removed |', '|---|---:|---:|---:|'] groups = [('W4', result['evaluation'])] if 'historical_source' in result: groups.append(('Historical source FP16', result['historical_source'])) for group, value in groups: for arm, data in value['arms'].items(): normal, removed = data['mk']['normal'], data['mk']['target_removed'] suffix = 'base' if arm == 'baseline' else '+ Resurface' lines.append(f"| {group} {suffix} | {data['ppl']:.6f} | " f"{normal['correct']}/{normal['count']} ({normal['accuracy_percent']:.2f}%) | " f"{removed['correct']}/{removed['count']} |") lines += ['', 'Full WikiText-2 validation: 130 reset windows, 264,764 next-token targets. ' 'MK: 384 normal plus 384 target-removed cases; previously observed template family and instances.'] if result['errors']: lines += ['', '## Audit errors', ''] + ['- ' + value for value in result['errors']] lines += ['', '## Scope', ''] + ['- ' + value for value in result['scope']] return '\n'.join(lines) + '\n' def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--evaluation', type=Path, required=True) parser.add_argument('--training', type=Path, required=True) parser.add_argument('--package-manifest', type=Path, required=True) parser.add_argument('--adapter', type=Path, required=True) parser.add_argument('--package-dir', type=Path, help='Optional actual packed tensor directory for independent file hashing') parser.add_argument('--historical', type=Path) parser.add_argument('--train-manifest', type=Path) parser.add_argument('--eval-manifest', type=Path) parser.add_argument('--prose-manifest', type=Path, default=ROOT / 'docs' / 'prose_train_manifest.json') parser.add_argument('--protocol', type=Path, default=ROOT / 'docs' / 'PROTOCOL.md') parser.add_argument('--tokenizer', type=Path, help='Optional pinned SentencePiece file for generated-ID decoding') parser.add_argument('--output', type=Path, required=True) parser.add_argument('--markdown', type=Path) args = parser.parse_args() if args.markdown and args.output.resolve() == args.markdown.resolve(): parser.error('JSON and Markdown outputs must be different files') for path in (args.output, args.markdown): if path and path.exists(): parser.error(f'Preserve existing audit: {path}') audit = Audit() result = {'format': 'MAMBA2_W4_RESURFACE_SEMANTIC_AUDIT_V1', 'complete': False, 'integrity_pass': False, 'inputs': {}, 'scope': [ 'CPU semantic audit of supplied reports and actual serialized adapter; no GPU replay.', 'PPL is recomputed from raw window NLL and target counts; recall from recorded generated text.', 'Training schedule and reported frozen-base status are checked; optimizer execution and all base bytes are not independently replayed.', 'Previously observed CONFIRM instances and WikiText validation; no untouched-generalization claim.', 'Packed W4 file size is separate from the decoded FP16 reference runtime memory.']} try: paths = {key: getattr(args, key) for key in ('evaluation', 'training', 'package_manifest', 'adapter', 'historical', 'train_manifest', 'eval_manifest', 'prose_manifest', 'protocol', 'tokenizer')} for key, path in paths.items(): if path is not None: result['inputs'][key] = {'path': str(path), 'bytes': path.stat().st_size, 'sha256': sha_file(path)} evaluation, training, package, prose = [load_json(path) for path in (args.evaluation, args.training, args.package_manifest, args.prose_manifest)] audit.equal(sha_file(args.prose_manifest), PROSE_MANIFEST_SHA, 'inputs: pinned prose selection manifest') audit.check(package['complete'] is True, 'package: complete manifest') audit.equal(package['source_checkpoint_sha256'], SOURCE_SHA, 'package: original NVIDIA source SHA') package_sha = sha_file(args.package_manifest) audit.equal(evaluation['w4_manifest_sha256'], package_sha, 'binding: evaluation/package manifest') audit.equal(training['binding']['w4_manifest_sha256'], package_sha, 'binding: training/package manifest') protocol_sha = sha_file(args.protocol) audit.equal(evaluation['protocol_sha256'], protocol_sha, 'binding: evaluation/current protocol') audit.equal(training['binding']['protocol_sha256'], protocol_sha, 'binding: training/current protocol') if args.train_manifest: audit.equal(training['binding']['train_manifest_sha256'], sha_file(args.train_manifest), 'binding: TRAIN manifest') if args.eval_manifest: audit.equal(evaluation['data_manifest_sha256'], sha_file(args.eval_manifest), 'binding: CONFIRM manifest') tokenizer = None if args.tokenizer: audit.equal(sha_file(args.tokenizer), TOKENIZER_SHA, 'inputs: actual tokenizer hash') import sentencepiece as spm tokenizer = spm.SentencePieceProcessor(model_file=str(args.tokenizer)) result['package'] = audit_package(package, args.package_manifest, training, evaluation, protocol_sha, audit, args.package_dir) result['training'] = audit_training(training, prose, audit) result['adapter'] = audit_adapter(args.adapter, training, evaluation, audit) result['evaluation'], windows, prompts = audit_evaluation(evaluation, 'W4', audit, tokenizer) audit.check(finite(training['finished_unix']) and finite(evaluation['started_unix']) and training['finished_unix'] <= evaluation['started_unix'], 'timeline: evaluation follows final training') if args.historical: historical = load_json(args.historical) result['historical_source'], old_windows, old_prompts = audit_evaluation( historical, 'historical source', audit, tokenizer, historical=True) audit.equal(windows, old_windows, 'historical comparison: all 130 window hashes/targets/starts') audit.equal(prompts, old_prompts, 'historical comparison: all 768 prompt hashes/answers/geometry') for key in ('text_sha256', 'token_stream_sha256_int64le', 'total_tokens', 'tokenizer_sha256'): audit.equal(evaluation['dataset'][key], historical['dataset'][key], 'historical comparison: corpus ' + key) result['historical_comparison_scope'] = 'Matched recorded prompts/windows; historical source execution was not rerun.' result['w4_vs_historical_source'] = {} for arm in ('baseline', 'adapter'): new = result['evaluation']['arms'][arm] old = result['historical_source']['arms'][arm] result['w4_vs_historical_source'][arm] = { 'ppl_delta_percent': 100 * (new['ppl'] / old['ppl'] - 1), 'normal_recall_delta_percentage_points': new['mk']['normal']['accuracy_percent'] - old['mk']['normal']['accuracy_percent']} base = result['evaluation']['arms']['baseline'] adapted = result['evaluation']['arms']['adapter'] result['quality_observations'] = { 'adapter_ppl_not_worse_than_w4_base': adapted['ppl'] <= base['ppl'], 'adapter_normal_recall_improved': adapted['mk']['normal']['correct'] > base['mk']['normal']['correct'], 'adapter_target_removed_correct': adapted['mk']['target_removed']['correct']} result['generated_ids_decoded_with_pinned_tokenizer'] = tokenizer is not None if args.package_dir is None: result['scope'].append('Packed file identity is bound through the manifest and loader receipts; actual packed files were not rehashed without --package-dir.') if tokenizer is None: result['scope'].append('Generated-ID range/length checked; ID-to-text decoding was not checked without --tokenizer.') result['complete'] = True except Exception as error: audit.check(False, f'Audit could not finish: {type(error).__name__}: {error}') result.update(integrity_pass=result['complete'] and not audit.errors, checks=audit.checks, errors=audit.errors) args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open('x') as stream: stream.write(json.dumps(result, indent=2, allow_nan=False) + '\n') if args.markdown: args.markdown.parent.mkdir(parents=True, exist_ok=True) with args.markdown.open('x') as stream: stream.write(markdown(result)) print(json.dumps({'integrity_pass': result['integrity_pass'], 'checks': result['checks'], 'errors': result['errors'], 'output': str(args.output)})) return 0 if result['integrity_pass'] else 1 if __name__ == '__main__': sys.exit(main())