| """Preserve the audited native scratch initializer across CPU host math paths. |
| |
| The initial audit and its independent CPU reproduction agree on every tensor. |
| An H100 worker's CPU differed by one float32 ULP in 14 A_log/dt_bias values. |
| Only these native, seed-1 vectors may be normalized, with a strict one-ULP |
| bound. Shared weights, RNG state, parameter objects and optimizer flags remain |
| untouched. The independent whole-model fingerprint must still match exactly. |
| """ |
| from functools import lru_cache |
| import hashlib |
| import json |
| import math |
| from pathlib import Path |
| import struct |
|
|
|
|
| @lru_cache(maxsize=1) |
| def reference(): |
| result = json.loads(Path(__file__).with_name('native_initialization_seed1.json').read_text()) |
| names = {f'layers.{i}.attn.{kind}' for i in (0, 1, 2, 3, 24, 25, 26, 27) |
| for kind in ('A_log', 'dt_bias')} |
| if result['maximum_allowed_ulp'] != 1 or set(result['vectors']) != names: |
| raise ValueError('Native initialization reference coverage differs') |
| for name, row in result['vectors'].items(): |
| values = row['values'] |
| if len(values) != 32 or not all(math.isfinite(v) for v in values): |
| raise ValueError('Invalid native initialization vector: ' + name) |
| if hashlib.sha256(struct.pack('<32f', *values)).hexdigest() != row['sha256']: |
| raise ValueError('Native initialization reference bytes differ: ' + name) |
| return result |
|
|
|
|
| def normalize_vector(parameter, values): |
| """Fail on meaningful changes; preserve native Parameter identity and flags.""" |
| import torch |
|
|
| if parameter.device.type != 'cpu' or parameter.dtype != torch.float32: |
| raise ValueError('Native initialization normalization requires CPU float32') |
| expected = torch.tensor(values, dtype=torch.float32) |
| if parameter.shape != expected.shape or not bool(torch.isfinite(parameter).all()): |
| raise ValueError('Native initialization vector shape/finiteness differs') |
| |
| |
| if not torch.equal(torch.signbit(parameter), torch.signbit(expected)): |
| raise ValueError('Native initialization vector sign differs') |
| ulp = (parameter.detach().contiguous().view(torch.int32).to(torch.int64) |
| - expected.view(torch.int32).to(torch.int64)).abs() |
| maximum = int(ulp.max().item()) |
| if maximum > 1: |
| raise ValueError(f'Native initialization differs by {maximum} ULP; maximum is one') |
| receipt = {'changed_elements': int((ulp != 0).sum().item()), 'max_ulp': maximum} |
| with torch.no_grad(): |
| parameter.copy_(expected) |
| return receipt |
|
|
|
|
| def preserve_audited_initialization(cfg, index, module): |
| frozen = reference() |
| if any(getattr(cfg, key) != value for key, value in frozen['config'].items()): |
| return {'applied': False, 'reason': 'not the audited scientific configuration'} |
| result = {'applied': True, 'vectors': {}} |
| for kind in ('A_log', 'dt_bias'): |
| name = f'layers.{index}.attn.{kind}' |
| result['vectors'][kind] = normalize_vector(getattr(module, kind), frozen['vectors'][name]['values']) |
| return result |
|
|
