File size: 3,215 Bytes
5ff5661 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | """Preserve the audited native scratch initializer across CPU host math paths.
The initial audit and its independent CPU reproduction agree on every tensor.
An H100 worker's CPU differed by one float32 ULP in 14 A_log/dt_bias values.
Only these native, seed-1 vectors may be normalized, with a strict one-ULP
bound. Shared weights, RNG state, parameter objects and optimizer flags remain
untouched. The independent whole-model fingerprint must still match exactly.
"""
from functools import lru_cache
import hashlib
import json
import math
from pathlib import Path
import struct
@lru_cache(maxsize=1)
def reference():
result = json.loads(Path(__file__).with_name('native_initialization_seed1.json').read_text())
names = {f'layers.{i}.attn.{kind}' for i in (0, 1, 2, 3, 24, 25, 26, 27)
for kind in ('A_log', 'dt_bias')}
if result['maximum_allowed_ulp'] != 1 or set(result['vectors']) != names:
raise ValueError('Native initialization reference coverage differs')
for name, row in result['vectors'].items():
values = row['values']
if len(values) != 32 or not all(math.isfinite(v) for v in values):
raise ValueError('Invalid native initialization vector: ' + name)
if hashlib.sha256(struct.pack('<32f', *values)).hexdigest() != row['sha256']:
raise ValueError('Native initialization reference bytes differ: ' + name)
return result
def normalize_vector(parameter, values):
"""Fail on meaningful changes; preserve native Parameter identity and flags."""
import torch
if parameter.device.type != 'cpu' or parameter.dtype != torch.float32:
raise ValueError('Native initialization normalization requires CPU float32')
expected = torch.tensor(values, dtype=torch.float32)
if parameter.shape != expected.shape or not bool(torch.isfinite(parameter).all()):
raise ValueError('Native initialization vector shape/finiteness differs')
# All A_log values are positive; all dt_bias values are negative. Under
# matching signs, adjacent signed bit patterns are exactly one float32 ULP.
if not torch.equal(torch.signbit(parameter), torch.signbit(expected)):
raise ValueError('Native initialization vector sign differs')
ulp = (parameter.detach().contiguous().view(torch.int32).to(torch.int64)
- expected.view(torch.int32).to(torch.int64)).abs()
maximum = int(ulp.max().item())
if maximum > 1:
raise ValueError(f'Native initialization differs by {maximum} ULP; maximum is one')
receipt = {'changed_elements': int((ulp != 0).sum().item()), 'max_ulp': maximum}
with torch.no_grad():
parameter.copy_(expected)
return receipt
def preserve_audited_initialization(cfg, index, module):
frozen = reference()
if any(getattr(cfg, key) != value for key, value in frozen['config'].items()):
return {'applied': False, 'reason': 'not the audited scientific configuration'}
result = {'applied': True, 'vectors': {}}
for kind in ('A_log', 'dt_bias'):
name = f'layers.{index}.attn.{kind}'
result['vectors'][kind] = normalize_vector(getattr(module, kind), frozen['vectors'][name]['values'])
return result
|