File size: 6,920 Bytes
e1acc15 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | """Frozen PP4 Round5 paired whole-group estimates; never resample speculative rounds."""
from __future__ import annotations
import math
import random
from collections.abc import Sequence
from fractions import Fraction
from typing import cast
from offload.launch.windows import up2_checks as ck
from offload.prefill_maxlen.evidence import Json, number, record
REGISTRATION = 'pp4-r6-512-aba-no-measurement-profiles-v1'
WAVES = 4
FAMILIES = ('opinions', 'filings')
RESAMPLES, SEED = 2000, 4103
def group_count(name: str) -> int:
ck.require(name in ('short-b1', '32k-b1', '32k-b4'), 'unknown Round5 cell')
return 2 if name == '32k-b4' else 4
def pool(rows: Sequence[dict[str, Json]]) -> dict[str, Json]:
ck.require(bool(rows), 'no independent request-wave observations')
rounds = accepted = seconds = 0.0
for row in rows:
r, a, t = (number(row[k]) for k in ('spec_request_rounds', 'accepted_tokens', 'server_decode_seconds'))
ck.require(all(math.isfinite(v) for v in (r, a, t)) and r > 0 and t > 0
and r.is_integer() and a.is_integer() and 0 <= a <= 5*r, 'invalid measured round/acceptance/time totals')
rounds += r
accepted += a
seconds += t
ms = 1000*seconds/rounds
return {'spec_request_rounds': rounds, 'accepted_tokens': accepted, 'server_decode_seconds': seconds,
'acceptance_per_round': accepted/rounds, 'server_decode_ms_per_spec_request_round': ms,
'expected_tps': 1000*(1+accepted/rounds)/ms}
def waves(cell: dict[str, Json], batch: int) -> list[dict[str, Json]]:
ck.require(cell.get('metric_registration') == REGISTRATION and cell.get('batch') == batch,
'Round5 metric registration/batch differs')
rows = ck.rows(cell['waves'])
ck.require(len(rows) == group_count(str(cell['cell'])) and [r.get('wave') for r in rows] == list(range(group_count(str(cell['cell'])))),
'registered independent whole groups required in every cell per start')
hashes: list[str] = []
for i, row in enumerate(rows):
ck.require(row.get('family') == FAMILIES[i % 2] and row.get('batch') == batch,
'frozen source-family/wave order differs')
streams = ck.rows(row['streams'])
ck.require(len(streams) == batch and row.get('completed_requests') == batch, 'completed request count differs')
salts: list[str] = []
for stream in streams:
ck.require(stream.get('sampling') == {'temperature': 1, 'top_p': 1, 'seed': 42+i,
'priority': 10, 'max_tokens': 512, 'ignore_eos': True}, 'registered512 sampling differs')
ck.require(record(stream['usage']).get('completion_tokens') == 512 and stream.get('finish') == 'length',
'exact512 output/length finish required')
salts.append(str(stream['cache_salt_sha256']))
hashes.append(str(stream['prompt_sha256']))
ck.require(len(set(salts)) == len(salts), 'duplicate stream salt within group')
pool([row])
ck.require(len(set(hashes)) == len(hashes), 'independent requests require distinct frozen prompt hashes')
return rows
def interval(values: Sequence[float]) -> list[Json]:
ordered = sorted(values)
return [ordered[math.ceil(.025*len(ordered))-1], ordered[math.ceil(.975*len(ordered))-1]]
def start_interval(rows: Sequence[dict[str, Json]]) -> dict[str, Json]:
ck.require(len(rows) in (2, 4), 'two B4 or four B1 whole groups required')
point = pool(rows)
rng = random.Random(SEED)
draws: dict[str, list[float]] = {key: [] for key in (
'acceptance_per_round', 'server_decode_ms_per_spec_request_round', 'expected_tps')}
for _ in range(RESAMPLES):
values = pool(rng.choices(rows, k=len(rows)))
for key, draw in draws.items():
draw.append(number(values[key]))
return {**point, 'pooled_95pct': {key: interval(values) for key, values in draws.items()},
'cluster': 'whole independent request groups; B4 streams intact; one start only',
'bootstrap_seed': SEED, 'resamples': RESAMPLES}
def rate(rows: Sequence[dict[str, Json]]) -> Fraction:
"""Exact decimal counter arithmetic preserves strict-win/inclusive-loss boundaries."""
return (sum((Fraction(str(r['spec_request_rounds']))+Fraction(str(r['accepted_tokens'])) for r in rows), Fraction())
/sum((Fraction(str(r['server_decode_seconds'])) for r in rows), Fraction()))
def compare(starts: list[list[dict[str, Json]]]) -> dict[str, Json]:
ck.require(len(starts) == 3 and len({len(rows) for rows in starts}) == 1 and len(starts[0]) in (2, 4),
'three matched OFF/ON/OFF starts required')
rates = [rate(rows) for rows in starts]
reference = (rates[0]+rates[2])/2
spread = abs(rates[0]-rates[2])
need = max(reference/50, spread)
delta = rates[1]-reference
rng = random.Random(SEED)
cache: dict[tuple[int, ...], tuple[list[dict[str, Json]], Fraction, Fraction]] = {}
differences: list[Fraction] = []
gains: list[float] = []
samples: list[list[dict[str, Json]]] = [[], [], []]
count = len(starts[0])
for _ in range(RESAMPLES):
indices = tuple(rng.randrange(count) for _ in range(count))
if indices not in cache:
selected = [[rows[i] for i in indices] for rows in starts]
draw_rates = [rate(rows) for rows in selected]
ref = (draw_rates[0]+draw_rates[2])/2
cache[indices] = ([pool(rows) for rows in selected], draw_rates[1]-ref, draw_rates[1]/ref-1)
points, difference, gain = cache[indices]
differences.append(difference)
gains.append(float(gain))
for sample, point in zip(samples, points):
sample.append(point)
lower = sorted(differences)[math.ceil(.025*RESAMPLES)-1]
return {'point_rates_tps': cast(Json, [float(r) for r in rates]), 'reference_tps': float(reference),
'off_spread_tps': float(spread), 'point_delta_tps': float(delta), 'required_delta_tps': float(need),
'point_gain': float(delta/reference), 'paired_gain_95pct': interval(gains),
'paired_delta_95pct_tps': interval([float(d) for d in differences]),
'win': delta > need and lower > need, 'no_loss_beyond_spread': delta >= -spread and lower >= -spread,
'starts': [{**pool(rows), 'pooled_95pct': {key: interval([number(p[key]) for p in sample]) for key in (
'acceptance_per_round', 'server_decode_ms_per_spec_request_round', 'expected_tps')}}
for rows, sample in zip(starts, samples)],
'bootstrap_seed': SEED, 'resamples': RESAMPLES, 'independent_groups': count,
'interval_scope': 'paired whole-group percentile95%; same indices across starts; '
'conditional on fixed cuts and starts; one ON start has no restart replication',
'metric_registration': REGISTRATION}
|