| { |
| "schema_version": 1, |
| "paper": "https://arxiv.org/html/2605.16343v1", |
| "profile": "ouro_local_explicit_v1", |
| "exact_author_protocol": false, |
| "seeds": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9 |
| ], |
| "repeat_policy": "LoopQ: independently calibrate each seed and pair evaluation seed. BF16 zero-shot repeats can be identical; do not call them independent training runs. Paper repeat mechanism is unpublished.", |
| "calibration_sample_policy": "first nonempty texts of pinned Pile stream; nested prefixes for sample-size studies; fixed across seeds", |
| "optimizer": { |
| "name": "adamw", |
| "learning_rate": 0.005, |
| "cta_learning_rate": 5e-05, |
| "weight_decay": 0.01, |
| "betas": [ |
| 0.9, |
| 0.999 |
| ], |
| "epsilon": 1e-08, |
| "final_steps": 1500, |
| "slt_round_steps": 25 |
| }, |
| "algorithm": { |
| "hidden_factors": "32,64", |
| "intermediate_factors": "64,88", |
| "activation_ste": "rounding", |
| "kl_mode": "conditional_topk", |
| "statistics_coordinate": "svd_parameters", |
| "fisher_estimator": "trajectory_gradient_square" |
| }, |
| "parity": { |
| "max_logprob_error": 1.25, |
| "max_hidden_error": 0.06, |
| "tie_threshold": 0.25, |
| "policy": "Thresholds are derived from a measured BF16-vs-BF16 backend floor, not chosen a priori. The earlier a priori values (0.01 / 0.001) were below that floor and were unattainable by any implementation; they are recorded here so the change is visible rather than silent. Failures above these values require diagnosis, not further relaxation.", |
| "derivation": { |
| "measurement": "gate_bf16_prefill, 2026-09-16 KST, A40, torch 2.8.0+cu128, HF sdpa vs vLLM 0.10.2 FlashAttention, BF16 both sides", |
| "observed_max_abs_logprob_error_over_top1000": 1.0592, |
| "observed_max_loop_relative_hidden_error": 0.03779, |
| "margin": "max_logprob_error ~1.18x floor; max_hidden_error ~1.59x floor", |
| "superseded_a_priori_values": { |
| "max_logprob_error": 0.01, |
| "max_hidden_error": 0.001 |
| }, |
| "tie_threshold_source": "Track B convention, docs/cowork/jun/0827_parity_gate_results.md; reference top-2 gap at or below 0.25 is a near-tie the reference itself does not break consistently" |
| }, |
| "scope": "single-sequence prefill only", |
| "caveat": "A max over top-1000 logprobs is dominated by tail tokens at BF16 spacing, so this gate is loose against subtle numeric drift. Its job is to catch gross deployment breakage (wrong weights, wrong loop routing, unfolded transform); artifact identity is enforced separately by sha256 and readiness checks." |
| }, |
| "evaluation": { |
| "hellaswag": { |
| "fewshot": 0, |
| "metric": "acc_norm,none", |
| "expected_samples": 10042 |
| }, |
| "winogrande": { |
| "fewshot": 0, |
| "metric": "acc,none", |
| "expected_samples": 1267 |
| }, |
| "lambada_openai": { |
| "fewshot": 0, |
| "metric": "acc,none", |
| "secondary_metric": "perplexity,none", |
| "expected_samples": 5153 |
| }, |
| "arc_challenge": { |
| "fewshot": 0, |
| "metric": "acc_norm,none", |
| "expected_samples": 1172 |
| }, |
| "mmlu": { |
| "fewshot": 5, |
| "metric": "acc,none", |
| "expected_samples": 14042, |
| "subjects": 57 |
| }, |
| "wikitext": { |
| "fewshot": 0, |
| "metric": "word_perplexity,none", |
| "expected_samples": 62 |
| } |
| }, |
| "protocol_note": "Explicit local protocol: zero-shot except MMLU 5-shot; plain likelihood scoring, no chat template; installed pinned harness prompts. WikiText word PPL and LAMBADA last-word PPL are not established as the authors' exact PPL definitions.", |
| "environment": { |
| "torch": "2.8.0", |
| "vllm": "0.10.2", |
| "transformers": "4.56.2", |
| "datasets": "5.0.1", |
| "lm_eval_commit": "64f3d0924fc695efd6d776a5ac91f97138085516" |
| }, |
| "paper_environment_difference": "Paper: RTX 5090, torch 2.9.0, CUDA 12.8. Local target: A40, repository-pinned torch 2.8.0; no speed equivalence claim.", |
| "study_model_assumption": "Figure 4/6 local sweeps use Ouro-1.4B W4A4. Do not label exact figure reproduction without resolving underspecified author settings." |
| } |
|
|
