"""Prespecified paired analysis. No threshold or method selection on confirmation.""" from __future__ import annotations import json from pathlib import Path import numpy as np import pandas as pd import torch from .core import encode, plan, push from .study import PROTOCOL, ROOT, verify_lock from .worlds import write def paired_interval(values, families, seed, draws=10000, alpha=.05): rng = np.random.default_rng(seed) estimates = np.zeros(draws) groups = sorted(set(families)) for group in groups: x = values[np.asarray(families) == group] estimates += x[rng.integers(len(x), size=(draws, len(x)))].mean(1)/len(groups) return [float(np.quantile(estimates, alpha/2)), float(np.quantile(estimates, 1-alpha/2))] def mechanism_control(device): joint = torch.zeros((1, 2, 2, 4), device=device, dtype=torch.float64) for e in (0, 1): for h in (0, 1): for nuisance in (0, 1): joint[0, e, h, 2*(h ^ e)+nuisance] = .5*(.75 if h else .25)*.5 methods = ['identity', 'posterior_bins', 'query_joint'] return {method: plan(push(joint, *encode(joint, method)))['expected'][0].tolist() for method in methods} def run(): verify_lock() protocol = json.loads(PROTOCOL.read_text()) destination = ROOT/'analysis' assert not destination.exists() destination.mkdir() metrics = ['utility', 'queries', 'abstention', 'full_oracle', 'available_oracle', 'encoded_oracle', 'information_loss', 'representation_loss', 'model_loss', 'total_regret', 'posterior_brier', 'posterior_log_loss', 'posterior_absolute_error', 'joint_forecast_log_loss', 'estimated_conditional_dispersion', 'true_conditional_dispersion'] keys = ['world', 'family', 'availability', 'sample_stream', 'label_budget', 'model', 'method', 'query_budget', 'query_cost'] frame = pd.read_csv(ROOT/'confirmation_evaluation/predictions.csv.gz', usecols=keys+metrics) primary = frame[(frame.model == protocol['primary']['model']) & (frame.label_budget == protocol['primary']['label_budget']) & (frame.availability == protocol['primary']['availability']) & (frame.query_budget > 0)] per_world = primary.groupby(['world', 'family', 'method'])[metrics].mean().reset_index() table = per_world.groupby('method')[metrics].mean().reset_index() table.to_csv(destination/'primary.csv', index=False) per_world.to_csv(destination/'primary_worlds.csv', index=False) family = per_world.groupby(['family', 'method'])[metrics].mean().reset_index() family.to_csv(destination/'primary_families.csv', index=False) pivot = per_world.pivot(index=['world', 'family'], columns='method', values='utility') comparisons = [] controls = protocol['primary']['compact_controls'] for control in controls+['identity']: difference = (pivot['query_joint']-pivot[control]).to_numpy() interval = paired_interval(difference, pivot.index.get_level_values('family').to_numpy(), protocol['primary']['bootstrap_seed'], protocol['primary']['bootstrap_draws'], alpha=.05/len(controls) if control != 'identity' else .05) comparisons.append(dict(control=control, mean_gain=float(difference.mean()), paired_interval=interval, simultaneous_compact_familywise_95=control != 'identity')) compact = [x for x in comparisons if x['control'] != 'identity'] compact_gate = all(x['mean_gain'] >= protocol['primary']['mean_utility_gain_at_least'] and x['paired_interval'][0] > protocol['primary']['all_paired_lower_95_bounds_above'] for x in compact) full = frame[(frame.method == 'identity') & (frame.label_budget == protocol['primary']['label_budget']) & (frame.availability == 'full') & (frame.query_budget > 0)] full_values = full.groupby(['world', 'family', 'model']).utility.mean().unstack('model') full_comparisons = [] for model in protocol['estimation']['models']: difference = (pivot['query_joint']-full_values[model]).to_numpy() ci = paired_interval(difference, pivot.index.get_level_values('family').to_numpy(), protocol['primary']['bootstrap_seed'], alpha=.05/len(protocol['estimation']['models'])) full_comparisons.append(dict(model=model, mean_gain=float(difference.mean()), paired_interval=ci)) quality_ni = all(x['paired_interval'][0] > -protocol['primary']['full_information_noninferiority_margin'] for x in full_comparisons) frontier = frame.groupby(['availability', 'label_budget', 'model', 'method', 'query_budget'])[metrics].mean().reset_index() frontier.to_csv(destination/'frontier.csv', index=False) # Preserve each modeled query cost as well; no favorable-cost aggregation. frame.groupby(['availability', 'label_budget', 'model', 'method', 'query_budget', 'query_cost'])[metrics].mean().to_csv(destination/'frontier_by_cost.csv') benchmark = json.loads((ROOT/'confirmation_benchmark.json').read_text()) timing = pd.DataFrame(benchmark['records']) savings = [] for n in protocol['cost_promotion']['repeated_request_counts']: candidate = timing[(timing.method == 'query_joint') & (timing.requests == n)].set_index('repetition') for model in protocol['estimation']['models']: reference = timing[(timing.method == 'identity') & (timing.model == model) & (timing.requests == n)].set_index('repetition') ratio = 1-candidate.complete_seconds/reference.complete_seconds interval = paired_interval(ratio.to_numpy(), np.zeros(len(ratio)), 18781, alpha=.05/(len(protocol['estimation']['models'])*4)) savings.append(dict(requests=n, reference_model=model, mean_saving_fraction=float(ratio.mean()), simultaneous_interval=interval)) cost_gate = all(x['mean_saving_fraction'] >= .20 and x['simultaneous_interval'][0] > 0 for x in savings) explicit = frame[(frame.model == 'faithful_explicit') & (frame.method == 'identity')] code_loss = frame[(frame.model == 'faithful_explicit') & (frame.query_budget > 0) & (frame.availability == 'full')].groupby('method')[metrics].mean().reset_index() code_loss.to_csv(destination/'faithful_representations.csv', index=False) code_counterexample = mechanism_control(protocol['execution']['device']) write(destination/'mechanism_control.json', dict( status='EXACT_CONSTRUCTED_COUNTEREXAMPLE_NOT_EMPIRICAL_NOVELTY', contexts_equal_posterior=.75, utilities_by_budget_then_cost=code_counterexample, current_posterior_sufficient=True, adaptive_posterior_sufficient=False, interpretation='The cause posterior does not preserve context that determines how an observation is related to the cause. Query_joint retains that relationship.')) decision = dict(status='CONTROLLED_CONFIRMATION_COMPLETE', compact_representation_advantage=compact_gate, strongest_full_information_noninferiority=quality_ni, complete_cost_saving_gate=cost_gate, controlled_quality_cost_gate=bool(compact_gate and quality_ni and cost_gate), practical_algorithm_promotion=False, promotion_requires_independent_external_confirmation=True, external_generalization_established=False, learned_control_advantage_tested=False, independent_external_authorship=False, worlds=len(pivot), confirmation_families=sorted(pivot.index.get_level_values('family').unique()), held_mechanisms=protocol['worlds']['held_mechanisms'], compact_comparisons=comparisons, full_model_comparisons=full_comparisons, complete_cost_comparisons=savings, threshold_changes_after_opening=False, primary_sample_budget=protocol['primary']['label_budget'], explicit_information_loss=float(explicit[explicit.availability == 'three_available'].information_loss.mean()), primary_metrics=table.to_dict('records')) write(destination/'decision.json', decision) plot(table, family, destination) report(decision, code_loss, benchmark, destination) print(json.dumps({k: decision[k] for k in ['status', 'compact_representation_advantage', 'strongest_full_information_noninferiority', 'complete_cost_saving_gate', 'practical_algorithm_promotion']})) def plot(table, family, destination): import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt order = ['identity', 'query_joint', 'posterior_clusters', 'posterior_bins', 'future_marginal', 'raw_information', 'raw_entropy', 'hash_code'] values = table.set_index('method').loc[order] fig, ax = plt.subplots(1, 2, figsize=(12, 4.8), layout='constrained') y = np.arange(len(order)) ax[0].barh(y, values.utility, color=['#627282', '#237b67']+['#a9b7c4']*6) ax[0].set_yticks(y, [x.replace('_', ' ') for x in order]); ax[0].invert_yaxis() ax[0].set_xlabel('Expected utility, including query cost') ax[0].set_title('Same estimated law and exact planner') ax[1].barh(y, values.representation_loss, label='Representation loss', color='#c58c3b') ax[1].barh(y, values.model_loss, left=values.representation_loss, label='Model loss', color='#ae5e66') ax[1].set_yticks(y, [x.replace('_', ' ') for x in order]); ax[1].invert_yaxis() ax[1].set_xlabel('Utility loss against the raw-information oracle') ax[1].set_title('Information loss and inference error are zero here') ax[1].legend(frameon=False) for extension in ('png', 'svg'): fig.savefig(destination/f'decision_sufficiency.{extension}', dpi=180) plt.close(fig) def report(decision, faithful, benchmark, destination): table = pd.DataFrame(decision['primary_metrics']).set_index('method') lines = ['# Decision sufficient representations', '', 'The study separates missing information, representation loss and model error while holding exact bounded inference fixed. ' 'Fresh generated worlds provide controlled confirmation; they do not provide independent external or operational confirmation.', '', f"Registered matched-capacity advantage: **{decision['compact_representation_advantage']}**. " f"Noninferiority to all full-information statistical references: **{decision['strongest_full_information_noninferiority']}**. " f"Complete-cost saving gate: **{decision['complete_cost_saving_gate']}**. " f"Practical algorithm promotion: **{decision['practical_algorithm_promotion']}**.", '', '## Same evidence and exact inference', '', 'The primary uses 1,024 labeled records per world, three independent nested sample streams, eight representation symbols, ' 'two positive query budgets and all three fixed query costs. Each of 192 worlds receives equal weight; all six families receive equal weight. ' 'No method receives the generating probability law. The estimator uses counts, the encoder uses its estimated law, and the planner uses the pushed-forward law. ' 'Only the evaluator uses the true law after policy construction.', '', '| Representation | Utility | Representation loss | Model loss | Expected queries |', '|---|---:|---:|---:|---:|'] for name, row in table.iterrows(): lines.append(f"| {name.replace('_', ' ')} | {row.utility:.5f} | {row.representation_loss:.5f} | {row.model_loss:.5f} | {row.queries:.3f} |") lines += ['', 'The uncompressed identity reference retains 64 raw states; all other methods have an eight-symbol ceiling. ' 'Joint forecasting clusters sqrt P(H,Y0,Y1 given E); posterior clustering retains only P(H given E), while the marginal forecast control retains only P(Y0,Y1 given E). ' 'All clustering uses the same deterministic initialization and 32 iterations. Feature controls retain three raw bits selected by individual hypothesis information or marginal entropy.', '', '![Representation and model losses](../../artifacts/decision_sufficiency_01/analysis/decision_sufficiency.png)', '', '## Prespecified paired comparisons', '', '| Compact control | Query joint minus control | Simultaneous interval |', '|---|---:|---:|'] for item in decision['compact_comparisons']: if item['control'] == 'identity': continue lo, hi = item['paired_interval'] lines.append(f"| {item['control'].replace('_', ' ')} | {item['mean_gain']:+.5f} | [{lo:+.5f}, {hi:+.5f}] |") lines += ['', 'Intervals use 10,000 paired world bootstraps within the six fixed families, with Bonferroni adjustment across all six eligible compact controls. ' 'Costs, derived observations and sample streams are not treated as independent cases. The gate requires a mean utility gain of at least 0.01 against every compact control and positive simultaneous lower bounds. ' 'Held mechanisms are redundant observations and aliased orientation; their results remain in the complete aggregate.', '', '## Where value is lost', '', 'For each world, availability, encoder, sample budget and cost, the evaluator verifies:', '', '`full raw oracle − realized value = information loss + representation loss + model loss + inference loss`', '', 'The first term compares full raw evidence with actually available raw evidence. The second compares available raw evidence with its encoded information. ' 'The third compares a true-law encoded oracle with the realized policy planned under the estimated encoded law. ' 'Inference loss is zero in the main comparison because every arm uses exact bounded planning, independently checked by scalar recursion. ' 'This is an error decomposition, not a claim that representation and model errors are independent causes.', '', 'The faithful explicit-law arms isolate compression loss; the three-available-channel condition isolates unavailable information. ' 'The support-mixture estimate is compared with a half-count joint table and a naive-Bayes estimate under identical labeled records. ' 'Posterior Brier scores, log loss, absolute error and hypothesis/future joint predictive log loss are retained separately from decision utility.', '', '## A precise sufficiency boundary', '', 'In the exact constructed control, two contexts have identical P(H1 given E)=0.75. They support the same present action with value 0.25. ' 'A future sensor equals H xor E. Retaining E makes one observation worth 0.99 after its 0.01 cost; merging the contexts leaves value 0.25. ' 'A representation sufficient for present actions need not be sufficient for future information acquisition. ' 'The query-joint representation retains the context and recovers 0.99. This is a known finite decision-theoretic mechanism, not a new theorem or evidence for a neural controller.', '', 'The required representation depends on the admissible actions, utilities and future observation kernels. ' 'Preserving the joint conditional law of the hypothesis and remaining query outcomes is sufficient for this bounded task. ' 'Preserving only a cause posterior is sufficient only when it also determines the relevant future observation kernels. ' 'These conclusions are consistent with established [belief-state planning](https://www.cassandra.org/arc/papers/aij98.pdf) and ' '[predictive state representations](https://proceedings.neurips.cc/paper/2001/file/1e4d36177d71bbb3558e43af9577d70e-Paper.pdf).', '', 'The evaluator also verifies a bounded-utility sufficiency certificate: representation loss is at most ' '`(3 + query_budget × query_cost) × expected within-code conditional total variation`. ' 'This compares each raw conditional law of H and remaining query outcomes with its encoded-cell law. ' 'The same quantity computed under an estimated model describes only that model; it cannot certify true-world adequacy without controlling model error.', '', '## Complete computational cost', '', 'The separate benchmark charges archive import, CUDA initialization, warmup, empirical counting, statistical fitting, encoding, probability-model materialization, exact planning and repeated decisions. ' 'Eleven alternating-order repetitions cover 1, 32, 1,024 and 65,536 requests across the same 192 worlds. ' 'Warmup is charged explicitly rather than treated as free. Request outcomes are read only when their query is selected. ' 'These are local batched decision workloads, not operational telemetry transport or engineer time.', '', '| Model and representation | Requests | Complete seconds | Construction seconds | Replay seconds |', '|---|---:|---:|---:|---:|---:|'] for item in benchmark['medians']: lines.append(f"| {item['model']} / {item['method']} | {item['requests']:,} | {item['median_complete_seconds']:.5f} | " f"{item['median_construction_seconds']:.5f} | {item['median_replay_seconds']:.5f} |") lines += ['', 'Eight encoded states require a 64-byte uint8 evidence lookup plus 512 bytes of float64 joint probabilities per world, versus 4,096 probability bytes for the raw-state model. ' 'This excludes neither the fitted count table nor the cached planner: temporary count/model tables and int64 planner choices are recorded separately in the computation ledger. ' 'A smaller resident representation does not establish a complete-workload speed advantage.', '', '## Frozen history and scope', '', 'The prior external study remains faithful-interface behavioral reuse supported, strict coordinate invariance failed, learning-specific advantage unestablished, ' 'and exact planning computationally preferable in its tested regime. Its ten residual coordinate cases comprise six improved and four worsened root decisions; none has both root choices nonoptimal. ' 'The original gates and scores are unchanged.', '', 'All foundational cognitive results, controller checkpoints, diagnostic negative controls and evidence-backend improvements remain frozen. ' 'This study adds no backbone execution, neural module or controller training. Newly generated parameter samples test controlled generalization; ' 'only two of six structural families are held mechanisms, and none is independently authored external confirmation. ' 'The same statistical procedure is refit per world; this does not test zero-shot transfer of one frozen learned encoder. ' 'No operational diagnostic reliability or engineer productivity claim follows.', '', 'The evidence engine needs no change for this experiment. Retain its separately measured provenance, preparation and repeat-use benefits; ' 'they are independent of the representation comparison.', '', '[Locked protocol](../../configs/decision_sufficiency_protocol_01.json) · ' '[Preconfirmation hashes](../../artifacts/decision_sufficiency_01/preconfirmation_lock.json) · ' '[Decision and all comparisons](../../artifacts/decision_sufficiency_01/analysis/decision.json) · ' '[All budgets and availability conditions](../../artifacts/decision_sufficiency_01/analysis/frontier.csv) · ' '[All six families](../../artifacts/decision_sufficiency_01/analysis/primary_families.csv) · ' '[Ten coordinate cases](../../artifacts/decision_sufficiency_01/coordinate_ten_cases.json) · ' '[Complete timing records](../../artifacts/decision_sufficiency_01/confirmation_benchmark.json)', '', 'Reproduce in an isolated checkout with a fresh artifact directory: ' '`python -m unionmind.decision_sufficiency.study generate --role development`, ' '`evaluate --role development`, `register`, `generate --role confirmation`, `evaluate --role confirmation`, ' 'then `python -m unionmind.decision_sufficiency.benchmark` and ' '`python -m unionmind.decision_sufficiency.analysis`. Existing outputs are never overwritten.', ''] path = Path('docs/experiments/decision-sufficient-representations.md') assert not path.exists() path.write_text('\n'.join(lines)) if __name__ == '__main__': run()