Download unionmind/decision_sufficiency/analysis.py from nur-dev/unionmind-predictive-representations: direct link, hf CLI and curl.
- Browser
- Download file 20.5 kB
-
https://huggingface.co/nur-dev/unionmind-predictive-representations/resolve/main/unionmind/decision_sufficiency/analysis.py
- Command line
-
hf download hf://nur-dev/unionmind-predictive-representations/unionmind/decision_sufficiency/analysis.py
-
curl -L -o analysis.py https://huggingface.co/nur-dev/unionmind-predictive-representations/resolve/main/unionmind/decision_sufficiency/analysis.py
20.5 kB
| """Prespecified paired analysis. No threshold or method selection on confirmation.""" | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from .core import encode, plan, push | |
| from .study import PROTOCOL, ROOT, verify_lock | |
| from .worlds import write | |
| def paired_interval(values, families, seed, draws=10000, alpha=.05): | |
| rng = np.random.default_rng(seed) | |
| estimates = np.zeros(draws) | |
| groups = sorted(set(families)) | |
| for group in groups: | |
| x = values[np.asarray(families) == group] | |
| estimates += x[rng.integers(len(x), size=(draws, len(x)))].mean(1)/len(groups) | |
| return [float(np.quantile(estimates, alpha/2)), | |
| float(np.quantile(estimates, 1-alpha/2))] | |
| def mechanism_control(device): | |
| joint = torch.zeros((1, 2, 2, 4), device=device, dtype=torch.float64) | |
| for e in (0, 1): | |
| for h in (0, 1): | |
| for nuisance in (0, 1): | |
| joint[0, e, h, 2*(h ^ e)+nuisance] = .5*(.75 if h else .25)*.5 | |
| methods = ['identity', 'posterior_bins', 'query_joint'] | |
| return {method: plan(push(joint, *encode(joint, method)))['expected'][0].tolist() | |
| for method in methods} | |
| def run(): | |
| verify_lock() | |
| protocol = json.loads(PROTOCOL.read_text()) | |
| destination = ROOT/'analysis' | |
| assert not destination.exists() | |
| destination.mkdir() | |
| metrics = ['utility', 'queries', 'abstention', 'full_oracle', 'available_oracle', | |
| 'encoded_oracle', 'information_loss', 'representation_loss', 'model_loss', | |
| 'total_regret', 'posterior_brier', 'posterior_log_loss', | |
| 'posterior_absolute_error', 'joint_forecast_log_loss', | |
| 'estimated_conditional_dispersion', 'true_conditional_dispersion'] | |
| keys = ['world', 'family', 'availability', 'sample_stream', 'label_budget', | |
| 'model', 'method', 'query_budget', 'query_cost'] | |
| frame = pd.read_csv(ROOT/'confirmation_evaluation/predictions.csv.gz', usecols=keys+metrics) | |
| primary = frame[(frame.model == protocol['primary']['model']) & | |
| (frame.label_budget == protocol['primary']['label_budget']) & | |
| (frame.availability == protocol['primary']['availability']) & (frame.query_budget > 0)] | |
| per_world = primary.groupby(['world', 'family', 'method'])[metrics].mean().reset_index() | |
| table = per_world.groupby('method')[metrics].mean().reset_index() | |
| table.to_csv(destination/'primary.csv', index=False) | |
| per_world.to_csv(destination/'primary_worlds.csv', index=False) | |
| family = per_world.groupby(['family', 'method'])[metrics].mean().reset_index() | |
| family.to_csv(destination/'primary_families.csv', index=False) | |
| pivot = per_world.pivot(index=['world', 'family'], columns='method', values='utility') | |
| comparisons = [] | |
| controls = protocol['primary']['compact_controls'] | |
| for control in controls+['identity']: | |
| difference = (pivot['query_joint']-pivot[control]).to_numpy() | |
| interval = paired_interval(difference, pivot.index.get_level_values('family').to_numpy(), | |
| protocol['primary']['bootstrap_seed'], protocol['primary']['bootstrap_draws'], | |
| alpha=.05/len(controls) if control != 'identity' else .05) | |
| comparisons.append(dict(control=control, mean_gain=float(difference.mean()), | |
| paired_interval=interval, simultaneous_compact_familywise_95=control != 'identity')) | |
| compact = [x for x in comparisons if x['control'] != 'identity'] | |
| compact_gate = all(x['mean_gain'] >= protocol['primary']['mean_utility_gain_at_least'] and | |
| x['paired_interval'][0] > protocol['primary']['all_paired_lower_95_bounds_above'] | |
| for x in compact) | |
| full = frame[(frame.method == 'identity') & (frame.label_budget == protocol['primary']['label_budget']) & | |
| (frame.availability == 'full') & (frame.query_budget > 0)] | |
| full_values = full.groupby(['world', 'family', 'model']).utility.mean().unstack('model') | |
| full_comparisons = [] | |
| for model in protocol['estimation']['models']: | |
| difference = (pivot['query_joint']-full_values[model]).to_numpy() | |
| ci = paired_interval(difference, pivot.index.get_level_values('family').to_numpy(), | |
| protocol['primary']['bootstrap_seed'], alpha=.05/len(protocol['estimation']['models'])) | |
| full_comparisons.append(dict(model=model, mean_gain=float(difference.mean()), paired_interval=ci)) | |
| quality_ni = all(x['paired_interval'][0] > -protocol['primary']['full_information_noninferiority_margin'] | |
| for x in full_comparisons) | |
| frontier = frame.groupby(['availability', 'label_budget', 'model', 'method', 'query_budget'])[metrics].mean().reset_index() | |
| frontier.to_csv(destination/'frontier.csv', index=False) | |
| # Preserve each modeled query cost as well; no favorable-cost aggregation. | |
| frame.groupby(['availability', 'label_budget', 'model', 'method', 'query_budget', 'query_cost'])[metrics].mean().to_csv(destination/'frontier_by_cost.csv') | |
| benchmark = json.loads((ROOT/'confirmation_benchmark.json').read_text()) | |
| timing = pd.DataFrame(benchmark['records']) | |
| savings = [] | |
| for n in protocol['cost_promotion']['repeated_request_counts']: | |
| candidate = timing[(timing.method == 'query_joint') & (timing.requests == n)].set_index('repetition') | |
| for model in protocol['estimation']['models']: | |
| reference = timing[(timing.method == 'identity') & (timing.model == model) & | |
| (timing.requests == n)].set_index('repetition') | |
| ratio = 1-candidate.complete_seconds/reference.complete_seconds | |
| interval = paired_interval(ratio.to_numpy(), np.zeros(len(ratio)), 18781, | |
| alpha=.05/(len(protocol['estimation']['models'])*4)) | |
| savings.append(dict(requests=n, reference_model=model, mean_saving_fraction=float(ratio.mean()), | |
| simultaneous_interval=interval)) | |
| cost_gate = all(x['mean_saving_fraction'] >= .20 and x['simultaneous_interval'][0] > 0 | |
| for x in savings) | |
| explicit = frame[(frame.model == 'faithful_explicit') & (frame.method == 'identity')] | |
| code_loss = frame[(frame.model == 'faithful_explicit') & (frame.query_budget > 0) & | |
| (frame.availability == 'full')].groupby('method')[metrics].mean().reset_index() | |
| code_loss.to_csv(destination/'faithful_representations.csv', index=False) | |
| code_counterexample = mechanism_control(protocol['execution']['device']) | |
| write(destination/'mechanism_control.json', dict( | |
| status='EXACT_CONSTRUCTED_COUNTEREXAMPLE_NOT_EMPIRICAL_NOVELTY', | |
| contexts_equal_posterior=.75, utilities_by_budget_then_cost=code_counterexample, | |
| current_posterior_sufficient=True, adaptive_posterior_sufficient=False, | |
| interpretation='The cause posterior does not preserve context that determines how an observation is related to the cause. Query_joint retains that relationship.')) | |
| decision = dict(status='CONTROLLED_CONFIRMATION_COMPLETE', | |
| compact_representation_advantage=compact_gate, | |
| strongest_full_information_noninferiority=quality_ni, | |
| complete_cost_saving_gate=cost_gate, | |
| controlled_quality_cost_gate=bool(compact_gate and quality_ni and cost_gate), | |
| practical_algorithm_promotion=False, | |
| promotion_requires_independent_external_confirmation=True, | |
| external_generalization_established=False, learned_control_advantage_tested=False, | |
| independent_external_authorship=False, worlds=len(pivot), | |
| confirmation_families=sorted(pivot.index.get_level_values('family').unique()), | |
| held_mechanisms=protocol['worlds']['held_mechanisms'], | |
| compact_comparisons=comparisons, full_model_comparisons=full_comparisons, | |
| complete_cost_comparisons=savings, threshold_changes_after_opening=False, | |
| primary_sample_budget=protocol['primary']['label_budget'], | |
| explicit_information_loss=float(explicit[explicit.availability == 'three_available'].information_loss.mean()), | |
| primary_metrics=table.to_dict('records')) | |
| write(destination/'decision.json', decision) | |
| plot(table, family, destination) | |
| report(decision, code_loss, benchmark, destination) | |
| print(json.dumps({k: decision[k] for k in ['status', 'compact_representation_advantage', | |
| 'strongest_full_information_noninferiority', 'complete_cost_saving_gate', 'practical_algorithm_promotion']})) | |
| def plot(table, family, destination): | |
| import matplotlib | |
| matplotlib.use('Agg') | |
| import matplotlib.pyplot as plt | |
| order = ['identity', 'query_joint', 'posterior_clusters', 'posterior_bins', | |
| 'future_marginal', 'raw_information', 'raw_entropy', 'hash_code'] | |
| values = table.set_index('method').loc[order] | |
| fig, ax = plt.subplots(1, 2, figsize=(12, 4.8), layout='constrained') | |
| y = np.arange(len(order)) | |
| ax[0].barh(y, values.utility, color=['#627282', '#237b67']+['#a9b7c4']*6) | |
| ax[0].set_yticks(y, [x.replace('_', ' ') for x in order]); ax[0].invert_yaxis() | |
| ax[0].set_xlabel('Expected utility, including query cost') | |
| ax[0].set_title('Same estimated law and exact planner') | |
| ax[1].barh(y, values.representation_loss, label='Representation loss', color='#c58c3b') | |
| ax[1].barh(y, values.model_loss, left=values.representation_loss, | |
| label='Model loss', color='#ae5e66') | |
| ax[1].set_yticks(y, [x.replace('_', ' ') for x in order]); ax[1].invert_yaxis() | |
| ax[1].set_xlabel('Utility loss against the raw-information oracle') | |
| ax[1].set_title('Information loss and inference error are zero here') | |
| ax[1].legend(frameon=False) | |
| for extension in ('png', 'svg'): | |
| fig.savefig(destination/f'decision_sufficiency.{extension}', dpi=180) | |
| plt.close(fig) | |
| def report(decision, faithful, benchmark, destination): | |
| table = pd.DataFrame(decision['primary_metrics']).set_index('method') | |
| lines = ['# Decision sufficient representations', '', | |
| 'The study separates missing information, representation loss and model error while holding exact bounded inference fixed. ' | |
| 'Fresh generated worlds provide controlled confirmation; they do not provide independent external or operational confirmation.', '', | |
| f"Registered matched-capacity advantage: **{decision['compact_representation_advantage']}**. " | |
| f"Noninferiority to all full-information statistical references: **{decision['strongest_full_information_noninferiority']}**. " | |
| f"Complete-cost saving gate: **{decision['complete_cost_saving_gate']}**. " | |
| f"Practical algorithm promotion: **{decision['practical_algorithm_promotion']}**.", '', | |
| '## Same evidence and exact inference', '', | |
| 'The primary uses 1,024 labeled records per world, three independent nested sample streams, eight representation symbols, ' | |
| 'two positive query budgets and all three fixed query costs. Each of 192 worlds receives equal weight; all six families receive equal weight. ' | |
| 'No method receives the generating probability law. The estimator uses counts, the encoder uses its estimated law, and the planner uses the pushed-forward law. ' | |
| 'Only the evaluator uses the true law after policy construction.', '', | |
| '| Representation | Utility | Representation loss | Model loss | Expected queries |', | |
| '|---|---:|---:|---:|---:|'] | |
| for name, row in table.iterrows(): | |
| lines.append(f"| {name.replace('_', ' ')} | {row.utility:.5f} | {row.representation_loss:.5f} | {row.model_loss:.5f} | {row.queries:.3f} |") | |
| lines += ['', 'The uncompressed identity reference retains 64 raw states; all other methods have an eight-symbol ceiling. ' | |
| 'Joint forecasting clusters sqrt P(H,Y0,Y1 given E); posterior clustering retains only P(H given E), while the marginal forecast control retains only P(Y0,Y1 given E). ' | |
| 'All clustering uses the same deterministic initialization and 32 iterations. Feature controls retain three raw bits selected by individual hypothesis information or marginal entropy.', '', | |
| '', '', | |
| '## Prespecified paired comparisons', '', | |
| '| Compact control | Query joint minus control | Simultaneous interval |', '|---|---:|---:|'] | |
| for item in decision['compact_comparisons']: | |
| if item['control'] == 'identity': | |
| continue | |
| lo, hi = item['paired_interval'] | |
| lines.append(f"| {item['control'].replace('_', ' ')} | {item['mean_gain']:+.5f} | [{lo:+.5f}, {hi:+.5f}] |") | |
| lines += ['', 'Intervals use 10,000 paired world bootstraps within the six fixed families, with Bonferroni adjustment across all six eligible compact controls. ' | |
| 'Costs, derived observations and sample streams are not treated as independent cases. The gate requires a mean utility gain of at least 0.01 against every compact control and positive simultaneous lower bounds. ' | |
| 'Held mechanisms are redundant observations and aliased orientation; their results remain in the complete aggregate.', '', | |
| '## Where value is lost', '', | |
| 'For each world, availability, encoder, sample budget and cost, the evaluator verifies:', '', | |
| '`full raw oracle − realized value = information loss + representation loss + model loss + inference loss`', '', | |
| 'The first term compares full raw evidence with actually available raw evidence. The second compares available raw evidence with its encoded information. ' | |
| 'The third compares a true-law encoded oracle with the realized policy planned under the estimated encoded law. ' | |
| 'Inference loss is zero in the main comparison because every arm uses exact bounded planning, independently checked by scalar recursion. ' | |
| 'This is an error decomposition, not a claim that representation and model errors are independent causes.', '', | |
| 'The faithful explicit-law arms isolate compression loss; the three-available-channel condition isolates unavailable information. ' | |
| 'The support-mixture estimate is compared with a half-count joint table and a naive-Bayes estimate under identical labeled records. ' | |
| 'Posterior Brier scores, log loss, absolute error and hypothesis/future joint predictive log loss are retained separately from decision utility.', '', | |
| '## A precise sufficiency boundary', '', | |
| 'In the exact constructed control, two contexts have identical P(H1 given E)=0.75. They support the same present action with value 0.25. ' | |
| 'A future sensor equals H xor E. Retaining E makes one observation worth 0.99 after its 0.01 cost; merging the contexts leaves value 0.25. ' | |
| 'A representation sufficient for present actions need not be sufficient for future information acquisition. ' | |
| 'The query-joint representation retains the context and recovers 0.99. This is a known finite decision-theoretic mechanism, not a new theorem or evidence for a neural controller.', '', | |
| 'The required representation depends on the admissible actions, utilities and future observation kernels. ' | |
| 'Preserving the joint conditional law of the hypothesis and remaining query outcomes is sufficient for this bounded task. ' | |
| 'Preserving only a cause posterior is sufficient only when it also determines the relevant future observation kernels. ' | |
| 'These conclusions are consistent with established [belief-state planning](https://www.cassandra.org/arc/papers/aij98.pdf) and ' | |
| '[predictive state representations](https://proceedings.neurips.cc/paper/2001/file/1e4d36177d71bbb3558e43af9577d70e-Paper.pdf).', '', | |
| 'The evaluator also verifies a bounded-utility sufficiency certificate: representation loss is at most ' | |
| '`(3 + query_budget × query_cost) × expected within-code conditional total variation`. ' | |
| 'This compares each raw conditional law of H and remaining query outcomes with its encoded-cell law. ' | |
| 'The same quantity computed under an estimated model describes only that model; it cannot certify true-world adequacy without controlling model error.', '', | |
| '## Complete computational cost', '', | |
| 'The separate benchmark charges archive import, CUDA initialization, warmup, empirical counting, statistical fitting, encoding, probability-model materialization, exact planning and repeated decisions. ' | |
| 'Eleven alternating-order repetitions cover 1, 32, 1,024 and 65,536 requests across the same 192 worlds. ' | |
| 'Warmup is charged explicitly rather than treated as free. Request outcomes are read only when their query is selected. ' | |
| 'These are local batched decision workloads, not operational telemetry transport or engineer time.', '', | |
| '| Model and representation | Requests | Complete seconds | Construction seconds | Replay seconds |', | |
| '|---|---:|---:|---:|---:|---:|'] | |
| for item in benchmark['medians']: | |
| lines.append(f"| {item['model']} / {item['method']} | {item['requests']:,} | {item['median_complete_seconds']:.5f} | " | |
| f"{item['median_construction_seconds']:.5f} | {item['median_replay_seconds']:.5f} |") | |
| lines += ['', 'Eight encoded states require a 64-byte uint8 evidence lookup plus 512 bytes of float64 joint probabilities per world, versus 4,096 probability bytes for the raw-state model. ' | |
| 'This excludes neither the fitted count table nor the cached planner: temporary count/model tables and int64 planner choices are recorded separately in the computation ledger. ' | |
| 'A smaller resident representation does not establish a complete-workload speed advantage.', '', | |
| '## Frozen history and scope', '', | |
| 'The prior external study remains faithful-interface behavioral reuse supported, strict coordinate invariance failed, learning-specific advantage unestablished, ' | |
| 'and exact planning computationally preferable in its tested regime. Its ten residual coordinate cases comprise six improved and four worsened root decisions; none has both root choices nonoptimal. ' | |
| 'The original gates and scores are unchanged.', '', | |
| 'All foundational cognitive results, controller checkpoints, diagnostic negative controls and evidence-backend improvements remain frozen. ' | |
| 'This study adds no backbone execution, neural module or controller training. Newly generated parameter samples test controlled generalization; ' | |
| 'only two of six structural families are held mechanisms, and none is independently authored external confirmation. ' | |
| 'The same statistical procedure is refit per world; this does not test zero-shot transfer of one frozen learned encoder. ' | |
| 'No operational diagnostic reliability or engineer productivity claim follows.', '', | |
| 'The evidence engine needs no change for this experiment. Retain its separately measured provenance, preparation and repeat-use benefits; ' | |
| 'they are independent of the representation comparison.', '', | |
| '[Locked protocol](../../configs/decision_sufficiency_protocol_01.json) · ' | |
| '[Preconfirmation hashes](../../artifacts/decision_sufficiency_01/preconfirmation_lock.json) · ' | |
| '[Decision and all comparisons](../../artifacts/decision_sufficiency_01/analysis/decision.json) · ' | |
| '[All budgets and availability conditions](../../artifacts/decision_sufficiency_01/analysis/frontier.csv) · ' | |
| '[All six families](../../artifacts/decision_sufficiency_01/analysis/primary_families.csv) · ' | |
| '[Ten coordinate cases](../../artifacts/decision_sufficiency_01/coordinate_ten_cases.json) · ' | |
| '[Complete timing records](../../artifacts/decision_sufficiency_01/confirmation_benchmark.json)', '', | |
| 'Reproduce in an isolated checkout with a fresh artifact directory: ' | |
| '`python -m unionmind.decision_sufficiency.study generate --role development`, ' | |
| '`evaluate --role development`, `register`, `generate --role confirmation`, `evaluate --role confirmation`, ' | |
| 'then `python -m unionmind.decision_sufficiency.benchmark` and ' | |
| '`python -m unionmind.decision_sufficiency.analysis`. Existing outputs are never overwritten.', ''] | |
| path = Path('docs/experiments/decision-sufficient-representations.md') | |
| assert not path.exists() | |
| path.write_text('\n'.join(lines)) | |
| if __name__ == '__main__': | |
| run() | |