"""Generate every plotted value and table body from archived result records."""
from pathlib import Path
import json,csv,statistics,shutil
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
R=Path(__file__).resolve().parents[1];A=R/'paper';F=A/'figures';T=A/'tables';F.mkdir(exist_ok=True);T.mkdir(exist_ok=True);(A/'metadata').mkdir(exist_ok=True)
plt.rcParams.update({'font.family':'DejaVu Sans','font.size':11.5,'axes.spines.top':False,'axes.spines.right':False,'pdf.fonttype':42,'savefig.bbox':'tight'})
NAMES={'additive_full':'Additive Gaussian','additive_diag':'Diagonal Gaussian','factorized_neural':'Factorized forward MLP','joint_neural':'Joint-state forward MLP','factorized_nb':'Factorized Gaussian NB','factorized_lda':'Factorized Gaussian LDA','factorized_kernel':'Factorized kernel','factorized_ridge':'Factorized ridge','joint_gaussian':'Joint-state Gaussian','factorized_logistic':'Factorized logistic','factorized_classifier':'Direct factorized MLP'}
METHODS={'query_joint':'Joint predictive clustering','posterior_clusters':'Posterior clustering','posterior_bins':'Posterior bins','future_marginal':'Observation clustering','raw_information':'Three features by mutual information','raw_entropy':'Three features by entropy','hash_code':'Fixed hash','identity':'Uncompressed mixture'}
load=lambda p:json.loads((R/p).read_text())
control=load('results/controlled/decision.json');day=load('provenance/verification/independent_statistics.json');hydro=load('provenance/hydraulic/opened_hydraulic_summary.json');summary=load('provenance/temperature/summary.json')
primary={r['method']:r for r in control['primary_metrics']}
with (T/'controlled.tex').open('w') as f:
 for m in ['query_joint','raw_information','raw_entropy','posterior_clusters','posterior_bins','future_marginal','hash_code','identity']:
  r=primary[m];f.write(f"{METHODS[m]} & {r['utility']:.4f} & {r['representation_loss']:.4f} & {r['model_loss']:.4f} " + chr(92)*2 + chr(10))
front=list(csv.DictReader((R/'results/controlled/frontier.csv').open()))
fig,ax=plt.subplots(1,2,figsize=(8,3.7))
for m,c in [('query_joint','#176b91'),('raw_information','#ba682e'),('posterior_clusters','#767676'),('identity','#277a50')]:
 xs=[64,256,1024,4096];ys=[]
 for n in xs:
  rows=[r for r in front if r['availability']=='full' and r['model']=='support_mixture' and r['method']==m and int(r['label_budget'])==n and int(r['query_budget'])>0]
  assert len(rows)==2,(m,n,len(rows));ys.append(statistics.mean(float(r['utility']) for r in rows))
 ax[0].plot(xs,ys,'o-',color=c,label=METHODS[m])
ax[0].set_xscale('log',base=2);ax[0].set_xticks(xs,[str(x) for x in xs]);ax[0].set_xlabel('Estimation records');ax[0].set_ylabel('Expected utility');ax[0].set_title('(a) Sample dependence');ax[0].legend(fontsize=9,loc='lower right')
fam=list(csv.DictReader((R/'results/controlled/primary_families.csv').open()));families=sorted(set(r['family'] for r in fam))
for m,dx,c in [('query_joint',-.24,'#176b91'),('raw_information',0,'#ba682e'),('posterior_clusters',.24,'#767676')]:
 ys=[float(next(r['utility'] for r in fam if r['family']==fa and r['method']==m)) for fa in families]
 ax[1].bar(np.arange(6)+dx,ys,.23,color=c,label=METHODS[m])
ax[1].set_xticks(np.arange(6),[x.replace('_','\n') for x in families],rotation=30,ha='right',fontsize=9);ax[1].set_ylabel('Expected utility');ax[1].set_title('(b) All six declared mechanism families');fig.tight_layout();fig.savefig(F/'controlled.pdf');plt.close(fig)
# Matrix uses independent verification's equal-day scores, never the pooled-episode scores.
def dscore(model,n):return next(r['top3_equal_day'] for r in day['summary'] if r['model']==model and r['samples']==n)
def hscore(model,n):
 rows=[r for r in hydro['rows'] if r.get('model')==model and r.get('samples')==n]
 assert len(rows)==1,(model,n,rows);return rows[0]['top3']
with (T/'all_models.tex').open('w') as f:
 for m in NAMES:
  vals=[hscore(m,64),hscore(m,128),dscore(m,64),dscore(m,128)]
  f.write(NAMES[m]+' & '+' & '.join(f'{v*100:.2f}' for v in vals)+' '+chr(92)*2+chr(10))
fig,axes=plt.subplots(1,2,figsize=(8,5.2),sharey=True)
for axis,n in zip(axes,[64,128]):
 y=np.arange(len(NAMES));axis.barh(y-.16,[100*hscore(m,n) for m in NAMES],.3,label='Hydraulic: opened development',color='#ba682e');axis.barh(y+.16,[100*dscore(m,n) for m in NAMES],.3,label='Gas array: four held days',color='#176b91');axis.set_yticks(y,list(NAMES.values()),fontsize=11);axis.set_ylim(len(NAMES)-.5,-.5);axis.set_xlim(0,105);axis.set_xlabel('Unseen-combination Top-3 (%)');axis.set_title(f'{n} total fitting examples');axis.grid(axis='x',alpha=.2)
axes[1].legend(fontsize=9,loc='lower right');fig.tight_layout();fig.savefig(F/'models.pdf');plt.close(fig)
fig,axes=plt.subplots(1,3,figsize=(8,3))
for axis,reference in zip(axes,['factorized_classifier','additive_full','factorized_lda']):
 a=next(r['per_day'] for r in day['summary'] if r['model']=='factorized_neural' and r['samples']==64);b=next(r['per_day'] for r in day['summary'] if r['model']==reference and r['samples']==64);g=100*(np.array(a)-np.array(b));axis.axhline(0,color='black',lw=.8);axis.scatter(range(1,5),g,color='#176b91',s=45);axis.set_xticks(range(1,5));axis.set_ylim(-20,50);axis.set_xlabel('Held day (ordered)');axis.set_title(NAMES[reference],fontsize=10);axis.grid(alpha=.15)
axes[0].set_ylabel('Forward MLP gain (percentage points)');fig.tight_layout();fig.savefig(F/'days.pdf');plt.close(fig)
with (T/'seen_models.tex').open('w') as f:
 for m in NAMES:
  vals=[]
  for n in [64,128]:
   rows=[r for r in summary['rows'] if r.get('model')==m and r.get('samples')==n and r.get('stratum')=='seen' and r.get('day') is not None]
   assert len(rows)==4;vals.append(statistics.mean(r['top3'] for r in rows))
  f.write(NAMES[m]+' & '+' & '.join(f'{100*v:.2f}' for v in vals)+' '+chr(92)*2+chr(10))
proof={'controlled_gain':control['compact_comparisons'][3], 'temperature_scores':{m:dscore(m,64) for m in NAMES},'hydraulic_scores':{m:hscore(m,64) for m in NAMES},'figure_sources':['results/controlled/frontier.csv','results/controlled/primary_families.csv','provenance/verification/independent_statistics.json','provenance/hydraulic/opened_hydraulic_summary.json']}
(A/'metadata/figure_values.json').write_text(json.dumps(proof,indent=2)+'\n')
diagnostics=load('results/sensitivity_analysis/frozen_diagnostics.json')
with (T/'sensitivity.tex').open('w') as f:
 for cohort,label in [('hydraulic','Hydraulic'),('temperature','Gas array')]:
  for n in [64,128]:
   rows={r['model']:r for r in diagnostics['sensor']['summary'] if r['cohort']==cohort and r['samples']==n}
   values=[label,str(n),str(rows['factorized_neural']['observed_combinations'])]
   values += [f"{100*rows[m]['macro_combination_top3']:.2f}" for m in ['factorized_neural','factorized_classifier','additive_full','factorized_lda']]
   f.write(' & '.join(values)+' '+chr(92)*2+chr(10))
print('Generated 3 vector figures and 5 table bodies')

for filename,macro in [("controlled.tex","ControlledRows"),("all_models.tex","AllModelRows"),("seen_models.tex","SeenModelRows"),("sensitivity.tex","SensitivityRows")]:
 p=T/filename;p.write_text(chr(92)+"newcommand{"+chr(92)+macro+"}{%"+chr(10)+p.read_text()+"}%"+chr(10))

# The added control is descriptive on opened worlds, not a revised primary test.
import gzip
with gzip.open(R/'results/postconfirmation/controlled_frontiers.csv.gz','rt') as handle:
    additional=list(csv.DictReader(handle))
new_timing=load('results/postconfirmation/controlled_summary.json')['warm_batched_construction_medians']
order=[('identity','Uncompressed mixture'),('query_joint','Joint predictive clustering'),
       ('posterior_clusters','Posterior clustering'),('posterior_bins','Posterior bins'),
       ('future_marginal','Observation-only clustering'),('raw_information','Mutual-information selection'),
       ('raw_entropy','Entropy selection'),('hash_code','Fixed hash'),
       ('action_values','Action-value clustering (added)')]
rows=[chr(92)+'newcommand{'+chr(92)+'ActionValueRows}{']
for method,label in order:
    values=[]
    for n in [64,256,1024,4096]:
        selected=[float(r['utility']) for r in additional if int(r['available'])==6
                  and r['estimator']=='support_mixture' and r['method']==method
                  and int(r['labels'])==n and int(r['budget'])>0]
        assert len(selected)==6
        values.append(statistics.mean(selected))
    rows.append(label+' & '+' & '.join(f'{v:.4f}' for v in values)
                +f' & {1000*new_timing[method]:.2f}'+chr(92)*2)
rows.extend(['}',chr(92)+'newcommand{'+chr(92)+'CrossedCovarianceRows}{'])
cross=load('results/postconfirmation/sensor_diagnostics.json')['crossed_mean_covariance']
for m in [64,128]:
    for c in [64,128]:
        selected=[r for r in cross if r['mean_labels']==m and r['covariance_labels']==c]
        rows.append(f'{m} & {c} & {100*statistics.mean(r["top3"] for r in selected):.2f}'
                    +f' & {statistics.mean(r["hypothesis_log_loss"] for r in selected):.2f}'+chr(92)*2)
rows.append('}')
(T/'postconfirmation.tex').write_text('\n'.join(rows)+'\n')