File size: 8,689 Bytes
f1448ba 1e1dffa f1448ba 1e1dffa 38ac4d6 b10ddba 38ac4d6 1e1dffa f1448ba 38ac4d6 b10ddba 38ac4d6 f1448ba 1e1dffa f1448ba 1e1dffa 38ac4d6 1e1dffa f1448ba 1e1dffa b10ddba 1e1dffa f1448ba 1e1dffa f1448ba 1e1dffa 38ac4d6 1e1dffa 38ac4d6 1e1dffa 38ac4d6 b10ddba 38ac4d6 1e1dffa 38ac4d6 1e1dffa f1448ba b10ddba f1448ba 38ac4d6 1e1dffa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | """Three release metrics from the independently audited three-model cohort."""
import hashlib
import json
from pathlib import Path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
ROOT = Path(__file__).resolve().parents[1]
SOURCE = ROOT/'evaluation/released_daily/released_daily_benchmark.json'
DEST = ROOT/'evaluation/released_daily/model_1_2_benchmark.png'
PERIOD_SOURCE = ROOT/'evaluation/deepmind_daily/period_comparison.json'
POST_DEST = ROOT/'evaluation/released_daily/model_1_2_after_2024_benchmark.png'
PRE_DEST = ROOT/'evaluation/released_daily/model_1_2_before_2024_benchmark.png'
def metric_available(row):
"""A zero-support metric is unavailable, never an error of zero."""
values = list(row['values'].values())
coverage = row['coverage']
if coverage['daily_issues'] == 0 and coverage['storms'] == 0:
if not all(v is None for v in values):
raise ValueError('Zero support requires null scores, not numeric bars')
return False
if coverage['daily_issues'] <= 0 or coverage['storms'] <= 0 or any(
v is None or not np.isfinite(v) or v < 0 for v in values):
raise ValueError('Missing or invalid metric cannot become a zero bar')
return True
def period_metrics(period):
scores = period['scores']
pressure = scores['pressure']['JMA']
return {
'pressure_JMA_hpa': {
'values': {m: pressure['models'][m]['mae_hpa']['value'] for m in ('1.1', '1.2', 'deepmind')},
'coverage': {'daily_issues': pressure['daily_cases'], 'storms': pressure['storms']},
},
**{key: {'values': {m: scores['route'][m][key]['value'] for m in ('1.1', '1.2', 'deepmind')},
'coverage': {'daily_issues': period['daily_issues'], 'storms': period['storms']}}
for key in ('mean_track_error_km', 'direction_error_deg')},
}
def main():
source = json.loads(SOURCE.read_text())
if source['status'] != 'complete_verified' or source['cohort']['daily_issues'] != 1473:
raise ValueError('Chart requires verified daily-cohort results')
reference = next(m for m in source['models'] if m['key'] == 'deepmind')
audit = json.loads((ROOT/'evaluation/deepmind_daily/publication_audit.json').read_text())
if reference['status'] != 'complete_verified' or audit['snapshot_sha256'] != hashlib.sha256(SOURCE.read_bytes()).hexdigest():
raise ValueError('DeepMind bars require the matching independent publication audit')
periods = json.loads(PERIOD_SOURCE.read_text())
if periods['status'] != 'complete_verified' or periods['source_hashes']['snapshot_sha256'] != hashlib.sha256(SOURCE.read_bytes()).hexdigest():
raise ValueError('Period comparison requires the exact original audited snapshot')
if periods['model'] != reference or not periods['single_checkpoint_across_all_periods']:
raise ValueError('Period groups are not different checkpoint generations')
for key, dest, title in (('total', DEST, 'Total · mixed-year comparison'),
('before_2024', PRE_DEST, 'Before 2024 · historical comparison'),
('after_2024', POST_DEST, 'After 2024 · 2025–2026 comparison')):
metrics = period_metrics(periods['periods'][key])
draw_chart(metrics, reference, periods, key, dest, title)
def draw_chart(metrics, reference, periods, period_key, dest, title):
models = ('1.1', '1.2', 'deepmind')
plt.rcParams.update({'font.family': 'DejaVu Sans', 'font.size': 11,
'axes.spines.top': False, 'axes.spines.right': False,
'axes.spines.left': False, 'axes.spines.bottom': False})
fig, axes = plt.subplots(1, 3, figsize=(15.2, 6.1))
fig.set_facecolor('#f6f8fc')
specs = [('pressure_JMA_hpa', 'Pressure intensity', 'Central-pressure MAE · hPa', 20, 2),
('mean_track_error_km', 'Track position', 'Mean position error · km', 1200, 1),
('direction_error_deg', 'Track direction', 'Six-hour heading error · degrees', 75, 2)]
for ax, (key, panel_title, label, maximum, digits) in zip(axes, specs):
row = metrics[key]
values = [row['values'][k] for k in models]
ax.set_facecolor('#f6f8fc')
ax.set_title(panel_title, loc='left', fontsize=16, fontweight='bold', color='#21334c', pad=38)
ax.text(0, 1.03, label, transform=ax.transAxes, color='#53647c', fontsize=10)
if not metric_available(row):
ax.set_xticks([])
ax.set_yticks([])
ax.text(.5, .65, 'Not scored', transform=ax.transAxes, ha='center',
fontsize=22, fontweight='bold', color='#53647c')
ax.text(.5, .35, 'No shared valid JMA pressure cases.\nFrozen 1.1 intensity inputs unavailable.\nMissing is not a zero error.',
transform=ax.transAxes, ha='center', va='center',
fontsize=11, linespacing=1.65, color='#53647c')
ax.text(.5, -.33, '0 shared pressure starts / 0 storms · no bars',
transform=ax.transAxes, ha='center', color='#53647c', fontsize=10)
continue
ax.set_axisbelow(True)
ax.grid(axis='y', color='#dfe5ee', linewidth=.8)
ax.bar([0, 1, 2], values, width=.55, color=['#9aa9bc', '#3c67d6', '#9073cf'], zorder=3)
for x, value in enumerate(values):
ax.text(x, value+maximum*.038, f'{value:,.{digits}f}', ha='center',
fontsize=17, fontweight='bold', color='#21334c')
ax.set_ylim(0, maximum)
ax.set_xticks([0, 1, 2], ['1.1', '1.2\nmean of 50', 'DeepMind Mini\n<2024 · 1 member'])
ax.tick_params(axis='both', length=0, labelcolor='#53647c')
coverage = row['coverage']
ax.text(.5, -.33, f"{coverage['daily_issues']:,} daily starts / {coverage['storms']} storms",
transform=ax.transAxes, ha='center', color='#53647c', fontsize=10)
fig.text(.055, .945, title, fontsize=23, fontweight='bold', color='#182b45')
fig.text(.055, .89, 'DeepMind = WeatherNext Cyclones Mini <2024 · software v0.3.0 · 1° · one CUDA member', fontsize=12, color='#53647c')
fig.text(.055, .84, 'Same saved forecasts and valid times · +6 to +120 h · equal storm weight · lower is better', fontsize=11, color='#53647c')
if period_key == 'total':
note = 'Storm proportions: <2024 85.2% · 2024 8.1% · >2024 6.7%. Historical dates overlap Mini fitting years.'
elif period_key == 'before_2024':
note = 'Strict UTC issue year <2024: 1,336 starts / 230 storms, all 1980–1999 here. These dates overlap Mini fitting years.'
else:
note = 'Strict UTC issue year >2024: 61 starts / 18 storms. Calendar 2024 is excluded, not counted as >2024.'
fig.text(.055, .09, note, fontsize=10, color='#53647c')
fig.text(.055, .052, 'Pressure coverage is smaller than track coverage. Missing is not zero. Different pipelines/member counts; no significance or unused-holdout claim.', fontsize=9.5, color='#53647c')
fig.subplots_adjust(left=.055, right=.975, top=.705, bottom=.32, wspace=.36)
fig.savefig(dest, dpi=180, facecolor=fig.get_facecolor())
plt.close(fig)
receipt = {'source': PERIOD_SOURCE.relative_to(ROOT).as_posix(), 'source_sha256': hashlib.sha256(PERIOD_SOURCE.read_bytes()).hexdigest(),
'chart': dest.relative_to(ROOT).as_posix(), 'chart_sha256': hashlib.sha256(dest.read_bytes()).hexdigest(),
'period': period_key, 'period_definition': periods['period_definition'][period_key],
'metrics': [s[0] for s in specs], 'values': {k: metrics[k]['values'] for k, *_ in specs},
'deepmind_status': 'complete_verified', 'deepmind_model': reference,
'models': list(models),
'bars_per_panel': 3 if all(metric_available(r) for r in metrics.values()) else None,
'available_bars_per_panel': {k: 3 if metric_available(r) else 0 for k, r in metrics.items()},
'coverage': {k: r['coverage'] for k, r in metrics.items()},
'unavailable_metrics': [k for k, r in metrics.items() if not metric_available(r)],
'error_bars': False,
'zero_fill': False, 'model_forecasts_modified': False,
'publication_audit_sha256': hashlib.sha256((ROOT/'evaluation/deepmind_daily/publication_audit.json').read_bytes()).hexdigest()}
dest.with_suffix('.json').write_text(json.dumps(receipt, indent=2)+'\n')
print(json.dumps({'chart': receipt['chart'], 'period': period_key, 'values': receipt['values']}))
if __name__ == '__main__':
main()
|