Blame for refresh_data.py

#!/usr/bin/env python3
"""Reproducible public-data snapshot for ASCEND (all World Bank economies); no synthetic fallback.

Usage:
  python3 refresh_data.py            # rebuild data.json from live public APIs
  python3 refresh_data.py --validate # re-validate the committed data.json

Optional: if geo/access_summary.json exists (produced by access_pipeline.py), its
modelled geographic-access indicators are merged as a separate, clearly labelled group.
"""
from pathlib import Path
import argparse, concurrent.futures, datetime, gzip, html, json, math, re, time
import urllib.request, urllib.parse

BASE = Path(__file__).resolve().parent
DEST = BASE / 'data.json'
RAW = BASE / 'data_raw'
ACCESS = BASE / 'geo' / 'access_summary.json'
YEARS = (2000, 2024)
VERSION = '2.3.0'

WB_NOTE = 'World Bank open data (CC BY 4.0); indicator attribution retained. Check dataset terms before redistribution.'
WHO_NOTE = 'WHO Global Health Observatory open data (CC BY-NC-SA 3.0 IGO); attribution retained.'
EPMC_NOTE = 'Europe PMC bibliographic service. Article and abstract rights remain with their respective holders; linked records retain identifiers and attribution.'
CT_NOTE = 'ClinicalTrials.gov public registry data; sponsor submissions are not independent verification of efficacy. See ClinicalTrials.gov terms and conditions.'
GOV_NOTE = 'Official Brazilian Ministry of Health public API. Connector proof only; schema field names retained, no individual patient records distributed.'

# key -> definition. `source` is 'worldbank' or 'who'. `scale` guides chart axes. `target` is the
# Lancet Commission on Global Surgery 2030 target where one exists (Meara et al., Lancet 2015).
INDICATORS = {
 'spending':     {'id':'SH.XPD.CHEX.PC.CD','source':'worldbank','label':'Current health expenditure per capita','short':'Health spending / person','unit':'current US$','decimals':0,'scale':'log','better':'higher'},
 'spending_ppp': {'id':'SH.XPD.CHEX.PP.CD','source':'worldbank','label':'Current health expenditure per capita, PPP','short':'Health spending / person (PPP)','unit':'international $','decimals':0,'scale':'log','better':'higher'},
 'physicians':   {'id':'SH.MED.PHYS.ZS','source':'worldbank','label':'Physicians','short':'Physicians','unit':'per 1,000 people','decimals':2,'scale':'linear','better':'higher'},
 'life':         {'id':'SP.DYN.LE00.IN','source':'worldbank','label':'Life expectancy at birth','short':'Life expectancy','unit':'years','decimals':1,'scale':'linear','better':'higher'},
 'oop':          {'id':'SH.XPD.OOPC.CH.ZS','source':'worldbank','label':'Out-of-pocket expenditure','short':'Out-of-pocket share','unit':'% of current health expenditure','decimals':1,'scale':'linear','better':'lower'},
 'population':   {'id':'SP.POP.TOTL','source':'worldbank','label':'Population, total','short':'Population','unit':'people','decimals':0,'scale':'log','better':None},
 'sao':          {'id':'SH.MED.SAOP.P5','source':'worldbank','label':'Specialist surgical workforce','short':'Surgical workforce (SAO)','unit':'per 100,000 population','decimals':1,'scale':'linear','better':'higher','target':20,'targetLabel':'LCoGS 2030 target: at least 20 per 100,000'},
 'volume':       {'id':'SH.SGR.PROC.P5','source':'worldbank','label':'Number of surgical procedures','short':'Surgical volume','unit':'per 100,000 population','decimals':0,'scale':'linear','better':'higher','target':5000,'targetLabel':'LCoGS 2030 target: at least 5,000 per 100,000'},
 'catastrophic': {'id':'SH.SGR.CRSK.ZS','source':'worldbank','label':'Risk of catastrophic expenditure for surgical care','short':'Catastrophic cost risk (surgery)','unit':'% of people at risk','decimals':1,'scale':'linear','better':'lower','target':0,'targetLabel':'LCoGS 2030 target: 100% protection (0% at risk)'},
 'impoverishing':{'id':'SH.SGR.IRSK.ZS','source':'worldbank','label':'Risk of impoverishing expenditure for surgical care','short':'Impoverishing cost risk (surgery)','unit':'% of people at risk','decimals':1,'scale':'linear','better':'lower','target':0,'targetLabel':'LCoGS 2030 target: 100% protection (0% at risk)'},
 'uhc':          {'id':'UHC_INDEX_REPORTED','source':'who','label':'UHC service coverage index (SDG 3.8.1)','short':'UHC service coverage index','unit':'index 0–100','decimals':0,'scale':'linear','better':'higher'},
 'cata10':       {'id':'FINPROTECTION_CATA_TOT_10_POP','source':'who','dim1':'RESIDENCEAREATYPE_TOTL','label':'Population with household health spending above 10% of total consumption or income (SDG 3.8.2)','short':'Catastrophic health spending (>10%)','unit':'% of population','decimals':1,'scale':'linear','better':'lower'},
}
GROUPS = [
 {'id':'context','label':'System context','indicators':['spending','spending_ppp','physicians','life','oop','population'],'note':'National averages; contextual observations, not measures of surgical access.'},
 {'id':'surgical','label':'Surgical system (LCoGS indicators)','indicators':['sao','volume','catastrophic','impoverishing'],'note':'Four of the six Lancet Commission on Global Surgery indicators as compiled in World Development Indicators. Sparse and irregular reporting years; the latest value can be a decade old.'},
 {'id':'coverage','label':'Coverage & financial protection','indicators':['uhc','cata10'],'note':'SDG 3.8 monitoring indicators from WHO and World Bank.'},
]
# UN System Data Commons (UNSD) counterparts used for source triangulation and live refresh. factor converts to ASCEND units.
UNDC_REST = 'https://unsd-datacommons.gcp.un-icc.cloud/core/api/v2'
UNDC_NOTE = 'UN System Data Commons (United Nations Statistics Division) knowledge graph; the underlying series retain their publishers\' terms (WHO, UN SDG Global Database). Retrieved via the public V2 observation endpoint.'
UNDC = {
 'uhc':        {'variable':'undata/sdg/SH_ACS_UNHC_25','factor':1,'unitNote':'index 0–100, SDG Global Database (2025 methodology)'},
 'cata10':     {'variable':'undata/who/FINP_CHE_SH.HE_THRESHOLD--T_10','factor':1,'unitNote':'% of population, WHO financial-protection series'},
 'physicians': {'variable':'undata/sdg/SH_MED_DEN.OCCUPATION--ISCO08_221','factor':0.1,'unitNote':'SDG 3.c.1 medical doctors per 10,000, converted to per 1,000'},
 'spending':   {'variable':'undata/who/GHED_PER_CAP.HE_CATEGORY--CHE','factor':1,'unitNote':'WHO Global Health Expenditure Database, current health expenditure per capita, current US$'},
 'oop':        {'variable':'undata/who/GHED_SH.HE_CATEGORY--OOP_HE__REF_EXP_CAT--CHE','factor':1,'unitNote':'WHO GHED, out-of-pocket expenditure as % of current health expenditure'},
 'life':       {'variable':'undata/unicef/DM_LIFE_EXP','factor':1,'unitNote':'UNICEF data warehouse (UN WPP basis), both sexes; the series extends to the current WPP estimate year'},
 'population': {'variable':'undata/unicef/DM_POP','factor':1,'unitNote':'UNICEF data warehouse (UN WPP basis), total population'},
 # Derived cross-check only (no single Commons variable): WHO NHWA head-counts of surgeons, anaesthesiologists and
 # obstetricians divided by World Bank population of the same year, per 100,000. One observation per economy, 2009–2015.
 'sao':        {'derived':True,'variables':['undata/who/HWF_CNT.HWF_TYPE--SURGEON','undata/who/HWF_CNT.HWF_TYPE--ANAESTH','undata/who/HWF_CNT.HWF_TYPE--OBSTETRICIAN'],'factor':1,'unitNote':'sum of WHO head-counts / World Bank population × 100,000; counts are single 2009–2015 observations'},
}
# Indicators searched in the Commons (MCP search_indicators anchored on Brazil and Kenya, 20 Oct 2026) with no counterpart.
UNDC_NONE = {
 'spending_ppp':'GHED per-capita slice is published in current US$ only; no PPP series in this deployment.',
 'volume':'No surgical-volume series; nearest hits are trachoma surgeries and caesarean-section shares, which are not comparable.',
 'catastrophic':'No surgery-specific catastrophic-expenditure series; the Commons holds general catastrophic (FINP_CHE_SH) and impoverishing (FINP_IMP_SH) health-spending shares.',
 'impoverishing':'No surgery-specific impoverishing-expenditure series; general FINP_IMP_SH poverty-line series exist but measure all health spending.',
}
ACCESS_GROUP = {'id':'access','label':'Geographic access (modelled)','indicators':['access2h','access1h','traveltime'],'note':'Population-weighted travel time to the nearest healthcare facility, computed by ASCEND at 1 km from the Malaria Atlas Project 2019 travel-time surfaces (motorised and walking-only) and WorldPop 2020 counts. Any facility, not Bellwether surgical capacity: a modelled upper bound on geographic surgical access.'}


def utcnow():
    return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')

def fetch(job):
    name,url,note = job
    record = {'id':name,'name':name.replace('_',' ').title(),'url':url,'accessedAt':utcnow(),'licenseNote':note}
    attempts=6 if 'ghoapi' in url else 4
    for attempt in range(attempts):
        try:
            req = urllib.request.Request(url,headers={'User-Agent':'ASCEND-research-prototype/2.0','Accept':'application/json','Accept-Encoding':'gzip'})
            with urllib.request.urlopen(req,timeout=90) as response:
                body=response.read()
                if body[:2] == b'\x1f\x8b': body=gzip.decompress(body)
                payload=json.loads(body.decode('utf-8'))
                record.update(status='success',httpStatus=response.status,bytes=len(body),attempts=attempt+1)
                (RAW/f'{name}.json').write_text(json.dumps(payload,ensure_ascii=False))
                return name,payload,record
        except Exception as exc:
            record.update(status='failed',error=str(exc)[:300],attempts=attempt+1)
            time.sleep(2+3*attempt)
    cached=RAW/f'{name}.json'
    if cached.exists():
        try:
            payload=json.loads(cached.read_text())
            record.update(status='cached',note='Live retrieval failed; the previously retrieved raw response in data_raw/ was reused unchanged. Timestamp is the file modification time.',
                          accessedAt=datetime.datetime.fromtimestamp(cached.stat().st_mtime,datetime.timezone.utc).isoformat(timespec='seconds'))
            return name,payload,record
        except Exception: pass
    return name,None,record

def clean_text(text):
    return html.unescape(re.sub('<[^>]+>', ' ', text or '')).strip()

def wb_rows(payload):
    return payload[1] if isinstance(payload,list) and len(payload)>1 and isinstance(payload[1],list) else []

def to_series(values):
    """{year:value} -> sorted [[year,value],...] with nulls dropped and finite check."""
    out=[]
    for year in range(YEARS[0],YEARS[1]+1):
        v=values.get(year)
        if v is None: continue
        v=float(v)
        if not math.isfinite(v): continue
        out.append([year,round(v,4)])
    return out

def validate(data):
    assert data['kind'] == 'observed/retrieved'
    assert data['version'] == VERSION
    ids=[c['id'] for c in data['countries']]
    assert len(ids)==len(set(ids)) and len(ids) >= 190, len(ids)
    keys=set(data['indicators'])
    for g in data['indicatorGroups']:
        assert set(g['indicators']) <= keys
    observations=0; nonempty=0
    for c in data['countries']:
        assert re.fullmatch(r'[A-Z]{3}',c['id']) and c['name'] and c['region'] and c['income']
        assert c['lat'] is None or -90 <= c['lat'] <= 90
        assert c['lon'] is None or -180 <= c['lon'] <= 180
        assert set(c['series']) == keys
        for key,series in c['series'].items():
            years=[y for y,_ in series]
            assert years==sorted(years) and len(set(years))==len(years)
            assert all(YEARS[0] <= y <= YEARS[1] for y in years)
            assert all(isinstance(v,(int,float)) and math.isfinite(v) for _,v in series)
            observations+=len(series); nonempty+=bool(series)
    for source in data['sources']:
        assert source['url'].startswith('https://') and source['accessedAt'] and source['licenseNote']
    assert data['evidence']['classification'] == 'related literature; not model effect estimates'
    for p in data['evidence']['publications']:
        assert p['title'] and p['url'].startswith('https://')
    assert data['trials']['classification'] == 'registered studies; not evidence of efficacy or surgery relevance'
    for p in data['trials']['studies']:
        assert p['nctId'].startswith('NCT') and p['url'].startswith('https://clinicaltrials.gov/study/')
    for c in data['connectorProofs']:
        assert 'schemaFields' in c and 'patientData' not in c
    coverage={k:sum(1 for c in data['countries'] if c['series'][k]) for k in keys}
    return {'countries':len(ids),'indicators':len(keys),'series':len(ids)*len(keys),'nonEmptySeries':nonempty,'observations':observations,
            'countriesWithAnyValue':coverage,'publications':len(data['evidence']['publications']),'trials':len(data['trials']['studies'])}


def build():
    RAW.mkdir(parents=True,exist_ok=True)
    jobs=[('worldbank_countries','https://api.worldbank.org/v2/country?format=json&per_page=400',WB_NOTE)]
    for key,meta in INDICATORS.items():
        if meta['source']=='worldbank':
            jobs.append((f'worldbank_{key}',f"https://api.worldbank.org/v2/country/all/indicator/{meta['id']}?format=json&date={YEARS[0]}:{YEARS[1]}&per_page=20000",WB_NOTE))
        else:
            jobs.append((f'who_{key}',f"https://ghoapi.azureedge.net/api/{meta['id']}",WHO_NOTE))
    queries={
      'europe_pmc_access':'(TITLE_ABS:"surgical access" OR TITLE_ABS:"access to surgery" OR TITLE_ABS:"access to surgical care") AND (TITLE_ABS:"global surgery" OR TITLE_ABS:"health system" OR TITLE_ABS:"low- and middle-income")',
      'europe_pmc_capacity':'(TITLE_ABS:"surgical capacity" OR TITLE_ABS:"surgical workforce") AND (TITLE_ABS:financing OR TITLE_ABS:access OR TITLE_ABS:"health system")',
      'europe_pmc_indicators':'TITLE_ABS:"Lancet Commission on Global Surgery" AND TITLE_ABS:indicators'
    }
    for key,m in UNDC.items():
        for i,var in enumerate(m.get('variables') or [m['variable']]):
            jobs.append((f'undc_{key}' if not m.get('derived') else f'undc_{key}_{i}',UNDC_REST+'/observation?'+urllib.parse.urlencode([('variable.dcids',var),('entity.expression','Earth<-containedInPlace+{typeOf:Country}'),('date',''),('select','entity'),('select','variable'),('select','value'),('select','date')]),UNDC_NOTE))
    for key,q in queries.items():
        jobs.append((key,'https://www.ebi.ac.uk/europepmc/webservices/rest/search?'+urllib.parse.urlencode({'query':q,'format':'json','pageSize':6,'resultType':'core','sort':'CITED desc'}),EPMC_NOTE))
    jobs.extend([
      ('demas_cnes','https://apidadosabertos.saude.gov.br/cnes/estabelecimentos?codigo_uf=35&limit=1&offset=0',GOV_NOTE),
      ('demas_sih','https://apidadosabertos.saude.gov.br/assistencia-a-saude/sih-procedimentos-hospitalares?nu_ano=2023&limit=1&offset=0',GOV_NOTE),
      ('clinical_trials','https://clinicaltrials.gov/api/v2/studies?'+urllib.parse.urlencode({'query.term':'AREA[BriefTitle]("surgical access" OR "access to surgery" OR "access to surgical care" OR "global surgery" OR "surgical capacity" OR "surgical workforce" OR "surgical care delivery" OR "surgical outreach") AND AREA[StudyType]INTERVENTIONAL','pageSize':6,'format':'json'}),CT_NOTE)
    ])
    responses={}; sources=[]
    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
        for name,payload,record in pool.map(fetch,jobs):
            responses[name]=payload;sources.append(record)
            print(name,record['status'],record.get('bytes',''),flush=True)
    (RAW/'request_manifest.json').write_text(json.dumps(sources,indent=2))
    metadata=wb_rows(responses['worldbank_countries'])
    if not metadata:
        raise RuntimeError('Country metadata unavailable; no substitute snapshot was fabricated. Inspect raw request manifest.')
    economies=[m for m in metadata if m.get('region',{}).get('value') and m['region']['value']!='Aggregates']
    # indicator tables keyed by iso3
    tables={}
    for key,meta in INDICATORS.items():
        if meta['source']=='worldbank':
            rows=wb_rows(responses.get(f'worldbank_{key}'))
            table={}
            for row in rows:
                iso=row.get('countryiso3code')
                if not iso or row.get('value') is None: continue
                table.setdefault(iso,{})[int(row['date'])]=row['value']
            tables[key]=(table,'success' if rows else 'unavailable',f'worldbank_{key}')
        else:
            payload=responses.get(f'who_{key}') or {}
            table={}
            for row in payload.get('value',[]):
                if row.get('SpatialDimType')!='COUNTRY' or row.get('NumericValue') is None: continue
                if (row.get('Dim1') or None)!=meta.get('dim1'): continue
                table.setdefault(row['SpatialDim'],{})[int(row['TimeDim'])]=row['NumericValue']
            tables[key]=(table,'success' if table else 'unavailable',f'who_{key}')
    access=None
    if ACCESS.exists():
        access=json.loads(ACCESS.read_text())
        for k,meta in access['indicators'].items():
            INDICATORS[k]={**meta,'source':'ascend-geo'}
        order=['access2h','access1h','traveltime','access2h_walk','access1h_walk','traveltime_walk']
        GROUPS.append({**ACCESS_GROUP,'indicators':[k for k in order if k in access['indicators']]})
        sources.append({'id':'ascend_geo_access','name':'ASCEND geographic-access pipeline','url':access['provenance']['methodUrl'],'accessedAt':access['generatedAt'],'licenseNote':access['provenance']['licenseNote'],'status':'computed','note':access['provenance']['summary']})
        for s in access['provenance']['inputs']:
            sources.append({'id':s['id'],'name':s['name'],'url':s['url'],'accessedAt':access['generatedAt'],'licenseNote':s['licenseNote'],'status':'success'})
    undc_tables={}
    def undc_table(payload,var,factor):
        table={}
        for entity,rec in (payload.get('byVariable',{}).get(var,{}).get('byEntity',{}) or {}).items():
            mt=re.fullmatch(r'country/([A-Z]{3})',entity)
            if not mt: continue
            facets=rec.get('orderedFacets') or []
            if not facets: continue
            vals={}
            for o in facets[0].get('observations',[]):
                try: y=int(str(o['date'])[:4]); v=float(o['value'])*factor
                except Exception: continue
                if math.isfinite(v): vals[y]=v
            if vals: table[mt.group(1)]=sorted((y,round(v,4)) for y,v in vals.items())
        return table
    for key,m in UNDC.items():
        if m.get('derived'):
            parts=[undc_table(responses.get(f'undc_{key}_{i}') or {},var,1) for i,var in enumerate(m['variables'])]
            undc_tables[key]=('derived',parts); continue
        payload=responses.get(f'undc_{key}') or {}
        table={}
        for entity,rec in (payload.get('byVariable',{}).get(m['variable'],{}).get('byEntity',{}) or {}).items():
            mt=re.fullmatch(r'country/([A-Z]{3})',entity)
            if not mt: continue
            facets=rec.get('orderedFacets') or []
            if not facets: continue
            vals={}
            for o in facets[0].get('observations',[]):
                try: y=int(str(o['date'])[:4]); v=float(o['value'])*m['factor']
                except Exception: continue
                if math.isfinite(v): vals[y]=v
            if vals: table[mt.group(1)]=sorted((y,round(v,4)) for y,v in vals.items())
        undc_tables[key]=table
    for key,val in list(undc_tables.items()):
        if isinstance(val,tuple) and val[0]=='derived':
            parts=val[1]; table={}
            poptab=tables['population'][0]
            for iso in set.intersection(*[set(p) for p in parts]) if parts else set():
                years=set.intersection(*[{y for y,_ in p[iso]} for p in parts])
                out=[]
                for y in sorted(years):
                    pop=poptab.get(iso,{}).get(y)
                    if not pop: continue
                    total=sum(dict(p[iso])[y] for p in parts)
                    out.append((y,round(total/float(pop)*1e5,4)))
                if out: table[iso]=out
            undc_tables[key]=table
    countries=[]
    for meta in sorted(economies,key=lambda m:m['name']):
        iso=meta['id']
        c={'id':iso,'iso2':meta.get('iso2Code'),'name':meta['name'],'region':meta['region']['value'].strip(),'income':meta['incomeLevel']['value'].strip(),
           'capital':meta.get('capitalCity') or None,
           'lat':float(meta['latitude']) if meta.get('latitude') else None,'lon':float(meta['longitude']) if meta.get('longitude') else None,
           'series':{}}
        for key in INDICATORS:
            if key in tables:
                c['series'][key]=to_series(tables[key][0].get(iso,{}))
            elif access:
                rec=access['countries'].get(iso,{}); v=rec.get(key)
                # Small-island artefact guard: a microstate whose nearest mapped facility is across open water
                # (mean travel time > 4 h with < 100,000 people) reflects a gap in the facility database, not access.
                suffix='_walk' if key.endswith('_walk') else ''
                artefact=rec.get('popTotal2020',0)<100000 and (rec.get('traveltime'+suffix) or 0)>240
                c['series'][key]=[[access['referenceYear'],round(float(v),4)]] if v is not None and math.isfinite(float(v)) and not artefact else []
        c['crosscheck']={}
        for key,table in undc_tables.items():
            ser=table.get(iso)
            if ser: c['crosscheck'][key]=[list(ser[-1]),len(ser)]  # latest [year,value] and number of observations held by UN Data Commons
        countries.append(c)
    # agreement summary at the latest common year, per indicator
    agreement={}
    for key,table in undc_tables.items():
        n=agree=0;diffs=[]
        for c in countries:
            ser=table.get(c['id']);snap=dict(c['series'].get(key,[]))
            if not ser or not snap: continue
            common=[p for p in ser if p[0] in snap]
            if not common: continue
            y,v=common[-1];a=snap[y];n+=1;rel=abs(a-v)/max(abs(a),abs(v),1e-9);diffs.append(rel)
            if rel<=0.05: agree+=1
        agreement[key]={'economiesCompared':n,'withinFivePercent':agree,'medianRelativeDifference':round(sorted(diffs)[len(diffs)//2],4) if diffs else None,'undcEconomies':len(table)}
    indicator_defs={}
    for key,meta in INDICATORS.items():
        status=tables[key][1] if key in tables else ('computed' if access else 'unavailable')
        indicator_defs[key]={**meta,'sourceId':tables[key][2] if key in tables else 'ascend_geo_access','retrievalStatus':status,
                             'countriesWithData':sum(1 for c in countries if c['series'][key])}
        if key in UNDC:
            m=UNDC[key]; first=m.get('variable') or m['variables'][0]
            indicator_defs[key]['undc']={**m,'rest':UNDC_REST+'/node?'+urllib.parse.urlencode({'nodes':first,'property':'->name'}),'status':'success' if undc_tables.get(key) else 'unavailable','agreement':agreement.get(key)}
        elif key in UNDC_NONE:
            indicator_defs[key]['undc']={'status':'no-counterpart','note':UNDC_NONE[key]}
    publications=[]; seen=set()
    for key,q in queries.items():
        response=responses[key] or {}
        for p in response.get('resultList',{}).get('result',[]):
            ident=p.get('doi') or p.get('pmid') or p['id']
            if ident in seen: continue
            seen.add(ident)
            source=p.get('source','MED');id_=p['id']
            publications.append({'id':id_,'title':clean_text(p.get('title')),'authors':p.get('authorString',''),'year':int(p['pubYear']) if p.get('pubYear') else None,
              'doi':p.get('doi'),'pmid':p.get('pmid'),'url':f'https://europepmc.org/article/{source}/{id_}',
              'abstract':clean_text(p.get('abstractText')),'journal':p.get('journalInfo',{}).get('journal',{}).get('title',''),
              'citedBy':p.get('citedByCount'),
              'classification':'related literature; not model effect estimates','query':q,'sourceId':key})
    studies=[]
    for row in (responses['clinical_trials'] or {}).get('studies',[]):
        p=row['protocolSection']; ident=p['identificationModule']; design=p.get('designModule',{}); status=p.get('statusModule',{})
        locations=p.get('contactsLocationsModule',{}).get('locations',[])
        countries_in=sorted({loc.get('country') for loc in locations if loc.get('country')})
        studies.append({'nctId':ident['nctId'],'title':ident.get('briefTitle',''),'status':status.get('overallStatus',''),
          'phase':design.get('phases',[]),'studyType':design.get('studyType',''),'briefSummary':p.get('descriptionModule',{}).get('briefSummary',''),
          'url':'https://clinicaltrials.gov/study/'+ident['nctId'],'countries':countries_in,'locations':[{k:loc[k] for k in ['facility','city','country'] if k in loc} for loc in locations[:6]],
          'sourceId':'clinical_trials','lastUpdate':status.get('lastUpdatePostDateStruct',{}).get('date')})
    proofs=[]
    for name,key in [('demas_cnes','estabelecimentos'),('demas_sih','sih_procedimentos_hospitalares')]:
        response=responses[name] or {}; rows=response.get(key,[])
        proofs.append({'id':name,'label':'CNES establishment schema' if name=='demas_cnes' else 'SIH hospital-procedure schema',
          'status':'success' if rows else 'unavailable','schemaFields':sorted(rows[0].keys()) if rows else [],'sampleRowsRetrieved':len(rows),
          'sourceId':name,'note':'Connectivity and schema proof only (example national registry). One response row does not establish data completeness, historical coverage, surgical relevance, or analytic validity.'})
    data={'version':VERSION,'kind':'observed/retrieved','accessedAt':utcnow(),'yearRange':list(YEARS),
      'description':'All World Bank economies: contextual health-system indicators, Lancet Commission on Global Surgery indicators, SDG 3.8 coverage and protection indicators'+(', and ASCEND-modelled geographic access' if access else '')+'. Observational comparisons do not establish intervention effects.',
      'indicatorGroups':GROUPS,'indicators':indicator_defs,'countries':countries,
      'sources':sources,'apiRegistry':[{k:v for k,v in r.items() if k not in ['bytes']} for r in sources],
      'evidence':{'classification':'related literature; not model effect estimates','selection':'Three explicit Europe PMC queries on surgical access, capacity and LCoGS indicators, six most-cited results per query, deduplicated. Convenience discovery set, not a systematic review. No relevance or risk-of-bias adjudication.','status':'success' if publications else 'unavailable','publications':publications},
      'trials':{'classification':'registered studies; not evidence of efficacy or surgery relevance','selection':'First six ClinicalTrials.gov API results for interventional studies with surgical access, capacity, workforce or delivery terms in the title. No country filter; registry coverage of health-system interventions is thin.','status':'success' if studies else 'unavailable','studies':studies},
      'connectorProofs':proofs,
      'limitations':['All World Bank economies are included; small states and territories often lack observations and remain visibly missing.',
        'Latest values can refer to different years; the observation year is always shown next to the value.',
        'Expenditure is shown in current US dollars and, separately, in PPP international dollars; neither is inflation-adjusted across years.',
        'Missing values remain null; no interpolation, imputation or carry-forward in fixed-year mode.',
        'Lancet Commission on Global Surgery indicators are compiled from heterogeneous national reports with irregular years and definitions.',
        'National ecological indicators cannot estimate effects of a surgical intervention.',
        'Bibliographic and registry retrieval is not a systematic evidence synthesis.',
        'No patient-level records are included in this distributable snapshot.']+
        (['Geographic access is modelled from a travel-time surface to any healthcare facility and 2020 population counts; it is not the LCoGS Bellwether 2-hour indicator and overstates surgical access.',
          'Access values are omitted where fewer than half the population lies on valid travel-time pixels, and for microstates (<100,000 people) whose mean travel time exceeds four hours, which indicates missing facility points rather than inaccessibility.'] if access else [])}
    data['undc']={'rest':UNDC_REST,'mcp':'https://unsd-datacommons.gcp.un-icc.cloud/mcp','documentation':'https://projects.officialstatistics.org/undata2/undatacommons-mcp/','portal':'https://data.un.org/undatacommons/','agreement':agreement,
      'note':'Eight ASCEND indicators carry a UN System Data Commons counterpart (seven single variables plus a derived SAO density) (DCID in indicators[*].undc). The snapshot stores each economy\'s latest UN Data Commons value for triangulation; the app can also refresh these series live from the browser because the endpoint allows cross-origin requests.'}
    data['validation']=validate(data)
    DEST.parent.mkdir(parents=True,exist_ok=True)
    DEST.write_text(json.dumps(data,ensure_ascii=False,separators=(',',':')))
    print(json.dumps(data['validation'],indent=2))
    print('bytes',DEST.stat().st_size)

if __name__ == '__main__':
    parser=argparse.ArgumentParser();parser.add_argument('--validate',action='store_true');args=parser.parse_args()
    if args.validate: print(json.dumps(validate(json.loads(DEST.read_text())),indent=2))
    else: build()