_tools/audit_next_batch_manifests.py

"""Enumerate lazy inventories independently of gameplay; repair them before final local tests."""
import os,json,ast,re,hashlib,tempfile,collections,concurrent.futures
from pathlib import Path
from urllib.parse import urljoin,urlparse,unquote
import requests
from huggingface_hub import HfApi
H=Path('/home/user/game-archive');R=H/'reports';B='smodusermc/offline-games';api=HfApi(token=os.environ['HF_TOKEN']);ids=[int(x) for x in os.environ.get('AUDIT_GAME_IDS','182,183,359,373,374,376,575,582').split(',')]
source=ast.parse((H/'tools/collect_next.py').read_text());nodes=[]
for n in source.body:
 if isinstance(n,(ast.Import,ast.ImportFrom)):nodes.append(n)
 elif isinstance(n,ast.FunctionDef) and n.name in ['normalized','dest','extract']:nodes.append(n)
 elif isinstance(n,ast.Assign) and any(isinstance(x,ast.Name) and x.id in ['EXT','PATH_RE','LITERAL','IGNORE_HOSTS'] for x in n.targets):nodes.append(n)
ns={'BASE':'https://garbsoftball.com'};exec(compile(ast.Module(body=nodes,type_ignores=[]),'collector_rules','exec'),ns)
allreports=[]
for ident in ids:
 mp=R/f'game-{ident}.json';m=json.loads(mp.read_text());records={x['path']:x for x in m['files']};base=urljoin(m['entry'],'play/' if ident==182 else './');expected={};processed=set();added=[];failed=[];inputs=[]
 def expect(url,reason):
  u=ns['normalized'](url)
  if not u:return
  key=ns['dest'](u);expected.setdefault(key,{'path':key,'url':u,'reasons':[]})['reasons'].append(reason)
 if ident in [180,181,182,183]:
  expect(urljoin(base,'game.json'),'temple configuration');expect(urljoin(base,'index.html'),'game entry')
  expect(urljoin(base,'fireboy-and-watergirl-'+({180:'forest',181:'light',182:'ice',183:'crystal'}[ident])+'-temple.min.js'),'game loader')
 if ident in [373,374,582]:expect(urljoin(base,'data.js' if ident==373 else 'data.json'),'Construct project inventory')
 for f in records.values():
  if f['path'].endswith(('offline.js','offline.json','data.js','data.json','game.json','temple.json')):expect(f.get('url') or 'https://garbsoftball.com/'+f['path'][5:],'stored inventory')
 with tempfile.TemporaryDirectory(prefix='game-manifest-deep-') as temp:
  stage=Path(temp)
  # Only inventories / target loaders are parsed, avoiding false filenames in generic engine code.
  while True:
   candidates=[x for k,x in expected.items() if k not in processed]
   if not candidates:break
   def fetch(x):
    key=x['path'];local=stage/key;local.parent.mkdir(parents=True,exist_ok=True)
    if key in records:
     api.download_bucket_files(B,[(key,local)],raise_on_missing_files=True);return x,local,None
    try:
     r=requests.get(x['url'],timeout=25);r.raise_for_status();body=r.content
     if len(body)>64*1024**2:raise ValueError('safety size limit')
     if re.match(br'\s*(?:<!doctype html|<html)',body,re.I) and not key.endswith('.html'):raise ValueError('HTML fallback')
     local.write_bytes(body);return x,local,r.headers.get('content-type','')
    except Exception as e:return x,None,str(e)[:180]
   with concurrent.futures.ThreadPoolExecutor(max_workers=4) as ex:items=list(ex.map(fetch,candidates))
   uploads=[]
   for x,local,ctype in items:
    key=x['path'];processed.add(key)
    if local is None:failed.append({**x,'error':ctype});continue
    body=local.read_bytes();sha=hashlib.sha256(body).hexdigest()
    if key not in records:
     f={'path':key,'url':x['url'],'bytes':len(body),'sha256':sha,'content_type':ctype,'discovery':'independent manifest audit'};records[key]=f;added.append(f);uploads.append((local,key))
    name=key.split('/')[-1];text=body.decode('utf-8-sig',errors='replace');doc=None
    if name in ['data.json','data.js','offline.json','offline.js','game.json','temple.json']:
     try:doc=json.loads(text)
     except ValueError:pass
    if doc is not None:
     inputs.append({'path':key,'sha256':sha})
     if isinstance(doc,dict) and isinstance(doc.get('fileList'),list):
      for f in doc['fileList']:
       if isinstance(f,str):expect(urljoin(x['url'],f),'offline fileList')
     if name in ['data.js','data.json']:
      for ref,kind in ns['extract'](text,x['url'],urljoin(base,'index.html'))[0]:
       if '/images/' in ref or kind.startswith('construct'):expect(ref,kind)
     if name in ['game.json','temple.json']:
      for ref,kind in ns['extract'](text,x['url'],urljoin(base,'index.html'))[0]:
       if kind.startswith(('temple-','phaser-')):expect(ref,kind)
    if 'fireboy-and-watergirl-' in name and name.endswith('.js'):
     for ref,kind in ns['extract'](text,x['url'],urljoin(base,'index.html'))[0]:
      if kind=='phaser-audio-name-list':expect(ref,kind)
   for start in range(0,len(uploads),24):api.batch_bucket_files(B,add=uploads[start:start+24])
   # File bytes have been uploaded before removal; final verification downloads them again.
   for _,local,_ in items:
    if local:local.unlink(missing_ok=True)
 m['files']=list(records.values());m['lazy_manifest_audit']='_verification/next-batch-declared-assets.json';mp.write_text(json.dumps(m,indent=2));api.batch_bucket_files(B,add=[(mp,f'_reports/games/{ident:04d}.json')])
 report={'game_id':ident,'name':m['name'],'explicit_expected_paths':list(expected.values()),'manifest_inputs':inputs,'added':added,'unresolved':failed,'coverage_note':'Explicit project/offline/temple inventories and known dynamic sound naming; not a proof of undiscoverable assets or full gameplay.'};allreports.append(report)
 print(ident,'declared',len(expected),'added',len(added),'unresolved',len(failed),flush=True)
old=json.loads((R/'next-batch-declared-assets.json').read_text()) if (R/'next-batch-declared-assets.json').exists() else [];allreports=[x for x in old if x['game_id'] not in ids]+allreports
(R/'next-batch-declared-assets.json').write_text(json.dumps(allreports,indent=2));api.batch_bucket_files(B,add=[(R/'next-batch-declared-assets.json','_verification/next-batch-declared-assets.json')])