_tools/publish_heavy_batch.py

"""Publish an honest size/integrity report, keeping historical batches intact."""
import os,json,re,hashlib,collections,shutil
from pathlib import Path
from urllib.parse import urljoin,unquote,urlparse
from huggingface_hub import HfApi
import subprocess,sys
if not Path('/home/user/game-archive/reports/deltarune-declared-audio-recovery.json').exists():subprocess.run([sys.executable,'/home/user/game-archive/tools/restore_deltarune_declared_audio.py'],check=True)
H=Path('/home/user/game-archive');R=H/'reports';B='smodusermc/offline-games';api=HfApi(token=os.environ['HF_TOKEN']);plan=json.loads((R/'heavy-batch-plan.json').read_text());state=json.loads((R/'heavy-batch-validation.json').read_text());media=json.loads((R/'heavy-batch-media.json').read_text());rows=state['results'];assert len(rows)==20 and state['phase']=='checks_complete'
# Follow-up UI tests can supersede a report; retain/check all recorded browser repairs.
extra={}
removed_paths={f['path'] for f in json.loads((R/'heavy-invalid-artwork.json').read_text())} if (R/'heavy-invalid-artwork.json').exists() else set()
for i in plan['ids']:
 for vp in R.glob(f'verification-{i:04d}*.json'):
  for f in json.loads(vp.read_text()).get('repairs',[]):
   if isinstance(f,dict) and f.get('path') and f.get('sha256') and f['path'] not in removed_paths:extra[f['path']]=f
supp=[];tmp=H.parent/'.cache'/'heavy-final-hashes';tmp.mkdir(parents=True,exist_ok=True)
for key,f in extra.items():
 p=tmp/'file';api.download_bucket_files(B,[(key,p)],raise_on_missing_files=True)
 with p.open('rb') as stream:digest=hashlib.file_digest(stream,'sha256').hexdigest()
 check={'path':key,'source_sha256':f['sha256'],'stored_sha256':digest,'matches':digest==f['sha256'],'bytes':p.stat().st_size};supp.append(check);assert check['matches'];p.unlink()
shutil.rmtree(tmp,ignore_errors=True);(R/'heavy-supplemental-runtime-hashes.json').write_text(json.dumps(supp,indent=2))
bid='batch-'+hashlib.sha256(','.join(map(str,plan['ids'])).encode()).hexdigest()[:10];prefix='_verification/batches/'+bid;archive=R/'batches'/bid;archive.mkdir(parents=True,exist_ok=True)
api.download_bucket_files(B,[('_reports/inventory.json',R/'inventory.json')],raise_on_missing_files=True);inventory=json.loads((R/'inventory.json').read_text());byid={x['id']:x for x in rows};source_issues=[]
for x in rows:
 i=x['id'];visual=json.loads((R/'heavy-batch-visual-review.json').read_text()).get(str(i),{});vp=R/f'verification-{i:04d}.json';runtime=json.loads(vp.read_text());x['reviewed_runtime_outcome']=visual.get('outcome',x['runtime_status']);runtime.update(reviewed_outcome=x['reviewed_runtime_outcome'],review_note=visual.get('note','Host resource limit; not a gameplay pass.'));vp.write_text(json.dumps(runtime,indent=2));m=json.loads((R/f'game-{i}.json').read_text());files={f['path']:f for f in m['files']};x['final_files']=len(files);x['final_bytes']=sum(f['bytes'] for f in files.values());x['html_declared_core_missing']=[]
 html=Path(f'/tmp/large-source-survey/{i}.html')
 if html.exists():
  text=html.read_text();base=m['entry']
  # FEZ explicitly declares its six content packages; retain mismatches as warnings.
  if i==178:
   x['package_size_checks']=[c for c in x['package_size_checks'] if c.get('source')!='CONTENT_FILES']
   for name,n in re.findall(r"\{\s*url:\s*'([^']+)'[^}]*?size:\s*(\d+)",text):
    key='site/'+unquote(urlparse(urljoin(base,name)).path).lstrip('/');actual=files.get(key,{}).get('bytes',0);x['package_size_checks'].append({'path':key,'declared_bytes':int(n),'captured_bytes':actual,'matches':actual==int(n),'source':'CONTENT_FILES'})
 for check in x['package_size_checks']:
  if check['captured_bytes']==0:x['html_declared_core_missing'].append(check['path'])
  elif not check['matches']:source_issues.append({'id':i,'type':'Source declaration differs from captured source bytes; not by itself storage corruption',**check})
 x['declared_packages_passed']=not x['unresolved'] and not x['html_declared_core_missing'] and all(c['matches'] for c in x['package_size_checks'])
 x['scope_note']='Storage hash checks are not proof of exhaustive source completeness or playability. Literal-candidate 404s can include unused engine defaults; explicit missing packages and binary audio names require attention.'
 for r in inventory:
  if r['id']==i:r.update(captured_files=x['final_files'],captured_bytes=x['final_bytes'],verification_report=x['runtime_report'],verification_outcome=x['runtime_status'],heavy_batch_validation=prefix+'/validation.json')
for r in inventory:
 if r['id']==111:
  m=json.loads((R/'game-111.json').read_text());r.update(captured_files=len(m['files']),captured_bytes=sum(f['bytes'] for f in m['files']))
state['batch_id']=bid;(R/'heavy-batch-validation.json').write_text(json.dumps(state,indent=2));(R/'inventory.json').write_text(json.dumps(inventory,indent=2));(R/'heavy-batch-source-size-warnings.json').write_text(json.dumps(source_issues,indent=2))
flat=[f for x in media['results'] for f in x['files']];size=api.bucket_info(B).size;before=plan['initial_bucket_bytes'];sha=sum(x['sha_comparisons'] for x in rows);unbaselined=sum(len(x['unbaselined']) for x in rows);audio=sum(f.get('kind')=='audio' and f.get('decoded',False) for f in flat);images=sum(f.get('kind')=='image' and f.get('decoded',False) for f in flat);fail=[f for f in flat if f.get('decoded') is False or f.get('recorded_hash_match') is False];pending=sum(r['status']=='pending_not_captured' for r in inventory)
summary={'batch_id':bid,'entries_processed':len(rows),'bucket_bytes_before':before,'bucket_bytes_before_report_publication':size,'growth_bytes_before_report_publication':size-before,'sha_comparisons':sha,'unbaselined_files':unbaselined,'sha_mismatches':sum(len(x['sha_mismatches']) for x in rows),'size_mismatches':sum(len(x['size_mismatches']) for x in rows),'loose_audio_decoded':audio,'loose_images_decoded':images,'media_failures':len(fail),'pending_candidates':pending,'runtime_status_counts':dict(collections.Counter(x['runtime_status'] for x in rows)),'no_full_playthroughs':True}
(R/'heavy-batch-summary.json').write_text(json.dumps(summary,indent=2))
recovery=json.loads((R/'deltarune-declared-audio-recovery.json').read_text());gm=json.loads((R/'gamemaker-audio-audit.json').read_text());summary['declared_audio_paths_restored']=recovery['restored_count'];summary['declared_audio_manifest_unresolved']=recovery['still_unresolved_count'];summary['gamemaker_required_external_unresolved']=len(gm['required_external_unresolved']);(R/'heavy-batch-summary.json').write_text(json.dumps(summary,indent=2))
text=f'''# Large-port batch: 20 entries and a size audit

[Public bucket](https://huggingface.co/buckets/{B})

## Why growth slowed

The previous 30-entry batch grew the bucket by roughly **230 MB**, not 10–30 MB. It mostly contained smaller browser/Flash games. Several earlier individual captures were over 1 GB (Class of ’09 editions and Apotheon), so they accounted for much more storage per entry.

This batch processed the **next 20 pending catalog entries**, including larger ports. The bucket increased from **{before/1e9:.3f} GB to approximately {size/1e9:.3f} GB** before this report was published: **{(size-before)/1e9:.3f} GB added**. These are logical bucket bytes, not a claim about backend deduplicated physical storage.

## Problems found, not hidden

- **Crashout Crew:** its large data/WASM files were already stored, but an interrupted capture missed the 13,106-byte startup loader. Repaired from the source, uploaded and SHA-256 checked by reading it back. StreamingAssets and full runtime completeness remain unverified.
- The first attempt at this batch filled the 993 MiB temporary filesystem. It was stopped, staging moved to the larger disk, and all 20 entries retraversed. This was not treated as a successful launcher-only capture.
- Discovery now explicitly handles chapter launchers, numbered split files, AGS game/audio volumes, emulator disc images, .NET assets, and GameMaker sound metadata. SOND flags and group IDs distinguish embedded sounds from true external streams; raw filename-string misses are not counted as required dependencies. An independent closure pass follows captured scripts again. Executable-prefixed Godot audio worklets and workers receive a separate repair/read-back pass. All failures remain recorded.
- **Dead Tapes:** declared game package and WASM return source 404s. Its launcher is **not a complete game capture**.
- **FEZ:** its six declared content-file URLs returned HTML fallback pages instead of game data. The engine/framework capture is **not a complete FEZ archive**.
- Some source loader size declarations disagree with the actual downloaded bytes. These are listed in `source-size-warnings.json`; recorded source/bucket hash agreement is separate from stale or inaccurate metadata.

- **Deltarune:** {recovery['restored_count']} missing declared audio paths restored from same-title archived files matching the loader's expected MD5, with separate destination SHA-256 read-back. Their bytes match audio already decoded. All {recovery['declared_audio_paths']} audio entries in the available paired filename/MD5 manifest are now represented. Four case-only aliases address SOND spelling differences; {len(gm['required_external_unresolved'])} metadata-declared external sound remains unresolved. Other chapter catalogs are not claimed exhaustive.
- **Artwork:** five source-broken images (an empty icon and four plain-text 404 bodies mislabeled image/jpeg) were removed from asset storage. Their failures remain in the evidence; no invented replacements were substituted.

## Verification scope

- **{sha:,} recorded SHA-256 comparisons** in the main read-back pass; **{summary['sha_mismatches']} mismatches** and **{summary['size_mismatches']} stored-size mismatches**. {unbaselined} files lacked an original hash baseline and are separately identified.
- **{audio} loose audio files decoded fully**, **{images} loose images decoded**; {len(fail)} source-image decode failures (rejected as described above), with no media hash mismatches. Browser-added files also receive hash read-back there.
- Embedded media inside large game packages is covered by package-byte integrity, not individual media decoding.
- Runtime checks are separately reported. A startup label may be a loader/error page; it is **not** a playable-game pass. Large builds may exceed this host's memory budget. **No full playthroughs are claimed.**
- **{pending} catalog candidates remain pending.** Twenty entries processed does not mean twenty complete/playable games.

| Entry | Current recorded payload | Files | Explicit missing package count | Runtime result |
|---|---:|---:|---:|---|
'''
for x in rows:text+=f"| {x['name']} | {x['final_bytes']/1e6:.1f} MB | {x['final_files']} | {len(x['html_declared_core_missing'])} | {x['reviewed_runtime_outcome']} |\n"
text+='\nFull capture failures, unresolved metadata-declared external audio, split declarations and exact runtime observations are in `validation.json` and the individual game reports. An empty explicit-package count is not an exhaustive source-completeness pass.\n\nFiles retain original server-relative folders. Downloading only HTML is insufficient; HTTP hosting and vendor routing may be required. These are not automatically file:// or single-file releases. Original rights remain with the owners.\n'
(H/'HEAVY-BATCH.md').write_text(text)
readme=(H/'BUCKET-README.md').read_text();readme=re.sub(r'## Latest: 20 larger-port entries\n.*?(?=## Previous: 30-game batch)', '', readme, flags=re.S);readme=readme.replace('## Latest: 30-game batch','## Previous: 30-game batch',1);new=f"## Latest: 20 larger-port entries\n\nApproximately {(size-before)/1e9:.3f} GB added. {sha:,} recorded-hash comparisons, {audio} loose audio decodes. **Not 20 verified-playable games:** source gaps, size-declaration discrepancies and host runtime limits are explicit. See `_verification/heavy-batch-update.md` and stable archive `{prefix}/`. Crashout Crew’s omitted loader was repaired.\n\n";readme=readme.replace('## Previous: 30-game batch',new+'## Previous: 30-game batch',1);readme=readme.replace('**119 entries have verification reports; 190 candidates remain queued**',f'**{len(list(R.glob("verification-[0-9][0-9][0-9][0-9].json")))} entries have verification reports; {pending} candidates remain queued**');(H/'BUCKET-README.md').write_text(readme)
work=(H/'WORKLOG.md').read_text().replace('Stage game assets only under `/tmp/game-verify-stage`; remove after each game.','Stage game assets on disk under `.cache/` (not the small tmpfs); remove after each game.')
if bid not in work:work+='\n\n## 20-entry large-port batch and storage-growth investigation\n\n'+f'Stable batch {bid}. {sha} source-hash comparisons, {audio} loose audio decodes, {images} loose image decodes. Approximate bucket growth {(size-before)/1e9:.3f} GB. Crashout loader repaired; Dead Tapes missing source binaries explicitly incomplete; source declaration discrepancies and runtime limits retained. See HEAVY-BATCH.md.\n';(H/'WORKLOG.md').write_text(work)
adds=[(H/'HEAVY-BATCH.md','_verification/heavy-batch-update.md'),(H/'BUCKET-README.md','README.md'),(H/'WORKLOG.md','_reports/worklog.md'),(R/'inventory.json','_reports/inventory.json'),(R/'heavy-batch-summary.json','_verification/heavy-batch-summary.json')]
# Refresh the current status without erasing historical evidence.
old_summary=json.loads((R/'verification-summary.json').read_text())
if not (R/'verification-summary-before-heavy-batch.json').exists():(R/'verification-summary-before-heavy-batch.json').write_text(json.dumps(old_summary,indent=2))
updated={**old_summary,'entries_with_verification_reports':len(list(R.glob('verification-[0-9][0-9][0-9][0-9].json'))),'latest_batch_ids':plan['ids'],'latest_batch_archive':prefix,'pending_catalog_candidates':pending,'bucket_bytes_before_report_upload':size,'latest_batch_metrics':summary,'outcome_note':'Earlier reviewed outcomes retained separately; latest automated statuses do not imply visual review or gameplay.'}
updated['previous_reviewed_outcomes']=updated.pop('outcomes',updated.get('previous_reviewed_outcomes',{}))
(R/'verification-summary.json').write_text(json.dumps(updated,indent=2))
if not (R/'verification-before-heavy-batch.md').exists():shutil.copy2(H/'VERIFICATION.md',R/'verification-before-heavy-batch.md')
(H/'VERIFICATION.md').write_text(text+'\n## Earlier evidence\n\nPrevious 30-game batch: `_verification/batches/batch-18a03fb44e/`. Earlier lazy-audio/map repairs remain in their reports. No historical incomplete or host-limited result is upgraded by this batch.\n')
adds.extend([(R/'heavy-batch-validation.json','_verification/heavy-batch-validation.json'),(R/'verification-summary.json','_verification/verification-summary.json'),(R/'verification-summary.json','_reports/summary.json'),(H/'VERIFICATION.md','_verification/verification-summary.md'),(H/'VERIFICATION.md','_reports/status.md')])
for src,name in [(H/'HEAVY-BATCH.md','README.md'),(R/'heavy-batch-validation.json','validation.json'),(R/'heavy-batch-media.json','media.json'),(R/'heavy-batch-summary.json','summary.json'),(R/'heavy-batch-plan.json','plan.json'),(R/'heavy-batch-source-size-warnings.json','source-size-warnings.json'),(R/'heavy-batch-visual-review.json','visual-review.json'),(R/'heavy-invalid-artwork.json','invalid-artwork.json'),(R/'heavy-supplemental-runtime-hashes.json','supplemental-runtime-hashes.json'),(R/'deltarune-declared-audio-recovery.json','deltarune-declared-audio-recovery.json'),(R/'godot-support-repair.json','godot-support-repair.json'),(R/'gamemaker-audio-audit.json','gamemaker-audio-audit.json'),(R/'crashout-loader-repair.json','crashout-loader-repair.json'),(R/'large-capture-limit-audit.json','older-capture-limit-audit.json')]:
 shutil.copy2(src,archive/name);adds.append((src,prefix+'/'+name))
for i in plan['ids']:
 for src,dest in [(R/f'game-{i}.json',f'capture/{i:04d}.json'),(R/f'verification-{i:04d}.json',f'runtime/{i:04d}.json')]:shutil.copy2(src,archive/dest.replace('/','-'));adds.append((src,prefix+'/'+dest))
 adds.append((R/f'verification-{i:04d}.json',f'_verification/games/{i:04d}.json'))
 for p in (R/'evidence').glob(f'{i:04d}-*.jpg'):adds.append((p,prefix+'/evidence/'+p.name))
for p in (H/'tools').glob('*.py'):adds.append((p,'_tools/'+p.name))
for start in range(0,len(adds),24):api.batch_bucket_files(B,add=adds[start:start+24])
print('PUBLISHED',json.dumps(summary),flush=True)