Buckets:
| """Enumerate Thirty Dollar Website's lazy sound URLs from its real sound catalog. | |
| Fetch in small batches, upload, download back for SHA256 verification, delete payloads. | |
| No credentials are saved. The _pause item is a control, not an expected WAV. | |
| """ | |
| import os,json,hashlib,requests,concurrent.futures,tempfile,shutil,time | |
| from pathlib import Path | |
| from urllib.parse import quote | |
| from huggingface_hub import HfApi | |
| ROOT=Path('/home/user/game-archive');R=ROOT/'reports';B='smodusermc/offline-games';BASE='https://garbsoftball.com/games/30dolar/' | |
| api=HfApi(token=os.environ['HF_TOKEN']) | |
| api.download_bucket_files(B,[('_reports/games/0006.json',R/'game-6.json')],raise_on_missing_files=True) | |
| m=json.loads((R/'game-6.json').read_text());records={f['path']:f for f in m['files']} | |
| cat=requests.get(BASE+'sounds.json?a=1',timeout=25);cat.raise_for_status();catalog=cat.json() | |
| (R/'thirty-dollar-sound-catalog.json').write_text(json.dumps(catalog,ensure_ascii=False,indent=2)) | |
| existing={f.path:f for f in api.list_bucket_tree(B,prefix='site/games/30dolar',recursive=True) if hasattr(f,'size')} | |
| expected=[{'id':x['id'],'name':x['name'],'relative':'sounds/'+x['id']+'.wav','role':'catalog_sound'} for x in catalog if x['id']!='_pause'] | |
| expected += [{'id':'intro-'+x,'relative':'assets/'+x+'.wav','role':'intro'} for x in ['dont_you_lecture_me','you_deadass_built_like_a','white_people_be_like']] | |
| report={'game_id':6,'name':'Thirty Dollar Website','catalog_entries':len(catalog),'expected_playable_sounds':len(catalog)-1,'silence_control':'_pause intentionally has no audio file; playSound returns immediately for it.','files_present_before':sum('site/games/30dolar/'+x['relative'] in existing for x in expected if x['role']=='catalog_sound'),'files':[],'failures':[],'uploaded_files':0,'download_back_sha256_verified':0} | |
| with tempfile.TemporaryDirectory(prefix='thirty-dollar-fetch-') as temp: | |
| tmp=Path(temp) | |
| def fetch(x): | |
| url=BASE+quote(x['relative'],safe='/');out={**x,'url':url,'path':'site/games/30dolar/'+x['relative']};p=tmp/hashlib.sha256(x['relative'].encode()).hexdigest() | |
| try: | |
| with requests.get(url,stream=True,timeout=(8,30)) as r: | |
| r.raise_for_status();h=hashlib.sha256();size=0;head=b'' | |
| with p.open('wb') as f: | |
| for c in r.iter_content(256*1024): | |
| size+=len(c) | |
| if size>30*1024**2:raise ValueError('sound exceeds 30 MiB safety limit') | |
| if not head:head=c[:128] | |
| f.write(c);h.update(c) | |
| if not size or head.lstrip().lower().startswith((b'<!doctype',b'<html')):raise ValueError('not an audio asset; empty or HTML fallback') | |
| out.update(bytes=size,sha256=h.hexdigest(),content_type=r.headers.get('Content-Type','audio/wav'),header_hex=head[:16].hex(),local=str(p));return out | |
| except Exception as e:p.unlink(missing_ok=True);out['error']=str(e)[:200];return out | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: | |
| for i in range(0,len(expected),12): | |
| batch=list(pool.map(fetch,expected[i:i+12]));ok=[x for x in batch if 'error' not in x] | |
| report['failures'].extend({k:v for k,v in x.items() if k!='local'} for x in batch if 'error' in x) | |
| if ok: | |
| api.batch_bucket_files(B,add=[(x['local'],x['path']) for x in ok]);report['uploaded_files']+=len(ok) | |
| back=tmp/'verify';back.mkdir(exist_ok=True) | |
| api.download_bucket_files(B,[(x['path'],back/str(n)) for n,x in enumerate(ok)],raise_on_missing_files=True) | |
| for n,x in enumerate(ok): | |
| with (back/str(n)).open('rb') as f:got=hashlib.file_digest(f,'sha256').hexdigest() | |
| assert got==x['sha256'],x['path']+' remote checksum mismatch' | |
| report['download_back_sha256_verified']+=1 | |
| record={k:v for k,v in x.items() if k not in ['local','relative','role','id','name','header_hex']};records[x['path']]=record | |
| report['files'].append({k:v for k,v in x.items() if k!='local'});Path(x['local']).unlink(missing_ok=True) | |
| shutil.rmtree(back) | |
| print('Checked',min(i+12,len(expected)),'/',len(expected),'audio paths;',len(report['files']),'verified,',len(report['failures']),'source failures',flush=True) | |
| m['files']=list(records.values());m['lazy_audio_audit']={'catalog_entries':len(catalog),'expected_sounds':len(catalog)-1,'report':'_verification/thirty-dollar-audio-audit.json','note':'Startup-only capture missed sounds loaded by ID. All explicit sound URLs audited; see report for failures and browser decode checks.'} | |
| (R/'game-6.json').write_text(json.dumps(m,indent=2));(R/'thirty-dollar-audio-audit.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)) | |
| api.batch_bucket_files(B,add=[(R/'game-6.json','_reports/games/0006.json'),(R/'thirty-dollar-audio-audit.json','_verification/thirty-dollar-audio-audit.json'),(R/'thirty-dollar-sound-catalog.json','_verification/thirty-dollar-sound-catalog.json')]) | |
| print('SUMMARY',json.dumps({k:v for k,v in report.items() if k not in ('files','failures')},ensure_ascii=False),flush=True) | |
| print('FAILURES',json.dumps(report['failures'],ensure_ascii=False),flush=True) | |
Xet Storage Details
- Size:
- 4.96 kB
- Xet hash:
- 73630385f44bbc9dd0682ce7976b05fdbce677abecff04566e1822b55f70e8c2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.