Archive CIDM-v2 reproducible b2_r5_official_data_integrity_fix1.py
Browse files
cidm_v2/source/B2_R5_P0_FIX1/b2_r5_official_data_integrity_fix1.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CIDM-v2 B2-R5/P0 FIX1: verifiable DaRUS file resolver and safe downloads.
|
| 2 |
+
This is data IO only. It NEVER changes benchmark TASKS, dataset MD5, model
|
| 3 |
+
weights, validation split, model forward, oracle gradients, or train budget.
|
| 4 |
+
"""
|
| 5 |
+
from __future__ import annotations
|
| 6 |
+
import os, re, json, time, hashlib, shutil
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import requests
|
| 9 |
+
|
| 10 |
+
HDF5_MAGIC=b'\x89HDF\r\n\x1a\n'
|
| 11 |
+
|
| 12 |
+
def file_md5(path):
|
| 13 |
+
h=hashlib.md5()
|
| 14 |
+
with open(path,'rb') as f:
|
| 15 |
+
for piece in iter(lambda:f.read(4<<20),b''):
|
| 16 |
+
h.update(piece)
|
| 17 |
+
return h.hexdigest()
|
| 18 |
+
|
| 19 |
+
def log_event(audit, record):
|
| 20 |
+
if audit is None:return
|
| 21 |
+
audit.append(record)
|
| 22 |
+
|
| 23 |
+
def search_verified_previous(task, name, expected, dest, old_dirs, audit):
|
| 24 |
+
seen=set()
|
| 25 |
+
for directory in old_dirs:
|
| 26 |
+
p=Path(directory).expanduser()
|
| 27 |
+
candidates=[p/name]
|
| 28 |
+
if p.is_file() and p.name==name:candidates=[p]
|
| 29 |
+
for candidate in candidates:
|
| 30 |
+
if str(candidate) in seen:continue
|
| 31 |
+
seen.add(str(candidate))
|
| 32 |
+
if not candidate.is_file():continue
|
| 33 |
+
actual=file_md5(candidate)
|
| 34 |
+
log_event(audit,dict(task=task,stage='candidate',file=str(candidate),md5=actual,expected=expected,bytes=candidate.stat().st_size,pass_md5=(actual==expected)))
|
| 35 |
+
if actual!=expected:continue
|
| 36 |
+
if candidate.resolve()!=dest.resolve():
|
| 37 |
+
dest.parent.mkdir(parents=True,exist_ok=True)
|
| 38 |
+
if dest.exists() or dest.is_symlink():
|
| 39 |
+
dest.unlink()
|
| 40 |
+
try:
|
| 41 |
+
os.link(candidate,dest)
|
| 42 |
+
linktype='hardlink'
|
| 43 |
+
except OSError:
|
| 44 |
+
dest.symlink_to(candidate.resolve())
|
| 45 |
+
linktype='symlink'
|
| 46 |
+
else:linktype='same'
|
| 47 |
+
log_event(audit,dict(task=task,stage='verified_reuse',file=str(candidate),reuse=linktype))
|
| 48 |
+
return dest
|
| 49 |
+
return None
|
| 50 |
+
|
| 51 |
+
def strict_download(url, part, *, session, allow_resume=True, audit=None, task='unknown'):
|
| 52 |
+
part=Path(part);pos=part.stat().st_size if (allow_resume and part.exists()) else 0
|
| 53 |
+
headers={'Accept-Encoding':'identity','User-Agent':'CIDM-v2-PDEBench-integrity/1.0'}
|
| 54 |
+
if pos:headers['Range']=f'bytes={pos}-'
|
| 55 |
+
with session.get(url,stream=True,timeout=(30,180),headers=headers,allow_redirects=True) as r:
|
| 56 |
+
r.raise_for_status()
|
| 57 |
+
code=r.status_code
|
| 58 |
+
if code not in (200,206):raise RuntimeError(f'UNEXPECTED_HTTP_STATUS_{code}')
|
| 59 |
+
cr=r.headers.get('Content-Range','')
|
| 60 |
+
if code==206:
|
| 61 |
+
match=re.fullmatch(r'bytes (\d+)-(\d+)/(\d+|\*)',cr)
|
| 62 |
+
if not match:raise RuntimeError(f'INVALID_HTTP_CONTENT_RANGE:{cr}')
|
| 63 |
+
start,end,total=match.groups()
|
| 64 |
+
if int(start)!=pos or int(end)<int(start):
|
| 65 |
+
raise RuntimeError(f'RANGE_OFFSET_MISMATCH expected={pos} got={cr}')
|
| 66 |
+
expected_chunk=int(end)-int(start)+1
|
| 67 |
+
elif code==200:
|
| 68 |
+
# Server ignored Range. Reset before writing rather than appending!
|
| 69 |
+
pos=0;expected_chunk=None
|
| 70 |
+
clen=r.headers.get('Content-Length')
|
| 71 |
+
if clen and int(clen)!=expected_chunk and expected_chunk is not None:
|
| 72 |
+
raise RuntimeError(f'HTTP_CONTENT_LENGTH_RANGE_MISMATCH {clen}/{expected_chunk}')
|
| 73 |
+
written=0
|
| 74 |
+
with open(part,'ab' if pos else 'wb') as f:
|
| 75 |
+
for buf in r.iter_content(chunk_size=4<<20):
|
| 76 |
+
if buf:
|
| 77 |
+
f.write(buf);written+=len(buf)
|
| 78 |
+
if clen and written!=int(clen):
|
| 79 |
+
raise RuntimeError(f'INCOMPLETE_DOWNLOAD_BYTES {written}/{clen}')
|
| 80 |
+
if expected_chunk is not None and written!=expected_chunk:
|
| 81 |
+
raise RuntimeError(f'INCOMPLETE_HTTP_RANGE {written}/{expected_chunk}')
|
| 82 |
+
if code==206 and total!='*' and part.stat().st_size!=int(total):
|
| 83 |
+
raise RuntimeError(f'HTTP_RANGE_FINAL_SIZE_MISMATCH {part.stat().st_size}/{total}')
|
| 84 |
+
log_event(audit,dict(task=task,stage='http_download',status=code,range=cr,content_length=clen,bytes_written=written,total_bytes=part.stat().st_size,url=url))
|
| 85 |
+
return part
|
| 86 |
+
|
| 87 |
+
def fetch_official_verified(root,task,*, tasks_spec,previous_dirs=(),audit=None,session=None,max_fresh_attempts=1):
|
| 88 |
+
if task not in tasks_spec:raise RuntimeError(f'UNKNOWN_OFFICIAL_TASK {task}')
|
| 89 |
+
name,fileid,expected,_=tasks_spec[task]
|
| 90 |
+
root=Path(root);root.mkdir(parents=True,exist_ok=True)
|
| 91 |
+
dest=root/name
|
| 92 |
+
dirs=[root,*[Path(p) for p in previous_dirs]]
|
| 93 |
+
existing=search_verified_previous(task,name,expected,dest,dirs,audit)
|
| 94 |
+
if existing:return existing
|
| 95 |
+
# Discard corrupted dest only after recording original checksum; no false positives.
|
| 96 |
+
if dest.exists() or dest.is_symlink():
|
| 97 |
+
log_event(audit,dict(task=task,stage='reject_invalid_destination',file=str(dest)))
|
| 98 |
+
dest.unlink()
|
| 99 |
+
part=Path(str(dest)+'.part')
|
| 100 |
+
url=f'https://darus.uni-stuttgart.de/api/access/datafile/{fileid}'
|
| 101 |
+
if session is None:session=requests.Session()
|
| 102 |
+
# Pass 0 resumes existing .part if applicable; on MD5 mismatch delete and do ONE fresh pass.
|
| 103 |
+
last=None
|
| 104 |
+
for pass_index in range(1+max_fresh_attempts):
|
| 105 |
+
if pass_index>0:
|
| 106 |
+
part.unlink(missing_ok=True)
|
| 107 |
+
for attempt in range(3):
|
| 108 |
+
try:
|
| 109 |
+
strict_download(url,part,session=session,allow_resume=(pass_index==0),audit=audit,task=task)
|
| 110 |
+
break
|
| 111 |
+
except (requests.RequestException,OSError,RuntimeError) as e:
|
| 112 |
+
last=repr(e)
|
| 113 |
+
log_event(audit,dict(task=task,stage='retry_http',pass_index=pass_index,attempt=attempt,error=last))
|
| 114 |
+
if attempt==2:raise RuntimeError(f'DOWNLOAD_INTEGRITY_BLOCKED_{task}: {last}') from e
|
| 115 |
+
time.sleep(min(2*(attempt+1),5))
|
| 116 |
+
if not part.is_file():raise RuntimeError('NO_DOWNLOADED_FILE_'+task)
|
| 117 |
+
actual=file_md5(part)
|
| 118 |
+
magic=part.open('rb').read(8)
|
| 119 |
+
good=(actual==expected and magic==HDF5_MAGIC)
|
| 120 |
+
log_event(audit,dict(task=task,stage='final_md5',pass_index=pass_index,bytes=part.stat().st_size,md5=actual,expected=expected,hdf5_magic=magic.hex(),pass_integrity=good))
|
| 121 |
+
if good:
|
| 122 |
+
part.replace(dest)
|
| 123 |
+
return dest
|
| 124 |
+
last=f'MD5_OR_HDF5_MISMATCH actual={actual} expected={expected}, size={part.stat().st_size}, magic={magic.hex()}'
|
| 125 |
+
part.unlink(missing_ok=True)
|
| 126 |
+
raise RuntimeError(f'BLOCKED_OFFICIAL_DATA_INTEGRITY_{task}: {last}; no training or oracle permitted')
|