wuff-mann commited on
Commit
ab8afe2
·
verified ·
1 Parent(s): 5d43369

Archive CIDM-v2 reproducible b2_r5_official_data_integrity_fix1.py

Browse files
cidm_v2/source/B2_R5_P0_FIX1/b2_r5_official_data_integrity_fix1.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CIDM-v2 B2-R5/P0 FIX1: verifiable DaRUS file resolver and safe downloads.
2
+ This is data IO only. It NEVER changes benchmark TASKS, dataset MD5, model
3
+ weights, validation split, model forward, oracle gradients, or train budget.
4
+ """
5
+ from __future__ import annotations
6
+ import os, re, json, time, hashlib, shutil
7
+ from pathlib import Path
8
+ import requests
9
+
10
+ HDF5_MAGIC=b'\x89HDF\r\n\x1a\n'
11
+
12
+ def file_md5(path):
13
+ h=hashlib.md5()
14
+ with open(path,'rb') as f:
15
+ for piece in iter(lambda:f.read(4<<20),b''):
16
+ h.update(piece)
17
+ return h.hexdigest()
18
+
19
+ def log_event(audit, record):
20
+ if audit is None:return
21
+ audit.append(record)
22
+
23
+ def search_verified_previous(task, name, expected, dest, old_dirs, audit):
24
+ seen=set()
25
+ for directory in old_dirs:
26
+ p=Path(directory).expanduser()
27
+ candidates=[p/name]
28
+ if p.is_file() and p.name==name:candidates=[p]
29
+ for candidate in candidates:
30
+ if str(candidate) in seen:continue
31
+ seen.add(str(candidate))
32
+ if not candidate.is_file():continue
33
+ actual=file_md5(candidate)
34
+ log_event(audit,dict(task=task,stage='candidate',file=str(candidate),md5=actual,expected=expected,bytes=candidate.stat().st_size,pass_md5=(actual==expected)))
35
+ if actual!=expected:continue
36
+ if candidate.resolve()!=dest.resolve():
37
+ dest.parent.mkdir(parents=True,exist_ok=True)
38
+ if dest.exists() or dest.is_symlink():
39
+ dest.unlink()
40
+ try:
41
+ os.link(candidate,dest)
42
+ linktype='hardlink'
43
+ except OSError:
44
+ dest.symlink_to(candidate.resolve())
45
+ linktype='symlink'
46
+ else:linktype='same'
47
+ log_event(audit,dict(task=task,stage='verified_reuse',file=str(candidate),reuse=linktype))
48
+ return dest
49
+ return None
50
+
51
+ def strict_download(url, part, *, session, allow_resume=True, audit=None, task='unknown'):
52
+ part=Path(part);pos=part.stat().st_size if (allow_resume and part.exists()) else 0
53
+ headers={'Accept-Encoding':'identity','User-Agent':'CIDM-v2-PDEBench-integrity/1.0'}
54
+ if pos:headers['Range']=f'bytes={pos}-'
55
+ with session.get(url,stream=True,timeout=(30,180),headers=headers,allow_redirects=True) as r:
56
+ r.raise_for_status()
57
+ code=r.status_code
58
+ if code not in (200,206):raise RuntimeError(f'UNEXPECTED_HTTP_STATUS_{code}')
59
+ cr=r.headers.get('Content-Range','')
60
+ if code==206:
61
+ match=re.fullmatch(r'bytes (\d+)-(\d+)/(\d+|\*)',cr)
62
+ if not match:raise RuntimeError(f'INVALID_HTTP_CONTENT_RANGE:{cr}')
63
+ start,end,total=match.groups()
64
+ if int(start)!=pos or int(end)<int(start):
65
+ raise RuntimeError(f'RANGE_OFFSET_MISMATCH expected={pos} got={cr}')
66
+ expected_chunk=int(end)-int(start)+1
67
+ elif code==200:
68
+ # Server ignored Range. Reset before writing rather than appending!
69
+ pos=0;expected_chunk=None
70
+ clen=r.headers.get('Content-Length')
71
+ if clen and int(clen)!=expected_chunk and expected_chunk is not None:
72
+ raise RuntimeError(f'HTTP_CONTENT_LENGTH_RANGE_MISMATCH {clen}/{expected_chunk}')
73
+ written=0
74
+ with open(part,'ab' if pos else 'wb') as f:
75
+ for buf in r.iter_content(chunk_size=4<<20):
76
+ if buf:
77
+ f.write(buf);written+=len(buf)
78
+ if clen and written!=int(clen):
79
+ raise RuntimeError(f'INCOMPLETE_DOWNLOAD_BYTES {written}/{clen}')
80
+ if expected_chunk is not None and written!=expected_chunk:
81
+ raise RuntimeError(f'INCOMPLETE_HTTP_RANGE {written}/{expected_chunk}')
82
+ if code==206 and total!='*' and part.stat().st_size!=int(total):
83
+ raise RuntimeError(f'HTTP_RANGE_FINAL_SIZE_MISMATCH {part.stat().st_size}/{total}')
84
+ log_event(audit,dict(task=task,stage='http_download',status=code,range=cr,content_length=clen,bytes_written=written,total_bytes=part.stat().st_size,url=url))
85
+ return part
86
+
87
+ def fetch_official_verified(root,task,*, tasks_spec,previous_dirs=(),audit=None,session=None,max_fresh_attempts=1):
88
+ if task not in tasks_spec:raise RuntimeError(f'UNKNOWN_OFFICIAL_TASK {task}')
89
+ name,fileid,expected,_=tasks_spec[task]
90
+ root=Path(root);root.mkdir(parents=True,exist_ok=True)
91
+ dest=root/name
92
+ dirs=[root,*[Path(p) for p in previous_dirs]]
93
+ existing=search_verified_previous(task,name,expected,dest,dirs,audit)
94
+ if existing:return existing
95
+ # Discard corrupted dest only after recording original checksum; no false positives.
96
+ if dest.exists() or dest.is_symlink():
97
+ log_event(audit,dict(task=task,stage='reject_invalid_destination',file=str(dest)))
98
+ dest.unlink()
99
+ part=Path(str(dest)+'.part')
100
+ url=f'https://darus.uni-stuttgart.de/api/access/datafile/{fileid}'
101
+ if session is None:session=requests.Session()
102
+ # Pass 0 resumes existing .part if applicable; on MD5 mismatch delete and do ONE fresh pass.
103
+ last=None
104
+ for pass_index in range(1+max_fresh_attempts):
105
+ if pass_index>0:
106
+ part.unlink(missing_ok=True)
107
+ for attempt in range(3):
108
+ try:
109
+ strict_download(url,part,session=session,allow_resume=(pass_index==0),audit=audit,task=task)
110
+ break
111
+ except (requests.RequestException,OSError,RuntimeError) as e:
112
+ last=repr(e)
113
+ log_event(audit,dict(task=task,stage='retry_http',pass_index=pass_index,attempt=attempt,error=last))
114
+ if attempt==2:raise RuntimeError(f'DOWNLOAD_INTEGRITY_BLOCKED_{task}: {last}') from e
115
+ time.sleep(min(2*(attempt+1),5))
116
+ if not part.is_file():raise RuntimeError('NO_DOWNLOADED_FILE_'+task)
117
+ actual=file_md5(part)
118
+ magic=part.open('rb').read(8)
119
+ good=(actual==expected and magic==HDF5_MAGIC)
120
+ log_event(audit,dict(task=task,stage='final_md5',pass_index=pass_index,bytes=part.stat().st_size,md5=actual,expected=expected,hdf5_magic=magic.hex(),pass_integrity=good))
121
+ if good:
122
+ part.replace(dest)
123
+ return dest
124
+ last=f'MD5_OR_HDF5_MISMATCH actual={actual} expected={expected}, size={part.stat().st_size}, magic={magic.hex()}'
125
+ part.unlink(missing_ok=True)
126
+ raise RuntimeError(f'BLOCKED_OFFICIAL_DATA_INTEGRITY_{task}: {last}; no training or oracle permitted')