rem3di-scratch-pcqm-cosmo / source /scratch_prepare.py
Queimo's picture
Verify random initialization, real-data gradients and exact next-step resume before scratch training
04a3ade verified
Raw History Blame Contribute Delete
9.04 kB
"""Restore data and architecture metadata, never pretrained Rem3Di parameters."""
from pathlib import Path
import os,sys,json,subprocess,hashlib,shutil,tarfile,time,fcntl
BASE=Path(__file__).resolve().parent.parent
OUT=BASE/'scratch_pcqm_cosmo';TRAINING=Path('/marimo/rem3di_training')
AUDIT=TRAINING/'corrected_20261027/evaluation/paper_audit'
os.environ.update(CUDA_VISIBLE_DEVICES='0',OMP_NUM_THREADS='8',MKL_NUM_THREADS='8',
TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD='1',CUBLAS_WORKSPACE_CONFIG=':4096:8',
PYTHONPATH='/marimo/Rem3Di',WANDB_MODE='online')
from dotenv import load_dotenv
load_dotenv('/marimo/.env',override=False)
from huggingface_hub import HfApi,hf_hub_download,snapshot_download
from huggingface_hub.errors import RepositoryNotFoundError
def sha(path,algorithm='sha256'):
h=hashlib.new(algorithm)
with Path(path).open('rb') as f:
for b in iter(lambda:f.read(8<<20),b''):h.update(b)
return h.hexdigest()
def status(phase,**extra):
OUT.mkdir(parents=True,exist_ok=True)
value=dict(phase=phase,pid=os.getpid(),time=time.time(),**extra)
p=OUT/'preparation_status.json';tmp=p.with_suffix('.partial');tmp.write_text(json.dumps(value,indent=2));tmp.replace(p)
print('SCRATCH_PREPARATION',json.dumps(value),flush=True)
def copy(repo,name,target,revision,kind='model'):
source=Path(hf_hub_download(repo,name,revision=revision,repo_type=kind))
target.parent.mkdir(parents=True,exist_ok=True)
if not target.exists() or sha(source)!=sha(target):shutil.copyfile(source,target)
def download(url,target,digest,algorithm='sha256'):
if target.exists() and sha(target,algorithm)==digest:return
temp=target.with_suffix(target.suffix+'.download')
subprocess.run(['curl','-fL','--retry','4','--connect-timeout','20','-o',str(temp),url],check=True)
assert sha(temp,algorithm)==digest,(target,'Source digest mismatch')
temp.replace(target)
def main():
OUT.mkdir(parents=True,exist_ok=True)
lock=(OUT/'prepare.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
status('restoring_structures_and_frozen_foundation')
restored=False
try:prepared=HfApi().dataset_info('Queimo/rem3di-scratch-pcqm-cosmo-data')
except RepositoryNotFoundError:prepared=None
if prepared is not None:
snap=Path(snapshot_download(prepared.id,revision=prepared.sha,repo_type='dataset'))
for name in ['pcqm.parquet','cosmo.parquet','bigsoldb_fixed_geometries.parquet','cosmo_label_transform.json','normalization.pt','normalization.json','pcqm_identity_quarantine.csv']:
if not (snap/name).exists():
assert name in ['normalization.pt','normalization.json']
continue
dest=OUT/'data'/name;dest.parent.mkdir(exist_ok=True);shutil.copyfile(snap/name,dest)
shutil.copyfile(snap/'split_validation.json',OUT/'split_validation.json')
restored=True
else:
archive=TRAINING/'pcqm4m-v2-train.sdf.tar.gz'
download('https://huggingface.co/datasets/sdSDDDDDDD/drugs/resolve/main/pcqm4mv2/pcqm4m-v2-train.sdf.tar.gz',
archive,'fd72bce606e7ddf36c2a832badeec6ab','md5')
sdf=TRAINING/'pcqm4m-v2-train.sdf'
if not sdf.exists():
with tarfile.open(archive,'r:gz') as tf:
member=next(m for m in tf if m.name.endswith('.sdf'))
with tf.extractfile(member) as src,sdf.with_suffix('.partial').open('wb') as dst:shutil.copyfileobj(src,dst,8<<20)
sdf.with_suffix('.partial').replace(sdf)
download('https://github.com/ACEsuit/mace-foundations/releases/download/mace_polar_1/MACE-POLAR-1-M.model',
TRAINING/'MACE-POLAR-1-M.model','fab8b8713c832f31a2a853aaa22fd638be8a369cbf5095e6b3e982a18d10e93a')
old='Queimo/rem3di-pcqm500k-corrected';rev='2f5c3b40b36259a13adc8412689254054bb93288'
prefix='runs/corrected_20261027/evaluation/paper_audit/author_pcqm500k_5ep'
copy(old,prefix+'/post_training_architecture_config.yaml',OUT/'architecture.yaml',rev)
architecture=AUDIT/'author_checkpoint/exp138/polar/post_training_architecture_config.yaml'
architecture.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(OUT/'architecture.yaml',architecture)
copy(old,prefix+'/source/build_author_mace_runtime.py',AUDIT/'build_author_mace_runtime.py',rev)
# Architecture-only download above: no encoder, decoder or preprocessor weights.
provenance=Path(hf_hub_download('Queimo/rem3di-pcqm-epoch2-cosmo','provenance.json',revision='9266a41fbe3b36afe0615e0c3ba232ca74bdeef7'))
expected=json.loads(provenance.read_text())['legacy_source_sha256']
for name,digest in expected.items():assert sha(AUDIT/'author_legacy_source'/name)==digest,name
status('restoring_cosmo_and_exact_bigsoldb_inputs')
snap=Path(snapshot_download('Queimo/combisolv-qm-rem3di-multitask',repo_type='dataset',
revision='de87254deae3bbdc031e3c2185df7c3ad63945ac',
allow_patterns=['prepared/cosmo-*.parquet','prepared/bigsoldb_fixed_geometries.parquet','prepared/manifest.json']))
target=OUT/'data';target.mkdir(exist_ok=True)
for p in (snap/'prepared').iterdir():shutil.copyfile(p,target/p.name)
prior=Path(snapshot_download('Queimo/rem3di-pcqm-epoch2-cosmo',revision='9266a41fbe3b36afe0615e0c3ba232ca74bdeef7',
allow_patterns=['bigsoldb_evaluation/data/*','bigsoldb_evaluation/features/pair_indices.npz',
'bigsoldb_evaluation/features/shared_graphs.parquet','bigsoldb_evaluation/protocol.json','bigsoldb_evaluation/results.csv']))
for p in (prior/'bigsoldb_evaluation').rglob('*'):
if p.is_file():
dest=BASE/'bigsoldb_evaluation'/p.relative_to(prior/'bigsoldb_evaluation');dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest)
status('building_original_pcqm500k_structures')
pcqm=TRAINING/'pcqm500k_std'
if not restored and not (TRAINING/'pcqm500k_std_complete.json').exists():
subprocess.run([sys.executable,'-u',str(TRAINING/'build_dataset.py'),'--count','500000','--output',str(pcqm)],check=True)
status('preparing_splits_and_frozen_features')
while not (BASE/'source/scratch_data.py').exists():time.sleep(2)
subprocess.run([sys.executable,'-u',str(BASE/'source/scratch_data.py')],check=True)
# Resume only this run's own random-initialized lineage after a sandbox loss.
try:info=HfApi().model_info('Queimo/rem3di-scratch-pcqm-cosmo')
except RepositoryNotFoundError:info=None
if info is not None:
points=sorted({int(s.rfilename.split('/')[0].split('_')[1]) for s in info.siblings
if s.rfilename.startswith('probe_') and s.rfilename.endswith('/manifest.json')})
snap=Path(snapshot_download(info.id,revision=info.sha,allow_patterns=['evaluation/*','*.json','README.md','prior_comparison.csv']))
for p in snap.rglob('*'):
rel=p.relative_to(snap)
if not p.is_file() or rel.parts[0].startswith('probe_') or rel.name in ['preparation_complete.json','preparation_status.json','status.json','preparation_process.json','driver_process.json','active_process.json']:continue
dest=OUT/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest)
if points:
latest=points[-1]
weights=Path(snapshot_download(info.id,revision=info.sha,allow_patterns=[f'probe_{latest:03d}/*']))
source=weights/f'probe_{latest:03d}';dest=OUT/'checkpoints'/source.name;dest.mkdir(parents=True,exist_ok=True)
manifest=json.loads((source/'manifest.json').read_text())
for name,h in manifest['files'].items():assert sha(source/name)==h['sha256'];shutil.copyfile(source/name,dest/name)
shutil.copyfile(source/'manifest.json',dest/'manifest.json')
(dest/'hub_verified.json').write_text(json.dumps(dict(verified=True,repository=info.id,revision=info.sha,point=latest)))
# Published results are authoritative even if the subsequent completion-marker commit was interrupted.
results=OUT/'evaluation/results.csv'
if results.exists():
import pandas as pd,numpy as np
test=pd.read_csv(BASE/'bigsoldb_evaluation/data/test.csv')
for row in pd.read_csv(results).itertuples():
prediction=np.load(OUT/f'evaluation/predictions/probe_{row.point:03d}.npz')
assert np.array_equal(prediction['test_clean_rows'],test['_clean_row'])
assert np.allclose(prediction['y_true'],test['LogS(mol/L)'],rtol=0,atol=1e-14)
assert np.isclose(np.sqrt(np.mean((prediction['y_pred']-prediction['y_true'])**2)),row.rmse,rtol=1e-9)
(OUT/f'evaluation/probe_{row.point:03d}_complete.json').write_text(json.dumps(dict(completed=True,revision=info.sha,restored=True)))
status('ready_for_training')
if __name__=='__main__':
try:main()
except Exception:
import traceback
status('failed',error=traceback.format_exc());raise