Download source/scratch_prepare.py from Queimo/rem3di-scratch-pcqm-cosmo: direct link, hf CLI and curl.
- Browser
- Download file 9.04 kB
-
https://huggingface.co/Queimo/rem3di-scratch-pcqm-cosmo/resolve/main/source/scratch_prepare.py
- Command line
-
hf download hf://Queimo/rem3di-scratch-pcqm-cosmo/source/scratch_prepare.py
-
curl -L -o scratch_prepare.py https://huggingface.co/Queimo/rem3di-scratch-pcqm-cosmo/resolve/main/source/scratch_prepare.py
9.04 kB
| """Restore data and architecture metadata, never pretrained Rem3Di parameters.""" | |
| from pathlib import Path | |
| import os,sys,json,subprocess,hashlib,shutil,tarfile,time,fcntl | |
| BASE=Path(__file__).resolve().parent.parent | |
| OUT=BASE/'scratch_pcqm_cosmo';TRAINING=Path('/marimo/rem3di_training') | |
| AUDIT=TRAINING/'corrected_20261027/evaluation/paper_audit' | |
| os.environ.update(CUDA_VISIBLE_DEVICES='0',OMP_NUM_THREADS='8',MKL_NUM_THREADS='8', | |
| TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD='1',CUBLAS_WORKSPACE_CONFIG=':4096:8', | |
| PYTHONPATH='/marimo/Rem3Di',WANDB_MODE='online') | |
| from dotenv import load_dotenv | |
| load_dotenv('/marimo/.env',override=False) | |
| from huggingface_hub import HfApi,hf_hub_download,snapshot_download | |
| from huggingface_hub.errors import RepositoryNotFoundError | |
| def sha(path,algorithm='sha256'): | |
| h=hashlib.new(algorithm) | |
| with Path(path).open('rb') as f: | |
| for b in iter(lambda:f.read(8<<20),b''):h.update(b) | |
| return h.hexdigest() | |
| def status(phase,**extra): | |
| OUT.mkdir(parents=True,exist_ok=True) | |
| value=dict(phase=phase,pid=os.getpid(),time=time.time(),**extra) | |
| p=OUT/'preparation_status.json';tmp=p.with_suffix('.partial');tmp.write_text(json.dumps(value,indent=2));tmp.replace(p) | |
| print('SCRATCH_PREPARATION',json.dumps(value),flush=True) | |
| def copy(repo,name,target,revision,kind='model'): | |
| source=Path(hf_hub_download(repo,name,revision=revision,repo_type=kind)) | |
| target.parent.mkdir(parents=True,exist_ok=True) | |
| if not target.exists() or sha(source)!=sha(target):shutil.copyfile(source,target) | |
| def download(url,target,digest,algorithm='sha256'): | |
| if target.exists() and sha(target,algorithm)==digest:return | |
| temp=target.with_suffix(target.suffix+'.download') | |
| subprocess.run(['curl','-fL','--retry','4','--connect-timeout','20','-o',str(temp),url],check=True) | |
| assert sha(temp,algorithm)==digest,(target,'Source digest mismatch') | |
| temp.replace(target) | |
| def main(): | |
| OUT.mkdir(parents=True,exist_ok=True) | |
| lock=(OUT/'prepare.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| status('restoring_structures_and_frozen_foundation') | |
| restored=False | |
| try:prepared=HfApi().dataset_info('Queimo/rem3di-scratch-pcqm-cosmo-data') | |
| except RepositoryNotFoundError:prepared=None | |
| if prepared is not None: | |
| snap=Path(snapshot_download(prepared.id,revision=prepared.sha,repo_type='dataset')) | |
| for name in ['pcqm.parquet','cosmo.parquet','bigsoldb_fixed_geometries.parquet','cosmo_label_transform.json','normalization.pt','normalization.json','pcqm_identity_quarantine.csv']: | |
| if not (snap/name).exists(): | |
| assert name in ['normalization.pt','normalization.json'] | |
| continue | |
| dest=OUT/'data'/name;dest.parent.mkdir(exist_ok=True);shutil.copyfile(snap/name,dest) | |
| shutil.copyfile(snap/'split_validation.json',OUT/'split_validation.json') | |
| restored=True | |
| else: | |
| archive=TRAINING/'pcqm4m-v2-train.sdf.tar.gz' | |
| download('https://huggingface.co/datasets/sdSDDDDDDD/drugs/resolve/main/pcqm4mv2/pcqm4m-v2-train.sdf.tar.gz', | |
| archive,'fd72bce606e7ddf36c2a832badeec6ab','md5') | |
| sdf=TRAINING/'pcqm4m-v2-train.sdf' | |
| if not sdf.exists(): | |
| with tarfile.open(archive,'r:gz') as tf: | |
| member=next(m for m in tf if m.name.endswith('.sdf')) | |
| with tf.extractfile(member) as src,sdf.with_suffix('.partial').open('wb') as dst:shutil.copyfileobj(src,dst,8<<20) | |
| sdf.with_suffix('.partial').replace(sdf) | |
| download('https://github.com/ACEsuit/mace-foundations/releases/download/mace_polar_1/MACE-POLAR-1-M.model', | |
| TRAINING/'MACE-POLAR-1-M.model','fab8b8713c832f31a2a853aaa22fd638be8a369cbf5095e6b3e982a18d10e93a') | |
| old='Queimo/rem3di-pcqm500k-corrected';rev='2f5c3b40b36259a13adc8412689254054bb93288' | |
| prefix='runs/corrected_20261027/evaluation/paper_audit/author_pcqm500k_5ep' | |
| copy(old,prefix+'/post_training_architecture_config.yaml',OUT/'architecture.yaml',rev) | |
| architecture=AUDIT/'author_checkpoint/exp138/polar/post_training_architecture_config.yaml' | |
| architecture.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(OUT/'architecture.yaml',architecture) | |
| copy(old,prefix+'/source/build_author_mace_runtime.py',AUDIT/'build_author_mace_runtime.py',rev) | |
| # Architecture-only download above: no encoder, decoder or preprocessor weights. | |
| provenance=Path(hf_hub_download('Queimo/rem3di-pcqm-epoch2-cosmo','provenance.json',revision='9266a41fbe3b36afe0615e0c3ba232ca74bdeef7')) | |
| expected=json.loads(provenance.read_text())['legacy_source_sha256'] | |
| for name,digest in expected.items():assert sha(AUDIT/'author_legacy_source'/name)==digest,name | |
| status('restoring_cosmo_and_exact_bigsoldb_inputs') | |
| snap=Path(snapshot_download('Queimo/combisolv-qm-rem3di-multitask',repo_type='dataset', | |
| revision='de87254deae3bbdc031e3c2185df7c3ad63945ac', | |
| allow_patterns=['prepared/cosmo-*.parquet','prepared/bigsoldb_fixed_geometries.parquet','prepared/manifest.json'])) | |
| target=OUT/'data';target.mkdir(exist_ok=True) | |
| for p in (snap/'prepared').iterdir():shutil.copyfile(p,target/p.name) | |
| prior=Path(snapshot_download('Queimo/rem3di-pcqm-epoch2-cosmo',revision='9266a41fbe3b36afe0615e0c3ba232ca74bdeef7', | |
| allow_patterns=['bigsoldb_evaluation/data/*','bigsoldb_evaluation/features/pair_indices.npz', | |
| 'bigsoldb_evaluation/features/shared_graphs.parquet','bigsoldb_evaluation/protocol.json','bigsoldb_evaluation/results.csv'])) | |
| for p in (prior/'bigsoldb_evaluation').rglob('*'): | |
| if p.is_file(): | |
| dest=BASE/'bigsoldb_evaluation'/p.relative_to(prior/'bigsoldb_evaluation');dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest) | |
| status('building_original_pcqm500k_structures') | |
| pcqm=TRAINING/'pcqm500k_std' | |
| if not restored and not (TRAINING/'pcqm500k_std_complete.json').exists(): | |
| subprocess.run([sys.executable,'-u',str(TRAINING/'build_dataset.py'),'--count','500000','--output',str(pcqm)],check=True) | |
| status('preparing_splits_and_frozen_features') | |
| while not (BASE/'source/scratch_data.py').exists():time.sleep(2) | |
| subprocess.run([sys.executable,'-u',str(BASE/'source/scratch_data.py')],check=True) | |
| # Resume only this run's own random-initialized lineage after a sandbox loss. | |
| try:info=HfApi().model_info('Queimo/rem3di-scratch-pcqm-cosmo') | |
| except RepositoryNotFoundError:info=None | |
| if info is not None: | |
| points=sorted({int(s.rfilename.split('/')[0].split('_')[1]) for s in info.siblings | |
| if s.rfilename.startswith('probe_') and s.rfilename.endswith('/manifest.json')}) | |
| snap=Path(snapshot_download(info.id,revision=info.sha,allow_patterns=['evaluation/*','*.json','README.md','prior_comparison.csv'])) | |
| for p in snap.rglob('*'): | |
| rel=p.relative_to(snap) | |
| if not p.is_file() or rel.parts[0].startswith('probe_') or rel.name in ['preparation_complete.json','preparation_status.json','status.json','preparation_process.json','driver_process.json','active_process.json']:continue | |
| dest=OUT/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest) | |
| if points: | |
| latest=points[-1] | |
| weights=Path(snapshot_download(info.id,revision=info.sha,allow_patterns=[f'probe_{latest:03d}/*'])) | |
| source=weights/f'probe_{latest:03d}';dest=OUT/'checkpoints'/source.name;dest.mkdir(parents=True,exist_ok=True) | |
| manifest=json.loads((source/'manifest.json').read_text()) | |
| for name,h in manifest['files'].items():assert sha(source/name)==h['sha256'];shutil.copyfile(source/name,dest/name) | |
| shutil.copyfile(source/'manifest.json',dest/'manifest.json') | |
| (dest/'hub_verified.json').write_text(json.dumps(dict(verified=True,repository=info.id,revision=info.sha,point=latest))) | |
| # Published results are authoritative even if the subsequent completion-marker commit was interrupted. | |
| results=OUT/'evaluation/results.csv' | |
| if results.exists(): | |
| import pandas as pd,numpy as np | |
| test=pd.read_csv(BASE/'bigsoldb_evaluation/data/test.csv') | |
| for row in pd.read_csv(results).itertuples(): | |
| prediction=np.load(OUT/f'evaluation/predictions/probe_{row.point:03d}.npz') | |
| assert np.array_equal(prediction['test_clean_rows'],test['_clean_row']) | |
| assert np.allclose(prediction['y_true'],test['LogS(mol/L)'],rtol=0,atol=1e-14) | |
| assert np.isclose(np.sqrt(np.mean((prediction['y_pred']-prediction['y_true'])**2)),row.rmse,rtol=1e-9) | |
| (OUT/f'evaluation/probe_{row.point:03d}_complete.json').write_text(json.dumps(dict(completed=True,revision=info.sha,restored=True))) | |
| status('ready_for_training') | |
| if __name__=='__main__': | |
| try:main() | |
| except Exception: | |
| import traceback | |
| status('failed',error=traceback.format_exc());raise | |