Download source/bigsoldb_evaluate.py from Queimo/rem3di-cosmo10-combisolv-multitask: direct link, hf CLI and curl.
- Browser
- Download file 17.6 kB
-
https://huggingface.co/Queimo/rem3di-cosmo10-combisolv-multitask/resolve/main/source/bigsoldb_evaluate.py
- Command line
-
hf download hf://Queimo/rem3di-cosmo10-combisolv-multitask/source/bigsoldb_evaluate.py
-
curl -L -o bigsoldb_evaluate.py https://huggingface.co/Queimo/rem3di-cosmo10-combisolv-multitask/resolve/main/source/bigsoldb_evaluate.py
17.6 kB
| """Concatenate frozen solute/solvent embeddings and temperature for Kumo L.""" | |
| from pathlib import Path | |
| import sys | |
| sys.path.insert(0,str(Path(__file__).resolve().parent)) | |
| import eval_common as common | |
| from eval_common import WORK,write,sha | |
| import fcntl,functools,gc,hashlib,json,os,re,shutil,subprocess,time,traceback | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| import wandb | |
| from rdkit import Chem | |
| from huggingface_hub import HfApi,CommitOperationAdd | |
| OUT=WORK/'bigsoldb_evaluation' | |
| UPSTREAM=WORK/'external/geomaw-solv' | |
| UPSTREAM_REVISION='8d0a6695afee1b8723f06122008b50ab3a25e144' | |
| REPOSITORY='Queimo/rem3di-pcqm-epoch2-cosmo' | |
| PAPER_URL='https://arxiv.org/html/2511.09774v1#S3.T1' | |
| RAW_MD5='1e6d8a510387746e9faab5f39dc46317' | |
| PUBLISHED={ | |
| 'XGBoost-DFT':[.621,.650,.806,.499,.722,.722], | |
| 'XGBoost-CFP':[.749,.889,.943,.315,.592,.561], | |
| 'XGBoost-MMB':[.710,.831,.912,.360,.616,.591], | |
| 'SolvBERT':[.871,1.231,1.110,.052,.247,.222], | |
| 'MPNN':[.668,.746,.864,.425,.724,.693], | |
| 'EquiformerV2':[.691,.858,.926,.345,.626,.606], | |
| 'Solvaformer':[.643,.700,.837,.460,.694,.677], | |
| 'MPNN+Charges':[.629,.667,.817,.486,.721,.710], | |
| } | |
| METRICS=['mae','mse','rmse','r2','pearson','spearman'] | |
| PROTOCOL=dict(dataset='BigSolDB 2.0',upstream_revision=UPSTREAM_REVISION,paper_table=PAPER_URL, | |
| target='LogS(mol/L): supplied log10 molar solubility; no additional label transform', | |
| input='concat(frozen solute embedding, frozen solvent embedding, Temperature_K)', | |
| representations=['cosmo_epoch10','monroe'],raw_widths={'cosmo_epoch10':1281,'monroe':1441}, | |
| geometry='One cached Monroe-native conformer per unique molecule; shared by both encoders', | |
| split='Exact held-out row IDs and measurements from committed MPNN+Charges predictions; remaining curated rows form the non-test training pool', | |
| training_pool='All 75731 non-test measurements; fixed Kumo recipe without validation-based tuning. Neural baselines may reserve a subset for validation.', | |
| kumo=dict(size='large',seed=42,num_estimators=4,feature_cap='Full concatenated width; no random feature truncation', | |
| precision='float16 autocast',estimator_batch_size=1,query_chunk_size=128), | |
| context_fallback='Try the entire training context first. Only upon CUDA OOM, use a matched, deterministic disjoint-context ensemble covering every training row; choose using training computation alone, before test scoring.', | |
| metrics=METRICS) | |
| def status(phase,**kw): | |
| write(OUT/'status.json',dict(phase=phase,pid=os.getpid(),updated_unix=time.time(),**kw)) | |
| print('BIGSOLDB_STATUS',phase,json.dumps(kw),flush=True) | |
| def metrics(y,p): | |
| from scipy.stats import pearsonr,spearmanr | |
| mse=float(np.mean((y-p)**2)) | |
| return dict(**common.metric(y,p),mse=mse,pearson=float(pearsonr(y,p).statistic),spearman=float(spearmanr(y,p).statistic)) | |
| def prepare_data(): | |
| status('recovering_published_split') | |
| env={**os.environ,'GIT_LFS_SKIP_SMUDGE':'1'} | |
| UPSTREAM.parent.mkdir(exist_ok=True) | |
| if not (UPSTREAM/'.git').exists(): | |
| subprocess.run(['git','clone','--depth','1','https://github.com/Sanofi-Public/geomaw-solv.git',str(UPSTREAM)],check=True,env=env) | |
| assert subprocess.check_output(['git','-C',str(UPSTREAM),'rev-parse','HEAD'],text=True).strip()==UPSTREAM_REVISION | |
| path=OUT/'data/BigSolDBv2.0.csv' | |
| if not path.exists(): | |
| subprocess.run(['curl','-fL','--retry','4','-o',str(path),'https://zenodo.org/records/15094979/files/BigSolDBv2.0.csv?download=1'],check=True) | |
| assert hashlib.md5(path.read_bytes()).hexdigest()==RAW_MD5 | |
| nb_path=UPSTREAM/'nbs/1_filter_and_split_data.ipynb' | |
| source=''.join(json.loads(nb_path.read_text())['cells'][4]['source']) | |
| source=source[source.index('from rdkit import RDLogger'):source.index('# Find duplicates')] | |
| # Memoization changes parsing cost only; preserve the author's exact filtering code. | |
| source=source.replace("df['SMILES_Solute'] = df['SMILES_Solute'].apply(canonicalize_smiles)", | |
| "canonicalize_smiles = functools.lru_cache(None)(canonicalize_smiles)\ndf['SMILES_Solute'] = df['SMILES_Solute'].apply(canonicalize_smiles)") | |
| raw=pd.read_csv(path);raw['_raw_row']=np.arange(len(raw));assert len(raw)==103944 | |
| namespace=dict(df=raw,np=np,pd=pd,re=re,Chem=Chem,functools=functools) | |
| exec(compile(source,str(nb_path)+':cell4','exec'),namespace) | |
| keys=['SMILES_Solute','SMILES_Solvent','Temperature_K'] | |
| filtered=namespace['df'];assert len(filtered)==92008 | |
| clean=filtered.drop_duplicates(keys,keep=False).reset_index(drop=True) | |
| assert len(clean)==85417 and clean.SMILES_Solute.nunique()==1268 and clean.SMILES_Solvent.nunique()==70 | |
| clean['_clean_row']=np.arange(len(clean)) | |
| prediction_path=UPSTREAM/'mpnn_charges/evaluation_results_test_run2/predictions.csv' | |
| author=pd.read_csv(prediction_path);ids=author['Unnamed: 0'].to_numpy(dtype=np.int64) | |
| assert len(ids)==9686 and len(np.unique(ids))==len(ids) and ids.min()>=0 and ids.max()<len(clean) | |
| test=clean.iloc[ids].copy().reset_index(drop=True) | |
| canon=functools.lru_cache(None)(lambda s:Chem.MolToSmiles(Chem.MolFromSmiles(str(s)),isomericSmiles=True)) | |
| for col in ['SMILES_Solute','SMILES_Solvent']: | |
| assert test[col].map(canon).tolist()==author[col].map(canon).tolist(),col | |
| assert np.array_equal(test.Temperature_K.to_numpy(),author.Temperature_K.to_numpy()) | |
| assert np.allclose(test['LogS(mol/L)'],author['LogS(mol/L)'],atol=1e-12,rtol=0) | |
| mask=np.ones(len(clean),dtype=bool);mask[ids]=False | |
| train=clean.loc[mask].copy().reset_index(drop=True) | |
| assert len(train)==75731 and not set(train.SMILES_Solute)&set(test.SMILES_Solute) | |
| assert train.SMILES_Solute.nunique()==1142 and test.SMILES_Solute.nunique()==126 | |
| assert not clean.duplicated(keys).any() and np.isfinite(clean['LogS(mol/L)']).all() | |
| reference=metrics(author.True_LogS.to_numpy(),author.Predicted_LogS.to_numpy()) | |
| assert [round(reference[k],3) for k in METRICS]==PUBLISHED['MPNN+Charges'],reference | |
| for name,df in [('train',train),('test',test)]:df.to_csv(OUT/'data'/f'{name}.csv',index=False) | |
| shutil.copyfile(prediction_path,OUT/'data/author_predictions.csv') | |
| write(OUT/'data/split_validation.json',dict(passed=True,curated_rows=85417,train_rows=75731,test_rows=9686, | |
| train_solutes=1142,test_solutes=126,all_author_row_ids_match=True,all_author_labels_match=True, | |
| author_metric_reproduction=reference,paper_count_discrepancy='Paper/notebook prose says 9250 test rows; the committed predictions yielding its published metrics contain 9686.', | |
| upstream_predictions_sha256=sha(prediction_path),raw_md5=RAW_MD5)) | |
| PROTOCOL['checkpoint_sha256']={name:sha(common.checkpoint(10)/name) for name in | |
| ['encoder.pth','atomic_preprocessor.pth','geometric_preprocessor.pth']} | |
| manifest=json.loads((common.checkpoint(10)/'manifest.json').read_text()) | |
| for name,digest in PROTOCOL['checkpoint_sha256'].items():assert digest==manifest['files'][name]['sha256'] | |
| PROTOCOL['versions']={name:__import__('importlib.metadata',fromlist=['version']).version(name) for name in | |
| ['torch','numpy','pandas','rdkit','scikit-learn','structured-data-models']} | |
| write(OUT/'protocol.json',PROTOCOL) | |
| smiles=[str(s) for s in pd.unique(pd.concat([train.SMILES_Solute,train.SMILES_Solvent,test.SMILES_Solute,test.SMILES_Solvent]))] | |
| assert len(smiles)==1334 | |
| return train,test,smiles | |
| def feature_arrays(train,test,smiles): | |
| import openadmet_evaluate as runtime | |
| runtime.OUT=OUT;runtime.status=status | |
| runtime.PROTOCOL['checkpoint_sha256']=PROTOCOL['checkpoint_sha256'] | |
| runtime.restore_monroe() | |
| built=runtime.prepare_graphs(smiles) | |
| embeddings={'monroe':runtime.monroe_embeddings(built,smiles),'cosmo_epoch10':runtime.cosmo_embeddings(built,smiles)} | |
| del built;gc.collect() | |
| lookup={s:i for i,s in enumerate(smiles)};rows=pd.concat([train,test],ignore_index=True) | |
| solute=np.asarray([lookup[s] for s in rows.SMILES_Solute]);solvent=np.asarray([lookup[s] for s in rows.SMILES_Solvent]) | |
| temperature=rows.Temperature_K.to_numpy(dtype=np.float32) | |
| np.savez_compressed(OUT/'features/pair_indices.npz',solute=solute,solvent=solvent,temperature_K=temperature, | |
| train_clean_rows=train['_clean_row'].to_numpy(),test_clean_rows=test['_clean_row'].to_numpy()) | |
| return embeddings,solute,solvent,temperature | |
| def table(x): | |
| import sdm | |
| return sdm.TableTensor.from_tensor(torch.as_tensor(np.ascontiguousarray(x),dtype=torch.float32,device='cuda')) | |
| def context_indices(n,blocks): | |
| if blocks==1:return None | |
| pieces=np.array_split(np.random.default_rng(42).permutation(n),blocks) | |
| width=max(map(len,pieces)) | |
| # Pad only the shorter blocks by repeating their first row; retain every row. | |
| return np.stack([np.pad(v,(0,width-len(v)),mode='wrap') for v in pieces]) | |
| def fit_context(model,X,y,ntrain,blocks): | |
| import sdm.processing as sp | |
| from sdm.models.callback import Callback | |
| recipe=model.default_recipe() | |
| selectors=[m for m in recipe.features.modules() if isinstance(m,sp.SelectColumns)] | |
| assert len(selectors)==1;selectors[0].max_columns=X.shape[1] | |
| shapes=[] | |
| class Audit(Callback): | |
| def on_context_preprocessing_end(self,model,x,y,related_tables): | |
| shapes.append(list(x.shape));status('fitting_kumo_context',blocks=blocks,processed_context_shapes=shapes) | |
| return x,y,related_tables | |
| indices=context_indices(ntrain,blocks) | |
| if indices is None:x,yc=table(X[:ntrain]),table(y[:ntrain,None]);estimators=4 | |
| else: | |
| assert set(np.unique(indices))==set(range(ntrain)) | |
| x,yc=table(X[indices]),table(y[indices,None]);estimators=None | |
| with torch.inference_mode(),torch.autocast('cuda',dtype=torch.float16): | |
| model.fit(x=x,y=yc,num_estimators=estimators,estimator_batch_size=1,recipe=recipe, | |
| generator=torch.Generator(device='cuda').manual_seed(42),callbacks=[Audit()]) | |
| del x,yc;gc.collect();torch.cuda.empty_cache() | |
| assert shapes and max(s[-1] for s in shapes)<=X.shape[1] | |
| return shapes | |
| def predict(model,X): | |
| values=[] | |
| with torch.inference_mode(),torch.autocast('cuda',dtype=torch.float16): | |
| for start in range(0,len(X),128): | |
| q=model.predict(table(X[start:start+128])).numerical.float().cpu().numpy() | |
| values.append(q.mean(1,dtype=np.float64)) | |
| if start%1024==0:status('predicting_test',completed=min(start+128,len(X)),total=len(X)) | |
| p=np.concatenate(values);assert np.isfinite(p).all();return p | |
| def evaluate(run,embeddings,solute,solvent,temperature,train,test): | |
| import sdm | |
| y=pd.concat([train,test])['LogS(mol/L)'].to_numpy(dtype=np.float64);n=len(train) | |
| records=pd.read_csv(OUT/'results.csv').to_dict('records') if (OUT/'results.csv').exists() else [] | |
| plan_path=OUT/'context_plan.json' | |
| plan=json.loads(plan_path.read_text()) if plan_path.exists() else None | |
| model=sdm.models.KumoTabular(task='regression',size='large',device='cuda') | |
| # Use the wider representation first to set any memory-only fallback equally for both. | |
| for representation in ['monroe','cosmo_epoch10']: | |
| if any(r['representation']==representation for r in records):continue | |
| E=embeddings[representation] | |
| X=np.concatenate([E[solute],E[solvent],temperature[:,None]],axis=1).astype(np.float32) | |
| assert X.shape[1]==PROTOCOL['raw_widths'][representation] and np.isfinite(X).all() | |
| assert np.array_equal(X[:,-1],temperature) and np.ptp(X[:n,-1])>0 | |
| status('fitting_representation',representation=representation,raw_columns=X.shape[1],train_rows=n,test_rows=len(test)) | |
| started=time.perf_counter() | |
| candidates=[plan['blocks']] if plan else [1,4,8,16,32] | |
| for blocks in candidates: | |
| try: | |
| shapes=fit_context(model,X,y,n,blocks) | |
| if plan is None: | |
| plan=dict(blocks=blocks,num_estimators=4 if blocks==1 else blocks, | |
| all_training_rows_covered=True,selection='Training-only CUDA memory feasibility; no test metrics used') | |
| write(plan_path,plan) | |
| ids=context_indices(n,blocks) | |
| np.savez_compressed(OUT/'features/train_context_indices.npz',indices=np.arange(n)[None] if ids is None else ids) | |
| break | |
| except torch.cuda.OutOfMemoryError: | |
| print('CONTEXT_CUDA_OOM',blocks,flush=True);model.clear();gc.collect();torch.cuda.empty_cache() | |
| if plan is not None or blocks==candidates[-1]:raise | |
| p=predict(model,X[n:]) | |
| result=dict(representation=representation,method='solute_solvent_temperature_concat_kumo_large', | |
| n_train=n,n_test=len(test),raw_columns=X.shape[1],context_blocks=plan['blocks'], | |
| num_estimators=plan['num_estimators'],seed=42,**metrics(y[n:],p),seconds=time.perf_counter()-started) | |
| pred_path=OUT/'predictions'/f'{representation}.npz' | |
| np.savez_compressed(pred_path,y_true=y[n:],y_pred=p,test_clean_rows=test['_clean_row'].to_numpy(),test_raw_rows=test['_raw_row'].to_numpy()) | |
| write(pred_path.with_suffix('.json'),{**result,'processed_context_shapes':shapes,'context_plan':plan}) | |
| records.append(result);tmp=OUT/'results.partial.csv';pd.DataFrame(records).to_csv(tmp,index=False);tmp.replace(OUT/'results.csv') | |
| write(OUT/'progress.json',dict(completed=len(records),total=2,latest=result));run.log(result) | |
| artifact=wandb.Artifact('cosmo-bigsoldb-prediction-backups',type='evaluation',metadata=result) | |
| for pth in [pred_path,pred_path.with_suffix('.json'),OUT/'results.csv',plan_path]:artifact.add_file(str(pth),name=str(pth.relative_to(OUT))) | |
| run.log_artifact(artifact,aliases=['latest',representation]).wait(timeout=60) | |
| print('BIGSOLDB_RESULT',json.dumps(result),flush=True) | |
| model.clear();del X;gc.collect();torch.cuda.empty_cache() | |
| assert len(records)==2 | |
| del model;gc.collect();torch.cuda.empty_cache() | |
| def publish(run): | |
| artifact=wandb.Artifact('cosmo-bigsoldb-evaluation-results',type='evaluation',metadata=dict(results=2,protocol=PROTOCOL)) | |
| files=[p for p in OUT.rglob('*') if p.is_file() and 'wandb' not in p.relative_to(OUT).parts and p.suffix in ['.csv','.json','.npz','.parquet','.png','.svg','.md']] | |
| for p in files:artifact.add_file(str(p),name='bigsoldb_evaluation/'+str(p.relative_to(OUT))) | |
| sources=[Path(__file__).with_name(n) for n in ['bigsoldb_evaluate.py','bigsoldb_report.py','openadmet_evaluate.py','eval_common.py']] | |
| for p in sources:artifact.add_file(str(p),name='source/'+p.name) | |
| combined=WORK/'expanded_evaluation/analysis/REPORT.md';artifact.add_file(str(combined),name='expanded_evaluation/analysis/REPORT.md') | |
| run.log_artifact(artifact,aliases=['latest','complete']).wait(timeout=60) | |
| ops=[CommitOperationAdd(path_in_repo='bigsoldb_evaluation/'+str(p.relative_to(OUT)),path_or_fileobj=str(p)) for p in files] | |
| ops += [CommitOperationAdd(path_in_repo='source/'+p.name,path_or_fileobj=str(p)) for p in sources] | |
| ops.append(CommitOperationAdd(path_in_repo='expanded_evaluation/analysis/REPORT.md',path_or_fileobj=str(combined))) | |
| commit=HfApi().create_commit(repo_id=REPOSITORY,operations=ops,commit_message='Add concatenated COSMO and Monroe Kumo L solubility benchmarks on the published BigSolDB test rows') | |
| write(OUT/'hub_verified.json',dict(completed=True,revision=commit.oid,repository=REPOSITORY));return commit.oid | |
| def main(): | |
| OUT.mkdir(exist_ok=True) | |
| with (OUT/'driver.lock').open('a') as lock: | |
| try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| except BlockingIOError:print('BIGSOLDB_ALREADY_RUNNING',flush=True);return | |
| for folder in ['data','features','predictions','analysis']:(OUT/folder).mkdir(exist_ok=True) | |
| train,test,smiles=prepare_data() | |
| # RDKit featurization forks CPU workers; initialize W&B after they exit. | |
| embeddings,solute,solvent,temperature=feature_arrays(train,test,smiles) | |
| saved=json.loads((OUT/'wandb.json').read_text()) if (OUT/'wandb.json').exists() else {} | |
| run=wandb.init(entity='queimo',project='rem3di-reproduction',id=saved.get('run_id') or wandb.util.generate_id(),resume='allow', | |
| name='bigsoldb-solute-solvent-temperature-concat-kumo-large',group='cosmo-bigsoldb',job_type='published-split-comparison', | |
| config=PROTOCOL,dir=str(OUT),settings=wandb.Settings(init_timeout=90)) | |
| write(OUT/'wandb.json',dict(run_id=run.id,url=run.url)) | |
| try: | |
| evaluate(run,embeddings,solute,solvent,temperature,train,test) | |
| from bigsoldb_report import build_report | |
| build_report(OUT,train,test,run) | |
| write(OUT/'complete.json',dict(completed=True,results=2,train_rows=75731,test_rows=9686,shared_geometry_rows=1334,wandb_url=run.url)) | |
| revision=publish(run);run.summary.update(dict(all_evaluations_complete=True,results=2,hub_revision=revision));run.finish() | |
| status('complete',results=2,hub_revision=revision) | |
| HfApi().create_commit(repo_id=REPOSITORY,operations=[CommitOperationAdd(path_in_repo='bigsoldb_evaluation/'+p.name,path_or_fileobj=str(p)) for p in [OUT/'status.json',OUT/'hub_verified.json']],commit_message='Verify completed BigSolDB concatenation benchmark') | |
| except Exception: | |
| status('failed',error=traceback.format_exc());run.finish(exit_code=1);raise | |
| if __name__=='__main__': | |
| try:main() | |
| except Exception: | |
| OUT.mkdir(exist_ok=True);status('failed',error=traceback.format_exc());raise | |