Queimo's picture
Pin exact evaluated starting checkpoint, objective equivalence tests, source and joint training protocol
31dee3b verified
Raw History Blame Contribute Delete
17.6 kB
"""Concatenate frozen solute/solvent embeddings and temperature for Kumo L."""
from pathlib import Path
import sys
sys.path.insert(0,str(Path(__file__).resolve().parent))
import eval_common as common
from eval_common import WORK,write,sha
import fcntl,functools,gc,hashlib,json,os,re,shutil,subprocess,time,traceback
import numpy as np
import pandas as pd
import torch
import wandb
from rdkit import Chem
from huggingface_hub import HfApi,CommitOperationAdd
OUT=WORK/'bigsoldb_evaluation'
UPSTREAM=WORK/'external/geomaw-solv'
UPSTREAM_REVISION='8d0a6695afee1b8723f06122008b50ab3a25e144'
REPOSITORY='Queimo/rem3di-pcqm-epoch2-cosmo'
PAPER_URL='https://arxiv.org/html/2511.09774v1#S3.T1'
RAW_MD5='1e6d8a510387746e9faab5f39dc46317'
PUBLISHED={
'XGBoost-DFT':[.621,.650,.806,.499,.722,.722],
'XGBoost-CFP':[.749,.889,.943,.315,.592,.561],
'XGBoost-MMB':[.710,.831,.912,.360,.616,.591],
'SolvBERT':[.871,1.231,1.110,.052,.247,.222],
'MPNN':[.668,.746,.864,.425,.724,.693],
'EquiformerV2':[.691,.858,.926,.345,.626,.606],
'Solvaformer':[.643,.700,.837,.460,.694,.677],
'MPNN+Charges':[.629,.667,.817,.486,.721,.710],
}
METRICS=['mae','mse','rmse','r2','pearson','spearman']
PROTOCOL=dict(dataset='BigSolDB 2.0',upstream_revision=UPSTREAM_REVISION,paper_table=PAPER_URL,
target='LogS(mol/L): supplied log10 molar solubility; no additional label transform',
input='concat(frozen solute embedding, frozen solvent embedding, Temperature_K)',
representations=['cosmo_epoch10','monroe'],raw_widths={'cosmo_epoch10':1281,'monroe':1441},
geometry='One cached Monroe-native conformer per unique molecule; shared by both encoders',
split='Exact held-out row IDs and measurements from committed MPNN+Charges predictions; remaining curated rows form the non-test training pool',
training_pool='All 75731 non-test measurements; fixed Kumo recipe without validation-based tuning. Neural baselines may reserve a subset for validation.',
kumo=dict(size='large',seed=42,num_estimators=4,feature_cap='Full concatenated width; no random feature truncation',
precision='float16 autocast',estimator_batch_size=1,query_chunk_size=128),
context_fallback='Try the entire training context first. Only upon CUDA OOM, use a matched, deterministic disjoint-context ensemble covering every training row; choose using training computation alone, before test scoring.',
metrics=METRICS)
def status(phase,**kw):
write(OUT/'status.json',dict(phase=phase,pid=os.getpid(),updated_unix=time.time(),**kw))
print('BIGSOLDB_STATUS',phase,json.dumps(kw),flush=True)
def metrics(y,p):
from scipy.stats import pearsonr,spearmanr
mse=float(np.mean((y-p)**2))
return dict(**common.metric(y,p),mse=mse,pearson=float(pearsonr(y,p).statistic),spearman=float(spearmanr(y,p).statistic))
def prepare_data():
status('recovering_published_split')
env={**os.environ,'GIT_LFS_SKIP_SMUDGE':'1'}
UPSTREAM.parent.mkdir(exist_ok=True)
if not (UPSTREAM/'.git').exists():
subprocess.run(['git','clone','--depth','1','https://github.com/Sanofi-Public/geomaw-solv.git',str(UPSTREAM)],check=True,env=env)
assert subprocess.check_output(['git','-C',str(UPSTREAM),'rev-parse','HEAD'],text=True).strip()==UPSTREAM_REVISION
path=OUT/'data/BigSolDBv2.0.csv'
if not path.exists():
subprocess.run(['curl','-fL','--retry','4','-o',str(path),'https://zenodo.org/records/15094979/files/BigSolDBv2.0.csv?download=1'],check=True)
assert hashlib.md5(path.read_bytes()).hexdigest()==RAW_MD5
nb_path=UPSTREAM/'nbs/1_filter_and_split_data.ipynb'
source=''.join(json.loads(nb_path.read_text())['cells'][4]['source'])
source=source[source.index('from rdkit import RDLogger'):source.index('# Find duplicates')]
# Memoization changes parsing cost only; preserve the author's exact filtering code.
source=source.replace("df['SMILES_Solute'] = df['SMILES_Solute'].apply(canonicalize_smiles)",
"canonicalize_smiles = functools.lru_cache(None)(canonicalize_smiles)\ndf['SMILES_Solute'] = df['SMILES_Solute'].apply(canonicalize_smiles)")
raw=pd.read_csv(path);raw['_raw_row']=np.arange(len(raw));assert len(raw)==103944
namespace=dict(df=raw,np=np,pd=pd,re=re,Chem=Chem,functools=functools)
exec(compile(source,str(nb_path)+':cell4','exec'),namespace)
keys=['SMILES_Solute','SMILES_Solvent','Temperature_K']
filtered=namespace['df'];assert len(filtered)==92008
clean=filtered.drop_duplicates(keys,keep=False).reset_index(drop=True)
assert len(clean)==85417 and clean.SMILES_Solute.nunique()==1268 and clean.SMILES_Solvent.nunique()==70
clean['_clean_row']=np.arange(len(clean))
prediction_path=UPSTREAM/'mpnn_charges/evaluation_results_test_run2/predictions.csv'
author=pd.read_csv(prediction_path);ids=author['Unnamed: 0'].to_numpy(dtype=np.int64)
assert len(ids)==9686 and len(np.unique(ids))==len(ids) and ids.min()>=0 and ids.max()<len(clean)
test=clean.iloc[ids].copy().reset_index(drop=True)
canon=functools.lru_cache(None)(lambda s:Chem.MolToSmiles(Chem.MolFromSmiles(str(s)),isomericSmiles=True))
for col in ['SMILES_Solute','SMILES_Solvent']:
assert test[col].map(canon).tolist()==author[col].map(canon).tolist(),col
assert np.array_equal(test.Temperature_K.to_numpy(),author.Temperature_K.to_numpy())
assert np.allclose(test['LogS(mol/L)'],author['LogS(mol/L)'],atol=1e-12,rtol=0)
mask=np.ones(len(clean),dtype=bool);mask[ids]=False
train=clean.loc[mask].copy().reset_index(drop=True)
assert len(train)==75731 and not set(train.SMILES_Solute)&set(test.SMILES_Solute)
assert train.SMILES_Solute.nunique()==1142 and test.SMILES_Solute.nunique()==126
assert not clean.duplicated(keys).any() and np.isfinite(clean['LogS(mol/L)']).all()
reference=metrics(author.True_LogS.to_numpy(),author.Predicted_LogS.to_numpy())
assert [round(reference[k],3) for k in METRICS]==PUBLISHED['MPNN+Charges'],reference
for name,df in [('train',train),('test',test)]:df.to_csv(OUT/'data'/f'{name}.csv',index=False)
shutil.copyfile(prediction_path,OUT/'data/author_predictions.csv')
write(OUT/'data/split_validation.json',dict(passed=True,curated_rows=85417,train_rows=75731,test_rows=9686,
train_solutes=1142,test_solutes=126,all_author_row_ids_match=True,all_author_labels_match=True,
author_metric_reproduction=reference,paper_count_discrepancy='Paper/notebook prose says 9250 test rows; the committed predictions yielding its published metrics contain 9686.',
upstream_predictions_sha256=sha(prediction_path),raw_md5=RAW_MD5))
PROTOCOL['checkpoint_sha256']={name:sha(common.checkpoint(10)/name) for name in
['encoder.pth','atomic_preprocessor.pth','geometric_preprocessor.pth']}
manifest=json.loads((common.checkpoint(10)/'manifest.json').read_text())
for name,digest in PROTOCOL['checkpoint_sha256'].items():assert digest==manifest['files'][name]['sha256']
PROTOCOL['versions']={name:__import__('importlib.metadata',fromlist=['version']).version(name) for name in
['torch','numpy','pandas','rdkit','scikit-learn','structured-data-models']}
write(OUT/'protocol.json',PROTOCOL)
smiles=[str(s) for s in pd.unique(pd.concat([train.SMILES_Solute,train.SMILES_Solvent,test.SMILES_Solute,test.SMILES_Solvent]))]
assert len(smiles)==1334
return train,test,smiles
def feature_arrays(train,test,smiles):
import openadmet_evaluate as runtime
runtime.OUT=OUT;runtime.status=status
runtime.PROTOCOL['checkpoint_sha256']=PROTOCOL['checkpoint_sha256']
runtime.restore_monroe()
built=runtime.prepare_graphs(smiles)
embeddings={'monroe':runtime.monroe_embeddings(built,smiles),'cosmo_epoch10':runtime.cosmo_embeddings(built,smiles)}
del built;gc.collect()
lookup={s:i for i,s in enumerate(smiles)};rows=pd.concat([train,test],ignore_index=True)
solute=np.asarray([lookup[s] for s in rows.SMILES_Solute]);solvent=np.asarray([lookup[s] for s in rows.SMILES_Solvent])
temperature=rows.Temperature_K.to_numpy(dtype=np.float32)
np.savez_compressed(OUT/'features/pair_indices.npz',solute=solute,solvent=solvent,temperature_K=temperature,
train_clean_rows=train['_clean_row'].to_numpy(),test_clean_rows=test['_clean_row'].to_numpy())
return embeddings,solute,solvent,temperature
def table(x):
import sdm
return sdm.TableTensor.from_tensor(torch.as_tensor(np.ascontiguousarray(x),dtype=torch.float32,device='cuda'))
def context_indices(n,blocks):
if blocks==1:return None
pieces=np.array_split(np.random.default_rng(42).permutation(n),blocks)
width=max(map(len,pieces))
# Pad only the shorter blocks by repeating their first row; retain every row.
return np.stack([np.pad(v,(0,width-len(v)),mode='wrap') for v in pieces])
def fit_context(model,X,y,ntrain,blocks):
import sdm.processing as sp
from sdm.models.callback import Callback
recipe=model.default_recipe()
selectors=[m for m in recipe.features.modules() if isinstance(m,sp.SelectColumns)]
assert len(selectors)==1;selectors[0].max_columns=X.shape[1]
shapes=[]
class Audit(Callback):
def on_context_preprocessing_end(self,model,x,y,related_tables):
shapes.append(list(x.shape));status('fitting_kumo_context',blocks=blocks,processed_context_shapes=shapes)
return x,y,related_tables
indices=context_indices(ntrain,blocks)
if indices is None:x,yc=table(X[:ntrain]),table(y[:ntrain,None]);estimators=4
else:
assert set(np.unique(indices))==set(range(ntrain))
x,yc=table(X[indices]),table(y[indices,None]);estimators=None
with torch.inference_mode(),torch.autocast('cuda',dtype=torch.float16):
model.fit(x=x,y=yc,num_estimators=estimators,estimator_batch_size=1,recipe=recipe,
generator=torch.Generator(device='cuda').manual_seed(42),callbacks=[Audit()])
del x,yc;gc.collect();torch.cuda.empty_cache()
assert shapes and max(s[-1] for s in shapes)<=X.shape[1]
return shapes
def predict(model,X):
values=[]
with torch.inference_mode(),torch.autocast('cuda',dtype=torch.float16):
for start in range(0,len(X),128):
q=model.predict(table(X[start:start+128])).numerical.float().cpu().numpy()
values.append(q.mean(1,dtype=np.float64))
if start%1024==0:status('predicting_test',completed=min(start+128,len(X)),total=len(X))
p=np.concatenate(values);assert np.isfinite(p).all();return p
def evaluate(run,embeddings,solute,solvent,temperature,train,test):
import sdm
y=pd.concat([train,test])['LogS(mol/L)'].to_numpy(dtype=np.float64);n=len(train)
records=pd.read_csv(OUT/'results.csv').to_dict('records') if (OUT/'results.csv').exists() else []
plan_path=OUT/'context_plan.json'
plan=json.loads(plan_path.read_text()) if plan_path.exists() else None
model=sdm.models.KumoTabular(task='regression',size='large',device='cuda')
# Use the wider representation first to set any memory-only fallback equally for both.
for representation in ['monroe','cosmo_epoch10']:
if any(r['representation']==representation for r in records):continue
E=embeddings[representation]
X=np.concatenate([E[solute],E[solvent],temperature[:,None]],axis=1).astype(np.float32)
assert X.shape[1]==PROTOCOL['raw_widths'][representation] and np.isfinite(X).all()
assert np.array_equal(X[:,-1],temperature) and np.ptp(X[:n,-1])>0
status('fitting_representation',representation=representation,raw_columns=X.shape[1],train_rows=n,test_rows=len(test))
started=time.perf_counter()
candidates=[plan['blocks']] if plan else [1,4,8,16,32]
for blocks in candidates:
try:
shapes=fit_context(model,X,y,n,blocks)
if plan is None:
plan=dict(blocks=blocks,num_estimators=4 if blocks==1 else blocks,
all_training_rows_covered=True,selection='Training-only CUDA memory feasibility; no test metrics used')
write(plan_path,plan)
ids=context_indices(n,blocks)
np.savez_compressed(OUT/'features/train_context_indices.npz',indices=np.arange(n)[None] if ids is None else ids)
break
except torch.cuda.OutOfMemoryError:
print('CONTEXT_CUDA_OOM',blocks,flush=True);model.clear();gc.collect();torch.cuda.empty_cache()
if plan is not None or blocks==candidates[-1]:raise
p=predict(model,X[n:])
result=dict(representation=representation,method='solute_solvent_temperature_concat_kumo_large',
n_train=n,n_test=len(test),raw_columns=X.shape[1],context_blocks=plan['blocks'],
num_estimators=plan['num_estimators'],seed=42,**metrics(y[n:],p),seconds=time.perf_counter()-started)
pred_path=OUT/'predictions'/f'{representation}.npz'
np.savez_compressed(pred_path,y_true=y[n:],y_pred=p,test_clean_rows=test['_clean_row'].to_numpy(),test_raw_rows=test['_raw_row'].to_numpy())
write(pred_path.with_suffix('.json'),{**result,'processed_context_shapes':shapes,'context_plan':plan})
records.append(result);tmp=OUT/'results.partial.csv';pd.DataFrame(records).to_csv(tmp,index=False);tmp.replace(OUT/'results.csv')
write(OUT/'progress.json',dict(completed=len(records),total=2,latest=result));run.log(result)
artifact=wandb.Artifact('cosmo-bigsoldb-prediction-backups',type='evaluation',metadata=result)
for pth in [pred_path,pred_path.with_suffix('.json'),OUT/'results.csv',plan_path]:artifact.add_file(str(pth),name=str(pth.relative_to(OUT)))
run.log_artifact(artifact,aliases=['latest',representation]).wait(timeout=60)
print('BIGSOLDB_RESULT',json.dumps(result),flush=True)
model.clear();del X;gc.collect();torch.cuda.empty_cache()
assert len(records)==2
del model;gc.collect();torch.cuda.empty_cache()
def publish(run):
artifact=wandb.Artifact('cosmo-bigsoldb-evaluation-results',type='evaluation',metadata=dict(results=2,protocol=PROTOCOL))
files=[p for p in OUT.rglob('*') if p.is_file() and 'wandb' not in p.relative_to(OUT).parts and p.suffix in ['.csv','.json','.npz','.parquet','.png','.svg','.md']]
for p in files:artifact.add_file(str(p),name='bigsoldb_evaluation/'+str(p.relative_to(OUT)))
sources=[Path(__file__).with_name(n) for n in ['bigsoldb_evaluate.py','bigsoldb_report.py','openadmet_evaluate.py','eval_common.py']]
for p in sources:artifact.add_file(str(p),name='source/'+p.name)
combined=WORK/'expanded_evaluation/analysis/REPORT.md';artifact.add_file(str(combined),name='expanded_evaluation/analysis/REPORT.md')
run.log_artifact(artifact,aliases=['latest','complete']).wait(timeout=60)
ops=[CommitOperationAdd(path_in_repo='bigsoldb_evaluation/'+str(p.relative_to(OUT)),path_or_fileobj=str(p)) for p in files]
ops += [CommitOperationAdd(path_in_repo='source/'+p.name,path_or_fileobj=str(p)) for p in sources]
ops.append(CommitOperationAdd(path_in_repo='expanded_evaluation/analysis/REPORT.md',path_or_fileobj=str(combined)))
commit=HfApi().create_commit(repo_id=REPOSITORY,operations=ops,commit_message='Add concatenated COSMO and Monroe Kumo L solubility benchmarks on the published BigSolDB test rows')
write(OUT/'hub_verified.json',dict(completed=True,revision=commit.oid,repository=REPOSITORY));return commit.oid
def main():
OUT.mkdir(exist_ok=True)
with (OUT/'driver.lock').open('a') as lock:
try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError:print('BIGSOLDB_ALREADY_RUNNING',flush=True);return
for folder in ['data','features','predictions','analysis']:(OUT/folder).mkdir(exist_ok=True)
train,test,smiles=prepare_data()
# RDKit featurization forks CPU workers; initialize W&B after they exit.
embeddings,solute,solvent,temperature=feature_arrays(train,test,smiles)
saved=json.loads((OUT/'wandb.json').read_text()) if (OUT/'wandb.json').exists() else {}
run=wandb.init(entity='queimo',project='rem3di-reproduction',id=saved.get('run_id') or wandb.util.generate_id(),resume='allow',
name='bigsoldb-solute-solvent-temperature-concat-kumo-large',group='cosmo-bigsoldb',job_type='published-split-comparison',
config=PROTOCOL,dir=str(OUT),settings=wandb.Settings(init_timeout=90))
write(OUT/'wandb.json',dict(run_id=run.id,url=run.url))
try:
evaluate(run,embeddings,solute,solvent,temperature,train,test)
from bigsoldb_report import build_report
build_report(OUT,train,test,run)
write(OUT/'complete.json',dict(completed=True,results=2,train_rows=75731,test_rows=9686,shared_geometry_rows=1334,wandb_url=run.url))
revision=publish(run);run.summary.update(dict(all_evaluations_complete=True,results=2,hub_revision=revision));run.finish()
status('complete',results=2,hub_revision=revision)
HfApi().create_commit(repo_id=REPOSITORY,operations=[CommitOperationAdd(path_in_repo='bigsoldb_evaluation/'+p.name,path_or_fileobj=str(p)) for p in [OUT/'status.json',OUT/'hub_verified.json']],commit_message='Verify completed BigSolDB concatenation benchmark')
except Exception:
status('failed',error=traceback.format_exc());run.finish(exit_code=1);raise
if __name__=='__main__':
try:main()
except Exception:
OUT.mkdir(exist_ok=True);status('failed',error=traceback.format_exc());raise