Convert large Excel data files to Parquet for faster startup

- Add convert_to_parquet.py one-time conversion script
- Convert 4 large xlsx files (~42 MB total) to parquet (~11 MB total):
qrf/db-D-interp-allT-semiclean: 12MB -> 2.4MB
qrf/mordred-descriptors: 9.8MB -> 6.7MB
quantity_module/data/db-D-interp-allT-clean: 10MB -> 2.0MB
quantity_module/data/data-descriptors-mordred-numconfs51: 9.8MB -> 23KB
quantity_module/data/solvent-viscosity, vrentas-duda-params (both sheets)
- Update qrf/functions.py and quantity_module/functions.py to use parquet
- In qrf/functions.py: replace mordred Calculator init for column names with
direct column lookup from df_desc; share a single _calc_all/_desc_by_name
at module level instead of re-creating Calculator per request in QRF_Apply
- Add pyarrow<16 to requirements.txt

Co-Authored-By: Claude Code <noreply@anthropic.com>

convert_to_parquet.py

@@ -0,0 +1,28 @@
+"""
+One-time script to convert large Excel data files to Parquet format for faster loading.
+Run from the repo root: python convert_to_parquet.py
+"""
+import pandas as pd
+
+files = [
+    ('qrf/db-D-interp-allT-semiclean.xlsx', 'qrf/db-D-interp-allT-semiclean.parquet', {}),
+    ('qrf/mordred-descriptors.xlsx',         'qrf/mordred-descriptors.parquet',         {}),
+    ('quantity_module/data/db-D-interp-allT-clean.xlsx',              'quantity_module/data/db-D-interp-allT-clean.parquet',              {}),
+    ('quantity_module/data/data-descriptors-mordred-numconfs51.xlsx', 'quantity_module/data/data-descriptors-mordred-numconfs51.parquet', {'usecols': ['Solute_InChIKey', 'Vabc', 'VMcGowan']}),
+    ('quantity_module/data/solvent-viscosity.xlsx',                   'quantity_module/data/solvent-viscosity.parquet',                   {}),
+    ('quantity_module/data/vrentas-duda-params.xlsx',                 'quantity_module/data/vrentas-duda-params-solutes.parquet',         {'sheet_name': 'Solutes'}),
+    ('quantity_module/data/vrentas-duda-params.xlsx',                 'quantity_module/data/vrentas-duda-params-polymers.parquet',        {'sheet_name': 'Polymers'}),
+]
+
+for src, dst, kwargs in files:
+    print(f'Converting {src} -> {dst} ...')
+    df = pd.read_excel(src, **kwargs)
+    # Coerce mixed-type object columns to numeric where possible so parquet can handle them
+    for col in df.select_dtypes(include='object').columns:
+        converted = pd.to_numeric(df[col], errors='coerce')
+        if converted.notna().sum() > 0:
+            df[col] = converted
+    df.to_parquet(dst, index=False)
+    print(f'  done ({len(df)} rows)')
+
+print('All conversions complete.')

qrf/db-D-interp-allT-semiclean.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53f0371780a12bb9c70397b32e4633c8913c3439312ef1686d3f57d2854d3a08
+size 2533547

qrf/functions.py

@@ -10,12 +10,17 @@ import mordred.descriptors
 import rdkit
 from rdkit import Chem
 
+# Build a single shared Calculator and descriptor lookup once at startup.
+# This is cheaper than re-instantiating mordred.Calculator on every request.
+_calc_all = mordred.Calculator(mordred.descriptors)
+_desc_by_name = {str(d): d for d in _calc_all.descriptors}
+
 QRF_T_list = np.array([25,30,35,37,40,45,50,55,60,65,70,75])
 QRF_T_cut = 2.5
-df_QRF = pd.read_excel('qrf/db-D-interp-allT-semiclean.xlsx')
-df_desc =  pd.read_excel('qrf/mordred-descriptors.xlsx')
-calc = mordred.Calculator(mordred.descriptors)
-colnames_mordred = [str(d) for d in calc.descriptors]
+df_QRF = pd.read_parquet('qrf/db-D-interp-allT-semiclean.parquet')
+df_desc = pd.read_parquet('qrf/mordred-descriptors.parquet')
+# Use column names from df_desc directly instead of initializing a full mordred Calculator
+colnames_mordred = [c for c in df_desc.columns if c != 'Solute_InChIKey']
 df_QRF = pd.merge(df_QRF, df_desc[['Solute_InChIKey',*colnames_mordred]], how='left', on='Solute_InChIKey', suffixes=('', '_dupe'))
 
 @functools.lru_cache(maxsize=None)
@@ -66,9 +71,7 @@ def QRF_Apply(density, polytg, smiles, quantiles=[0.03,0.5,0.97], T=37):
         solute_desc_list.remove('Polymer_Tg')
     if 'Polymer_Density' in solute_desc_list:
         solute_desc_list.remove('Polymer_Density')
-    calc_all = mordred.Calculator(mordred.descriptors)
-    desc_strs = [str(d) for d in calc_all.descriptors]
-    desc_funcs = {d1:df for d1 in solute_desc_list for d2,df in zip(desc_strs,calc_all.descriptors) if d1 == d2}
+    desc_funcs = {d: _desc_by_name[d] for d in solute_desc_list if d in _desc_by_name}
     calc = mordred.Calculator(list(desc_funcs.values()))
     m = Chem.MolFromSmiles(smiles)
     m = Chem.AddHs(m)

qrf/mordred-descriptors.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29fb9c32fb13f69d87ecc62f092d0f394946f6fe1a05ff9917c0f9d98e7e0795
+size 7064094

quantity_module/data/data-descriptors-mordred-numconfs51.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db1d1cc6a5d310e2e2846101fb1791387ecaa6d7647dbb9cae5cac9c42671a5a
+size 23818

quantity_module/data/db-D-interp-allT-clean.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3df5c08bcdcfafd3f37d4404f401f855cb6217dc2c7f1624a9bf00519663c733
+size 2082818

quantity_module/data/solvent-viscosity.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01a256c5c70979308dc15b7c433f860f72813a5859bc7720ddebad8831d6ac0d
+size 9268

quantity_module/data/vrentas-duda-params-polymers.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02cbc3415294076b364f2a1eac63b68ca2785ebf07c3b9ac203ed6c3fc8a67a2
+size 10887

quantity_module/data/vrentas-duda-params-solutes.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3810ed12e5c113a7423d0769d870bd710a830808739930a7c0bb1cac30ff8ddd
+size 11141

quantity_module/functions.py

@@ -42,8 +42,8 @@ else:
     with open('quantity_module/data/param_distribution_allT.json','r') as fp:
         param_dists = json.load(fp)
 # other stuff
-df_visc = pd.read_excel('quantity_module/data/solvent-viscosity.xlsx')
-df_desc = pd.read_excel(f'quantity_module/data/data-descriptors-mordred-numconfs51.xlsx', usecols=['Solute_InChIKey', 'Vabc','VMcGowan'])
+df_visc = pd.read_parquet('quantity_module/data/solvent-viscosity.parquet')
+df_desc = pd.read_parquet('quantity_module/data/data-descriptors-mordred-numconfs51.parquet', columns=['Solute_InChIKey', 'Vabc','VMcGowan'])
 if not use_new:
     ## clean data
     df_final_37 = pd.read_excel('quantity_module/data/db-D-interp-37-clean.xlsx')
@@ -60,7 +60,7 @@ if not use_new:
     df_final_50 = pd.merge(df_final_50, df_desc[['Solute_InChIKey', 'Vabc', 'VMcGowan']], how='left', on='Solute_InChIKey', suffixes=('', '_dupe'))
 else:
     ## clean data
-    df_final = pd.read_excel('quantity_module/data/db-D-interp-allT-clean.xlsx')
+    df_final = pd.read_parquet('quantity_module/data/db-D-interp-allT-clean.parquet')
     # convert all T to K
     df_final['T'] = df_final['T'] + 273.15
     df_final['Polymer_Tg'] = df_final['Polymer_Tg'] + 273.15
@@ -179,9 +179,9 @@ def get_V(smiles):
     return Vabc
 
 #### Vrentas-Duda setup
-df_vd_solv = pd.read_excel('quantity_module/data/vrentas-duda-params.xlsx', sheet_name='Solutes')
+df_vd_solv = pd.read_parquet('quantity_module/data/vrentas-duda-params-solutes.parquet')
 df_vd_solv.drop_duplicates(keep='first', inplace=True, ignore_index=True) # drop exact duplicates
-df_vd_poly = pd.read_excel('quantity_module/data/vrentas-duda-params.xlsx', sheet_name='Polymers')
+df_vd_poly = pd.read_parquet('quantity_module/data/vrentas-duda-params-polymers.parquet')
 df_vd_poly.drop_duplicates(keep='first', inplace=True, ignore_index=True) # drop exact duplicates
 df_props = pd.read_excel('data/db-polymer-properties-and-categories.xlsx')
 df_vd_poly = pd.merge(df_vd_poly, df_props[['Polymer_Name','Polymer_Tg','Polymer_Tm', 'CHRIS Class', 'New Class']], how='left', on='Polymer_Name')

requirements.txt

@@ -22,4 +22,5 @@ mordred==1.2.0
 beautifulsoup4==4.12.3
 setuptools==72.1.0
 openpyxl==3.1.5
+pyarrow<16