Upload feature_extraction.py with huggingface_hub
Browse files- feature_extraction.py +258 -0
feature_extraction.py
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Feature extraction for the hERG Phase-5 models — end-to-end from SMILES, RDKit only.
|
| 2 |
+
|
| 3 |
+
Reproduces exactly the feature blocks used by the Phase-5 XGBoost models (Approach A
|
| 4 |
+
and the XGBoost legs of Approaches B/C), and is backwards-compatible with the Phase-3/4
|
| 5 |
+
blocks (Morgan / MACCS / 217 RDKit descriptors are computed with the same settings as
|
| 6 |
+
work/step0_lock_splits_features.py, so Phase-5 models can be compared like-for-like).
|
| 7 |
+
|
| 8 |
+
Blocks (in canonical assembly order):
|
| 9 |
+
morgan ECFP4 Morgan bits, radius 2, 2048 bits (Phase 3/4)
|
| 10 |
+
maccs MACCS keys, 166 bits (167-bit vector with placeholder bit 0 dropped) (Phase 3/4)
|
| 11 |
+
desc 217 RDKit molecular descriptors (Descriptors.descList, the exact
|
| 12 |
+
set/order of the Phase-3/4 cache, work/desc_names.json) (Phase 3/4)
|
| 13 |
+
physchem_extra [QED] — the one standard physicochemical property absent from the
|
| 14 |
+
217-descriptor set (verified programmatically; TPSA, MolLogP,
|
| 15 |
+
FractionCSP3, NumAromaticRings etc. are already inside `desc`,
|
| 16 |
+
so they are deliberately NOT duplicated here) (Phase 5)
|
| 17 |
+
morgan_counts Morgan radius-2 atom-environment COUNTS folded to 2048
|
| 18 |
+
(assembly applies log1p) (Phase 5)
|
| 19 |
+
ecfp6 Morgan bits, radius 3, 2048 bits (Phase 5)
|
| 20 |
+
fcfp4 Morgan bits, radius 2, 2048 bits, feature-class atom invariants
|
| 21 |
+
(FCFP4) (Phase 5)
|
| 22 |
+
herg hERG-targeted features (13, continuous) — see featurize_herg() (Phase 5)
|
| 23 |
+
pharm 2D pharmacophore pair counts (60, counts) — see featurize_pharm() (Phase 5)
|
| 24 |
+
morgan1024 Morgan ECFP4 bits folded to 1024 (added 2026-11-13 at Hossain's
|
| 25 |
+
instruction, alongside the 2048-bit default) (Phase 5)
|
| 26 |
+
morgan1024_counts Morgan ECFP4 counts folded to 1024 (assembly applies log1p) (Phase 5)
|
| 27 |
+
cdk OPTIONAL, not computed here: CDK fingerprints come from
|
| 28 |
+
PaDEL-Descriptor (separate Java tool); see PHASE5_RESULTS.md.
|
| 29 |
+
|
| 30 |
+
Usage:
|
| 31 |
+
from feature_extraction import featurize_smiles, BLOCK_ORDER
|
| 32 |
+
X, blocks = featurize_smiles("CCOc1ccc(cc1)...") # blocks: dict name -> 1-D np.array
|
| 33 |
+
|
| 34 |
+
Preprocessing applied at model time (NOT in this module; fit on training folds only):
|
| 35 |
+
- desc / physchem_extra / herg: drop near-constant & >5%-NaN columns (fit on train),
|
| 36 |
+
median-impute (train medians), standardize (train mean/SD) — Phase 3/4 DescPrep logic.
|
| 37 |
+
- morgan_counts / pharm: log1p, then the same standardisation as above.
|
| 38 |
+
- bit fingerprints (morgan / maccs / ecfp6 / fcfp4 / cdk): used raw.
|
| 39 |
+
"""
|
| 40 |
+
import os
|
| 41 |
+
import numpy as np
|
| 42 |
+
from rdkit import Chem, RDConfig, RDLogger
|
| 43 |
+
from rdkit.Chem import AllChem, MACCSkeys, rdMolDescriptors, rdFingerprintGenerator
|
| 44 |
+
from rdkit.Chem import ChemicalFeatures, QED
|
| 45 |
+
from rdkit.Chem.Descriptors import MolLogP
|
| 46 |
+
|
| 47 |
+
RDLogger.DisableLog("rdApp.*")
|
| 48 |
+
|
| 49 |
+
from rdkit.Chem import Descriptors as _Desc
|
| 50 |
+
DESC_NAMES = [n for n, _ in _Desc.descList] # 217, fixed order (== Phase-3/4 desc_names.json)
|
| 51 |
+
_DESC_FNS = dict(_Desc.descList)
|
| 52 |
+
BLOCK_ORDER = ["morgan", "maccs", "desc", "physchem_extra", "morgan_counts",
|
| 53 |
+
"ecfp6", "fcfp4", "herg", "pharm", "morgan1024", "morgan1024_counts"]
|
| 54 |
+
BLOCK_SIZES = {"morgan": 2048, "maccs": 166, "desc": 217, "physchem_extra": 1,
|
| 55 |
+
"morgan_counts": 2048, "ecfp6": 2048, "fcfp4": 2048, "herg": 13, "pharm": 60,
|
| 56 |
+
"morgan1024": 1024, "morgan1024_counts": 1024}
|
| 57 |
+
|
| 58 |
+
# ---------------------------------------------------------------- hERG block ----
|
| 59 |
+
# Method (documented approximation, no fitted parameters):
|
| 60 |
+
# * Protonatable / ionizable centres are identified with RDKit's BaseFeatures
|
| 61 |
+
# feature factory (PosIonizable / NegIonizable families).
|
| 62 |
+
# * Each PosIonizable centre is assigned a class pKa by atom environment:
|
| 63 |
+
# aromatic pyridine-like N ......... 5.2
|
| 64 |
+
# imidazole-like aromatic N (5-ring, >=2 ring N) ... 7.0
|
| 65 |
+
# aniline-like N (attached to aromatic C) ......... 4.6
|
| 66 |
+
# other (aliphatic amine etc.) .................... 9.5
|
| 67 |
+
# Fraction protonated at pH 7.4 for a centre: f = 1 / (1 + 10^(7.4 - pKa)).
|
| 68 |
+
# * logD(7.4) estimate: MolLogP - log10(1 + 10^(pKa_base - 7.4)) using the strongest
|
| 69 |
+
# basic centre, and additionally - log10(1 + 10^(7.4 - 4.5)) if a carboxylic-acid
|
| 70 |
+
# group is present (assumed acid pKa 4.5). This is a crude class-pKa approximation
|
| 71 |
+
# of logD, NOT a measured or rigorously predicted logD — it is labelled as such
|
| 72 |
+
# everywhere it is used.
|
| 73 |
+
_CLASS_PKA = {"pyridine": 5.2, "imidazole": 7.0, "aniline": 4.6, "aliphatic": 9.5}
|
| 74 |
+
_ACID_PKA = 4.5
|
| 75 |
+
_PH = 7.4
|
| 76 |
+
_factory = None
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _get_factory():
|
| 80 |
+
global _factory
|
| 81 |
+
if _factory is None:
|
| 82 |
+
cands = [os.path.join(RDConfig.RDDataDir, "BaseFeatures.fdef"),
|
| 83 |
+
os.path.join(RDConfig.RDDataDir, "Code", "GraphMol", "ChemicalFeatures",
|
| 84 |
+
"BaseFeatures.fdef")]
|
| 85 |
+
fdef = next(p for p in cands if os.path.exists(p))
|
| 86 |
+
_factory = ChemicalFeatures.BuildFeatureFactory(fdef)
|
| 87 |
+
return _factory
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _classify_base(atom, mol):
|
| 91 |
+
if atom.GetIsAromatic():
|
| 92 |
+
ring = mol.GetRingInfo()
|
| 93 |
+
for r in ring.AtomRings():
|
| 94 |
+
if atom.GetIdx() in r and len(r) == 5 and sum(1 for i in r if mol.GetAtomWithIdx(i).GetAtomicNum() == 7) >= 2:
|
| 95 |
+
return "imidazole"
|
| 96 |
+
return "pyridine"
|
| 97 |
+
for nb in atom.GetNeighbors():
|
| 98 |
+
if nb.GetIsAromatic():
|
| 99 |
+
return "aniline"
|
| 100 |
+
return "aliphatic"
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _frac_prot(pka, ph=_PH):
|
| 104 |
+
return 1.0 / (1.0 + 10.0 ** (ph - pka))
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
HERG_NAMES = ["n_pos_ionizable", "n_neg_ionizable", "expected_protonated_pH74",
|
| 108 |
+
"strongest_base_pKa", "logD74_est", "formal_charge", "abs_formal_charge",
|
| 109 |
+
"n_aromatic_rings", "frac_aromatic_atoms", "n_amine_like_N",
|
| 110 |
+
"n_carboxylic_acid", "has_base", "has_acid"]
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def featurize_herg(mol):
|
| 114 |
+
feats = _get_factory().GetFeaturesForMol(mol)
|
| 115 |
+
pos, neg = [], []
|
| 116 |
+
for f in feats:
|
| 117 |
+
if f.GetFamily() == "PosIonizable":
|
| 118 |
+
pos.append(f)
|
| 119 |
+
elif f.GetFamily() == "NegIonizable":
|
| 120 |
+
neg.append(f)
|
| 121 |
+
pkas = []
|
| 122 |
+
exp_prot = 0.0
|
| 123 |
+
for f in pos:
|
| 124 |
+
aidx = f.GetAtomIds()[0]
|
| 125 |
+
cls = _classify_base(mol.GetAtomWithIdx(aidx), mol)
|
| 126 |
+
pka = _CLASS_PKA[cls]
|
| 127 |
+
pkas.append(pka)
|
| 128 |
+
exp_prot += _frac_prot(pka)
|
| 129 |
+
logp = MolLogP(mol)
|
| 130 |
+
logd = logp
|
| 131 |
+
if pkas:
|
| 132 |
+
logd -= np.log10(1.0 + 10.0 ** (max(pkas) - _PH))
|
| 133 |
+
n_acid = len(mol.GetSubstructMatches(Chem.MolFromSmarts("[CX3](=O)[OX2H1]")))
|
| 134 |
+
if n_acid:
|
| 135 |
+
logd -= np.log10(1.0 + 10.0 ** (_PH - _ACID_PKA))
|
| 136 |
+
n_ar = sum(1 for a in mol.GetAtoms() if a.GetIsAromatic())
|
| 137 |
+
amine_like = len(mol.GetSubstructMatches(
|
| 138 |
+
Chem.MolFromSmarts("[NX3;!$(NC=O);!$(NS(=O)(=O));!$(N=*);!$(N#*)]")))
|
| 139 |
+
fc = Chem.GetFormalCharge(mol)
|
| 140 |
+
return np.array([
|
| 141 |
+
len(pos), len(neg), exp_prot,
|
| 142 |
+
max(pkas) if pkas else 0.0,
|
| 143 |
+
logd, fc, abs(fc),
|
| 144 |
+
rdMolDescriptors.CalcNumAromaticRings(mol),
|
| 145 |
+
n_ar / max(mol.GetNumAtoms(), 1),
|
| 146 |
+
amine_like, n_acid,
|
| 147 |
+
1.0 if pkas else 0.0, 1.0 if n_acid else 0.0,
|
| 148 |
+
], dtype=float)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
# ---------------------------------------------------------------- pharm block ---
|
| 152 |
+
# 2D pharmacophore pair counts. Feature points from RDKit's BaseFeatures factory,
|
| 153 |
+
# families mapped to 5 classes: Donor, Acceptor, Hydrophobe (Hydrophobe +
|
| 154 |
+
# LumpedHydrophobe + AromaticRing), PosIonizable, NegIonizable. For every pair of
|
| 155 |
+
# points (same point excluded; pairs taken with repetition over classes), the
|
| 156 |
+
# minimum topological distance between the two points' atoms is binned as
|
| 157 |
+
# <=2, 3-4, 5-6, >=7, giving 15 class-pairs x 4 bins = 60 counts.
|
| 158 |
+
_PHARM_FAMS = ["Donor", "Acceptor", "Hydrophobe", "PosIonizable", "NegIonizable"]
|
| 159 |
+
_FAM_MAP = {"Donor": "Donor", "Acceptor": "Acceptor", "Hydrophobe": "Hydrophobe",
|
| 160 |
+
"LumpedHydrophobe": "Hydrophobe", "AromaticRing": "Hydrophobe",
|
| 161 |
+
"PosIonizable": "PosIonizable", "NegIonizable": "NegIonizable",
|
| 162 |
+
"RingDonor": "Donor", "RingAcceptor": "Acceptor"}
|
| 163 |
+
PHARM_NAMES = [f"{a}|{b}|d{db}" for i, a in enumerate(_PHARM_FAMS) for b in _PHARM_FAMS[i:]
|
| 164 |
+
for db in ["<=2", "3-4", "5-6", ">=7"]]
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def _dist_bin(d):
|
| 168 |
+
return 0 if d <= 2 else 1 if d <= 4 else 2 if d <= 6 else 3
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def featurize_pharm(mol):
|
| 172 |
+
feats = _get_factory().GetFeaturesForMol(mol)
|
| 173 |
+
pts = []
|
| 174 |
+
for f in feats:
|
| 175 |
+
fam = _FAM_MAP.get(f.GetFamily())
|
| 176 |
+
if fam is not None:
|
| 177 |
+
pts.append((fam, set(f.GetAtomIds())))
|
| 178 |
+
out = np.zeros(60, dtype=float)
|
| 179 |
+
if len(pts) < 2:
|
| 180 |
+
return out
|
| 181 |
+
dm = Chem.GetDistanceMatrix(mol)
|
| 182 |
+
pair_index = {}
|
| 183 |
+
k = 0
|
| 184 |
+
for i, a in enumerate(_PHARM_FAMS):
|
| 185 |
+
for b in _PHARM_FAMS[i:]:
|
| 186 |
+
pair_index[(a, b)] = k
|
| 187 |
+
k += 4
|
| 188 |
+
for i in range(len(pts)):
|
| 189 |
+
for j in range(i + 1, len(pts)):
|
| 190 |
+
fa, aa = pts[i]
|
| 191 |
+
fb, ab = pts[j]
|
| 192 |
+
d = min(dm[x, y] for x in aa for y in ab)
|
| 193 |
+
key = (fa, fb) if (fa, fb) in pair_index else (fb, fa)
|
| 194 |
+
out[pair_index[key] + _dist_bin(int(d))] += 1.0
|
| 195 |
+
return out
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
# ---------------------------------------------------------------- main entry ----
|
| 199 |
+
_morgan_gen = rdFingerprintGenerator.GetMorganGenerator(radius=2, fpSize=2048)
|
| 200 |
+
_morgan_gen_1024 = rdFingerprintGenerator.GetMorganGenerator(radius=2, fpSize=1024)
|
| 201 |
+
_morgan_gen_r3 = rdFingerprintGenerator.GetMorganGenerator(radius=3, fpSize=2048)
|
| 202 |
+
_fcfp_gen = rdFingerprintGenerator.GetMorganGenerator(
|
| 203 |
+
radius=2, fpSize=2048,
|
| 204 |
+
atomInvariantsGenerator=rdFingerprintGenerator.GetMorganFeatureAtomInvGen())
|
| 205 |
+
from rdkit.ML.Descriptors.MoleculeDescriptors import MolecularDescriptorCalculator
|
| 206 |
+
_CALC = MolecularDescriptorCalculator(DESC_NAMES)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _desc_vector(mol):
|
| 210 |
+
# C++-loop calculator over the descList functions; identical values/order to
|
| 211 |
+
# calling each descList function in Python, several-fold faster.
|
| 212 |
+
try:
|
| 213 |
+
return np.array(_CALC.CalcDescriptors(mol), dtype=float)
|
| 214 |
+
except Exception:
|
| 215 |
+
out = np.empty(len(DESC_NAMES), dtype=float)
|
| 216 |
+
for i, n in enumerate(DESC_NAMES):
|
| 217 |
+
try:
|
| 218 |
+
out[i] = _DESC_FNS[n](mol)
|
| 219 |
+
except Exception:
|
| 220 |
+
out[i] = np.nan
|
| 221 |
+
return out
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def featurize_mol(mol):
|
| 225 |
+
"""mol: RDKit Mol -> dict block name -> 1-D float32 array (NaN-free except desc)."""
|
| 226 |
+
out = {}
|
| 227 |
+
out["morgan"] = np.asarray(_morgan_gen.GetFingerprint(mol), dtype=np.float32)
|
| 228 |
+
out["maccs"] = np.asarray(MACCSkeys.GenMACCSKeys(mol), dtype=np.float32)[1:] # drop bit 0
|
| 229 |
+
out["desc"] = _desc_vector(mol)
|
| 230 |
+
out["physchem_extra"] = np.array([QED.qed(mol)], dtype=float)
|
| 231 |
+
counts = _morgan_gen.GetCountFingerprint(mol)
|
| 232 |
+
arr = np.zeros(2048, dtype=np.float32)
|
| 233 |
+
for idx, c in counts.GetNonzeroElements().items():
|
| 234 |
+
arr[idx % 2048] += c
|
| 235 |
+
out["morgan_counts"] = arr
|
| 236 |
+
out["ecfp6"] = np.asarray(_morgan_gen_r3.GetFingerprint(mol), dtype=np.float32)
|
| 237 |
+
out["fcfp4"] = np.asarray(_fcfp_gen.GetFingerprint(mol), dtype=np.float32)
|
| 238 |
+
out["herg"] = featurize_herg(mol)
|
| 239 |
+
out["pharm"] = featurize_pharm(mol)
|
| 240 |
+
# Morgan ECFP4 at 1024 bits (bits + counts) — added to the Phase-5 ablation by
|
| 241 |
+
# Hossain's instruction of 2026-11-13 (tests whether the smaller, denser folding
|
| 242 |
+
# beats the 2048-bit default).
|
| 243 |
+
out["morgan1024"] = np.asarray(_morgan_gen_1024.GetFingerprint(mol), dtype=np.float32)
|
| 244 |
+
counts = _morgan_gen_1024.GetCountFingerprint(mol)
|
| 245 |
+
arr = np.zeros(1024, dtype=np.float32)
|
| 246 |
+
for idx, c in counts.GetNonzeroElements().items():
|
| 247 |
+
arr[idx % 1024] += c
|
| 248 |
+
out["morgan1024_counts"] = arr
|
| 249 |
+
return out
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def featurize_smiles(smiles):
|
| 253 |
+
"""SMILES -> (assembled float32 vector in BLOCK_ORDER, blocks dict). None if unparseable."""
|
| 254 |
+
mol = Chem.MolFromSmiles(smiles)
|
| 255 |
+
if mol is None:
|
| 256 |
+
return None, None
|
| 257 |
+
blocks = featurize_mol(mol)
|
| 258 |
+
return np.concatenate([blocks[b] for b in BLOCK_ORDER]).astype(np.float32), blocks
|