""" Formulation Tokenizer Converts formulation recipes into token sequences for the transformer. Tokenization strategy: - Protein descriptor tokens: MW, pI, Tm, hydrophobicity (quantized) - Buffer tokens: buffer species (categorical) - pH tokens: pH value (quantized to bins) - Ionic strength tokens: IS value (quantized) - Stabilizer tokens: identity + concentration pairs - Outcome tokens: stability score (quantized) Special tokens: - : padding (id=0) - : classification/start (id=1) - : separator between input/output (id=2) - : end-of-sequence (id=3) - : unknown (id=4) Vocabulary size: ~1024 tokens total """ import numpy as np import json from typing import Dict, List, Tuple, Optional from dataclasses import dataclass from pathlib import Path import logging logger = logging.getLogger(__name__) @dataclass class TokenizerConfig: """Tokenizer configuration.""" # Quantization bins mw_bins: int = 20 # discretize MW into 20 bins pi_bins: int = 16 # discretize pI into 16 bins ph_bins: int = 20 # discretize pH into 20 bins tm_bins: int = 20 # discretize Tm into 20 bins is_bins: int = 20 # discretize ionic strength into 20 bins osmol_bins: int = 16 temp_bins: int = 12 stability_bins: int = 20 # quantize stability score into bins # Ranges for quantization mw_range: Tuple = (1, 500) pi_range: Tuple = (2, 12) ph_range: Tuple = (2, 12) tm_range: Tuple = (20, 100) is_range: Tuple = (1, 1000) osmol_range: Tuple = (50, 500) temp_range: Tuple = (-20, 50) stability_range: Tuple = (0, 1) # Vocabulary buffer_types: List[str] = None stabilizer_types: List[str] = None def __post_init__(self): if self.buffer_types is None: self.buffer_types = [ "histidine", "acetate", "phosphate", "tris", "citrate", "succinate" ] if self.stabilizer_types is None: self.stabilizer_types = [ "sucrose", "sorbitol", "trehalose", "polysorbate_20", "polysorbate_80", "glycerol", "bsa", "gelatin" ] class FormulationTokenizer: """Tokenizes formulation data for transformer input.""" # Special tokens PAD_TOKEN = "" CLS_TOKEN = "" SEP_TOKEN = "" EOS_TOKEN = "" UNK_TOKEN = "" PAD_ID = 0 CLS_ID = 1 SEP_ID = 2 EOS_ID = 3 UNK_ID = 4 def __init__(self, config: Optional[TokenizerConfig] = None): """Initialize tokenizer.""" self.config = config or TokenizerConfig() self.vocab = self._build_vocab() self.vocab_size = len(self.vocab) logger.info(f"FormulationTokenizer initialized: vocab_size={self.vocab_size}") def _build_vocab(self) -> Dict[str, int]: """Build vocabulary for all possible tokens.""" vocab = {} token_id = 5 # Start after special tokens # Special tokens vocab[self.PAD_TOKEN] = self.PAD_ID vocab[self.CLS_TOKEN] = self.CLS_ID vocab[self.SEP_TOKEN] = self.SEP_ID vocab[self.EOS_TOKEN] = self.EOS_ID vocab[self.UNK_TOKEN] = self.UNK_ID # Protein descriptor quantized tokens for i in range(self.config.mw_bins): vocab[f"MW_{i}"] = token_id token_id += 1 for i in range(self.config.pi_bins): vocab[f"PI_{i}"] = token_id token_id += 1 for i in range(self.config.tm_bins): vocab[f"TM_{i}"] = token_id token_id += 1 # Buffer tokens for buffer_type in self.config.buffer_types: vocab[f"BUFFER_{buffer_type.upper()}"] = token_id token_id += 1 # Buffer concentration bins for i in range(15): # 0-15 mM bins vocab[f"BUFCONC_{i}"] = token_id token_id += 1 # pH tokens for i in range(self.config.ph_bins): vocab[f"PH_{i}"] = token_id token_id += 1 # Ionic strength tokens for i in range(self.config.is_bins): vocab[f"IS_{i}"] = token_id token_id += 1 # Osmolarity tokens for i in range(self.config.osmol_bins): vocab[f"OSMOL_{i}"] = token_id token_id += 1 # Temperature tokens for i in range(self.config.temp_bins): vocab[f"TEMP_{i}"] = token_id token_id += 1 # Stabilizer identity tokens for stab_type in self.config.stabilizer_types: vocab[f"STAB_{stab_type.upper()}"] = token_id token_id += 1 # Stabilizer concentration bins (0-20%, in 1% increments) for i in range(21): vocab[f"STABCONC_{i}"] = token_id token_id += 1 # Outcome/stability tokens (prediction targets) for i in range(self.config.stability_bins): vocab[f"STABILITY_{i}"] = token_id token_id += 1 return vocab def encode_protein(self, mw: float, pi: float, tm: float) -> List[int]: """Encode protein descriptor as tokens.""" tokens = [] # MW (quantized) mw_bin = self._quantize(mw, self.config.mw_range, self.config.mw_bins) tokens.append(self.vocab[f"MW_{mw_bin}"]) # pI (quantized) pi_bin = self._quantize(pi, self.config.pi_range, self.config.pi_bins) tokens.append(self.vocab[f"PI_{pi_bin}"]) # Tm (quantized) tm_bin = self._quantize(tm, self.config.tm_range, self.config.tm_bins) tokens.append(self.vocab[f"TM_{tm_bin}"]) return tokens def encode_formulation(self, formulation_dict: Dict) -> List[int]: """Encode formulation composition as tokens.""" tokens = [] # Buffer species buffer = formulation_dict.get("buffer_species", "histidine").upper() buffer_token = f"BUFFER_{buffer}" tokens.append(self.vocab.get(buffer_token, self.UNK_ID)) # Buffer concentration buf_conc = min(formulation_dict.get("buffer_conc_mm", 20), 150) buf_conc_bin = int((buf_conc / 150.0) * 15) # 0-15 bins tokens.append(self.vocab[f"BUFCONC_{buf_conc_bin}"]) # pH ph = formulation_dict.get("ph", 7.0) ph_bin = self._quantize(ph, self.config.ph_range, self.config.ph_bins) tokens.append(self.vocab[f"PH_{ph_bin}"]) # Ionic strength is_val = formulation_dict.get("ionic_strength_mm", 150) is_bin = self._quantize(is_val, self.config.is_range, self.config.is_bins) tokens.append(self.vocab[f"IS_{is_bin}"]) # Osmolarity osmol = formulation_dict.get("osmolarity_mosm_kg", 300) osmol_bin = self._quantize(osmol, self.config.osmol_range, self.config.osmol_bins) tokens.append(self.vocab[f"OSMOL_{osmol_bin}"]) # Temperature temp = formulation_dict.get("temperature_c", 25.0) temp_bin = self._quantize(temp, self.config.temp_range, self.config.temp_bins) tokens.append(self.vocab[f"TEMP_{temp_bin}"]) # Stabilizers (variable length, up to 5) stabilizers = formulation_dict.get("stabilizers", {}) if isinstance(stabilizers, str): stabilizers = json.loads(stabilizers) for stab_name, stab_conc in list(stabilizers.items())[:5]: # max 5 stabilizers # Stabilizer identity stab_upper = stab_name.upper() stab_token = f"STAB_{stab_upper}" tokens.append(self.vocab.get(stab_token, self.UNK_ID)) # Stabilizer concentration (quantize to 0-20%) conc_bin = min(int(stab_conc), 20) tokens.append(self.vocab[f"STABCONC_{conc_bin}"]) return tokens def encode_stability_bin(self, stability_score: float) -> int: """Encode stability score as bin index (0-19).""" stab_bin = self._quantize( stability_score, self.config.stability_range, self.config.stability_bins, ) return stab_bin def encode_stability(self, stability_score: float) -> int: """Encode stability score as vocabulary token ID.""" stab_bin = self.encode_stability_bin(stability_score) return self.vocab[f"STABILITY_{stab_bin}"] def encode_sample( self, protein_dict: Dict, formulation_dict: Dict, outcome_dict: Optional[Dict] = None, ) -> Tuple[List[int], Optional[int]]: """ Encode full sample: (protein + formulation) → stability Returns: (input_tokens, target_bin) — target_bin is the stability class (0-19) """ input_tokens = [self.CLS_ID] # Protein protein_tokens = self.encode_protein( protein_dict["mw_kda"], protein_dict["pi"], protein_dict["tm_baseline_c"], ) input_tokens.extend(protein_tokens) # Separator input_tokens.append(self.SEP_ID) # Formulation form_tokens = self.encode_formulation(formulation_dict) input_tokens.extend(form_tokens) # Target (if provided): return bin index, not vocabulary token target_bin = None if outcome_dict is not None: target_bin = self.encode_stability_bin(outcome_dict.get("stability_score", 0.5)) return input_tokens, target_bin def decode_tokens(self, token_ids: List[int]) -> List[str]: """Decode token IDs back to token strings.""" # Build reverse vocab reverse_vocab = {v: k for k, v in self.vocab.items()} tokens = [] for token_id in token_ids: token_str = reverse_vocab.get(token_id, self.UNK_TOKEN) tokens.append(token_str) return tokens def _quantize( self, value: float, value_range: Tuple[float, float], num_bins: int, ) -> int: """Quantize continuous value into bin index.""" min_val, max_val = value_range value = np.clip(value, min_val, max_val) # Normalize to [0, 1] normalized = (value - min_val) / (max_val - min_val) # Map to bin bin_idx = int(normalized * (num_bins - 1)) return np.clip(bin_idx, 0, num_bins - 1) def save(self, path: Path) -> None: """Save tokenizer configuration.""" config_dict = { "vocab_size": self.vocab_size, "config": self.config.__dict__, "vocab": self.vocab, } with open(path, 'w') as f: json.dump(config_dict, f, indent=2) logger.info(f"Tokenizer saved to {path}") @classmethod def load(cls, path: Path) -> "FormulationTokenizer": """Load tokenizer configuration.""" with open(path, 'r') as f: config_dict = json.load(f) tokenizer = cls() tokenizer.vocab = config_dict["vocab"] tokenizer.vocab_size = config_dict["vocab_size"] logger.info(f"Tokenizer loaded from {path} (vocab_size={tokenizer.vocab_size})") return tokenizer if __name__ == "__main__": """Test tokenizer.""" logging.basicConfig(level=logging.INFO) tokenizer = FormulationTokenizer() # Test encoding protein = { "mw_kda": 100.0, "pi": 7.0, "tm_baseline_c": 65.0, } formulation = { "buffer_species": "histidine", "buffer_conc_mm": 20.0, "ph": 6.0, "ionic_strength_mm": 150.0, "osmolarity_mosm_kg": 310.0, "stabilizers": {"sucrose": 5.0, "polysorbate_80": 0.2}, "temperature_c": 25.0, } outcome = { "stability_score": 0.75, } input_tokens, target_token = tokenizer.encode_sample(protein, formulation, outcome) print("\n" + "=" * 70) print("TOKENIZER TEST") print("=" * 70) print(f"Vocab size: {tokenizer.vocab_size}") print(f"Input tokens: {input_tokens[:20]}... (total {len(input_tokens)})") print(f"Target token: {target_token}") # Decode decoded = tokenizer.decode_tokens(input_tokens[:10]) print(f"\nDecoded tokens: {decoded}") print("=" * 70)