quantppi / explainer /server.py
trizzhal9000's picture
Deploy QuantPPI Studio real-data 3D platform & API
5585d19 verified
Raw History Blame Contribute Delete
81.9 kB
#!/usr/bin/env python3 """ QuantPPI End-to-End Real-Data Backend Server ============================================ Serves real ESM-2 (650M, 150M, 35M) protein embeddings from HDF5 caches, real SCOPe ASTRAL-40 v2.08 (15,166 domains + 15,118 3D PDB .ent structures), real Swiss-Prot Human proteome (18,099 proteins + Gene Ontology annotations), live Campaign 4 data-oblivious quantization codecs (168B to 488B), live Theorem 1 & Theorem 2 2x2 factorial ablations, and the complete 56-method QuantPPI vs. Foldseek benchmark ledgers. """ import os import sys import re import json import time import math import urllib.parse from pathlib import Path from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Dict, List, Any, Optional, Tuple os.environ.setdefault("OMP_NUM_THREADS", "1") os.environ.setdefault("MKL_NUM_THREADS", "1") os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1") os.environ.setdefault("MPLCONFIGDIR", "/tmp") import h5py import numpy as np import pandas as pd THIS_DIR = Path(__file__).resolve().parent REPO_ROOT = THIS_DIR.parent CAMPAIGN4_DIR = REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_4_breakthrough" FOLDSEEK_RES_DIR = REPO_ROOT / "experiments" / "foldseek_comparison" / "results" DATA_DIR = REPO_ROOT / "data" CACHE_DIR = REPO_ROOT / "cache" PDB_DIR = DATA_DIR / "scope40_pdb" / "candidates_15118_model1" PDB_H5_PATH = CACHE_DIR / "scope40_ca_structures_15118.h5" _H5_STRUCT_INDEX: Optional[Dict[str, int]] = None def _get_h5_struct_index() -> Dict[str, int]: global _H5_STRUCT_INDEX if _H5_STRUCT_INDEX is not None: return _H5_STRUCT_INDEX _H5_STRUCT_INDEX = {} if PDB_H5_PATH.exists(): try: with h5py.File(PDB_H5_PATH, "r") as f: dom_ids = f["domain_ids"][:] for idx, raw_id in enumerate(dom_ids): did = raw_id.decode("ascii", errors="ignore").strip() if isinstance(raw_id, (bytes, np.bytes_)) else str(raw_id).strip() if did: _H5_STRUCT_INDEX[did] = idx except Exception as exc: print(f"[QuantPPI Server] Warning reading {PDB_H5_PATH}: {exc}") return _H5_STRUCT_INDEX def has_pdb_structure(dom_id: str) -> bool: if (PDB_DIR / f"{dom_id}.ent").exists(): return True return dom_id in _get_h5_struct_index() for p in [str(REPO_ROOT), str(CAMPAIGN4_DIR), str(CAMPAIGN4_DIR / "src")]: if p not in sys.path: sys.path.insert(0, p) from src.codecs import ( DynamicTopKOutlierCodec, AnalyticalSCMFCodec, TernaryDeadZoneCodec, MultiStageResidualCodec, HybridTopKResidualCodec, SphericalE8LatticeCodec, HuberTailCompandingCodec, SmoothCompandingCodec, TwoStageSearchIndex, ) from src.transforms.transforms import get_transform from src.evaluators.retrieval import ( MetricCalculator, ExactFP32Baseline, Circulant1BitBaseline, FWHT1BitBaseline, Analytic2BitBaseline, RaBitQ1BitBaseline, TurboQuant1BitBaseline, TurboQuant2BitBaseline, CalibratedNativeProtectionBaseline, CalibratedSCMFBaseline, compute_geometry_diagnostics, ) from src.evaluators.biological import ( compute_average_precision, compute_roc50, compute_go_jaccard, ) SCOP_CLASS_NAMES = { "a": "All alpha proteins", "b": "All beta proteins", "c": "Alpha and beta proteins (a/b)", "d": "Alpha and beta proteins (a+b)", "e": "Multi-domain proteins (alpha and beta)", "f": "Membrane and cell surface proteins and peptides", "g": "Small proteins", } CODEC_CATALOG = [ { "id": "track1_top2", "name": "QuantPPI Top-2 + 1-bit (168 B)", "short_name": "QuantPPI 168B Top-2", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 168, "compression_1280": 30.48, "k": 2, "trunk_mode": "1bit", "rho_q": 0.797885, "bussgang_beta": 1.253314, "description": "Dynamic per-vector Top-2 outlier exemption (uint16 indices + FP16 values) + 1-bit Circulant trunk with Theorem 1 bleed elimination and Theorem 2 Bussgang de-attenuation.", }, { "id": "track1_top4", "name": "QuantPPI Top-4 + 1-bit (176 B)", "short_name": "QuantPPI 176B Top-4", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 176, "compression_1280": 29.09, "k": 4, "trunk_mode": "1bit", "rho_q": 0.797885, "bussgang_beta": 1.253314, "description": "Dynamic per-vector Top-4 outlier exemption (16 B) + 1-bit Circulant trunk (160 B) with Pythagorean residual norm recovery and Theorems 1 & 2.", }, { "id": "track3_hybrid_k2_block224", "name": "QuantPPI Block-Order Argmax Hybrid (224 B)", "short_name": "QuantPPI 224B Block-Order", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 224, "compression_1280": 22.86, "k": 2, "trunk_mode": "block_order", "rho_q": 0.880710, "bussgang_beta": 1.135448, "description": "Top-2 outlier exemption (8 B) + 8D Block-Order-Statistic Argmax coding (3-bit argmax index + 8 sign bits per 8D block = 216 B). Surpasses Trained PQ M=224 and Calibrated Native Protection with zero training.", }, { "id": "track3_hybrid_k4_block256", "name": "QuantPPI Block-Order 4-bit Hybrid (256 B)", "short_name": "QuantPPI 256B Block-4b", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 256, "compression_1280": 20.00, "k": 4, "trunk_mode": "block_order_4b", "rho_q": 0.896700, "bussgang_beta": 1.115200, "description": "Top-4 outlier exemption (16 B) + 8D Block-Order 4-bit magnitude-tier trunk coding (240 B).", }, { "id": "track3_hybrid_k2_ternary", "name": "QuantPPI Radix-3 Ternary Hybrid (264 B)", "short_name": "QuantPPI 264B Ternary", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 264, "compression_1280": 19.39, "k": 2, "trunk_mode": "ternary", "rho_q": 0.880660, "bussgang_beta": 1.135512, "description": "Top-2 outlier exemption (8 B) + symmetric dead-zone ternary {-1, 0, +1} trunk packed at 5 trits per byte (3^5 = 243 < 256, 256 B trunk).", }, { "id": "track3_hybrid_k2_mixed320", "name": "QuantPPI Mixed 2-bit/Ternary Hybrid (320 B)", "short_name": "QuantPPI 320B Mixed", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 320, "compression_1280": 16.00, "k": 2, "trunk_mode": "mixed_2bit_ternary", "rho_q": 0.935500, "bussgang_beta": 1.068947, "description": "Top-2 outlier exemption (8 B) + mixed 2-bit / Radix-3 ternary spherical Lloyd-Max trunk (312 B). Beats the historical 320 B 2-bit ceiling by +11.80 percentage points.", }, { "id": "track3_hybrid_k4_2bit", "name": "QuantPPI 2-bit Spherical Hybrid (336 B)", "short_name": "QuantPPI 336B 2-bit", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 336, "compression_1280": 15.24, "k": 4, "trunk_mode": "2bit", "rho_q": 0.939400, "bussgang_beta": 1.064509, "description": "Top-4 outlier exemption (16 B) + 4-level spherical Gaussian Lloyd-Max 2-bit trunk (320 B).", }, { "id": "track3_hybrid_k2_quinary382", "name": "QuantPPI Base-5 Quinary Hybrid (382 B)", "short_name": "QuantPPI 382B Quinary", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 382, "compression_1280": 13.40, "k": 2, "trunk_mode": "quinary", "rho_q": 0.959197, "bussgang_beta": 1.042539, "description": "Top-2 outlier exemption (8 B) + 5-level Quinary (M=5) spherical Lloyd-Max trunk packing 3 quints into 7 bits (5^3 = 125 < 128). Surpasses Phase-5 Half-DB Trained OPQ M=224.", }, { "id": "track3_hybrid_k4_senary432", "name": "QuantPPI Base-6 Senary Hybrid (432 B)", "short_name": "QuantPPI 432B Senary", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 432, "compression_1280": 11.85, "k": 4, "trunk_mode": "senary", "rho_q": 0.970578, "bussgang_beta": 1.030314, "description": "Top-4 outlier exemption (16 B) + 6-level Senary (M=6) spherical Lloyd-Max trunk packing 5 hexits into 13 bits (6^5 = 7776 < 8192). Beats ALL Trained OPQ M=224 regimes with zero training.", }, { "id": "track3_hybrid_k2_3bit488", "name": "QuantPPI 3-bit Octal Hybrid (488 B)", "short_name": "QuantPPI 488B 3-bit", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 488, "compression_1280": 10.49, "k": 2, "trunk_mode": "3bit", "rho_q": 0.982574, "bussgang_beta": 1.017735, "description": "Top-2 outlier exemption (8 B) + 8-level 3-bit Octal (M=8) spherical Lloyd-Max trunk (480 B). Achieves 0.9993 Recall@20@30 on SCOPe and 0.9994 on Swiss-Prot.", }, { "id": "track5_huber_tail", "name": "Track 5: Huber Spike-Tail Companding (176 B)", "short_name": "Track 5 Huber (176B)", "family": "Campaign 4 Alternative Tracks (N_cal=0)", "bytes_1280": 176, "compression_1280": 29.09, "k": 4, "trunk_mode": "huber_1bit", "rho_q": 0.797885, "bussgang_beta": 1.253314, "description": "Native-basis Huber clipping with sparse spike residual (k=4) and deterministic boundary sign*tau recovery.", }, { "id": "track4_spherical_e8", "name": "Track 4: Spherical E8 Lattice (162 B)", "short_name": "Track 4 Spherical E8 (162B)", "family": "Campaign 4 Alternative Tracks (N_cal=0)", "bytes_1280": 162, "compression_1280": 31.60, "k": 0, "trunk_mode": "e8_lattice", "rho_q": 0.785000, "bussgang_beta": 1.0, "description": "256-codeword unit spherical E8 root-shell lattice codebook (1 byte per 8D block + 2B gain).", }, { "id": "circulant_1bit", "name": "Baseline: Uniform Circulant 1-bit (160 B)", "short_name": "Circulant 1-bit (160B)", "family": "Unprotected Data-Oblivious Baselines", "bytes_1280": 160, "compression_1280": 32.00, "k": 0, "trunk_mode": "1bit_unprotected", "rho_q": 0.797885, "bussgang_beta": 1.0, "description": "Standard 1-bit random-sign circulant rotation without outlier exemption. Suffers from rogue-coordinate energy smearing (Recall@20@30 = 0.7612).", }, { "id": "fwht_1bit", "name": "Baseline: Uniform FWHT 1-bit / TurboQuant 1-bit (160 B)", "short_name": "FWHT 1-bit (160B)", "family": "Unprotected Data-Oblivious Baselines", "bytes_1280": 160, "compression_1280": 32.00, "k": 0, "trunk_mode": "1bit_unprotected", "rho_q": 0.797885, "bussgang_beta": 1.0, "description": "Standard 1-bit Fast Walsh-Hadamard Transform without outlier exemption (Recall@20@30 = 0.7446).", }, { "id": "analytic_2bit", "name": "Baseline: Uniform Analytic 2-bit Lloyd-Max (320 B)", "short_name": "Analytic 2-bit (320B)", "family": "Unprotected Data-Oblivious Baselines", "bytes_1280": 320, "compression_1280": 16.00, "k": 0, "trunk_mode": "2bit_unprotected", "rho_q": 0.939400, "bussgang_beta": 1.0, "description": "Unprotected 2-bit Gaussian Lloyd-Max quantizer. Fails severely on PLM embeddings (Recall@20@30 = 0.4986) due to fixed threshold distortion from rogue spikes.", }, { "id": "calibrated_native_protection_k32", "name": "Baseline: Calibrated Native Protection (k=32, 224 B)", "short_name": "Calib Native Prot (224B)", "family": "Calibrated Baselines (N_cal=4096)", "bytes_1280": 224, "compression_1280": 22.86, "k": 32, "trunk_mode": "calibrated_1bit", "rho_q": 0.797885, "bussgang_beta": 1.0, "description": "Globally calibrated top-32 variance coordinates stored in FP16 (64 B) + 1-bit trunk (160 B) without Theorems 1 & 2 (Recall@20@30 = 0.8449).", }, ] class RealDataState: """Singleton state manager holding real embeddings, sequence metadata, and cached wire buffers.""" def __init__(self): print("[QuantPPI Server] Initializing RealDataState from authoritative repository files...") t0 = time.perf_counter() # 1. Load SCOPe ASTRAL-40 v2.08 FASTA (15,166 domains) self.scope_entries: List[Dict[str, Any]] = [] self.scope_id_to_raw_idx: Dict[str, int] = {} self._load_scope_fasta() # 2. Load Protocol P1 Canonical Manifest (500 queries, 15,166 candidates) p1_manifest_path = ( REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_3_paper_readiness" / "manifests" / "protocol_p1_scope_canonical.json" ) with open(p1_manifest_path, "r") as f: p1_m = json.load(f) self.p1_query_raw_ids: List[int] = p1_m["query_ids"] self.p1_cand_raw_ids: List[int] = p1_m["candidate_ids"] self.p1_query_set = set(self.p1_query_raw_ids) # Ordered candidate metadata matching p1_cand_raw_ids self.scope_p1_candidates: List[Dict[str, Any]] = [] self.scope_sid_to_p1_pos: Dict[str, int] = {} self.scope_sid_to_query_pos: Dict[str, int] = {} for pos, raw_i in enumerate(self.p1_cand_raw_ids): entry = dict(self.scope_entries[raw_i]) entry["p1_cand_pos"] = pos entry["raw_idx"] = raw_i entry["is_p1_query"] = pos < len(self.p1_query_raw_ids) entry["p1_query_pos"] = pos if pos < len(self.p1_query_raw_ids) else None self.scope_p1_candidates.append(entry) self.scope_sid_to_p1_pos[entry["id"]] = pos if pos < len(self.p1_query_raw_ids): self.scope_sid_to_query_pos[entry["id"]] = pos # Precompute SCOPe family/superfamily/fold arrays for fast biological evaluation self.scope_cand_fam = np.array([c["family"] for c in self.scope_p1_candidates]) self.scope_cand_sfam = np.array([c["superfamily"] for c in self.scope_p1_candidates]) self.scope_cand_fold = np.array([c["fold"] for c in self.scope_p1_candidates]) # 3. Load Swiss-Prot Human FASTA (18,099 proteins) & GO TSV self.go_map: Dict[str, List[str]] = {} self._load_swissprot_go() self.sp_entries: List[Dict[str, Any]] = [] self._load_swissprot_fasta() p5_manifest_path = ( REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_3_paper_readiness" / "manifests" / "protocol_p5_swissprot_locked.json" ) with open(p5_manifest_path, "r") as f: p5_m = json.load(f) self.p5_calib_raw_ids: List[int] = p5_m.get("calibration_ids", []) self.p5_query_raw_ids: List[int] = p5_m["query_ids"] self.p5_cand_raw_ids: List[int] = p5_m["candidate_ids"] self.sp_p5_queries: List[Dict[str, Any]] = [] self.sp_p5_candidates: List[Dict[str, Any]] = [] self.sp_acc_to_raw_idx: Dict[str, int] = {} self.sp_acc_to_query_pos: Dict[str, int] = {} self.sp_acc_to_cand_pos: Dict[str, int] = {} for raw_i, e in enumerate(self.sp_entries): self.sp_acc_to_raw_idx[e["id"]] = raw_i for q_pos, raw_i in enumerate(self.p5_query_raw_ids): e = self.sp_entries[raw_i] e["is_p5_query"] = True e["p5_query_pos"] = q_pos self.sp_p5_queries.append(e) self.sp_acc_to_query_pos[e["id"]] = q_pos for c_pos, raw_i in enumerate(self.p5_cand_raw_ids): e = self.sp_entries[raw_i] e["is_p5_candidate"] = True e["p5_cand_pos"] = c_pos self.sp_p5_candidates.append(e) self.sp_acc_to_cand_pos[e["id"]] = c_pos # 4. Embedding Caches (Lazy loaded per (dataset, model_scale)) self._emb_cache: Dict[Tuple[str, str], np.ndarray] = {} self._geom_cache: Dict[Tuple[str, str], Dict[str, Any]] = {} self._encoded_db_cache: Dict[Tuple[str, str, str], Tuple[Any, bytes, float]] = {} # Preload primary SCOPe 650M cache immediately self.get_embeddings("scope", "esm2_650m") # 5. Load Foldseek Top-20 Structural Alignments per query self.foldseek_by_query: Dict[str, List[Dict[str, Any]]] = {} self._load_foldseek_alignments() # 6. Load Benchmark Ledgers self.benchmarks_payload = self._load_benchmarks() dt = time.perf_counter() - t0 print( f"[QuantPPI Server] Ready in {dt:.2f}s | SCOPe: {len(self.scope_entries)} domains | " f"Swiss-Prot: {len(self.sp_entries)} human proteins | " f"3D PDB structures: {PDB_DIR.exists() or PDB_H5_PATH.exists()}" ) def _load_scope_fasta(self): fasta_path = DATA_DIR / "astral-scopedom-seqres-gd-sel-gs-bib-40-2.08.fa" header = None seq_parts: List[str] = [] def _commit(h: str, sp: List[str]): seq = "".join(sp).strip().upper() if not (20 <= len(seq) <= 1022): return parts = h[1:].split() if len(parts) < 2: return dom_id = parts[0] lbl = parts[1] lbl_parts = lbl.split(".") if len(lbl_parts) != 4: return chain_region = parts[2].strip("()") if len(parts) > 2 else "" rest = " ".join(parts[3:]) if len(parts) > 3 else "" org = "" desc = rest if "{" in rest and "}" in rest: idx = rest.find("{") desc = rest[:idx].strip() org = rest[idx + 1 : rest.rfind("}")].strip() raw_idx = len(self.scope_entries) entry = { "id": dom_id, "pdb_id": dom_id[1:5].upper() if len(dom_id) >= 5 else "", "chain": chain_region, "class": lbl_parts[0], "class_name": SCOP_CLASS_NAMES.get(lbl_parts[0], lbl_parts[0]), "fold": f"{lbl_parts[0]}.{lbl_parts[1]}", "superfamily": f"{lbl_parts[0]}.{lbl_parts[1]}.{lbl_parts[2]}", "family": lbl, "sccs": lbl, "description": desc or "SCOPe structural domain", "organism": org, "sequence": seq, "seq_len": len(seq), "has_pdb": has_pdb_structure(dom_id), "raw_idx": raw_idx, "is_p1_query": False, "p1_query_pos": None, "p1_cand_pos": None, } self.scope_entries.append(entry) self.scope_id_to_raw_idx[dom_id] = raw_idx with open(fasta_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue if line.startswith(">"): if header is not None: _commit(header, seq_parts) header = line seq_parts = [] else: seq_parts.append(line) if header is not None: _commit(header, seq_parts) def _load_swissprot_go(self): tsv_path = DATA_DIR / "swissprot_go.tsv" if not tsv_path.exists(): return with open(tsv_path, "r", encoding="utf-8") as f: for i, line in enumerate(f): if i == 0 and line.startswith("Entry"): continue parts = line.strip().split("\t") if len(parts) >= 2: self.go_map[parts[0]] = [t.strip() for t in parts[1].split(";") if t.strip()] def _load_swissprot_fasta(self): fasta_path = DATA_DIR / "swissprot_human_full.fasta" header = None seq_parts: List[str] = [] def _commit(h: str, sp: List[str]): seq = "".join(sp).strip().upper() if not (20 <= len(seq) <= 1022): return # Format: >sp|P04637|P53_HUMAN Cellular tumor antigen p53 OS=Homo sapiens OX=9606 GN=TP53 PE=1 SV=4 bar_parts = h[1:].split("|") acc = bar_parts[1] if len(bar_parts) >= 2 else h[1:].split()[0] rest = bar_parts[2] if len(bar_parts) >= 3 else h[1:] tokens = rest.split() entry_name = tokens[0] if tokens else acc text_after = " ".join(tokens[1:]) if len(tokens) > 1 else "" gene = "" m_gn = re.search(r"GN=([^\s]+)", text_after) if m_gn: gene = m_gn.group(1) desc = text_after.split(" OS=")[0].strip() if " OS=" in text_after else text_after go_terms = self.go_map.get(acc, []) raw_idx = len(self.sp_entries) self.sp_entries.append({ "id": acc, "entry_name": entry_name, "gene": gene or entry_name.replace("_HUMAN", ""), "description": desc or entry_name, "organism": "Homo sapiens (Human)", "go_terms": go_terms, "go_count": len(go_terms), "sequence": seq, "seq_len": len(seq), "raw_idx": raw_idx, "is_p5_query": False, "p5_query_pos": None, "is_p5_candidate": False, "p5_cand_pos": None, }) with open(fasta_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue if line.startswith(">"): if header is not None: _commit(header, seq_parts) header = line seq_parts = [] else: seq_parts.append(line) if header is not None: _commit(header, seq_parts) def _load_foldseek_alignments(self): csv_path = FOLDSEEK_RES_DIR / "foldseek_structural_alignments_top20.csv" if not csv_path.exists(): return df = pd.read_csv(csv_path) for _, row in df.iterrows(): qid = str(row["query_id"]) if qid not in self.foldseek_by_query: self.foldseek_by_query[qid] = [] self.foldseek_by_query[qid].append({ "rank": int(row["rank_non_self"]), "target_id": str(row["target_id"]), "target_sccs": str(row["target_sccs"]), "homology_tier": str(row["homology_tier"]), "paper_struct_score": float(row["paper_struct_score"]), "bits": float(row["bits"]), "evalue": float(row["evalue"]), "alntmscore": float(row["alntmscore"]), "lddt": float(row["lddt"]), "fident": float(row["fident"]), "alnlen": int(row["alnlen"]), }) def _load_benchmarks(self) -> Dict[str, Any]: out: Dict[str, Any] = {} mc_path = FOLDSEEK_RES_DIR / "method_comparison_table.csv" if mc_path.exists(): df = pd.read_csv(mc_path).replace({np.nan: None}) out["methods"] = df.to_dict(orient="records") ab_path = FOLDSEEK_RES_DIR / "ablation_studies_complete_15118.csv" if ab_path.exists(): df = pd.read_csv(ab_path).replace({np.nan: None}) out["ablations"] = df.to_dict(orient="records") ms_path = FOLDSEEK_RES_DIR / "multiseed_stability_15118.csv" if ms_path.exists(): df = pd.read_csv(ms_path).replace({np.nan: None}) out["multiseed"] = df.to_dict(orient="records") stat_path = FOLDSEEK_RES_DIR / "statistical_significance_tests.csv" if stat_path.exists(): df = pd.read_csv(stat_path).replace({np.nan: None}) out["statistical_tests"] = df.to_dict(orient="records") pr_path = FOLDSEEK_RES_DIR / "precision_recall_cutoffs_k.csv" if pr_path.exists(): df = pd.read_csv(pr_path).replace({np.nan: None}) out["pr_cutoffs"] = df.to_dict(orient="records") t1_path = FOLDSEEK_RES_DIR / "task1_d6a1ia1_investigation.json" if t1_path.exists(): with open(t1_path, "r") as f: out["d6a1ia1_forensic"] = json.load(f) return out def get_embeddings(self, dataset: str, model_scale: str) -> np.ndarray: key = (dataset, model_scale) if key in self._emb_cache: return self._emb_cache[key] file_map = { ("scope", "esm2_650m"): CACHE_DIR / "esm2_650m_mean_l2norm_final_15166.h5", ("scope", "esm2_150m"): CACHE_DIR / "esm2_150m_mean_l2norm_final_15166.h5", ("scope", "esm2_35m"): CACHE_DIR / "esm2_35m_mean_l2norm_final_15166.h5", ("swissprot", "esm2_650m"): CACHE_DIR / "esm2_650m_mean_sp_full_18099.h5", ("swissprot", "esm2_150m"): CACHE_DIR / "esm2_150m_mean_sp_full_18099.h5", ("swissprot", "esm2_35m"): CACHE_DIR / "esm2_35m_mean_sp_full_18099.h5", } h5_path = file_map.get(key) if not h5_path or not h5_path.exists(): raise ValueError(f"Unknown dataset/model_scale combination: {key}") with h5py.File(h5_path, "r") as f: emb = f["embeddings"][:].astype(np.float32) self._emb_cache[key] = emb return emb def get_corpus_geometry(self, dataset: str, model_scale: str) -> Dict[str, Any]: key = (dataset, model_scale) if key in self._geom_cache: return self._geom_cache[key] emb = self.get_embeddings(dataset, model_scale) N, D = emb.shape mean_sq = np.mean(emb ** 2, axis=0) mean_vec = np.mean(emb, axis=0) common_mode_norm = float(np.linalg.norm(mean_vec)) order_desc = np.argsort(-mean_sq) sorted_sq = mean_sq[order_desc] cum_sq = np.cumsum(sorted_sq) top_coords = [] for rank in range(min(32, D)): coord_idx = int(order_desc[rank]) top_coords.append({ "rank": rank + 1, "coord_index": coord_idx, "mean_energy": float(sorted_sq[rank]), "cumulative_energy": float(cum_sq[rank]), "mean_signed": float(mean_vec[coord_idx]), }) diag = compute_geometry_diagnostics(emb) diag.update({ "dataset": dataset, "model_scale": model_scale, "num_vectors": int(N), "dimension": int(D), "common_mode_l2_norm": common_mode_norm, "top_32_native_coordinates": top_coords, "spectrum_top_128": [float(x) for x in sorted_sq[:128]], "cumulative_top_128": [float(x) for x in cum_sq[:128]], }) self._geom_cache[key] = diag return diag def create_codec( self, codec_id: str, dimension: int = 1280, transform_name: str = "circulant", dataset: str = "scope", model_scale: str = "esm2_650m", ) -> Any: if codec_id == "track1_top2": return DynamicTopKOutlierCodec(dimension=dimension, k=2, transform_name=transform_name, seed=42) if codec_id == "track1_top4": return DynamicTopKOutlierCodec(dimension=dimension, k=4, transform_name=transform_name, seed=42) if codec_id == "track1_top16": return DynamicTopKOutlierCodec(dimension=dimension, k=16, transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_block224": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="block_order", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k4_block256": return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="block_order_4b", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_ternary": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="ternary", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_mixed320": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="mixed_2bit_ternary", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k4_2bit": return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="2bit", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_quinary382": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="quinary", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k4_senary432": return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="senary", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_3bit488": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="3bit", transform_name=transform_name, seed=42) if codec_id == "track5_huber_tail": return HuberTailCompandingCodec(dimension=dimension, max_outliers=4, transform_name=transform_name, seed=42) if codec_id == "track4_spherical_e8": return SphericalE8LatticeCodec(dimension=dimension, transform_name=transform_name, seed=42) if codec_id == "circulant_1bit": return Circulant1BitBaseline(dimension=dimension, seed=42) if codec_id == "fwht_1bit": return FWHT1BitBaseline(dimension=dimension, seed=42) if codec_id == "analytic_2bit": return Analytic2BitBaseline(dimension=dimension, seed=42) if codec_id == "calibrated_native_protection_k32": raw_emb = self.get_embeddings(dataset, model_scale) if dataset == "scope": calib = raw_emb[self.p1_cand_raw_ids[500:4596]] else: calib = raw_emb[self.p5_calib_raw_ids] return CalibratedNativeProtectionBaseline(dimension=dimension, k=32, calib_vectors=calib, seed=42) raise ValueError(f"Unsupported codec_id: {codec_id}") def get_encoded_candidates( self, dataset: str, model_scale: str, codec_id: str, ) -> Tuple[Any, Any, np.ndarray, float]: """Returns (codec, packed_bytes, candidate_embeddings, encode_time_s), caching encoded bytes in memory.""" raw_emb = self.get_embeddings(dataset, model_scale) if dataset == "scope": cand_emb = raw_emb[self.p1_cand_raw_ids] else: cand_emb = raw_emb[self.p5_cand_raw_ids] D = cand_emb.shape[1] cache_key = (dataset, model_scale, codec_id) if cache_key in self._encoded_db_cache: codec, packed_bytes, enc_time = self._encoded_db_cache[cache_key] return codec, packed_bytes, cand_emb, enc_time codec = self.create_codec(codec_id, dimension=D, transform_name="circulant", dataset=dataset, model_scale=model_scale) t0 = time.perf_counter() packed_bytes = codec.encode(cand_emb) # Warm ADC cache if present if hasattr(codec, "_get_prepared_adc"): codec._get_prepared_adc(packed_bytes, len(cand_emb)) enc_time = time.perf_counter() - t0 self._encoded_db_cache[cache_key] = (codec, packed_bytes, enc_time) return codec, packed_bytes, cand_emb, enc_time STATE = RealDataState() AA3_TO_1 = { "ALA": "A", "ARG": "R", "ASN": "N", "ASP": "D", "CYS": "C", "GLN": "Q", "GLU": "E", "GLY": "G", "HIS": "H", "ILE": "I", "LEU": "L", "LYS": "K", "MET": "M", "PHE": "F", "PRO": "P", "SER": "S", "THR": "T", "TRP": "W", "TYR": "Y", "VAL": "V", "MSE": "M", "SEC": "U", "PYL": "O", } KYTE_DOOLITTLE = { "I": 4.5, "V": 4.2, "L": 3.8, "F": 2.8, "C": 2.5, "M": 1.9, "A": 1.8, "G": -0.4, "T": -0.7, "S": -0.8, "W": -0.9, "Y": -1.3, "P": -1.6, "H": -3.2, "E": -3.5, "Q": -3.5, "D": -3.5, "N": -3.5, "K": -3.9, "R": -4.5, } _MINI_TRACE_CACHE: Dict[str, Any] = {} def _assign_ca_secondary_structure(coords: np.ndarray) -> List[str]: """ Assigns secondary structure ('H' = alpha-helix, 'E' = beta-strand, 'C' = coil) directly from experimental 3D C-alpha coordinates using P-SEA inter-atomic distance rules. """ n = len(coords) if n < 5: return ["C"] * n raw_ss = ["C"] * n for i in range(n - 4): d2 = float(np.linalg.norm(coords[i + 2] - coords[i])) d3 = float(np.linalg.norm(coords[i + 3] - coords[i])) d4 = float(np.linalg.norm(coords[i + 4] - coords[i])) # Alpha-helix: compact right-handed turn (d2 ~ 5.4A, d3 ~ 5.1A, d4 ~ 6.2A) if 4.8 <= d2 <= 6.1 and 4.4 <= d3 <= 6.0 and 5.2 <= d4 <= 7.1: for k in range(i, min(i + 4, n)): raw_ss[k] = "H" # Beta-strand: extended pleated backbone (d2 ~ 6.7A, d3 ~ 9.9A, d4 ~ 12.8A) elif 6.15 <= d2 <= 7.8 and 8.5 <= d3 <= 11.6: for k in range(i, min(i + 4, n)): if raw_ss[k] == "C": raw_ss[k] = "E" return raw_ss def parse_pdb_ca_atoms(ent_path: Path) -> Dict[str, Any]: """Extracts real 3D C-alpha atomic coordinates, secondary structure, and ASTRAL remarks from a SCOPe .ent file or HDF5 archive.""" ca_atoms = [] remarks = [] header_line = "" total_atom_records = 0 if isinstance(ent_path, str): dom_id = ent_path.replace(".ent", "").strip() ent_path = PDB_DIR / f"{dom_id}.ent" else: dom_id = ent_path.stem if ent_path.exists(): with open(ent_path, "r", encoding="utf-8", errors="replace") as f: for line in f: if line.startswith("HEADER"): header_line = line.strip() elif line.startswith("REMARK 99"): txt = line[10:].strip() if txt: remarks.append(txt) elif line.startswith("ATOM "): total_atom_records += 1 atom_name = line[12:16].strip() alt_loc = line[16:17].strip() if atom_name == "CA" and alt_loc in ("", "A", "1"): res_name = line[17:20].strip() chain_id = line[21:22].strip() res_seq_str = line[22:26].strip() try: res_seq = int(res_seq_str) except ValueError: res_seq = len(ca_atoms) + 1 try: x = float(line[30:38]) y = float(line[38:46]) z = float(line[46:54]) bfac = float(line[60:66]) if len(line) >= 66 and line[60:66].strip() else 0.0 aa1 = AA3_TO_1.get(res_name.upper(), "X") ca_atoms.append({ "res_seq": res_seq, "res_name": res_name, "aa": aa1, "hydropathy": KYTE_DOOLITTLE.get(aa1, 0.0), "chain": chain_id, "x": round(x, 3), "y": round(y, 3), "z": round(z, 3), "bfactor": round(bfac, 2), }) except ValueError: pass else: idx_map = _get_h5_struct_index() h5_idx = idx_map.get(dom_id) if h5_idx is not None and PDB_H5_PATH.exists(): with h5py.File(PDB_H5_PATH, "r") as f: s_idx = int(f["offsets"][h5_idx]) e_idx = int(f["offsets"][h5_idx + 1]) raw_hdr = f["headers"][h5_idx] hdr_str = raw_hdr.decode("utf-8", errors="ignore") if isinstance(raw_hdr, (bytes, np.bytes_)) else str(raw_hdr) if " | " in hdr_str: parts = hdr_str.split(" | ") header_line = parts[0] remarks = parts[1:] else: header_line = hdr_str xyz_arr = f["xyz"][s_idx:e_idx].astype(np.float64) bfac_arr = f["bfactor"][s_idx:e_idx].astype(np.float64) rseq_arr = f["res_seq"][s_idx:e_idx] rname_arr = f["res_name"][s_idx:e_idx] total_atom_records = int((e_idx - s_idx) * 8) chain_guess = dom_id[5:6].upper() if len(dom_id) >= 6 and dom_id[5:6] != "_" else "A" for i in range(e_idx - s_idx): rn = rname_arr[i].decode("ascii", errors="ignore").strip() if isinstance(rname_arr[i], (bytes, np.bytes_)) else str(rname_arr[i]).strip() aa1 = AA3_TO_1.get(rn.upper(), "X") ca_atoms.append({ "res_seq": int(rseq_arr[i]), "res_name": rn, "aa": aa1, "hydropathy": KYTE_DOOLITTLE.get(aa1, 0.0), "chain": chain_guess, "x": round(float(xyz_arr[i, 0]), 3), "y": round(float(xyz_arr[i, 1]), 3), "z": round(float(xyz_arr[i, 2]), 3), "bfactor": round(float(bfac_arr[i]), 2), }) ss_comp = {"helix_pct": 0.0, "sheet_pct": 0.0, "coil_pct": 100.0} if ca_atoms: coords = np.array([[a["x"], a["y"], a["z"]] for a in ca_atoms], dtype=np.float64) center = np.mean(coords, axis=0) centered = coords - center radius = float(np.max(np.linalg.norm(centered, axis=1))) if len(centered) > 0 else 1.0 ss_list = _assign_ca_secondary_structure(coords) n_res = len(ca_atoms) h_cnt = sum(1 for s in ss_list if s == "H") e_cnt = sum(1 for s in ss_list if s == "E") ss_comp = { "helix_pct": round((h_cnt / n_res) * 100.0, 1), "sheet_pct": round((e_cnt / n_res) * 100.0, 1), "coil_pct": round(((n_res - h_cnt - e_cnt) / n_res) * 100.0, 1), } for i, a in enumerate(ca_atoms): a["cx"] = round(float(centered[i, 0]), 3) a["cy"] = round(float(centered[i, 1]), 3) a["cz"] = round(float(centered[i, 2]), 3) a["ss"] = ss_list[i] else: radius = 1.0 return { "header": header_line, "remarks": remarks, "total_atom_records": total_atom_records, "num_ca_residues": len(ca_atoms), "bounding_radius_angstrom": round(radius, 2), "ss_composition": ss_comp, "ca_atoms": ca_atoms, } def get_mini_ca_trace(dom_id: str, max_points: int = 72) -> Optional[Dict[str, Any]]: """Returns a cached compact 3D C-alpha trace for inline 3D thumbnail rendering in protein cards.""" if dom_id in _MINI_TRACE_CACHE: return _MINI_TRACE_CACHE[dom_id] if not has_pdb_structure(dom_id): _MINI_TRACE_CACHE[dom_id] = None return None ent_path = PDB_DIR / f"{dom_id}.ent" parsed = parse_pdb_ca_atoms(ent_path) atoms = parsed["ca_atoms"] if not atoms: _MINI_TRACE_CACHE[dom_id] = None return None n = len(atoms) if n > max_points: idxs = np.linspace(0, n - 1, max_points).astype(int) sampled = [atoms[int(i)] for i in idxs] else: sampled = atoms ss_map = {"H": 1, "E": 2, "C": 0} pts = [[a["cx"], a["cy"], a["cz"], ss_map.get(a.get("ss", "C"), 0)] for a in sampled] res = { "radius": parsed["bounding_radius_angstrom"], "num_ca": n, "ss_composition": parsed["ss_composition"], "points": pts, } _MINI_TRACE_CACHE[dom_id] = res return res def kabsch_superpose(coords_ref: np.ndarray, coords_mov: np.ndarray) -> Tuple[np.ndarray, float]: """ Computes optimal rigid-body 3D Kabsch alignment of coords_mov onto coords_ref over their shared prefix/resampled C-alpha trace and returns (aligned_mov, rmsd). """ n_ref = len(coords_ref) n_mov = len(coords_mov) if n_ref < 3 or n_mov < 3: return coords_mov, 0.0 c_ref = coords_ref - np.mean(coords_ref, axis=0) c_mov = coords_mov - np.mean(coords_mov, axis=0) # Resample to common length L for correspondence estimation L = min(n_ref, n_mov) idx_ref = np.linspace(0, n_ref - 1, L).astype(int) idx_mov = np.linspace(0, n_mov - 1, L).astype(int) P = c_mov[idx_mov] Q = c_ref[idx_ref] H = P.T @ Q U, S, Vt = np.linalg.svd(H) d = np.sign(np.linalg.det(Vt.T @ U.T)) diag = np.diag([1.0, 1.0, d]) R = Vt.T @ diag @ U.T aligned_mov = c_mov @ R diff = (P @ R) - Q rmsd = float(np.sqrt(np.mean(np.sum(diff ** 2, axis=1)))) return aligned_mov, rmsd def run_single_protein_quantization_pipeline( dataset: str, model_scale: str, protein_id: str, codec_id: str, transform_name: str = "circulant", custom_sequence: Optional[str] = None, ) -> Dict[str, Any]: """ Executes the complete mathematical and binary serialization pipeline on a real ESM-2 protein embedding vector and returns all intermediate states. """ emb_matrix = STATE.get_embeddings(dataset, model_scale) N, D = emb_matrix.shape matched_note = None if custom_sequence: seq_clean = re.sub(r"[^A-Za-z]", "", custom_sequence).upper() entries = STATE.scope_entries if dataset == "scope" else STATE.sp_entries best_idx = 0 best_score = -1.0 # Exact or substring or 3-mer Jaccard match against real cached sequences kmer_q = set(seq_clean[i : i + 3] for i in range(max(1, len(seq_clean) - 2))) for idx, e in enumerate(entries): s = e["sequence"] if s == seq_clean: best_idx = idx best_score = 1.0 matched_note = f"Exact sequence match in {dataset.upper()} ({e['id']})" break elif seq_clean in s or s in seq_clean: score = min(len(seq_clean), len(s)) / max(len(seq_clean), len(s)) if score > best_score: best_score = score best_idx = idx matched_note = f"Subsequence match ({score*100:.1f}%) to {e['id']}" if best_score < 0.5 and len(kmer_q) > 0: for idx in range(0, len(entries), max(1, len(entries) // 1500)): s = entries[idx]["sequence"] kmer_s = set(s[i : i + 3] for i in range(max(1, len(s) - 2))) sc = len(kmer_q & kmer_s) / max(1, len(kmer_q | kmer_s)) if sc > best_score: best_score = sc best_idx = idx matched_note = f"Nearest real {dataset.upper()} sequence homolog ({entries[best_idx]['id']}, 3-mer Jaccard {best_score:.3f})" raw_idx = best_idx meta = dict(entries[raw_idx]) else: if dataset == "scope": raw_idx = STATE.scope_id_to_raw_idx.get(protein_id) if raw_idx is None: # Try case-insensitive or PDB match pid_low = protein_id.lower() for e in STATE.scope_entries: if e["id"].lower() == pid_low or e["pdb_id"].lower() == pid_low: raw_idx = e["raw_idx"] break if raw_idx is None: raw_idx = STATE.p1_query_raw_ids[0] meta = dict(STATE.scope_entries[raw_idx]) else: raw_idx = STATE.sp_acc_to_raw_idx.get(protein_id) if raw_idx is None: pid_up = protein_id.upper() for e in STATE.sp_entries: if e["id"].upper() == pid_up or e["gene"].upper() == pid_up or e["entry_name"].upper() == pid_up: raw_idx = e["raw_idx"] break if raw_idx is None: raw_idx = STATE.p5_query_raw_ids[0] meta = dict(STATE.sp_entries[raw_idx]) x = emb_matrix[raw_idx].astype(np.float32) x_2d = x.reshape(1, D) # Instantiate selected codec codec = STATE.create_codec( codec_id=codec_id, dimension=D, transform_name=transform_name, dataset=dataset, model_scale=model_scale, ) codec_info = next((c for c in CODEC_CATALOG if c["id"] == codec_id), CODEC_CATALOG[0]) k = getattr(codec, "k", getattr(codec, "max_outliers", codec_info["k"])) # Stage 0: Native Vector Diagnostics sq_energy = x ** 2 total_energy = float(np.sum(sq_energy)) order_desc = np.argsort(- np.abs(x)) top16_indices = order_desc[:16] top16_outliers = [ { "rank": r + 1, "coord_index": int(idx), "value": float(x[idx]), "fp16_value": float(np.float16(x[idx])), "energy_fraction": float((x[idx] ** 2) / max(total_energy, 1e-12)), } for r, idx in enumerate(top16_indices) ] # Single-vector participation ratio & kurtosis pr_vec = float((total_energy ** 2) / max(float(np.sum(sq_energy ** 2)), 1e-12)) x_mean = float(np.mean(x)) x_var = float(np.var(x)) kurt_native = float(np.mean(((x - x_mean) ** 4)) / (x_var ** 2) - 3.0) if x_var > 0 else 0.0 # Stage 1: Outlier Exemption & Residual Trunk Isolation if k > 0: if hasattr(codec, "protected_dims"): exempt_idx = np.sort(codec.protected_dims[:k]).astype(np.int64) else: exempt_idx = np.sort(order_desc[:k]).astype(np.int64) exempt_fp16 = x[exempt_idx].astype(np.float16) exempt_vals = exempt_fp16.astype(np.float32) z = x.copy() z[exempt_idx] = 0.0 outlier_energy = float(np.sum(exempt_vals ** 2)) pythagorean_res_norm = float(math.sqrt(max(1.0 - outlier_energy, 0.0))) else: exempt_idx = np.array([], dtype=np.int64) exempt_vals = np.array([], dtype=np.float32) z = x.copy() outlier_energy = 0.0 pythagorean_res_norm = float(np.linalg.norm(z)) true_res_norm = float(np.linalg.norm(z)) # Stage 2: Orthogonal Rotation (y = R * z) transform = getattr(codec, "transform", get_transform(transform_name, dimension=D, seed=42)) y_rot = transform.forward(z.reshape(1, D))[0].astype(np.float32) z_inv = transform.inverse(y_rot.reshape(1, D))[0].astype(np.float32) inv_err = float(np.max(np.abs(z_inv - z))) y_mean = float(np.mean(y_rot)) y_var = float(np.var(y_rot)) kurt_rotated = float(np.mean(((y_rot - y_mean) ** 4)) / (y_var ** 2) - 3.0) if y_var > 0 else 0.0 # Stage 3 & Stage 4: True Wire Encoding & Decoding t0_enc = time.perf_counter() packed_bytes = codec.encode(x_2d) enc_us = (time.perf_counter() - t0_enc) * 1e6 if hasattr(codec, "decode"): x_recon = codec.decode(packed_bytes, count=1)[0].astype(np.float32) else: # Reconstruct from asymmetric score against identity basis eye = np.eye(D, dtype=np.float32) x_recon = codec.compute_asymmetric_scores(eye, packed_bytes, count=1)[:, 0].astype(np.float32) # Compute quantized trunk in rotated domain for visualization x_recon_trunk = x_recon.copy() if len(exempt_idx) > 0: x_recon_trunk[exempt_idx] = 0.0 y_quant_rot = transform.forward(x_recon_trunk.reshape(1, D))[0].astype(np.float32) # Compute bleed energy that would have leaked onto S_i without Theorem 1 signs_rot = np.where(y_rot >= 0, 1.0, -1.0).astype(np.float32) / math.sqrt(D) u_unmasked = transform.inverse(signs_rot.reshape(1, D))[0] bleed_on_support = float(np.sum(u_unmasked[exempt_idx] ** 2)) if len(exempt_idx) > 0 else 0.0 # Self-score and reconstruction fidelity asym_self_score = float(codec.compute_asymmetric_scores(x_2d, packed_bytes, count=1)[0, 0]) recon_mse = float(np.mean((x - x_recon) ** 2)) cosine_sim = float(np.dot(x, x_recon) / max(float(np.linalg.norm(x) * np.linalg.norm(x_recon)), 1e-12)) # Hex wire dump and byte segments hex_bytes = [f"{b:02x}" for b in packed_bytes] total_bytes = len(packed_bytes) idx_bytes = int(k * 2) if k > 0 and codec_id != "calibrated_native_protection_k32" else 0 val_bytes = int(k * 2) if k > 0 else 0 trunk_bytes = total_bytes - idx_bytes - val_bytes segments = [] offset = 0 if idx_bytes > 0: segments.append({ "name": f"Outlier Indices (uint16 × {k})", "start": offset, "end": offset + idx_bytes, "bytes": idx_bytes, "color": "outlier-idx", "detail": f"Support S_i = {[int(i) for i in exempt_idx]}", }) offset += idx_bytes if val_bytes > 0: segments.append({ "name": f"Outlier Values (FP16 × {k})", "start": offset, "end": offset + val_bytes, "bytes": val_bytes, "color": "outlier-val", "detail": f"Exact FP16 amplitudes {[round(float(v), 4) for v in exempt_vals[:8]]}", }) offset += val_bytes segments.append({ "name": f"Compressed Rotated Trunk ({codec_info['trunk_mode']})", "start": offset, "end": total_bytes, "bytes": trunk_bytes, "color": "trunk-bits", "detail": f"{trunk_bytes} B packed trunk ({trunk_bytes * 8 / D:.3f} bits/dim)", }) return { "dataset": dataset, "model_scale": model_scale, "dimension": D, "matched_note": matched_note, "protein": meta, "codec": { **codec_info, "transform_name": transform_name, "actual_bytes": total_bytes, "fp32_bytes": D * 4, "actual_compression": round((D * 4) / total_bytes, 2), "encode_time_us": round(enc_us, 1), }, "stage0_native": { "vector": [round(float(v), 5) for v in x], "l2_norm": round(float(np.linalg.norm(x)), 6), "max_abs": round(float(np.max(np.abs(x))), 5), "participation_ratio": round(pr_vec, 2), "pr_ratio": round(pr_vec / D, 4), "kurtosis": round(kurt_native, 2), "top16_outliers": top16_outliers, "top_k_energy_share": { "k1": round(float(np.sum(np.sort(sq_energy)[::-1][:1]) / total_energy), 4), "k2": round(float(np.sum(np.sort(sq_energy)[::-1][:2]) / total_energy), 4), "k4": round(float(np.sum(np.sort(sq_energy)[::-1][:4]) / total_energy), 4), "k8": round(float(np.sum(np.sort(sq_energy)[::-1][:8]) / total_energy), 4), "k16": round(float(np.sum(np.sort(sq_energy)[::-1][:16]) / total_energy), 4), "k32": round(float(np.sum(np.sort(sq_energy)[::-1][:32]) / total_energy), 4), }, }, "stage1_outliers": { "k": int(k), "exempt_indices": [int(i) for i in exempt_idx], "exempt_fp16_values": [round(float(v), 5) for v in exempt_vals], "outlier_energy_fraction": round(outlier_energy, 5), "pythagorean_residual_norm": round(pythagorean_res_norm, 6), "true_residual_norm": round(true_res_norm, 6), "norm_recovery_error": round(abs(pythagorean_res_norm - true_res_norm), 7), "residual_trunk_vector": [round(float(v), 5) for v in z], "residual_max_abs": round(float(np.max(np.abs(z))), 5), }, "stage2_rotated": { "transform_name": transform_name, "rotated_vector": [round(float(v), 5) for v in y_rot], "max_abs": round(float(np.max(np.abs(y_rot))), 5), "kurtosis_after_rotation": round(kurt_rotated, 3), "invertibility_max_error": float(inv_err), }, "stage3_quantized": { "quantized_rotated_trunk": [round(float(v), 5) for v in y_quant_rot], "reconstructed_vector": [round(float(v), 5) for v in x_recon], "rho_q": codec_info["rho_q"], "bussgang_beta": codec_info["bussgang_beta"], "bleed_energy_eliminated": round(bleed_on_support, 6), "asymmetric_self_score": round(asym_self_score, 5), "reconstruction_mse": float(recon_mse), "cosine_similarity": round(cosine_sim, 5), }, "stage4_wire": { "total_bytes": total_bytes, "fp32_bytes": D * 4, "compression_ratio": round((D * 4) / total_bytes, 2), "hex_bytes": hex_bytes, "segments": segments, }, } def run_live_search( dataset: str, model_scale: str, query_id: str, codec_id: str, top_k: int = 30, ) -> Dict[str, Any]: """ Executes live asymmetric search across all 15,166 SCOPe domains or 13,003 Swiss-Prot proteins and compares QuantPPI against Exact FP32, Unprotected 1-bit Circulant, and Foldseek. """ codec, packed_bytes, cand_emb, enc_time = STATE.get_encoded_candidates(dataset, model_scale, codec_id) _, baseline_bytes, _, _ = STATE.get_encoded_candidates(dataset, model_scale, "circulant_1bit") baseline_codec = STATE._encoded_db_cache[(dataset, model_scale, "circulant_1bit")][0] raw_emb = STATE.get_embeddings(dataset, model_scale) N, D = cand_emb.shape if dataset == "scope": raw_idx = STATE.scope_id_to_raw_idx.get(query_id, STATE.p1_query_raw_ids[0]) query_meta = STATE.scope_entries[raw_idx] cand_pos_self = STATE.scope_sid_to_p1_pos.get(query_meta["id"], -1) else: raw_idx = STATE.sp_acc_to_raw_idx.get(query_id, STATE.p5_query_raw_ids[0]) query_meta = STATE.sp_entries[raw_idx] cand_pos_self = STATE.sp_acc_to_cand_pos.get(query_meta["id"], -1) q_vec = raw_emb[raw_idx : raw_idx + 1] # 1. Exact FP32 search t0_fp32 = time.perf_counter() exact_scores = (q_vec @ cand_emb.T)[0].copy() fp32_ms = (time.perf_counter() - t0_fp32) * 1000.0 # 2. Selected QuantPPI codec asymmetric search t0_q = time.perf_counter() pred_scores = codec.compute_asymmetric_scores(q_vec, packed_bytes, count=N)[0].copy() quant_ms = (time.perf_counter() - t0_q) * 1000.0 # 3. Unprotected 1-bit Circulant baseline search base_scores = baseline_codec.compute_asymmetric_scores(q_vec, baseline_bytes, count=N)[0].copy() # Mask self-match if query is in candidate pool valid_mask = np.ones(N, dtype=bool) if 0 <= cand_pos_self < N: valid_mask[cand_pos_self] = False ip_mse = float(np.mean((pred_scores[valid_mask] - exact_scores[valid_mask]) ** 2)) base_mse = float(np.mean((base_scores[valid_mask] - exact_scores[valid_mask]) ** 2)) if 0 <= cand_pos_self < N: exact_scores[cand_pos_self] = -1e9 pred_scores[cand_pos_self] = -1e9 base_scores[cand_pos_self] = -1e9 gt_order = np.argsort(-exact_scores, kind="stable") pred_order = np.argsort(-pred_scores, kind="stable") base_order = np.argsort(-base_scores, kind="stable") gt_rank_of = np.empty(N, dtype=np.int32) gt_rank_of[gt_order] = np.arange(1, N + 1) base_rank_of = np.empty(N, dtype=np.int32) base_rank_of[base_order] = np.arange(1, N + 1) gt_top20 = set(gt_order[:20].tolist()) r20_30_quant = len(set(pred_order[:30].tolist()) & gt_top20) / 20.0 r20_30_base = len(set(base_order[:30].tolist()) & gt_top20) / 20.0 r1_quant = 1.0 if pred_order[0] == gt_order[0] else 0.0 r10_quant = len(set(pred_order[:10].tolist()) & set(gt_order[:10].tolist())) / 10.0 r20_quant = len(set(pred_order[:20].tolist()) & gt_top20) / 20.0 # Spearman rank correlation on top 250 ground-truth pool pool_idx = gt_order[:250] rk_p = np.argsort(np.argsort(pred_scores[pool_idx])) rk_t = np.argsort(np.argsort(exact_scores[pool_idx])) n_p = len(pool_idx) spearman_rho = float(1.0 - (6.0 * np.sum((rk_p - rk_t) ** 2)) / (n_p * (n_p ** 2 - 1))) # Biological evaluation on this query bio_metrics: Dict[str, Any] = {} hits_table: List[Dict[str, Any]] = [] if dataset == "scope": q_fam = query_meta["family"] q_sfam = query_meta["superfamily"] q_fold = query_meta["fold"] order_no_self = pred_order[pred_order != cand_pos_self] if cand_pos_self >= 0 else pred_order gt_no_self = gt_order[gt_order != cand_pos_self] if cand_pos_self >= 0 else gt_order base_no_self = base_order[base_order != cand_pos_self] if cand_pos_self >= 0 else base_order is_fam_q = STATE.scope_cand_fam[order_no_self] == q_fam is_fam_gt = STATE.scope_cand_fam[gt_no_self] == q_fam is_fam_base = STATE.scope_cand_fam[base_no_self] == q_fam is_sfam_q = (STATE.scope_cand_sfam[order_no_self] == q_sfam) & (~is_fam_q) is_sfam_gt = (STATE.scope_cand_sfam[gt_no_self] == q_sfam) & (~is_fam_gt) bio_metrics = { "query_sccs": query_meta["sccs"], "family_homolog_count": int(np.sum(is_fam_q)), "superfamily_remote_count": int(np.sum(is_sfam_q)), "quant_family_ap": round(compute_average_precision(is_fam_q), 4), "fp32_family_ap": round(compute_average_precision(is_fam_gt), 4), "base_family_ap": round(compute_average_precision(is_fam_base), 4), "quant_superfamily_roc50": round(compute_roc50(is_sfam_q[STATE.scope_cand_fam[order_no_self] != q_fam]), 4), "fp32_superfamily_roc50": round(compute_roc50(is_sfam_gt[STATE.scope_cand_fam[gt_no_self] != q_fam]), 4), } fs_hits = STATE.foldseek_by_query.get(query_meta["id"], []) fs_map = {h["target_id"]: h for h in fs_hits} for rank_idx in range(min(top_k, len(order_no_self))): c_pos = int(order_no_self[rank_idx]) c_meta = STATE.scope_p1_candidates[c_pos] if c_meta["family"] == q_fam: tier = "Same Family" elif c_meta["superfamily"] == q_sfam: tier = "Same Superfamily (Remote)" elif c_meta["fold"] == q_fold: tier = "Same Fold" else: tier = "Different Fold" fs_entry = fs_map.get(c_meta["id"]) hits_table.append({ "quant_rank": rank_idx + 1, "fp32_rank": int(gt_rank_of[c_pos]), "base_1bit_rank": int(base_rank_of[c_pos]), "id": c_meta["id"], "pdb_id": c_meta["pdb_id"], "sccs": c_meta["sccs"], "description": c_meta["description"], "organism": c_meta["organism"], "seq_len": c_meta["seq_len"], "has_pdb": c_meta["has_pdb"], "homology_tier": tier, "quant_score": round(float(pred_scores[c_pos]), 5), "fp32_score": round(float(exact_scores[c_pos]), 5), "score_error": round(float(pred_scores[c_pos] - exact_scores[c_pos]), 5), "in_fp32_top20": int(gt_rank_of[c_pos]) <= 20, "foldseek": fs_entry, "mini_trace": get_mini_ca_trace(c_meta["id"]) if (rank_idx < 12 and c_meta["has_pdb"]) else None, "seq_preview": c_meta.get("sequence", "")[:120], }) else: q_go = set(query_meta.get("go_terms", [])) jacc_quant = [] jacc_fp32 = [] jacc_base = [] for idx in range(min(top_k, len(pred_order))): c_pos = int(pred_order[idx]) c_meta = STATE.sp_p5_candidates[c_pos] c_go = set(c_meta.get("go_terms", [])) shared_go = sorted(list(q_go & c_go)) jac = compute_go_jaccard(q_go, c_go) jacc_quant.append(jac) gt_c_go = set(STATE.sp_p5_candidates[int(gt_order[idx])].get("go_terms", [])) jacc_fp32.append(compute_go_jaccard(q_go, gt_c_go)) bs_c_go = set(STATE.sp_p5_candidates[int(base_order[idx])].get("go_terms", [])) jacc_base.append(compute_go_jaccard(q_go, bs_c_go)) hits_table.append({ "quant_rank": idx + 1, "fp32_rank": int(gt_rank_of[c_pos]), "base_1bit_rank": int(base_rank_of[c_pos]), "id": c_meta["id"], "gene": c_meta["gene"], "entry_name": c_meta["entry_name"], "description": c_meta["description"], "seq_len": c_meta["seq_len"], "go_jaccard": round(jac, 4), "shared_go_terms": shared_go[:8], "total_shared_go": len(shared_go), "candidate_go_count": len(c_go), "quant_score": round(float(pred_scores[c_pos]), 5), "fp32_score": round(float(exact_scores[c_pos]), 5), "score_error": round(float(pred_scores[c_pos] - exact_scores[c_pos]), 5), "in_fp32_top20": int(gt_rank_of[c_pos]) <= 20, "seq_preview": c_meta.get("sequence", "")[:120], }) bio_metrics = { "query_go_count": len(q_go), "quant_top1_go_jaccard": round(float(jacc_quant[0]) if jacc_quant else 0.0, 4), "quant_top5_go_jaccard": round(float(np.mean(jacc_quant[:5])) if jacc_quant else 0.0, 4), "quant_top20_go_jaccard": round(float(np.mean(jacc_quant[:20])) if jacc_quant else 0.0, 4), "fp32_top1_go_jaccard": round(float(jacc_fp32[0]) if jacc_fp32 else 0.0, 4), "fp32_top5_go_jaccard": round(float(np.mean(jacc_fp32[:5])) if jacc_fp32 else 0.0, 4), "base_top1_go_jaccard": round(float(jacc_base[0]) if jacc_base else 0.0, 4), } # Scatter plot points (top 60 + 60 evenly spaced across rank distribution) sample_positions = np.concatenate([ gt_order[:60], gt_order[np.linspace(60, N - 2, 60).astype(int)], ]) scatter_points = [ { "id": (STATE.scope_p1_candidates[int(p)]["id"] if dataset == "scope" else STATE.sp_p5_candidates[int(p)]["id"]), "fp32_score": round(float(exact_scores[int(p)]), 5), "quant_score": round(float(pred_scores[int(p)]), 5), "base_1bit_score": round(float(base_scores[int(p)]), 5), "is_top20": int(gt_rank_of[int(p)]) <= 20, } for p in sample_positions ] codec_info = next((c for c in CODEC_CATALOG if c["id"] == codec_id), CODEC_CATALOG[0]) actual_bytes = getattr(codec, "total_bytes_per_vector", D * 4) query_out = { **query_meta, "seq_preview": query_meta.get("sequence", "")[:180], "mini_trace": get_mini_ca_trace(query_meta["id"]) if dataset == "scope" else None, } return { "dataset": dataset, "model_scale": model_scale, "dimension": D, "num_candidates": N, "query": query_out, "codec": { **codec_info, "actual_bytes": actual_bytes, "db_size_mb": round((N * actual_bytes) / (1024 * 1024), 2), "fp32_db_size_mb": round((N * D * 4) / (1024 * 1024), 2), }, "metrics": { "recall_20_at_30": round(r20_30_quant, 4), "base_1bit_recall_20_at_30": round(r20_30_base, 4), "recall_1": round(r1_quant, 4), "recall_10": round(r10_quant, 4), "recall_20": round(r20_quant, 4), "inner_product_mse": float(ip_mse), "base_1bit_mse": float(base_mse), "mse_reduction_vs_1bit": round(base_mse / max(ip_mse, 1e-12), 1), "spearman_rho": round(spearman_rho, 5), "search_latency_ms": round(quant_ms, 3), "fp32_latency_ms": round(fp32_ms, 3), }, "biological": bio_metrics, "hits": hits_table, "scatter_points": scatter_points, "foldseek_top20": STATE.foldseek_by_query.get(query_meta["id"], []) if dataset == "scope" else [], } def run_live_theorem_ablation( query_id: str, k: int = 4, model_scale: str = "esm2_650m", ) -> Dict[str, Any]: """ Runs a live 2x2 factorial ablation of Theorem 1 (Support-Orthogonal Bleed Elimination) and Theorem 2 (Analytical Bussgang De-Attenuation) on the chosen real SCOPe protein. """ raw_emb = STATE.get_embeddings("scope", model_scale) cand_emb = raw_emb[STATE.p1_cand_raw_ids] N, D = cand_emb.shape raw_idx = STATE.scope_id_to_raw_idx.get(query_id, STATE.p1_query_raw_ids[0]) query_meta = STATE.scope_entries[raw_idx] cand_pos_self = STATE.scope_sid_to_p1_pos.get(query_meta["id"], 0) q_vec = raw_emb[raw_idx : raw_idx + 1] exact_scores = (q_vec @ cand_emb.T)[0].copy() valid_mask = np.ones(N, dtype=bool) if 0 <= cand_pos_self < N: valid_mask[cand_pos_self] = False exact_ranked = exact_scores.copy() if 0 <= cand_pos_self < N: exact_ranked[cand_pos_self] = -1e9 gt_order = np.argsort(-exact_ranked, kind="stable") gt_top20 = set(gt_order[:20].tolist()) sample_idx = np.concatenate([gt_order[:40], gt_order[np.linspace(40, N - 2, 40).astype(int)]]) arms_spec = [ ("naive", "1. Naive Hybrid (No Thm 1, No Thm 2)", False, False), ("thm2_only", "2. Theorem 2 Only (Bussgang Scaling Only)", False, True), ("thm1_only", "3. Theorem 1 Only (Bleed Elimination Only)", True, False), ("both", "4. Full QuantPPI (Theorem 1 + Theorem 2)", True, True), ] # Encode once (all 4 arms use the exact same wire buffer!) base_codec = DynamicTopKOutlierCodec(dimension=D, k=k, transform_name="circulant", seed=42) cache_key = ("scope", model_scale, f"ablation_wire_k{k}") if cache_key in STATE._encoded_db_cache: _, packed_bytes, _ = STATE._encoded_db_cache[cache_key] else: packed_bytes = base_codec.encode(cand_emb) STATE._encoded_db_cache[cache_key] = (base_codec, packed_bytes, 0.0) q_fam = query_meta["family"] arms_out = [] for arm_id, label, thm1, thm2 in arms_spec: arm_codec = DynamicTopKOutlierCodec( dimension=D, k=k, transform_name="circulant", seed=42, enable_bleed_elimination=thm1, enable_bussgang_scaling=thm2, ) scores = arm_codec.compute_asymmetric_scores(q_vec, packed_bytes, count=N)[0].copy() mse = float(np.mean((scores[valid_mask] - exact_scores[valid_mask]) ** 2)) mae = float(np.mean(np.abs(scores[valid_mask] - exact_scores[valid_mask]))) scores_ranked = scores.copy() if 0 <= cand_pos_self < N: scores_ranked[cand_pos_self] = -1e9 order = np.argsort(-scores_ranked, kind="stable") order_no_self = order[order != cand_pos_self] r20_30 = len(set(order[:30].tolist()) & gt_top20) / 20.0 r1 = 1.0 if order[0] == gt_order[0] else 0.0 r20 = len(set(order[:20].tolist()) & gt_top20) / 20.0 fam_ap = compute_average_precision(STATE.scope_cand_fam[order_no_self] == q_fam) arms_out.append({ "arm_id": arm_id, "label": label, "enable_bleed_elimination": thm1, "enable_bussgang_scaling": thm2, "wire_bytes": arm_codec.total_bytes_per_vector, "recall_20_at_30": round(r20_30, 4), "recall_1": round(r1, 4), "recall_20": round(r20, 4), "family_ap": round(fam_ap, 4), "inner_product_mse": float(mse), "inner_product_mae": float(mae), "scatter": [ { "fp32": round(float(exact_scores[int(idx)]), 5), "pred": round(float(scores[int(idx)]), 5), "is_top20": int(idx) in gt_top20, } for idx in sample_idx ], }) return { "query": query_meta, "model_scale": model_scale, "dimension": D, "k": k, "wire_bytes": base_codec.total_bytes_per_vector, "arms": arms_out, } class QuantPPIRequestHandler(BaseHTTPRequestHandler): def log_message(self, format: str, *args): # Concise request logging sys.stderr.write(f"[API] {self.address_string()} - {format % args}\n") def _send_json(self, payload: Any, status: int = 200): body = json.dumps(payload).encode("utf-8") self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") self.end_headers() self.wfile.write(body) def do_OPTIONS(self): self.send_response(204) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") self.end_headers() def do_GET(self): parsed = urllib.parse.urlparse(self.path) path = parsed.path qs = urllib.parse.parse_qs(parsed.query) def _p(name: str, default: str = "") -> str: vals = qs.get(name) return vals[0] if vals else default try: if path == "/api/overview": dataset = _p("dataset", "scope") model_scale = _p("model_scale", "esm2_650m") geom = STATE.get_corpus_geometry(dataset, model_scale) self._send_json({ "geometry": geom, "codecs": CODEC_CATALOG, "datasets": [ { "id": "scope", "name": "SCOPe 2.08 ASTRAL-40 Structural Domains", "total_sequences": len(STATE.scope_entries), "candidates": len(STATE.p1_cand_raw_ids), "canonical_queries": len(STATE.p1_query_raw_ids), "pdb_structures": 15118, "source_fasta": "data/astral-scopedom-seqres-gd-sel-gs-bib-40-2.08.fa", }, { "id": "swissprot", "name": "UniProtKB/Swiss-Prot Human Proteome", "total_sequences": len(STATE.sp_entries), "candidates": len(STATE.p5_cand_raw_ids), "canonical_queries": len(STATE.p5_query_raw_ids), "calibration_pool": len(STATE.p5_calib_raw_ids), "source_fasta": "data/swissprot_human_full.fasta", }, ], "model_scales": [ {"id": "esm2_650m", "name": "ESM-2 650M (33-layer, D=1280)", "dimension": 1280, "fp32_bytes": 5120}, {"id": "esm2_150m", "name": "ESM-2 150M (30-layer, D=640)", "dimension": 640, "fp32_bytes": 2560}, {"id": "esm2_35m", "name": "ESM-2 35M (12-layer, D=480)", "dimension": 480, "fp32_bytes": 1920}, ], }) return if path == "/api/proteins": dataset = _p("dataset", "scope") q = _p("q", "").strip().lower() queries_only = _p("queries_only", "false").lower() == "true" limit = min(int(_p("limit", "40")), 200) offset = max(int(_p("offset", "0")), 0) if dataset == "scope": source = [STATE.scope_p1_candidates[i] for i in range(len(STATE.p1_query_raw_ids))] if queries_only else STATE.scope_p1_candidates if q: filtered = [ e for e in source if q in e["id"].lower() or q in e["pdb_id"].lower() or q in e["sccs"].lower() or q in e["description"].lower() or q in e["organism"].lower() or (len(q) >= 4 and q.upper() in e["sequence"]) ] else: filtered = source else: source = STATE.sp_p5_queries if queries_only else STATE.sp_entries if q: filtered = [ e for e in source if q in e["id"].lower() or q in e["gene"].lower() or q in e["entry_name"].lower() or q in e["description"].lower() or any(q in go.lower() for go in e.get("go_terms", [])) or (len(q) >= 4 and q.upper() in e["sequence"]) ] else: filtered = source slice_items = [ {k: v for k, v in item.items() if k != "sequence"} for item in filtered[offset : offset + limit] ] self._send_json({ "dataset": dataset, "total_matches": len(filtered), "items": slice_items, }) return if path == "/api/quantize": dataset = _p("dataset", "scope") model_scale = _p("model_scale", "esm2_650m") protein_id = _p("protein_id", "d6a1ia1" if dataset == "scope" else "P04637") codec_id = _p("codec_id", "track3_hybrid_k2_block224") transform_name = _p("transform_name", "circulant") custom_seq = _p("custom_sequence", "") or None res = run_single_protein_quantization_pipeline( dataset=dataset, model_scale=model_scale, protein_id=protein_id, codec_id=codec_id, transform_name=transform_name, custom_sequence=custom_seq, ) self._send_json(res) return if path == "/api/search": dataset = _p("dataset", "scope") model_scale = _p("model_scale", "esm2_650m") query_id = _p("query_id", "d1dlwa_" if dataset == "scope" else "P68871") codec_id = _p("codec_id", "track3_hybrid_k2_block224") top_k = min(int(_p("top_k", "30")), 100) res = run_live_search( dataset=dataset, model_scale=model_scale, query_id=query_id, codec_id=codec_id, top_k=top_k, ) self._send_json(res) return if path == "/api/ablation": query_id = _p("query_id", "d1dlwa_") k = int(_p("k", "4")) model_scale = _p("model_scale", "esm2_650m") res = run_live_theorem_ablation(query_id=query_id, k=k, model_scale=model_scale) self._send_json(res) return if path == "/api/structure": dom_id = _p("id", "d6a1ia1").strip() compare_id = _p("compare_id", "").strip() ent_path = PDB_DIR / f"{dom_id}.ent" if not has_pdb_structure(dom_id): self._send_json({"error": f"3D PDB structure not found for {dom_id}"}, status=404) return primary = parse_pdb_ca_atoms(ent_path) raw_idx = STATE.scope_id_to_raw_idx.get(dom_id) primary["metadata"] = STATE.scope_entries[raw_idx] if raw_idx is not None else {"id": dom_id} comparison = None if compare_id and has_pdb_structure(compare_id): comp_path = PDB_DIR / f"{compare_id}.ent" comparison = parse_pdb_ca_atoms(comp_path) c_raw_idx = STATE.scope_id_to_raw_idx.get(compare_id) comparison["metadata"] = STATE.scope_entries[c_raw_idx] if c_raw_idx is not None else {"id": compare_id} if primary["ca_atoms"] and comparison["ca_atoms"]: ref_xyz = np.array([[a["cx"], a["cy"], a["cz"]] for a in primary["ca_atoms"]], dtype=np.float64) mov_xyz = np.array([[a["cx"], a["cy"], a["cz"]] for a in comparison["ca_atoms"]], dtype=np.float64) aligned_xyz, rmsd = kabsch_superpose(ref_xyz, mov_xyz) for i, a in enumerate(comparison["ca_atoms"]): a["ax"] = round(float(aligned_xyz[i, 0]), 3) a["ay"] = round(float(aligned_xyz[i, 1]), 3) a["az"] = round(float(aligned_xyz[i, 2]), 3) comparison["kabsch_rmsd_angstrom"] = round(rmsd, 3) self._send_json({ "primary": primary, "comparison": comparison, }) return if path == "/api/benchmarks": self._send_json(STATE.benchmarks_payload) return # Serve static frontend from dist/ if available dist_dir = THIS_DIR / "dist" if dist_dir.exists(): rel = path.lstrip("/") or "index.html" file_path = dist_dir / rel if not file_path.exists() or file_path.is_dir(): file_path = dist_dir / "index.html" if file_path.exists(): ext = file_path.suffix.lower() mime = { ".html": "text/html; charset=utf-8", ".js": "application/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".svg": "image/svg+xml", ".json": "application/json", ".png": "image/png", }.get(ext, "application/octet-stream") data = file_path.read_bytes() self.send_response(200) self.send_header("Content-Type", mime) self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) return self._send_json({"error": f"Not found: {path}"}, status=404) except Exception as e: import traceback traceback.print_exc() self._send_json({"error": str(e)}, status=500) def do_POST(self): parsed = urllib.parse.urlparse(self.path) path = parsed.path length = int(self.headers.get("Content-Length", "0")) raw_body = self.rfile.read(length).decode("utf-8") if length > 0 else "{}" try: body = json.loads(raw_body) if path == "/api/quantize": res = run_single_protein_quantization_pipeline( dataset=body.get("dataset", "scope"), model_scale=body.get("model_scale", "esm2_650m"), protein_id=body.get("protein_id", "d6a1ia1"), codec_id=body.get("codec_id", "track3_hybrid_k2_block224"), transform_name=body.get("transform_name", "circulant"), custom_sequence=body.get("custom_sequence"), ) self._send_json(res) return self._send_json({"error": f"Unknown POST endpoint: {path}"}, status=404) except Exception as e: import traceback traceback.print_exc() self._send_json({"error": str(e)}, status=500) def main(): import argparse parser = argparse.ArgumentParser(description="QuantPPI Real-Data Web Server") parser.add_argument("--host", default=os.environ.get("HOST", "0.0.0.0")) parser.add_argument("--port", type=int, default=int(os.environ.get("PORT", "8000"))) args = parser.parse_args() server = ThreadingHTTPServer((args.host, args.port), QuantPPIRequestHandler) print(f"[QuantPPI Server] Listening on http://{args.host}:{args.port}") try: server.serve_forever() except KeyboardInterrupt: print("\n[QuantPPI Server] Shutting down.") server.server_close() if __name__ == "__main__": main()
#!/usr/bin/env python3
"""
QuantPPI End-to-End Real-Data Backend Server
============================================
Serves real ESM-2 (650M, 150M, 35M) protein embeddings from HDF5 caches,
real SCOPe ASTRAL-40 v2.08 (15,166 domains + 15,118 3D PDB .ent structures),
real Swiss-Prot Human proteome (18,099 proteins + Gene Ontology annotations),
live Campaign 4 data-oblivious quantization codecs (168B to 488B),
live Theorem 1 & Theorem 2 2x2 factorial ablations, and the complete 56-method
QuantPPI vs. Foldseek benchmark ledgers.
"""
import os
import sys
import re
import json
import time
import math
import urllib.parse
from pathlib import Path
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Dict, List, Any, Optional, Tuple
os.environ.setdefault("OMP_NUM_THREADS", "1")
os.environ.setdefault("MKL_NUM_THREADS", "1")
os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1")
os.environ.setdefault("MPLCONFIGDIR", "/tmp")
import h5py
import numpy as np
import pandas as pd
THIS_DIR = Path(__file__).resolve().parent
REPO_ROOT = THIS_DIR.parent
CAMPAIGN4_DIR = REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_4_breakthrough"
FOLDSEEK_RES_DIR = REPO_ROOT / "experiments" / "foldseek_comparison" / "results"
DATA_DIR = REPO_ROOT / "data"
CACHE_DIR = REPO_ROOT / "cache"
PDB_DIR = DATA_DIR / "scope40_pdb" / "candidates_15118_model1"
PDB_H5_PATH = CACHE_DIR / "scope40_ca_structures_15118.h5"
_H5_STRUCT_INDEX: Optional[Dict[str, int]] = None
def _get_h5_struct_index() -> Dict[str, int]:
global _H5_STRUCT_INDEX
if _H5_STRUCT_INDEX is not None:
return _H5_STRUCT_INDEX
_H5_STRUCT_INDEX = {}
if PDB_H5_PATH.exists():
try:
with h5py.File(PDB_H5_PATH, "r") as f:
dom_ids = f["domain_ids"][:]
for idx, raw_id in enumerate(dom_ids):
did = raw_id.decode("ascii", errors="ignore").strip() if isinstance(raw_id, (bytes, np.bytes_)) else str(raw_id).strip()
if did:
_H5_STRUCT_INDEX[did] = idx
except Exception as exc:
print(f"[QuantPPI Server] Warning reading {PDB_H5_PATH}: {exc}")
return _H5_STRUCT_INDEX
def has_pdb_structure(dom_id: str) -> bool:
if (PDB_DIR / f"{dom_id}.ent").exists():
return True
return dom_id in _get_h5_struct_index()
for p in [str(REPO_ROOT), str(CAMPAIGN4_DIR), str(CAMPAIGN4_DIR / "src")]:
if p not in sys.path:
sys.path.insert(0, p)
from src.codecs import (
DynamicTopKOutlierCodec,
AnalyticalSCMFCodec,
TernaryDeadZoneCodec,
MultiStageResidualCodec,
HybridTopKResidualCodec,
SphericalE8LatticeCodec,
HuberTailCompandingCodec,
SmoothCompandingCodec,
TwoStageSearchIndex,
)
from src.transforms.transforms import get_transform
from src.evaluators.retrieval import (
MetricCalculator,
ExactFP32Baseline,
Circulant1BitBaseline,
FWHT1BitBaseline,
Analytic2BitBaseline,
RaBitQ1BitBaseline,
TurboQuant1BitBaseline,
TurboQuant2BitBaseline,
CalibratedNativeProtectionBaseline,
CalibratedSCMFBaseline,
compute_geometry_diagnostics,
)
from src.evaluators.biological import (
compute_average_precision,
compute_roc50,