Spaces:
Configuration error
Configuration error
Download explainer/server.py from trizzhal9000/quantppi: direct link, hf CLI and curl.
- Browser
- Download file 81.9 kB
-
https://huggingface.co/spaces/trizzhal9000/quantppi/resolve/main/explainer/server.py
- Command line
-
hf download hf://spaces/trizzhal9000/quantppi/explainer/server.py
-
curl -L -o server.py https://huggingface.co/spaces/trizzhal9000/quantppi/resolve/main/explainer/server.py
81.9 kB
#!/usr/bin/env python3
"""
QuantPPI End-to-End Real-Data Backend Server
============================================
Serves real ESM-2 (650M, 150M, 35M) protein embeddings from HDF5 caches,
real SCOPe ASTRAL-40 v2.08 (15,166 domains + 15,118 3D PDB .ent structures),
real Swiss-Prot Human proteome (18,099 proteins + Gene Ontology annotations),
live Campaign 4 data-oblivious quantization codecs (168B to 488B),
live Theorem 1 & Theorem 2 2x2 factorial ablations, and the complete 56-method
QuantPPI vs. Foldseek benchmark ledgers.
"""
import os
import sys
import re
import json
import time
import math
import urllib.parse
from pathlib import Path
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Dict, List, Any, Optional, Tuple
os.environ.setdefault("OMP_NUM_THREADS", "1")
os.environ.setdefault("MKL_NUM_THREADS", "1")
os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1")
os.environ.setdefault("MPLCONFIGDIR", "/tmp")
import h5py
import numpy as np
import pandas as pd
THIS_DIR = Path(__file__).resolve().parent
REPO_ROOT = THIS_DIR.parent
CAMPAIGN4_DIR = REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_4_breakthrough"
FOLDSEEK_RES_DIR = REPO_ROOT / "experiments" / "foldseek_comparison" / "results"
DATA_DIR = REPO_ROOT / "data"
CACHE_DIR = REPO_ROOT / "cache"
PDB_DIR = DATA_DIR / "scope40_pdb" / "candidates_15118_model1"
PDB_H5_PATH = CACHE_DIR / "scope40_ca_structures_15118.h5"
_H5_STRUCT_INDEX: Optional[Dict[str, int]] = None
def _get_h5_struct_index() -> Dict[str, int]:
global _H5_STRUCT_INDEX
if _H5_STRUCT_INDEX is not None:
return _H5_STRUCT_INDEX
_H5_STRUCT_INDEX = {}
if PDB_H5_PATH.exists():
try:
with h5py.File(PDB_H5_PATH, "r") as f:
dom_ids = f["domain_ids"][:]
for idx, raw_id in enumerate(dom_ids):
did = raw_id.decode("ascii", errors="ignore").strip() if isinstance(raw_id, (bytes, np.bytes_)) else str(raw_id).strip()
if did:
_H5_STRUCT_INDEX[did] = idx
except Exception as exc:
print(f"[QuantPPI Server] Warning reading {PDB_H5_PATH}: {exc}")
return _H5_STRUCT_INDEX
def has_pdb_structure(dom_id: str) -> bool:
if (PDB_DIR / f"{dom_id}.ent").exists():
return True
return dom_id in _get_h5_struct_index()
for p in [str(REPO_ROOT), str(CAMPAIGN4_DIR), str(CAMPAIGN4_DIR / "src")]:
if p not in sys.path:
sys.path.insert(0, p)
from src.codecs import (
DynamicTopKOutlierCodec,
AnalyticalSCMFCodec,
TernaryDeadZoneCodec,
MultiStageResidualCodec,
HybridTopKResidualCodec,
SphericalE8LatticeCodec,
HuberTailCompandingCodec,
SmoothCompandingCodec,
TwoStageSearchIndex,
)
from src.transforms.transforms import get_transform
from src.evaluators.retrieval import (
MetricCalculator,
ExactFP32Baseline,
Circulant1BitBaseline,
FWHT1BitBaseline,
Analytic2BitBaseline,
RaBitQ1BitBaseline,
TurboQuant1BitBaseline,
TurboQuant2BitBaseline,
CalibratedNativeProtectionBaseline,
CalibratedSCMFBaseline,
compute_geometry_diagnostics,
)
from src.evaluators.biological import (
compute_average_precision,
compute_roc50,
compute_go_jaccard,
)
SCOP_CLASS_NAMES = {
"a": "All alpha proteins",
"b": "All beta proteins",
"c": "Alpha and beta proteins (a/b)",
"d": "Alpha and beta proteins (a+b)",
"e": "Multi-domain proteins (alpha and beta)",
"f": "Membrane and cell surface proteins and peptides",
"g": "Small proteins",
}
CODEC_CATALOG = [
{
"id": "track1_top2",
"name": "QuantPPI Top-2 + 1-bit (168 B)",
"short_name": "QuantPPI 168B Top-2",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 168,
"compression_1280": 30.48,
"k": 2,
"trunk_mode": "1bit",
"rho_q": 0.797885,
"bussgang_beta": 1.253314,
"description": "Dynamic per-vector Top-2 outlier exemption (uint16 indices + FP16 values) + 1-bit Circulant trunk with Theorem 1 bleed elimination and Theorem 2 Bussgang de-attenuation.",
},
{
"id": "track1_top4",
"name": "QuantPPI Top-4 + 1-bit (176 B)",
"short_name": "QuantPPI 176B Top-4",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 176,
"compression_1280": 29.09,
"k": 4,
"trunk_mode": "1bit",
"rho_q": 0.797885,
"bussgang_beta": 1.253314,
"description": "Dynamic per-vector Top-4 outlier exemption (16 B) + 1-bit Circulant trunk (160 B) with Pythagorean residual norm recovery and Theorems 1 & 2.",
},
{
"id": "track3_hybrid_k2_block224",
"name": "QuantPPI Block-Order Argmax Hybrid (224 B)",
"short_name": "QuantPPI 224B Block-Order",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 224,
"compression_1280": 22.86,
"k": 2,
"trunk_mode": "block_order",
"rho_q": 0.880710,
"bussgang_beta": 1.135448,
"description": "Top-2 outlier exemption (8 B) + 8D Block-Order-Statistic Argmax coding (3-bit argmax index + 8 sign bits per 8D block = 216 B). Surpasses Trained PQ M=224 and Calibrated Native Protection with zero training.",
},
{
"id": "track3_hybrid_k4_block256",
"name": "QuantPPI Block-Order 4-bit Hybrid (256 B)",
"short_name": "QuantPPI 256B Block-4b",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 256,
"compression_1280": 20.00,
"k": 4,
"trunk_mode": "block_order_4b",
"rho_q": 0.896700,
"bussgang_beta": 1.115200,
"description": "Top-4 outlier exemption (16 B) + 8D Block-Order 4-bit magnitude-tier trunk coding (240 B).",
},
{
"id": "track3_hybrid_k2_ternary",
"name": "QuantPPI Radix-3 Ternary Hybrid (264 B)",
"short_name": "QuantPPI 264B Ternary",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 264,
"compression_1280": 19.39,
"k": 2,
"trunk_mode": "ternary",
"rho_q": 0.880660,
"bussgang_beta": 1.135512,
"description": "Top-2 outlier exemption (8 B) + symmetric dead-zone ternary {-1, 0, +1} trunk packed at 5 trits per byte (3^5 = 243 < 256, 256 B trunk).",
},
{
"id": "track3_hybrid_k2_mixed320",
"name": "QuantPPI Mixed 2-bit/Ternary Hybrid (320 B)",
"short_name": "QuantPPI 320B Mixed",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 320,
"compression_1280": 16.00,
"k": 2,
"trunk_mode": "mixed_2bit_ternary",
"rho_q": 0.935500,
"bussgang_beta": 1.068947,
"description": "Top-2 outlier exemption (8 B) + mixed 2-bit / Radix-3 ternary spherical Lloyd-Max trunk (312 B). Beats the historical 320 B 2-bit ceiling by +11.80 percentage points.",
},
{
"id": "track3_hybrid_k4_2bit",
"name": "QuantPPI 2-bit Spherical Hybrid (336 B)",
"short_name": "QuantPPI 336B 2-bit",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 336,
"compression_1280": 15.24,
"k": 4,
"trunk_mode": "2bit",
"rho_q": 0.939400,
"bussgang_beta": 1.064509,
"description": "Top-4 outlier exemption (16 B) + 4-level spherical Gaussian Lloyd-Max 2-bit trunk (320 B).",
},
{
"id": "track3_hybrid_k2_quinary382",
"name": "QuantPPI Base-5 Quinary Hybrid (382 B)",
"short_name": "QuantPPI 382B Quinary",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 382,
"compression_1280": 13.40,
"k": 2,
"trunk_mode": "quinary",
"rho_q": 0.959197,
"bussgang_beta": 1.042539,
"description": "Top-2 outlier exemption (8 B) + 5-level Quinary (M=5) spherical Lloyd-Max trunk packing 3 quints into 7 bits (5^3 = 125 < 128). Surpasses Phase-5 Half-DB Trained OPQ M=224.",
},
{
"id": "track3_hybrid_k4_senary432",
"name": "QuantPPI Base-6 Senary Hybrid (432 B)",
"short_name": "QuantPPI 432B Senary",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 432,
"compression_1280": 11.85,
"k": 4,
"trunk_mode": "senary",
"rho_q": 0.970578,
"bussgang_beta": 1.030314,
"description": "Top-4 outlier exemption (16 B) + 6-level Senary (M=6) spherical Lloyd-Max trunk packing 5 hexits into 13 bits (6^5 = 7776 < 8192). Beats ALL Trained OPQ M=224 regimes with zero training.",
},
{
"id": "track3_hybrid_k2_3bit488",
"name": "QuantPPI 3-bit Octal Hybrid (488 B)",
"short_name": "QuantPPI 488B 3-bit",
"family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)",
"bytes_1280": 488,
"compression_1280": 10.49,
"k": 2,
"trunk_mode": "3bit",
"rho_q": 0.982574,
"bussgang_beta": 1.017735,
"description": "Top-2 outlier exemption (8 B) + 8-level 3-bit Octal (M=8) spherical Lloyd-Max trunk (480 B). Achieves 0.9993 Recall@20@30 on SCOPe and 0.9994 on Swiss-Prot.",
},
{
"id": "track5_huber_tail",
"name": "Track 5: Huber Spike-Tail Companding (176 B)",
"short_name": "Track 5 Huber (176B)",
"family": "Campaign 4 Alternative Tracks (N_cal=0)",
"bytes_1280": 176,
"compression_1280": 29.09,
"k": 4,
"trunk_mode": "huber_1bit",
"rho_q": 0.797885,
"bussgang_beta": 1.253314,
"description": "Native-basis Huber clipping with sparse spike residual (k=4) and deterministic boundary sign*tau recovery.",
},
{
"id": "track4_spherical_e8",
"name": "Track 4: Spherical E8 Lattice (162 B)",
"short_name": "Track 4 Spherical E8 (162B)",
"family": "Campaign 4 Alternative Tracks (N_cal=0)",
"bytes_1280": 162,
"compression_1280": 31.60,
"k": 0,
"trunk_mode": "e8_lattice",
"rho_q": 0.785000,
"bussgang_beta": 1.0,
"description": "256-codeword unit spherical E8 root-shell lattice codebook (1 byte per 8D block + 2B gain).",
},
{
"id": "circulant_1bit",
"name": "Baseline: Uniform Circulant 1-bit (160 B)",
"short_name": "Circulant 1-bit (160B)",
"family": "Unprotected Data-Oblivious Baselines",
"bytes_1280": 160,
"compression_1280": 32.00,
"k": 0,
"trunk_mode": "1bit_unprotected",
"rho_q": 0.797885,
"bussgang_beta": 1.0,
"description": "Standard 1-bit random-sign circulant rotation without outlier exemption. Suffers from rogue-coordinate energy smearing (Recall@20@30 = 0.7612).",
},
{
"id": "fwht_1bit",
"name": "Baseline: Uniform FWHT 1-bit / TurboQuant 1-bit (160 B)",
"short_name": "FWHT 1-bit (160B)",
"family": "Unprotected Data-Oblivious Baselines",
"bytes_1280": 160,
"compression_1280": 32.00,
"k": 0,
"trunk_mode": "1bit_unprotected",
"rho_q": 0.797885,
"bussgang_beta": 1.0,
"description": "Standard 1-bit Fast Walsh-Hadamard Transform without outlier exemption (Recall@20@30 = 0.7446).",
},
{
"id": "analytic_2bit",
"name": "Baseline: Uniform Analytic 2-bit Lloyd-Max (320 B)",
"short_name": "Analytic 2-bit (320B)",
"family": "Unprotected Data-Oblivious Baselines",
"bytes_1280": 320,
"compression_1280": 16.00,
"k": 0,
"trunk_mode": "2bit_unprotected",
"rho_q": 0.939400,
"bussgang_beta": 1.0,
"description": "Unprotected 2-bit Gaussian Lloyd-Max quantizer. Fails severely on PLM embeddings (Recall@20@30 = 0.4986) due to fixed threshold distortion from rogue spikes.",
},
{
"id": "calibrated_native_protection_k32",
"name": "Baseline: Calibrated Native Protection (k=32, 224 B)",
"short_name": "Calib Native Prot (224B)",
"family": "Calibrated Baselines (N_cal=4096)",
"bytes_1280": 224,
"compression_1280": 22.86,
"k": 32,
"trunk_mode": "calibrated_1bit",
"rho_q": 0.797885,
"bussgang_beta": 1.0,
"description": "Globally calibrated top-32 variance coordinates stored in FP16 (64 B) + 1-bit trunk (160 B) without Theorems 1 & 2 (Recall@20@30 = 0.8449).",
},
]
class RealDataState:
"""Singleton state manager holding real embeddings, sequence metadata, and cached wire buffers."""
def __init__(self):
print("[QuantPPI Server] Initializing RealDataState from authoritative repository files...")
t0 = time.perf_counter()
# 1. Load SCOPe ASTRAL-40 v2.08 FASTA (15,166 domains)
self.scope_entries: List[Dict[str, Any]] = []
self.scope_id_to_raw_idx: Dict[str, int] = {}
self._load_scope_fasta()
# 2. Load Protocol P1 Canonical Manifest (500 queries, 15,166 candidates)
p1_manifest_path = (
REPO_ROOT
/ "experiments"
/ "data_oblivious_quantization"
/ "campaign_3_paper_readiness"
/ "manifests"
/ "protocol_p1_scope_canonical.json"
)
with open(p1_manifest_path, "r") as f:
p1_m = json.load(f)
self.p1_query_raw_ids: List[int] = p1_m["query_ids"]
self.p1_cand_raw_ids: List[int] = p1_m["candidate_ids"]
self.p1_query_set = set(self.p1_query_raw_ids)
# Ordered candidate metadata matching p1_cand_raw_ids
self.scope_p1_candidates: List[Dict[str, Any]] = []
self.scope_sid_to_p1_pos: Dict[str, int] = {}
self.scope_sid_to_query_pos: Dict[str, int] = {}
for pos, raw_i in enumerate(self.p1_cand_raw_ids):
entry = dict(self.scope_entries[raw_i])
entry["p1_cand_pos"] = pos
entry["raw_idx"] = raw_i
entry["is_p1_query"] = pos < len(self.p1_query_raw_ids)
entry["p1_query_pos"] = pos if pos < len(self.p1_query_raw_ids) else None
self.scope_p1_candidates.append(entry)
self.scope_sid_to_p1_pos[entry["id"]] = pos
if pos < len(self.p1_query_raw_ids):
self.scope_sid_to_query_pos[entry["id"]] = pos
# Precompute SCOPe family/superfamily/fold arrays for fast biological evaluation
self.scope_cand_fam = np.array([c["family"] for c in self.scope_p1_candidates])
self.scope_cand_sfam = np.array([c["superfamily"] for c in self.scope_p1_candidates])
self.scope_cand_fold = np.array([c["fold"] for c in self.scope_p1_candidates])
# 3. Load Swiss-Prot Human FASTA (18,099 proteins) & GO TSV
self.go_map: Dict[str, List[str]] = {}
self._load_swissprot_go()
self.sp_entries: List[Dict[str, Any]] = []
self._load_swissprot_fasta()
p5_manifest_path = (
REPO_ROOT
/ "experiments"
/ "data_oblivious_quantization"
/ "campaign_3_paper_readiness"
/ "manifests"
/ "protocol_p5_swissprot_locked.json"
)
with open(p5_manifest_path, "r") as f:
p5_m = json.load(f)
self.p5_calib_raw_ids: List[int] = p5_m.get("calibration_ids", [])
self.p5_query_raw_ids: List[int] = p5_m["query_ids"]
self.p5_cand_raw_ids: List[int] = p5_m["candidate_ids"]
self.sp_p5_queries: List[Dict[str, Any]] = []
self.sp_p5_candidates: List[Dict[str, Any]] = []
self.sp_acc_to_raw_idx: Dict[str, int] = {}
self.sp_acc_to_query_pos: Dict[str, int] = {}
self.sp_acc_to_cand_pos: Dict[str, int] = {}
for raw_i, e in enumerate(self.sp_entries):
self.sp_acc_to_raw_idx[e["id"]] = raw_i
for q_pos, raw_i in enumerate(self.p5_query_raw_ids):
e = self.sp_entries[raw_i]
e["is_p5_query"] = True
e["p5_query_pos"] = q_pos
self.sp_p5_queries.append(e)
self.sp_acc_to_query_pos[e["id"]] = q_pos
for c_pos, raw_i in enumerate(self.p5_cand_raw_ids):
e = self.sp_entries[raw_i]
e["is_p5_candidate"] = True
e["p5_cand_pos"] = c_pos
self.sp_p5_candidates.append(e)
self.sp_acc_to_cand_pos[e["id"]] = c_pos
# 4. Embedding Caches (Lazy loaded per (dataset, model_scale))
self._emb_cache: Dict[Tuple[str, str], np.ndarray] = {}
self._geom_cache: Dict[Tuple[str, str], Dict[str, Any]] = {}
self._encoded_db_cache: Dict[Tuple[str, str, str], Tuple[Any, bytes, float]] = {}
# Preload primary SCOPe 650M cache immediately
self.get_embeddings("scope", "esm2_650m")
# 5. Load Foldseek Top-20 Structural Alignments per query
self.foldseek_by_query: Dict[str, List[Dict[str, Any]]] = {}
self._load_foldseek_alignments()
# 6. Load Benchmark Ledgers
self.benchmarks_payload = self._load_benchmarks()
dt = time.perf_counter() - t0
print(
f"[QuantPPI Server] Ready in {dt:.2f}s | SCOPe: {len(self.scope_entries)} domains | "
f"Swiss-Prot: {len(self.sp_entries)} human proteins | "
f"3D PDB structures: {PDB_DIR.exists() or PDB_H5_PATH.exists()}"
)
def _load_scope_fasta(self):
fasta_path = DATA_DIR / "astral-scopedom-seqres-gd-sel-gs-bib-40-2.08.fa"
header = None
seq_parts: List[str] = []
def _commit(h: str, sp: List[str]):
seq = "".join(sp).strip().upper()
if not (20 <= len(seq) <= 1022):
return
parts = h[1:].split()
if len(parts) < 2:
return
dom_id = parts[0]
lbl = parts[1]
lbl_parts = lbl.split(".")
if len(lbl_parts) != 4:
return
chain_region = parts[2].strip("()") if len(parts) > 2 else ""
rest = " ".join(parts[3:]) if len(parts) > 3 else ""
org = ""
desc = rest
if "{" in rest and "}" in rest:
idx = rest.find("{")
desc = rest[:idx].strip()
org = rest[idx + 1 : rest.rfind("}")].strip()
raw_idx = len(self.scope_entries)
entry = {
"id": dom_id,
"pdb_id": dom_id[1:5].upper() if len(dom_id) >= 5 else "",
"chain": chain_region,
"class": lbl_parts[0],
"class_name": SCOP_CLASS_NAMES.get(lbl_parts[0], lbl_parts[0]),
"fold": f"{lbl_parts[0]}.{lbl_parts[1]}",
"superfamily": f"{lbl_parts[0]}.{lbl_parts[1]}.{lbl_parts[2]}",
"family": lbl,
"sccs": lbl,
"description": desc or "SCOPe structural domain",
"organism": org,
"sequence": seq,
"seq_len": len(seq),
"has_pdb": has_pdb_structure(dom_id),
"raw_idx": raw_idx,
"is_p1_query": False,
"p1_query_pos": None,
"p1_cand_pos": None,
}
self.scope_entries.append(entry)
self.scope_id_to_raw_idx[dom_id] = raw_idx
with open(fasta_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
if line.startswith(">"):
if header is not None:
_commit(header, seq_parts)
header = line
seq_parts = []
else:
seq_parts.append(line)
if header is not None:
_commit(header, seq_parts)
def _load_swissprot_go(self):
tsv_path = DATA_DIR / "swissprot_go.tsv"
if not tsv_path.exists():
return
with open(tsv_path, "r", encoding="utf-8") as f:
for i, line in enumerate(f):
if i == 0 and line.startswith("Entry"):
continue
parts = line.strip().split("\t")
if len(parts) >= 2:
self.go_map[parts[0]] = [t.strip() for t in parts[1].split(";") if t.strip()]
def _load_swissprot_fasta(self):
fasta_path = DATA_DIR / "swissprot_human_full.fasta"
header = None
seq_parts: List[str] = []
def _commit(h: str, sp: List[str]):
seq = "".join(sp).strip().upper()
if not (20 <= len(seq) <= 1022):
return
# Format: >sp|P04637|P53_HUMAN Cellular tumor antigen p53 OS=Homo sapiens OX=9606 GN=TP53 PE=1 SV=4
bar_parts = h[1:].split("|")
acc = bar_parts[1] if len(bar_parts) >= 2 else h[1:].split()[0]
rest = bar_parts[2] if len(bar_parts) >= 3 else h[1:]
tokens = rest.split()
entry_name = tokens[0] if tokens else acc
text_after = " ".join(tokens[1:]) if len(tokens) > 1 else ""
gene = ""
m_gn = re.search(r"GN=([^\s]+)", text_after)
if m_gn:
gene = m_gn.group(1)
desc = text_after.split(" OS=")[0].strip() if " OS=" in text_after else text_after
go_terms = self.go_map.get(acc, [])
raw_idx = len(self.sp_entries)
self.sp_entries.append({
"id": acc,
"entry_name": entry_name,
"gene": gene or entry_name.replace("_HUMAN", ""),
"description": desc or entry_name,
"organism": "Homo sapiens (Human)",
"go_terms": go_terms,
"go_count": len(go_terms),
"sequence": seq,
"seq_len": len(seq),
"raw_idx": raw_idx,
"is_p5_query": False,
"p5_query_pos": None,
"is_p5_candidate": False,
"p5_cand_pos": None,
})
with open(fasta_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
if line.startswith(">"):
if header is not None:
_commit(header, seq_parts)
header = line
seq_parts = []
else:
seq_parts.append(line)
if header is not None:
_commit(header, seq_parts)
def _load_foldseek_alignments(self):
csv_path = FOLDSEEK_RES_DIR / "foldseek_structural_alignments_top20.csv"
if not csv_path.exists():
return
df = pd.read_csv(csv_path)
for _, row in df.iterrows():
qid = str(row["query_id"])
if qid not in self.foldseek_by_query:
self.foldseek_by_query[qid] = []
self.foldseek_by_query[qid].append({
"rank": int(row["rank_non_self"]),
"target_id": str(row["target_id"]),
"target_sccs": str(row["target_sccs"]),
"homology_tier": str(row["homology_tier"]),
"paper_struct_score": float(row["paper_struct_score"]),
"bits": float(row["bits"]),
"evalue": float(row["evalue"]),
"alntmscore": float(row["alntmscore"]),
"lddt": float(row["lddt"]),
"fident": float(row["fident"]),
"alnlen": int(row["alnlen"]),
})
def _load_benchmarks(self) -> Dict[str, Any]:
out: Dict[str, Any] = {}
mc_path = FOLDSEEK_RES_DIR / "method_comparison_table.csv"
if mc_path.exists():
df = pd.read_csv(mc_path).replace({np.nan: None})
out["methods"] = df.to_dict(orient="records")
ab_path = FOLDSEEK_RES_DIR / "ablation_studies_complete_15118.csv"
if ab_path.exists():
df = pd.read_csv(ab_path).replace({np.nan: None})
out["ablations"] = df.to_dict(orient="records")
ms_path = FOLDSEEK_RES_DIR / "multiseed_stability_15118.csv"
if ms_path.exists():
df = pd.read_csv(ms_path).replace({np.nan: None})
out["multiseed"] = df.to_dict(orient="records")
stat_path = FOLDSEEK_RES_DIR / "statistical_significance_tests.csv"
if stat_path.exists():
df = pd.read_csv(stat_path).replace({np.nan: None})
out["statistical_tests"] = df.to_dict(orient="records")
pr_path = FOLDSEEK_RES_DIR / "precision_recall_cutoffs_k.csv"
if pr_path.exists():
df = pd.read_csv(pr_path).replace({np.nan: None})
out["pr_cutoffs"] = df.to_dict(orient="records")
t1_path = FOLDSEEK_RES_DIR / "task1_d6a1ia1_investigation.json"
if t1_path.exists():
with open(t1_path, "r") as f:
out["d6a1ia1_forensic"] = json.load(f)
return out
def get_embeddings(self, dataset: str, model_scale: str) -> np.ndarray:
key = (dataset, model_scale)
if key in self._emb_cache:
return self._emb_cache[key]
file_map = {
("scope", "esm2_650m"): CACHE_DIR / "esm2_650m_mean_l2norm_final_15166.h5",
("scope", "esm2_150m"): CACHE_DIR / "esm2_150m_mean_l2norm_final_15166.h5",
("scope", "esm2_35m"): CACHE_DIR / "esm2_35m_mean_l2norm_final_15166.h5",
("swissprot", "esm2_650m"): CACHE_DIR / "esm2_650m_mean_sp_full_18099.h5",
("swissprot", "esm2_150m"): CACHE_DIR / "esm2_150m_mean_sp_full_18099.h5",
("swissprot", "esm2_35m"): CACHE_DIR / "esm2_35m_mean_sp_full_18099.h5",
}
h5_path = file_map.get(key)
if not h5_path or not h5_path.exists():
raise ValueError(f"Unknown dataset/model_scale combination: {key}")
with h5py.File(h5_path, "r") as f:
emb = f["embeddings"][:].astype(np.float32)
self._emb_cache[key] = emb
return emb
def get_corpus_geometry(self, dataset: str, model_scale: str) -> Dict[str, Any]:
key = (dataset, model_scale)
if key in self._geom_cache:
return self._geom_cache[key]
emb = self.get_embeddings(dataset, model_scale)
N, D = emb.shape
mean_sq = np.mean(emb ** 2, axis=0)
mean_vec = np.mean(emb, axis=0)
common_mode_norm = float(np.linalg.norm(mean_vec))
order_desc = np.argsort(-mean_sq)
sorted_sq = mean_sq[order_desc]
cum_sq = np.cumsum(sorted_sq)
top_coords = []
for rank in range(min(32, D)):
coord_idx = int(order_desc[rank])
top_coords.append({
"rank": rank + 1,
"coord_index": coord_idx,
"mean_energy": float(sorted_sq[rank]),
"cumulative_energy": float(cum_sq[rank]),
"mean_signed": float(mean_vec[coord_idx]),
})
diag = compute_geometry_diagnostics(emb)
diag.update({
"dataset": dataset,
"model_scale": model_scale,
"num_vectors": int(N),
"dimension": int(D),
"common_mode_l2_norm": common_mode_norm,
"top_32_native_coordinates": top_coords,
"spectrum_top_128": [float(x) for x in sorted_sq[:128]],
"cumulative_top_128": [float(x) for x in cum_sq[:128]],
})
self._geom_cache[key] = diag
return diag
def create_codec(
self,
codec_id: str,
dimension: int = 1280,
transform_name: str = "circulant",
dataset: str = "scope",
model_scale: str = "esm2_650m",
) -> Any:
if codec_id == "track1_top2":
return DynamicTopKOutlierCodec(dimension=dimension, k=2, transform_name=transform_name, seed=42)
if codec_id == "track1_top4":
return DynamicTopKOutlierCodec(dimension=dimension, k=4, transform_name=transform_name, seed=42)
if codec_id == "track1_top16":
return DynamicTopKOutlierCodec(dimension=dimension, k=16, transform_name=transform_name, seed=42)
if codec_id == "track3_hybrid_k2_block224":
return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="block_order", transform_name=transform_name, seed=42)
if codec_id == "track3_hybrid_k4_block256":
return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="block_order_4b", transform_name=transform_name, seed=42)
if codec_id == "track3_hybrid_k2_ternary":
return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="ternary", transform_name=transform_name, seed=42)
if codec_id == "track3_hybrid_k2_mixed320":
return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="mixed_2bit_ternary", transform_name=transform_name, seed=42)
if codec_id == "track3_hybrid_k4_2bit":
return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="2bit", transform_name=transform_name, seed=42)
if codec_id == "track3_hybrid_k2_quinary382":
return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="quinary", transform_name=transform_name, seed=42)
if codec_id == "track3_hybrid_k4_senary432":
return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="senary", transform_name=transform_name, seed=42)
if codec_id == "track3_hybrid_k2_3bit488":
return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="3bit", transform_name=transform_name, seed=42)
if codec_id == "track5_huber_tail":
return HuberTailCompandingCodec(dimension=dimension, max_outliers=4, transform_name=transform_name, seed=42)
if codec_id == "track4_spherical_e8":
return SphericalE8LatticeCodec(dimension=dimension, transform_name=transform_name, seed=42)
if codec_id == "circulant_1bit":
return Circulant1BitBaseline(dimension=dimension, seed=42)
if codec_id == "fwht_1bit":
return FWHT1BitBaseline(dimension=dimension, seed=42)
if codec_id == "analytic_2bit":
return Analytic2BitBaseline(dimension=dimension, seed=42)
if codec_id == "calibrated_native_protection_k32":
raw_emb = self.get_embeddings(dataset, model_scale)
if dataset == "scope":
calib = raw_emb[self.p1_cand_raw_ids[500:4596]]
else:
calib = raw_emb[self.p5_calib_raw_ids]
return CalibratedNativeProtectionBaseline(dimension=dimension, k=32, calib_vectors=calib, seed=42)
raise ValueError(f"Unsupported codec_id: {codec_id}")
def get_encoded_candidates(
self,
dataset: str,
model_scale: str,
codec_id: str,
) -> Tuple[Any, Any, np.ndarray, float]:
"""Returns (codec, packed_bytes, candidate_embeddings, encode_time_s), caching encoded bytes in memory."""
raw_emb = self.get_embeddings(dataset, model_scale)
if dataset == "scope":
cand_emb = raw_emb[self.p1_cand_raw_ids]
else:
cand_emb = raw_emb[self.p5_cand_raw_ids]
D = cand_emb.shape[1]
cache_key = (dataset, model_scale, codec_id)
if cache_key in self._encoded_db_cache:
codec, packed_bytes, enc_time = self._encoded_db_cache[cache_key]
return codec, packed_bytes, cand_emb, enc_time
codec = self.create_codec(codec_id, dimension=D, transform_name="circulant", dataset=dataset, model_scale=model_scale)
t0 = time.perf_counter()
packed_bytes = codec.encode(cand_emb)
# Warm ADC cache if present
if hasattr(codec, "_get_prepared_adc"):
codec._get_prepared_adc(packed_bytes, len(cand_emb))
enc_time = time.perf_counter() - t0
self._encoded_db_cache[cache_key] = (codec, packed_bytes, enc_time)
return codec, packed_bytes, cand_emb, enc_time
STATE = RealDataState()
AA3_TO_1 = {
"ALA": "A", "ARG": "R", "ASN": "N", "ASP": "D", "CYS": "C",
"GLN": "Q", "GLU": "E", "GLY": "G", "HIS": "H", "ILE": "I",
"LEU": "L", "LYS": "K", "MET": "M", "PHE": "F", "PRO": "P",
"SER": "S", "THR": "T", "TRP": "W", "TYR": "Y", "VAL": "V",
"MSE": "M", "SEC": "U", "PYL": "O",
}
KYTE_DOOLITTLE = {
"I": 4.5, "V": 4.2, "L": 3.8, "F": 2.8, "C": 2.5,
"M": 1.9, "A": 1.8, "G": -0.4, "T": -0.7, "S": -0.8,
"W": -0.9, "Y": -1.3, "P": -1.6, "H": -3.2, "E": -3.5,
"Q": -3.5, "D": -3.5, "N": -3.5, "K": -3.9, "R": -4.5,
}
_MINI_TRACE_CACHE: Dict[str, Any] = {}
def _assign_ca_secondary_structure(coords: np.ndarray) -> List[str]:
"""
Assigns secondary structure ('H' = alpha-helix, 'E' = beta-strand, 'C' = coil)
directly from experimental 3D C-alpha coordinates using P-SEA inter-atomic distance rules.
"""
n = len(coords)
if n < 5:
return ["C"] * n
raw_ss = ["C"] * n
for i in range(n - 4):
d2 = float(np.linalg.norm(coords[i + 2] - coords[i]))
d3 = float(np.linalg.norm(coords[i + 3] - coords[i]))
d4 = float(np.linalg.norm(coords[i + 4] - coords[i]))
# Alpha-helix: compact right-handed turn (d2 ~ 5.4A, d3 ~ 5.1A, d4 ~ 6.2A)
if 4.8 <= d2 <= 6.1 and 4.4 <= d3 <= 6.0 and 5.2 <= d4 <= 7.1:
for k in range(i, min(i + 4, n)):
raw_ss[k] = "H"
# Beta-strand: extended pleated backbone (d2 ~ 6.7A, d3 ~ 9.9A, d4 ~ 12.8A)
elif 6.15 <= d2 <= 7.8 and 8.5 <= d3 <= 11.6:
for k in range(i, min(i + 4, n)):
if raw_ss[k] == "C":
raw_ss[k] = "E"
return raw_ss
def parse_pdb_ca_atoms(ent_path: Path) -> Dict[str, Any]:
"""Extracts real 3D C-alpha atomic coordinates, secondary structure, and ASTRAL remarks from a SCOPe .ent file or HDF5 archive."""
ca_atoms = []
remarks = []
header_line = ""
total_atom_records = 0
if isinstance(ent_path, str):
dom_id = ent_path.replace(".ent", "").strip()
ent_path = PDB_DIR / f"{dom_id}.ent"
else:
dom_id = ent_path.stem
if ent_path.exists():
with open(ent_path, "r", encoding="utf-8", errors="replace") as f:
for line in f:
if line.startswith("HEADER"):
header_line = line.strip()
elif line.startswith("REMARK 99"):
txt = line[10:].strip()
if txt:
remarks.append(txt)
elif line.startswith("ATOM "):
total_atom_records += 1
atom_name = line[12:16].strip()
alt_loc = line[16:17].strip()
if atom_name == "CA" and alt_loc in ("", "A", "1"):
res_name = line[17:20].strip()
chain_id = line[21:22].strip()
res_seq_str = line[22:26].strip()
try:
res_seq = int(res_seq_str)
except ValueError:
res_seq = len(ca_atoms) + 1
try:
x = float(line[30:38])
y = float(line[38:46])
z = float(line[46:54])
bfac = float(line[60:66]) if len(line) >= 66 and line[60:66].strip() else 0.0
aa1 = AA3_TO_1.get(res_name.upper(), "X")
ca_atoms.append({
"res_seq": res_seq,
"res_name": res_name,
"aa": aa1,
"hydropathy": KYTE_DOOLITTLE.get(aa1, 0.0),
"chain": chain_id,
"x": round(x, 3),
"y": round(y, 3),
"z": round(z, 3),
"bfactor": round(bfac, 2),
})
except ValueError:
pass
else:
idx_map = _get_h5_struct_index()
h5_idx = idx_map.get(dom_id)
if h5_idx is not None and PDB_H5_PATH.exists():
with h5py.File(PDB_H5_PATH, "r") as f:
s_idx = int(f["offsets"][h5_idx])
e_idx = int(f["offsets"][h5_idx + 1])
raw_hdr = f["headers"][h5_idx]
hdr_str = raw_hdr.decode("utf-8", errors="ignore") if isinstance(raw_hdr, (bytes, np.bytes_)) else str(raw_hdr)
if " | " in hdr_str:
parts = hdr_str.split(" | ")
header_line = parts[0]
remarks = parts[1:]
else:
header_line = hdr_str
xyz_arr = f["xyz"][s_idx:e_idx].astype(np.float64)
bfac_arr = f["bfactor"][s_idx:e_idx].astype(np.float64)
rseq_arr = f["res_seq"][s_idx:e_idx]
rname_arr = f["res_name"][s_idx:e_idx]
total_atom_records = int((e_idx - s_idx) * 8)
chain_guess = dom_id[5:6].upper() if len(dom_id) >= 6 and dom_id[5:6] != "_" else "A"
for i in range(e_idx - s_idx):
rn = rname_arr[i].decode("ascii", errors="ignore").strip() if isinstance(rname_arr[i], (bytes, np.bytes_)) else str(rname_arr[i]).strip()
aa1 = AA3_TO_1.get(rn.upper(), "X")
ca_atoms.append({
"res_seq": int(rseq_arr[i]),
"res_name": rn,
"aa": aa1,
"hydropathy": KYTE_DOOLITTLE.get(aa1, 0.0),
"chain": chain_guess,
"x": round(float(xyz_arr[i, 0]), 3),
"y": round(float(xyz_arr[i, 1]), 3),
"z": round(float(xyz_arr[i, 2]), 3),
"bfactor": round(float(bfac_arr[i]), 2),
})
ss_comp = {"helix_pct": 0.0, "sheet_pct": 0.0, "coil_pct": 100.0}
if ca_atoms:
coords = np.array([[a["x"], a["y"], a["z"]] for a in ca_atoms], dtype=np.float64)
center = np.mean(coords, axis=0)
centered = coords - center
radius = float(np.max(np.linalg.norm(centered, axis=1))) if len(centered) > 0 else 1.0
ss_list = _assign_ca_secondary_structure(coords)
n_res = len(ca_atoms)
h_cnt = sum(1 for s in ss_list if s == "H")
e_cnt = sum(1 for s in ss_list if s == "E")
ss_comp = {
"helix_pct": round((h_cnt / n_res) * 100.0, 1),
"sheet_pct": round((e_cnt / n_res) * 100.0, 1),
"coil_pct": round(((n_res - h_cnt - e_cnt) / n_res) * 100.0, 1),
}
for i, a in enumerate(ca_atoms):
a["cx"] = round(float(centered[i, 0]), 3)
a["cy"] = round(float(centered[i, 1]), 3)
a["cz"] = round(float(centered[i, 2]), 3)
a["ss"] = ss_list[i]
else:
radius = 1.0
return {
"header": header_line,
"remarks": remarks,
"total_atom_records": total_atom_records,
"num_ca_residues": len(ca_atoms),
"bounding_radius_angstrom": round(radius, 2),
"ss_composition": ss_comp,
"ca_atoms": ca_atoms,
}
def get_mini_ca_trace(dom_id: str, max_points: int = 72) -> Optional[Dict[str, Any]]:
"""Returns a cached compact 3D C-alpha trace for inline 3D thumbnail rendering in protein cards."""
if dom_id in _MINI_TRACE_CACHE:
return _MINI_TRACE_CACHE[dom_id]
if not has_pdb_structure(dom_id):
_MINI_TRACE_CACHE[dom_id] = None
return None
ent_path = PDB_DIR / f"{dom_id}.ent"
parsed = parse_pdb_ca_atoms(ent_path)
atoms = parsed["ca_atoms"]
if not atoms:
_MINI_TRACE_CACHE[dom_id] = None
return None
n = len(atoms)
if n > max_points:
idxs = np.linspace(0, n - 1, max_points).astype(int)
sampled = [atoms[int(i)] for i in idxs]
else:
sampled = atoms
ss_map = {"H": 1, "E": 2, "C": 0}
pts = [[a["cx"], a["cy"], a["cz"], ss_map.get(a.get("ss", "C"), 0)] for a in sampled]
res = {
"radius": parsed["bounding_radius_angstrom"],
"num_ca": n,
"ss_composition": parsed["ss_composition"],
"points": pts,
}
_MINI_TRACE_CACHE[dom_id] = res
return res
def kabsch_superpose(coords_ref: np.ndarray, coords_mov: np.ndarray) -> Tuple[np.ndarray, float]:
"""
Computes optimal rigid-body 3D Kabsch alignment of coords_mov onto coords_ref
over their shared prefix/resampled C-alpha trace and returns (aligned_mov, rmsd).
"""
n_ref = len(coords_ref)
n_mov = len(coords_mov)
if n_ref < 3 or n_mov < 3:
return coords_mov, 0.0
c_ref = coords_ref - np.mean(coords_ref, axis=0)
c_mov = coords_mov - np.mean(coords_mov, axis=0)
# Resample to common length L for correspondence estimation
L = min(n_ref, n_mov)
idx_ref = np.linspace(0, n_ref - 1, L).astype(int)
idx_mov = np.linspace(0, n_mov - 1, L).astype(int)
P = c_mov[idx_mov]
Q = c_ref[idx_ref]
H = P.T @ Q
U, S, Vt = np.linalg.svd(H)
d = np.sign(np.linalg.det(Vt.T @ U.T))
diag = np.diag([1.0, 1.0, d])
R = Vt.T @ diag @ U.T
aligned_mov = c_mov @ R
diff = (P @ R) - Q
rmsd = float(np.sqrt(np.mean(np.sum(diff ** 2, axis=1))))
return aligned_mov, rmsd
def run_single_protein_quantization_pipeline(
dataset: str,
model_scale: str,
protein_id: str,
codec_id: str,
transform_name: str = "circulant",
custom_sequence: Optional[str] = None,
) -> Dict[str, Any]:
"""
Executes the complete mathematical and binary serialization pipeline on a real
ESM-2 protein embedding vector and returns all intermediate states.
"""
emb_matrix = STATE.get_embeddings(dataset, model_scale)
N, D = emb_matrix.shape
matched_note = None
if custom_sequence:
seq_clean = re.sub(r"[^A-Za-z]", "", custom_sequence).upper()
entries = STATE.scope_entries if dataset == "scope" else STATE.sp_entries
best_idx = 0
best_score = -1.0
# Exact or substring or 3-mer Jaccard match against real cached sequences
kmer_q = set(seq_clean[i : i + 3] for i in range(max(1, len(seq_clean) - 2)))
for idx, e in enumerate(entries):
s = e["sequence"]
if s == seq_clean:
best_idx = idx
best_score = 1.0
matched_note = f"Exact sequence match in {dataset.upper()} ({e['id']})"
break
elif seq_clean in s or s in seq_clean:
score = min(len(seq_clean), len(s)) / max(len(seq_clean), len(s))
if score > best_score:
best_score = score
best_idx = idx
matched_note = f"Subsequence match ({score*100:.1f}%) to {e['id']}"
if best_score < 0.5 and len(kmer_q) > 0:
for idx in range(0, len(entries), max(1, len(entries) // 1500)):
s = entries[idx]["sequence"]
kmer_s = set(s[i : i + 3] for i in range(max(1, len(s) - 2)))
sc = len(kmer_q & kmer_s) / max(1, len(kmer_q | kmer_s))
if sc > best_score:
best_score = sc
best_idx = idx
matched_note = f"Nearest real {dataset.upper()} sequence homolog ({entries[best_idx]['id']}, 3-mer Jaccard {best_score:.3f})"
raw_idx = best_idx
meta = dict(entries[raw_idx])
else:
if dataset == "scope":
raw_idx = STATE.scope_id_to_raw_idx.get(protein_id)
if raw_idx is None:
# Try case-insensitive or PDB match
pid_low = protein_id.lower()
for e in STATE.scope_entries:
if e["id"].lower() == pid_low or e["pdb_id"].lower() == pid_low:
raw_idx = e["raw_idx"]
break
if raw_idx is None:
raw_idx = STATE.p1_query_raw_ids[0]
meta = dict(STATE.scope_entries[raw_idx])
else:
raw_idx = STATE.sp_acc_to_raw_idx.get(protein_id)
if raw_idx is None:
pid_up = protein_id.upper()
for e in STATE.sp_entries:
if e["id"].upper() == pid_up or e["gene"].upper() == pid_up or e["entry_name"].upper() == pid_up:
raw_idx = e["raw_idx"]
break
if raw_idx is None:
raw_idx = STATE.p5_query_raw_ids[0]
meta = dict(STATE.sp_entries[raw_idx])
x = emb_matrix[raw_idx].astype(np.float32)
x_2d = x.reshape(1, D)
# Instantiate selected codec
codec = STATE.create_codec(
codec_id=codec_id,
dimension=D,
transform_name=transform_name,
dataset=dataset,
model_scale=model_scale,
)
codec_info = next((c for c in CODEC_CATALOG if c["id"] == codec_id), CODEC_CATALOG[0])
k = getattr(codec, "k", getattr(codec, "max_outliers", codec_info["k"]))
# Stage 0: Native Vector Diagnostics
sq_energy = x ** 2
total_energy = float(np.sum(sq_energy))
order_desc = np.argsort(- np.abs(x))
top16_indices = order_desc[:16]
top16_outliers = [
{
"rank": r + 1,
"coord_index": int(idx),
"value": float(x[idx]),
"fp16_value": float(np.float16(x[idx])),
"energy_fraction": float((x[idx] ** 2) / max(total_energy, 1e-12)),
}
for r, idx in enumerate(top16_indices)
]
# Single-vector participation ratio & kurtosis
pr_vec = float((total_energy ** 2) / max(float(np.sum(sq_energy ** 2)), 1e-12))
x_mean = float(np.mean(x))
x_var = float(np.var(x))
kurt_native = float(np.mean(((x - x_mean) ** 4)) / (x_var ** 2) - 3.0) if x_var > 0 else 0.0
# Stage 1: Outlier Exemption & Residual Trunk Isolation
if k > 0:
if hasattr(codec, "protected_dims"):
exempt_idx = np.sort(codec.protected_dims[:k]).astype(np.int64)
else:
exempt_idx = np.sort(order_desc[:k]).astype(np.int64)
exempt_fp16 = x[exempt_idx].astype(np.float16)
exempt_vals = exempt_fp16.astype(np.float32)
z = x.copy()
z[exempt_idx] = 0.0
outlier_energy = float(np.sum(exempt_vals ** 2))
pythagorean_res_norm = float(math.sqrt(max(1.0 - outlier_energy, 0.0)))
else:
exempt_idx = np.array([], dtype=np.int64)
exempt_vals = np.array([], dtype=np.float32)
z = x.copy()
outlier_energy = 0.0
pythagorean_res_norm = float(np.linalg.norm(z))
true_res_norm = float(np.linalg.norm(z))
# Stage 2: Orthogonal Rotation (y = R * z)
transform = getattr(codec, "transform", get_transform(transform_name, dimension=D, seed=42))
y_rot = transform.forward(z.reshape(1, D))[0].astype(np.float32)
z_inv = transform.inverse(y_rot.reshape(1, D))[0].astype(np.float32)
inv_err = float(np.max(np.abs(z_inv - z)))
y_mean = float(np.mean(y_rot))
y_var = float(np.var(y_rot))
kurt_rotated = float(np.mean(((y_rot - y_mean) ** 4)) / (y_var ** 2) - 3.0) if y_var > 0 else 0.0
# Stage 3 & Stage 4: True Wire Encoding & Decoding
t0_enc = time.perf_counter()
packed_bytes = codec.encode(x_2d)
enc_us = (time.perf_counter() - t0_enc) * 1e6
if hasattr(codec, "decode"):
x_recon = codec.decode(packed_bytes, count=1)[0].astype(np.float32)
else:
# Reconstruct from asymmetric score against identity basis
eye = np.eye(D, dtype=np.float32)
x_recon = codec.compute_asymmetric_scores(eye, packed_bytes, count=1)[:, 0].astype(np.float32)
# Compute quantized trunk in rotated domain for visualization
x_recon_trunk = x_recon.copy()
if len(exempt_idx) > 0:
x_recon_trunk[exempt_idx] = 0.0
y_quant_rot = transform.forward(x_recon_trunk.reshape(1, D))[0].astype(np.float32)
# Compute bleed energy that would have leaked onto S_i without Theorem 1
signs_rot = np.where(y_rot >= 0, 1.0, -1.0).astype(np.float32) / math.sqrt(D)
u_unmasked = transform.inverse(signs_rot.reshape(1, D))[0]
bleed_on_support = float(np.sum(u_unmasked[exempt_idx] ** 2)) if len(exempt_idx) > 0 else 0.0
# Self-score and reconstruction fidelity
asym_self_score = float(codec.compute_asymmetric_scores(x_2d, packed_bytes, count=1)[0, 0])
recon_mse = float(np.mean((x - x_recon) ** 2))
cosine_sim = float(np.dot(x, x_recon) / max(float(np.linalg.norm(x) * np.linalg.norm(x_recon)), 1e-12))
# Hex wire dump and byte segments
hex_bytes = [f"{b:02x}" for b in packed_bytes]
total_bytes = len(packed_bytes)
idx_bytes = int(k * 2) if k > 0 and codec_id != "calibrated_native_protection_k32" else 0
val_bytes = int(k * 2) if k > 0 else 0
trunk_bytes = total_bytes - idx_bytes - val_bytes
segments = []
offset = 0
if idx_bytes > 0:
segments.append({
"name": f"Outlier Indices (uint16 × {k})",
"start": offset,
"end": offset + idx_bytes,
"bytes": idx_bytes,
"color": "outlier-idx",
"detail": f"Support S_i = {[int(i) for i in exempt_idx]}",
})
offset += idx_bytes
if val_bytes > 0:
segments.append({
"name": f"Outlier Values (FP16 × {k})",
"start": offset,
"end": offset + val_bytes,
"bytes": val_bytes,
"color": "outlier-val",
"detail": f"Exact FP16 amplitudes {[round(float(v), 4) for v in exempt_vals[:8]]}",
})
offset += val_bytes
segments.append({
"name": f"Compressed Rotated Trunk ({codec_info['trunk_mode']})",
"start": offset,
"end": total_bytes,
"bytes": trunk_bytes,
"color": "trunk-bits",
"detail": f"{trunk_bytes} B packed trunk ({trunk_bytes * 8 / D:.3f} bits/dim)",
})
return {
"dataset": dataset,
"model_scale": model_scale,
"dimension": D,
"matched_note": matched_note,
"protein": meta,
"codec": {
**codec_info,
"transform_name": transform_name,
"actual_bytes": total_bytes,
"fp32_bytes": D * 4,
"actual_compression": round((D * 4) / total_bytes, 2),
"encode_time_us": round(enc_us, 1),
},
"stage0_native": {
"vector": [round(float(v), 5) for v in x],
"l2_norm": round(float(np.linalg.norm(x)), 6),
"max_abs": round(float(np.max(np.abs(x))), 5),
"participation_ratio": round(pr_vec, 2),
"pr_ratio": round(pr_vec / D, 4),
"kurtosis": round(kurt_native, 2),
"top16_outliers": top16_outliers,
"top_k_energy_share": {
"k1": round(float(np.sum(np.sort(sq_energy)[::-1][:1]) / total_energy), 4),
"k2": round(float(np.sum(np.sort(sq_energy)[::-1][:2]) / total_energy), 4),
"k4": round(float(np.sum(np.sort(sq_energy)[::-1][:4]) / total_energy), 4),
"k8": round(float(np.sum(np.sort(sq_energy)[::-1][:8]) / total_energy), 4),
"k16": round(float(np.sum(np.sort(sq_energy)[::-1][:16]) / total_energy), 4),
"k32": round(float(np.sum(np.sort(sq_energy)[::-1][:32]) / total_energy), 4),
},
},
"stage1_outliers": {
"k": int(k),
"exempt_indices": [int(i) for i in exempt_idx],
"exempt_fp16_values": [round(float(v), 5) for v in exempt_vals],
"outlier_energy_fraction": round(outlier_energy, 5),
"pythagorean_residual_norm": round(pythagorean_res_norm, 6),
"true_residual_norm": round(true_res_norm, 6),
"norm_recovery_error": round(abs(pythagorean_res_norm - true_res_norm), 7),
"residual_trunk_vector": [round(float(v), 5) for v in z],
"residual_max_abs": round(float(np.max(np.abs(z))), 5),
},
"stage2_rotated": {
"transform_name": transform_name,
"rotated_vector": [round(float(v), 5) for v in y_rot],
"max_abs": round(float(np.max(np.abs(y_rot))), 5),
"kurtosis_after_rotation": round(kurt_rotated, 3),
"invertibility_max_error": float(inv_err),
},
"stage3_quantized": {
"quantized_rotated_trunk": [round(float(v), 5) for v in y_quant_rot],
"reconstructed_vector": [round(float(v), 5) for v in x_recon],
"rho_q": codec_info["rho_q"],
"bussgang_beta": codec_info["bussgang_beta"],
"bleed_energy_eliminated": round(bleed_on_support, 6),
"asymmetric_self_score": round(asym_self_score, 5),
"reconstruction_mse": float(recon_mse),
"cosine_similarity": round(cosine_sim, 5),
},
"stage4_wire": {
"total_bytes": total_bytes,
"fp32_bytes": D * 4,
"compression_ratio": round((D * 4) / total_bytes, 2),
"hex_bytes": hex_bytes,
"segments": segments,
},
}
def run_live_search(
dataset: str,
model_scale: str,
query_id: str,
codec_id: str,
top_k: int = 30,
) -> Dict[str, Any]:
"""
Executes live asymmetric search across all 15,166 SCOPe domains or 13,003 Swiss-Prot
proteins and compares QuantPPI against Exact FP32, Unprotected 1-bit Circulant, and Foldseek.
"""
codec, packed_bytes, cand_emb, enc_time = STATE.get_encoded_candidates(dataset, model_scale, codec_id)
_, baseline_bytes, _, _ = STATE.get_encoded_candidates(dataset, model_scale, "circulant_1bit")
baseline_codec = STATE._encoded_db_cache[(dataset, model_scale, "circulant_1bit")][0]
raw_emb = STATE.get_embeddings(dataset, model_scale)
N, D = cand_emb.shape
if dataset == "scope":
raw_idx = STATE.scope_id_to_raw_idx.get(query_id, STATE.p1_query_raw_ids[0])
query_meta = STATE.scope_entries[raw_idx]
cand_pos_self = STATE.scope_sid_to_p1_pos.get(query_meta["id"], -1)
else:
raw_idx = STATE.sp_acc_to_raw_idx.get(query_id, STATE.p5_query_raw_ids[0])
query_meta = STATE.sp_entries[raw_idx]
cand_pos_self = STATE.sp_acc_to_cand_pos.get(query_meta["id"], -1)
q_vec = raw_emb[raw_idx : raw_idx + 1]
# 1. Exact FP32 search
t0_fp32 = time.perf_counter()
exact_scores = (q_vec @ cand_emb.T)[0].copy()
fp32_ms = (time.perf_counter() - t0_fp32) * 1000.0
# 2. Selected QuantPPI codec asymmetric search
t0_q = time.perf_counter()
pred_scores = codec.compute_asymmetric_scores(q_vec, packed_bytes, count=N)[0].copy()
quant_ms = (time.perf_counter() - t0_q) * 1000.0
# 3. Unprotected 1-bit Circulant baseline search
base_scores = baseline_codec.compute_asymmetric_scores(q_vec, baseline_bytes, count=N)[0].copy()
# Mask self-match if query is in candidate pool
valid_mask = np.ones(N, dtype=bool)
if 0 <= cand_pos_self < N:
valid_mask[cand_pos_self] = False
ip_mse = float(np.mean((pred_scores[valid_mask] - exact_scores[valid_mask]) ** 2))
base_mse = float(np.mean((base_scores[valid_mask] - exact_scores[valid_mask]) ** 2))
if 0 <= cand_pos_self < N:
exact_scores[cand_pos_self] = -1e9
pred_scores[cand_pos_self] = -1e9
base_scores[cand_pos_self] = -1e9
gt_order = np.argsort(-exact_scores, kind="stable")
pred_order = np.argsort(-pred_scores, kind="stable")
base_order = np.argsort(-base_scores, kind="stable")
gt_rank_of = np.empty(N, dtype=np.int32)
gt_rank_of[gt_order] = np.arange(1, N + 1)
base_rank_of = np.empty(N, dtype=np.int32)
base_rank_of[base_order] = np.arange(1, N + 1)
gt_top20 = set(gt_order[:20].tolist())
r20_30_quant = len(set(pred_order[:30].tolist()) & gt_top20) / 20.0
r20_30_base = len(set(base_order[:30].tolist()) & gt_top20) / 20.0
r1_quant = 1.0 if pred_order[0] == gt_order[0] else 0.0
r10_quant = len(set(pred_order[:10].tolist()) & set(gt_order[:10].tolist())) / 10.0
r20_quant = len(set(pred_order[:20].tolist()) & gt_top20) / 20.0
# Spearman rank correlation on top 250 ground-truth pool
pool_idx = gt_order[:250]
rk_p = np.argsort(np.argsort(pred_scores[pool_idx]))
rk_t = np.argsort(np.argsort(exact_scores[pool_idx]))
n_p = len(pool_idx)
spearman_rho = float(1.0 - (6.0 * np.sum((rk_p - rk_t) ** 2)) / (n_p * (n_p ** 2 - 1)))
# Biological evaluation on this query
bio_metrics: Dict[str, Any] = {}
hits_table: List[Dict[str, Any]] = []
if dataset == "scope":
q_fam = query_meta["family"]
q_sfam = query_meta["superfamily"]
q_fold = query_meta["fold"]
order_no_self = pred_order[pred_order != cand_pos_self] if cand_pos_self >= 0 else pred_order
gt_no_self = gt_order[gt_order != cand_pos_self] if cand_pos_self >= 0 else gt_order
base_no_self = base_order[base_order != cand_pos_self] if cand_pos_self >= 0 else base_order
is_fam_q = STATE.scope_cand_fam[order_no_self] == q_fam
is_fam_gt = STATE.scope_cand_fam[gt_no_self] == q_fam
is_fam_base = STATE.scope_cand_fam[base_no_self] == q_fam
is_sfam_q = (STATE.scope_cand_sfam[order_no_self] == q_sfam) & (~is_fam_q)
is_sfam_gt = (STATE.scope_cand_sfam[gt_no_self] == q_sfam) & (~is_fam_gt)
bio_metrics = {
"query_sccs": query_meta["sccs"],
"family_homolog_count": int(np.sum(is_fam_q)),
"superfamily_remote_count": int(np.sum(is_sfam_q)),
"quant_family_ap": round(compute_average_precision(is_fam_q), 4),
"fp32_family_ap": round(compute_average_precision(is_fam_gt), 4),
"base_family_ap": round(compute_average_precision(is_fam_base), 4),
"quant_superfamily_roc50": round(compute_roc50(is_sfam_q[STATE.scope_cand_fam[order_no_self] != q_fam]), 4),
"fp32_superfamily_roc50": round(compute_roc50(is_sfam_gt[STATE.scope_cand_fam[gt_no_self] != q_fam]), 4),
}
fs_hits = STATE.foldseek_by_query.get(query_meta["id"], [])
fs_map = {h["target_id"]: h for h in fs_hits}
for rank_idx in range(min(top_k, len(order_no_self))):
c_pos = int(order_no_self[rank_idx])
c_meta = STATE.scope_p1_candidates[c_pos]
if c_meta["family"] == q_fam:
tier = "Same Family"
elif c_meta["superfamily"] == q_sfam:
tier = "Same Superfamily (Remote)"
elif c_meta["fold"] == q_fold:
tier = "Same Fold"
else:
tier = "Different Fold"
fs_entry = fs_map.get(c_meta["id"])
hits_table.append({
"quant_rank": rank_idx + 1,
"fp32_rank": int(gt_rank_of[c_pos]),
"base_1bit_rank": int(base_rank_of[c_pos]),
"id": c_meta["id"],
"pdb_id": c_meta["pdb_id"],
"sccs": c_meta["sccs"],
"description": c_meta["description"],
"organism": c_meta["organism"],
"seq_len": c_meta["seq_len"],
"has_pdb": c_meta["has_pdb"],
"homology_tier": tier,
"quant_score": round(float(pred_scores[c_pos]), 5),
"fp32_score": round(float(exact_scores[c_pos]), 5),
"score_error": round(float(pred_scores[c_pos] - exact_scores[c_pos]), 5),
"in_fp32_top20": int(gt_rank_of[c_pos]) <= 20,
"foldseek": fs_entry,
"mini_trace": get_mini_ca_trace(c_meta["id"]) if (rank_idx < 12 and c_meta["has_pdb"]) else None,
"seq_preview": c_meta.get("sequence", "")[:120],
})
else:
q_go = set(query_meta.get("go_terms", []))
jacc_quant = []
jacc_fp32 = []
jacc_base = []
for idx in range(min(top_k, len(pred_order))):
c_pos = int(pred_order[idx])
c_meta = STATE.sp_p5_candidates[c_pos]
c_go = set(c_meta.get("go_terms", []))
shared_go = sorted(list(q_go & c_go))
jac = compute_go_jaccard(q_go, c_go)
jacc_quant.append(jac)
gt_c_go = set(STATE.sp_p5_candidates[int(gt_order[idx])].get("go_terms", []))
jacc_fp32.append(compute_go_jaccard(q_go, gt_c_go))
bs_c_go = set(STATE.sp_p5_candidates[int(base_order[idx])].get("go_terms", []))
jacc_base.append(compute_go_jaccard(q_go, bs_c_go))
hits_table.append({
"quant_rank": idx + 1,
"fp32_rank": int(gt_rank_of[c_pos]),
"base_1bit_rank": int(base_rank_of[c_pos]),
"id": c_meta["id"],
"gene": c_meta["gene"],
"entry_name": c_meta["entry_name"],
"description": c_meta["description"],
"seq_len": c_meta["seq_len"],
"go_jaccard": round(jac, 4),
"shared_go_terms": shared_go[:8],
"total_shared_go": len(shared_go),
"candidate_go_count": len(c_go),
"quant_score": round(float(pred_scores[c_pos]), 5),
"fp32_score": round(float(exact_scores[c_pos]), 5),
"score_error": round(float(pred_scores[c_pos] - exact_scores[c_pos]), 5),
"in_fp32_top20": int(gt_rank_of[c_pos]) <= 20,
"seq_preview": c_meta.get("sequence", "")[:120],
})
bio_metrics = {
"query_go_count": len(q_go),
"quant_top1_go_jaccard": round(float(jacc_quant[0]) if jacc_quant else 0.0, 4),
"quant_top5_go_jaccard": round(float(np.mean(jacc_quant[:5])) if jacc_quant else 0.0, 4),
"quant_top20_go_jaccard": round(float(np.mean(jacc_quant[:20])) if jacc_quant else 0.0, 4),
"fp32_top1_go_jaccard": round(float(jacc_fp32[0]) if jacc_fp32 else 0.0, 4),
"fp32_top5_go_jaccard": round(float(np.mean(jacc_fp32[:5])) if jacc_fp32 else 0.0, 4),
"base_top1_go_jaccard": round(float(jacc_base[0]) if jacc_base else 0.0, 4),
}
# Scatter plot points (top 60 + 60 evenly spaced across rank distribution)
sample_positions = np.concatenate([
gt_order[:60],
gt_order[np.linspace(60, N - 2, 60).astype(int)],
])
scatter_points = [
{
"id": (STATE.scope_p1_candidates[int(p)]["id"] if dataset == "scope" else STATE.sp_p5_candidates[int(p)]["id"]),
"fp32_score": round(float(exact_scores[int(p)]), 5),
"quant_score": round(float(pred_scores[int(p)]), 5),
"base_1bit_score": round(float(base_scores[int(p)]), 5),
"is_top20": int(gt_rank_of[int(p)]) <= 20,
}
for p in sample_positions
]
codec_info = next((c for c in CODEC_CATALOG if c["id"] == codec_id), CODEC_CATALOG[0])
actual_bytes = getattr(codec, "total_bytes_per_vector", D * 4)
query_out = {
**query_meta,
"seq_preview": query_meta.get("sequence", "")[:180],
"mini_trace": get_mini_ca_trace(query_meta["id"]) if dataset == "scope" else None,
}
return {
"dataset": dataset,
"model_scale": model_scale,
"dimension": D,
"num_candidates": N,
"query": query_out,
"codec": {
**codec_info,
"actual_bytes": actual_bytes,
"db_size_mb": round((N * actual_bytes) / (1024 * 1024), 2),
"fp32_db_size_mb": round((N * D * 4) / (1024 * 1024), 2),
},
"metrics": {
"recall_20_at_30": round(r20_30_quant, 4),
"base_1bit_recall_20_at_30": round(r20_30_base, 4),
"recall_1": round(r1_quant, 4),
"recall_10": round(r10_quant, 4),
"recall_20": round(r20_quant, 4),
"inner_product_mse": float(ip_mse),
"base_1bit_mse": float(base_mse),
"mse_reduction_vs_1bit": round(base_mse / max(ip_mse, 1e-12), 1),
"spearman_rho": round(spearman_rho, 5),
"search_latency_ms": round(quant_ms, 3),
"fp32_latency_ms": round(fp32_ms, 3),
},
"biological": bio_metrics,
"hits": hits_table,
"scatter_points": scatter_points,
"foldseek_top20": STATE.foldseek_by_query.get(query_meta["id"], []) if dataset == "scope" else [],
}
def run_live_theorem_ablation(
query_id: str,
k: int = 4,
model_scale: str = "esm2_650m",
) -> Dict[str, Any]:
"""
Runs a live 2x2 factorial ablation of Theorem 1 (Support-Orthogonal Bleed Elimination)
and Theorem 2 (Analytical Bussgang De-Attenuation) on the chosen real SCOPe protein.
"""
raw_emb = STATE.get_embeddings("scope", model_scale)
cand_emb = raw_emb[STATE.p1_cand_raw_ids]
N, D = cand_emb.shape
raw_idx = STATE.scope_id_to_raw_idx.get(query_id, STATE.p1_query_raw_ids[0])
query_meta = STATE.scope_entries[raw_idx]
cand_pos_self = STATE.scope_sid_to_p1_pos.get(query_meta["id"], 0)
q_vec = raw_emb[raw_idx : raw_idx + 1]
exact_scores = (q_vec @ cand_emb.T)[0].copy()
valid_mask = np.ones(N, dtype=bool)
if 0 <= cand_pos_self < N:
valid_mask[cand_pos_self] = False
exact_ranked = exact_scores.copy()
if 0 <= cand_pos_self < N:
exact_ranked[cand_pos_self] = -1e9
gt_order = np.argsort(-exact_ranked, kind="stable")
gt_top20 = set(gt_order[:20].tolist())
sample_idx = np.concatenate([gt_order[:40], gt_order[np.linspace(40, N - 2, 40).astype(int)]])
arms_spec = [
("naive", "1. Naive Hybrid (No Thm 1, No Thm 2)", False, False),
("thm2_only", "2. Theorem 2 Only (Bussgang Scaling Only)", False, True),
("thm1_only", "3. Theorem 1 Only (Bleed Elimination Only)", True, False),
("both", "4. Full QuantPPI (Theorem 1 + Theorem 2)", True, True),
]
# Encode once (all 4 arms use the exact same wire buffer!)
base_codec = DynamicTopKOutlierCodec(dimension=D, k=k, transform_name="circulant", seed=42)
cache_key = ("scope", model_scale, f"ablation_wire_k{k}")
if cache_key in STATE._encoded_db_cache:
_, packed_bytes, _ = STATE._encoded_db_cache[cache_key]
else:
packed_bytes = base_codec.encode(cand_emb)
STATE._encoded_db_cache[cache_key] = (base_codec, packed_bytes, 0.0)
q_fam = query_meta["family"]
arms_out = []
for arm_id, label, thm1, thm2 in arms_spec:
arm_codec = DynamicTopKOutlierCodec(
dimension=D,
k=k,
transform_name="circulant",
seed=42,
enable_bleed_elimination=thm1,
enable_bussgang_scaling=thm2,
)
scores = arm_codec.compute_asymmetric_scores(q_vec, packed_bytes, count=N)[0].copy()
mse = float(np.mean((scores[valid_mask] - exact_scores[valid_mask]) ** 2))
mae = float(np.mean(np.abs(scores[valid_mask] - exact_scores[valid_mask])))
scores_ranked = scores.copy()
if 0 <= cand_pos_self < N:
scores_ranked[cand_pos_self] = -1e9
order = np.argsort(-scores_ranked, kind="stable")
order_no_self = order[order != cand_pos_self]
r20_30 = len(set(order[:30].tolist()) & gt_top20) / 20.0
r1 = 1.0 if order[0] == gt_order[0] else 0.0
r20 = len(set(order[:20].tolist()) & gt_top20) / 20.0
fam_ap = compute_average_precision(STATE.scope_cand_fam[order_no_self] == q_fam)
arms_out.append({
"arm_id": arm_id,
"label": label,
"enable_bleed_elimination": thm1,
"enable_bussgang_scaling": thm2,
"wire_bytes": arm_codec.total_bytes_per_vector,
"recall_20_at_30": round(r20_30, 4),
"recall_1": round(r1, 4),
"recall_20": round(r20, 4),
"family_ap": round(fam_ap, 4),
"inner_product_mse": float(mse),
"inner_product_mae": float(mae),
"scatter": [
{
"fp32": round(float(exact_scores[int(idx)]), 5),
"pred": round(float(scores[int(idx)]), 5),
"is_top20": int(idx) in gt_top20,
}
for idx in sample_idx
],
})
return {
"query": query_meta,
"model_scale": model_scale,
"dimension": D,
"k": k,
"wire_bytes": base_codec.total_bytes_per_vector,
"arms": arms_out,
}
class QuantPPIRequestHandler(BaseHTTPRequestHandler):
def log_message(self, format: str, *args):
# Concise request logging
sys.stderr.write(f"[API] {self.address_string()} - {format % args}\n")
def _send_json(self, payload: Any, status: int = 200):
body = json.dumps(payload).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
self.send_header("Access-Control-Allow-Headers", "Content-Type")
self.end_headers()
self.wfile.write(body)
def do_OPTIONS(self):
self.send_response(204)
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
self.send_header("Access-Control-Allow-Headers", "Content-Type")
self.end_headers()
def do_GET(self):
parsed = urllib.parse.urlparse(self.path)
path = parsed.path
qs = urllib.parse.parse_qs(parsed.query)
def _p(name: str, default: str = "") -> str:
vals = qs.get(name)
return vals[0] if vals else default
try:
if path == "/api/overview":
dataset = _p("dataset", "scope")
model_scale = _p("model_scale", "esm2_650m")
geom = STATE.get_corpus_geometry(dataset, model_scale)
self._send_json({
"geometry": geom,
"codecs": CODEC_CATALOG,
"datasets": [
{
"id": "scope",
"name": "SCOPe 2.08 ASTRAL-40 Structural Domains",
"total_sequences": len(STATE.scope_entries),
"candidates": len(STATE.p1_cand_raw_ids),
"canonical_queries": len(STATE.p1_query_raw_ids),
"pdb_structures": 15118,
"source_fasta": "data/astral-scopedom-seqres-gd-sel-gs-bib-40-2.08.fa",
},
{
"id": "swissprot",
"name": "UniProtKB/Swiss-Prot Human Proteome",
"total_sequences": len(STATE.sp_entries),
"candidates": len(STATE.p5_cand_raw_ids),
"canonical_queries": len(STATE.p5_query_raw_ids),
"calibration_pool": len(STATE.p5_calib_raw_ids),
"source_fasta": "data/swissprot_human_full.fasta",
},
],
"model_scales": [
{"id": "esm2_650m", "name": "ESM-2 650M (33-layer, D=1280)", "dimension": 1280, "fp32_bytes": 5120},
{"id": "esm2_150m", "name": "ESM-2 150M (30-layer, D=640)", "dimension": 640, "fp32_bytes": 2560},
{"id": "esm2_35m", "name": "ESM-2 35M (12-layer, D=480)", "dimension": 480, "fp32_bytes": 1920},
],
})
return
if path == "/api/proteins":
dataset = _p("dataset", "scope")
q = _p("q", "").strip().lower()
queries_only = _p("queries_only", "false").lower() == "true"
limit = min(int(_p("limit", "40")), 200)
offset = max(int(_p("offset", "0")), 0)
if dataset == "scope":
source = [STATE.scope_p1_candidates[i] for i in range(len(STATE.p1_query_raw_ids))] if queries_only else STATE.scope_p1_candidates
if q:
filtered = [
e for e in source
if q in e["id"].lower()
or q in e["pdb_id"].lower()
or q in e["sccs"].lower()
or q in e["description"].lower()
or q in e["organism"].lower()
or (len(q) >= 4 and q.upper() in e["sequence"])
]
else:
filtered = source
else:
source = STATE.sp_p5_queries if queries_only else STATE.sp_entries
if q:
filtered = [
e for e in source
if q in e["id"].lower()
or q in e["gene"].lower()
or q in e["entry_name"].lower()
or q in e["description"].lower()
or any(q in go.lower() for go in e.get("go_terms", []))
or (len(q) >= 4 and q.upper() in e["sequence"])
]
else:
filtered = source
slice_items = [
{k: v for k, v in item.items() if k != "sequence"}
for item in filtered[offset : offset + limit]
]
self._send_json({
"dataset": dataset,
"total_matches": len(filtered),
"items": slice_items,
})
return
if path == "/api/quantize":
dataset = _p("dataset", "scope")
model_scale = _p("model_scale", "esm2_650m")
protein_id = _p("protein_id", "d6a1ia1" if dataset == "scope" else "P04637")
codec_id = _p("codec_id", "track3_hybrid_k2_block224")
transform_name = _p("transform_name", "circulant")
custom_seq = _p("custom_sequence", "") or None
res = run_single_protein_quantization_pipeline(
dataset=dataset,
model_scale=model_scale,
protein_id=protein_id,
codec_id=codec_id,
transform_name=transform_name,
custom_sequence=custom_seq,
)
self._send_json(res)
return
if path == "/api/search":
dataset = _p("dataset", "scope")
model_scale = _p("model_scale", "esm2_650m")
query_id = _p("query_id", "d1dlwa_" if dataset == "scope" else "P68871")
codec_id = _p("codec_id", "track3_hybrid_k2_block224")
top_k = min(int(_p("top_k", "30")), 100)
res = run_live_search(
dataset=dataset,
model_scale=model_scale,
query_id=query_id,
codec_id=codec_id,
top_k=top_k,
)
self._send_json(res)
return
if path == "/api/ablation":
query_id = _p("query_id", "d1dlwa_")
k = int(_p("k", "4"))
model_scale = _p("model_scale", "esm2_650m")
res = run_live_theorem_ablation(query_id=query_id, k=k, model_scale=model_scale)
self._send_json(res)
return
if path == "/api/structure":
dom_id = _p("id", "d6a1ia1").strip()
compare_id = _p("compare_id", "").strip()
ent_path = PDB_DIR / f"{dom_id}.ent"
if not has_pdb_structure(dom_id):
self._send_json({"error": f"3D PDB structure not found for {dom_id}"}, status=404)
return
primary = parse_pdb_ca_atoms(ent_path)
raw_idx = STATE.scope_id_to_raw_idx.get(dom_id)
primary["metadata"] = STATE.scope_entries[raw_idx] if raw_idx is not None else {"id": dom_id}
comparison = None
if compare_id and has_pdb_structure(compare_id):
comp_path = PDB_DIR / f"{compare_id}.ent"
comparison = parse_pdb_ca_atoms(comp_path)
c_raw_idx = STATE.scope_id_to_raw_idx.get(compare_id)
comparison["metadata"] = STATE.scope_entries[c_raw_idx] if c_raw_idx is not None else {"id": compare_id}
if primary["ca_atoms"] and comparison["ca_atoms"]:
ref_xyz = np.array([[a["cx"], a["cy"], a["cz"]] for a in primary["ca_atoms"]], dtype=np.float64)
mov_xyz = np.array([[a["cx"], a["cy"], a["cz"]] for a in comparison["ca_atoms"]], dtype=np.float64)
aligned_xyz, rmsd = kabsch_superpose(ref_xyz, mov_xyz)
for i, a in enumerate(comparison["ca_atoms"]):
a["ax"] = round(float(aligned_xyz[i, 0]), 3)
a["ay"] = round(float(aligned_xyz[i, 1]), 3)
a["az"] = round(float(aligned_xyz[i, 2]), 3)
comparison["kabsch_rmsd_angstrom"] = round(rmsd, 3)
self._send_json({
"primary": primary,
"comparison": comparison,
})
return
if path == "/api/benchmarks":
self._send_json(STATE.benchmarks_payload)
return
# Serve static frontend from dist/ if available
dist_dir = THIS_DIR / "dist"
if dist_dir.exists():
rel = path.lstrip("/") or "index.html"
file_path = dist_dir / rel
if not file_path.exists() or file_path.is_dir():
file_path = dist_dir / "index.html"
if file_path.exists():
ext = file_path.suffix.lower()
mime = {
".html": "text/html; charset=utf-8",
".js": "application/javascript; charset=utf-8",
".css": "text/css; charset=utf-8",
".svg": "image/svg+xml",
".json": "application/json",
".png": "image/png",
}.get(ext, "application/octet-stream")
data = file_path.read_bytes()
self.send_response(200)
self.send_header("Content-Type", mime)
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
return
self._send_json({"error": f"Not found: {path}"}, status=404)
except Exception as e:
import traceback
traceback.print_exc()
self._send_json({"error": str(e)}, status=500)
def do_POST(self):
parsed = urllib.parse.urlparse(self.path)
path = parsed.path
length = int(self.headers.get("Content-Length", "0"))
raw_body = self.rfile.read(length).decode("utf-8") if length > 0 else "{}"
try:
body = json.loads(raw_body)
if path == "/api/quantize":
res = run_single_protein_quantization_pipeline(
dataset=body.get("dataset", "scope"),
model_scale=body.get("model_scale", "esm2_650m"),
protein_id=body.get("protein_id", "d6a1ia1"),
codec_id=body.get("codec_id", "track3_hybrid_k2_block224"),
transform_name=body.get("transform_name", "circulant"),
custom_sequence=body.get("custom_sequence"),
)
self._send_json(res)
return
self._send_json({"error": f"Unknown POST endpoint: {path}"}, status=404)
except Exception as e:
import traceback
traceback.print_exc()
self._send_json({"error": str(e)}, status=500)
def main():
import argparse
parser = argparse.ArgumentParser(description="QuantPPI Real-Data Web Server")
parser.add_argument("--host", default=os.environ.get("HOST", "0.0.0.0"))
parser.add_argument("--port", type=int, default=int(os.environ.get("PORT", "8000")))
args = parser.parse_args()
server = ThreadingHTTPServer((args.host, args.port), QuantPPIRequestHandler)
print(f"[QuantPPI Server] Listening on http://{args.host}:{args.port}")
try:
server.serve_forever()
except KeyboardInterrupt:
print("\n[QuantPPI Server] Shutting down.")
server.server_close()
if __name__ == "__main__":
main()
#!/usr/bin/env python3
"""
QuantPPI End-to-End Real-Data Backend Server
============================================
Serves real ESM-2 (650M, 150M, 35M) protein embeddings from HDF5 caches,
real SCOPe ASTRAL-40 v2.08 (15,166 domains + 15,118 3D PDB .ent structures),
real Swiss-Prot Human proteome (18,099 proteins + Gene Ontology annotations),
live Campaign 4 data-oblivious quantization codecs (168B to 488B),
live Theorem 1 & Theorem 2 2x2 factorial ablations, and the complete 56-method
QuantPPI vs. Foldseek benchmark ledgers.
"""
import os
import sys
import re
import json
import time
import math
import urllib.parse
from pathlib import Path
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Dict, List, Any, Optional, Tuple
os.environ.setdefault("OMP_NUM_THREADS", "1")
os.environ.setdefault("MKL_NUM_THREADS", "1")
os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1")
os.environ.setdefault("MPLCONFIGDIR", "/tmp")
import h5py
import numpy as np
import pandas as pd
THIS_DIR = Path(__file__).resolve().parent
REPO_ROOT = THIS_DIR.parent
CAMPAIGN4_DIR = REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_4_breakthrough"
FOLDSEEK_RES_DIR = REPO_ROOT / "experiments" / "foldseek_comparison" / "results"
DATA_DIR = REPO_ROOT / "data"
CACHE_DIR = REPO_ROOT / "cache"
PDB_DIR = DATA_DIR / "scope40_pdb" / "candidates_15118_model1"
PDB_H5_PATH = CACHE_DIR / "scope40_ca_structures_15118.h5"
_H5_STRUCT_INDEX: Optional[Dict[str, int]] = None
def _get_h5_struct_index() -> Dict[str, int]:
global _H5_STRUCT_INDEX
if _H5_STRUCT_INDEX is not None:
return _H5_STRUCT_INDEX
_H5_STRUCT_INDEX = {}
if PDB_H5_PATH.exists():
try:
with h5py.File(PDB_H5_PATH, "r") as f:
dom_ids = f["domain_ids"][:]
for idx, raw_id in enumerate(dom_ids):
did = raw_id.decode("ascii", errors="ignore").strip() if isinstance(raw_id, (bytes, np.bytes_)) else str(raw_id).strip()
if did:
_H5_STRUCT_INDEX[did] = idx
except Exception as exc:
print(f"[QuantPPI Server] Warning reading {PDB_H5_PATH}: {exc}")
return _H5_STRUCT_INDEX
def has_pdb_structure(dom_id: str) -> bool:
if (PDB_DIR / f"{dom_id}.ent").exists():
return True
return dom_id in _get_h5_struct_index()
for p in [str(REPO_ROOT), str(CAMPAIGN4_DIR), str(CAMPAIGN4_DIR / "src")]:
if p not in sys.path:
sys.path.insert(0, p)
from src.codecs import (
DynamicTopKOutlierCodec,
AnalyticalSCMFCodec,
TernaryDeadZoneCodec,
MultiStageResidualCodec,
HybridTopKResidualCodec,
SphericalE8LatticeCodec,
HuberTailCompandingCodec,
SmoothCompandingCodec,
TwoStageSearchIndex,
)
from src.transforms.transforms import get_transform
from src.evaluators.retrieval import (
MetricCalculator,
ExactFP32Baseline,
Circulant1BitBaseline,
FWHT1BitBaseline,
Analytic2BitBaseline,
RaBitQ1BitBaseline,
TurboQuant1BitBaseline,
TurboQuant2BitBaseline,
CalibratedNativeProtectionBaseline,
CalibratedSCMFBaseline,
compute_geometry_diagnostics,
)
from src.evaluators.biological import (
compute_average_precision,
compute_roc50,