projectxddi's picture
AI server: backend, start-up, status and admin pages
093748c verified
Raw History Blame Contribute Delete
3.92 kB
"""Residue-label CSVs → DataFrames, and the exact DevSet1010 / TestSet300 / TestSetNew46 splits."""
from __future__ import annotations
import ast
from pathlib import Path
import numpy as np
import pandas as pd
from ..constants import ALL_LABELS, DEV, SPLIT_SIZES, TEST46, TEST300
from ..paths import Paths
def parse_list(v):
if isinstance(v, str):
return list(ast.literal_eval(v))
return list(v)
def load_split_csv(path: str | Path, labels=ALL_LABELS, require_unique: bool = True) -> pd.DataFrame:
"""Read one grouped CSV; label columns become python lists; lengths are checked."""
df = pd.read_csv(path)
df["prot_id"] = df["prot_id"].astype(str)
df["sequence"] = df["sequence"].astype(str)
present = [c for c in labels if c in df.columns]
for c in present:
df[c] = df[c].apply(parse_list)
if require_unique and not df.prot_id.is_unique:
raise RuntimeError(f"{path}: duplicate prot_id")
for r in df.itertuples():
for c in present:
if len(getattr(r, c)) != len(r.sequence):
raise RuntimeError(f"{path}/{r.prot_id}/{c}: label length != sequence length")
return df
def load_raw_frames(paths: Paths) -> dict[str, pd.DataFrame]:
"""Raw CSVs. The raw dev CSV has 1014 proteins; the official DevSet1010 is a subset."""
return {
"raw_train": load_split_csv(paths.train_csv),
TEST300: load_split_csv(paths.test300_csv),
TEST46: load_split_csv(paths.test46_csv),
}
def backend_frames(paths: Paths, official_ids, labels=ALL_LABELS) -> dict[str, pd.DataFrame]:
"""Frames of the GEAR store ({"train", "test300", "test46"}): Dev = official IDs in CSV order (the original
backend's store order, which fixes the training-batch order), external sets as in their CSVs."""
frames = {"train": load_split_csv(paths.train_csv, labels), "test300": load_split_csv(paths.test300_csv, labels),
"test46": load_split_csv(paths.test46_csv, labels)}
official = {str(p) for p in official_ids}
frames["train"] = frames["train"][frames["train"].prot_id.isin(official)].reset_index(drop=True)
return frames
def build_splits(raw: dict[str, pd.DataFrame], official_order: list[str]) -> dict[str, pd.DataFrame]:
"""DevSet1010 in official fold order + the two external sets."""
raw_train = raw["raw_train"]
missing = set(official_order) - set(raw_train.prot_id)
if missing:
raise RuntimeError(f"Official Dev IDs missing from project CSV: {sorted(missing)[:20]}")
dev = raw_train.set_index("prot_id", drop=False).loc[official_order].reset_index(drop=True)
splits = {DEV: dev, TEST300: raw[TEST300], TEST46: raw[TEST46]}
for name, df in splits.items():
if len(df) != SPLIT_SIZES[name]:
raise RuntimeError(f"{name}: {len(df)} proteins, expected {SPLIT_SIZES[name]}")
return splits
# ------------------------------------------------------------------ residue bookkeeping
def residue_offsets(df: pd.DataFrame) -> tuple[dict[str, tuple[int, int]], int]:
"""prot_id -> (start, end) in the concatenated residue vector, and total length."""
out, cur = {}, 0
for r in df.itertuples():
L = len(r.sequence)
out[str(r.prot_id)] = (cur, cur + L)
cur += L
return out, cur
def concat_labels(df: pd.DataFrame, label: str, dtype=np.int8) -> np.ndarray:
return np.concatenate([np.asarray(x, dtype=dtype) for x in df[label]])
def residue_ids(df: pd.DataFrame) -> tuple[np.ndarray, np.ndarray]:
"""(prot_id per residue, residue index per residue)."""
pids, ridx = [], []
for r in df.itertuples():
L = len(r.sequence)
pids.extend([str(r.prot_id)] * L)
ridx.extend(range(L))
return np.asarray(pids, dtype=object), np.asarray(ridx, dtype=np.int32)
def n_residues(df: pd.DataFrame) -> int:
return int(df.sequence.str.len().sum())