File size: 3,916 Bytes
093748c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
"""Residue-label CSVs → DataFrames, and the exact DevSet1010 / TestSet300 / TestSetNew46 splits."""
from __future__ import annotations

import ast
from pathlib import Path

import numpy as np
import pandas as pd

from ..constants import ALL_LABELS, DEV, SPLIT_SIZES, TEST46, TEST300
from ..paths import Paths


def parse_list(v):
    if isinstance(v, str):
        return list(ast.literal_eval(v))
    return list(v)


def load_split_csv(path: str | Path, labels=ALL_LABELS, require_unique: bool = True) -> pd.DataFrame:
    """Read one grouped CSV; label columns become python lists; lengths are checked."""
    df = pd.read_csv(path)
    df["prot_id"] = df["prot_id"].astype(str)
    df["sequence"] = df["sequence"].astype(str)
    present = [c for c in labels if c in df.columns]
    for c in present:
        df[c] = df[c].apply(parse_list)
    if require_unique and not df.prot_id.is_unique:
        raise RuntimeError(f"{path}: duplicate prot_id")
    for r in df.itertuples():
        for c in present:
            if len(getattr(r, c)) != len(r.sequence):
                raise RuntimeError(f"{path}/{r.prot_id}/{c}: label length != sequence length")
    return df


def load_raw_frames(paths: Paths) -> dict[str, pd.DataFrame]:
    """Raw CSVs. The raw dev CSV has 1014 proteins; the official DevSet1010 is a subset."""
    return {
        "raw_train": load_split_csv(paths.train_csv),
        TEST300: load_split_csv(paths.test300_csv),
        TEST46: load_split_csv(paths.test46_csv),
    }


def backend_frames(paths: Paths, official_ids, labels=ALL_LABELS) -> dict[str, pd.DataFrame]:
    """Frames of the GEAR store ({"train", "test300", "test46"}): Dev = official IDs in CSV order (the original
    backend's store order, which fixes the training-batch order), external sets as in their CSVs."""
    frames = {"train": load_split_csv(paths.train_csv, labels), "test300": load_split_csv(paths.test300_csv, labels),
              "test46": load_split_csv(paths.test46_csv, labels)}
    official = {str(p) for p in official_ids}
    frames["train"] = frames["train"][frames["train"].prot_id.isin(official)].reset_index(drop=True)
    return frames


def build_splits(raw: dict[str, pd.DataFrame], official_order: list[str]) -> dict[str, pd.DataFrame]:
    """DevSet1010 in official fold order + the two external sets."""
    raw_train = raw["raw_train"]
    missing = set(official_order) - set(raw_train.prot_id)
    if missing:
        raise RuntimeError(f"Official Dev IDs missing from project CSV: {sorted(missing)[:20]}")
    dev = raw_train.set_index("prot_id", drop=False).loc[official_order].reset_index(drop=True)
    splits = {DEV: dev, TEST300: raw[TEST300], TEST46: raw[TEST46]}
    for name, df in splits.items():
        if len(df) != SPLIT_SIZES[name]:
            raise RuntimeError(f"{name}: {len(df)} proteins, expected {SPLIT_SIZES[name]}")
    return splits


# ------------------------------------------------------------------ residue bookkeeping
def residue_offsets(df: pd.DataFrame) -> tuple[dict[str, tuple[int, int]], int]:
    """prot_id -> (start, end) in the concatenated residue vector, and total length."""
    out, cur = {}, 0
    for r in df.itertuples():
        L = len(r.sequence)
        out[str(r.prot_id)] = (cur, cur + L)
        cur += L
    return out, cur


def concat_labels(df: pd.DataFrame, label: str, dtype=np.int8) -> np.ndarray:
    return np.concatenate([np.asarray(x, dtype=dtype) for x in df[label]])


def residue_ids(df: pd.DataFrame) -> tuple[np.ndarray, np.ndarray]:
    """(prot_id per residue, residue index per residue)."""
    pids, ridx = [], []
    for r in df.itertuples():
        L = len(r.sequence)
        pids.extend([str(r.prot_id)] * L)
        ridx.extend(range(L))
    return np.asarray(pids, dtype=object), np.asarray(ridx, dtype=np.int32)


def n_residues(df: pd.DataFrame) -> int:
    return int(df.sequence.str.len().sum())