File size: 6,861 Bytes
f1fe80f
 
 
a5b90b6
f1fe80f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a5b90b6
f1fe80f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a5b90b6
 
 
 
f1fe80f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a5b90b6
 
 
 
 
 
 
f1fe80f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
"""v24: exact-sequence label table with a source priority measured on the training set.

For every sequence in the disclosed external sources, the label is taken from the first source that has one, in the order
  BRENDA 2026.1 > BRENDA 2023.1 > CatPred-DB > CataPro > DLKcat > PDB/SIFTS > UniProtKB unreviewed (current EC) > ESP > ESP (current EC)
(on 14,122 training proteins that are not in Swiss-Prot, BRENDA's EC is right 95% of the time when it disagrees with the
current UniProt EC, which is right 2.5% of the time; this order scores 0.974 against the official training labels).
Swiss-Prot and train sequences are not in this table: identical train/Swiss-Prot sequences keep their own labels.

Output: <model>/weights/srchash.npz (h: sorted uint64 MD5 prefixes, s: uint32 EC-set id) + srchash_sets.json
Usage: python3 build_source_hash.py --esm ~/enzyme-esm --catapro <CataPro/datasets> --model ~/enzyme-model [--exclude FASTA]
"""
import argparse, collections, gzip, hashlib, json, re
from pathlib import Path
import numpy as np
import pandas as pd

ap = argparse.ArgumentParser()
ap.add_argument('--esm', required=True); ap.add_argument('--catapro', required=True); ap.add_argument('--model', required=True)
ap.add_argument('--exclude')
a = ap.parse_args()
E = Path(a.esm)
VOC = set(json.load(open(Path(a.model) / 'ec_vocabulary.json')))
ORDER = ['B6', 'BR', 'CP', 'CT', 'DK', 'PD', 'TR', 'ES', 'E2']


def ecs(x):
    return [e.strip() for e in re.split(r'[;,]', str(x)) if e.strip() in VOC]


def h(s):
    return int(hashlib.md5(s.encode()).hexdigest()[:16], 16)


lab = collections.defaultdict(lambda: collections.defaultdict(set))   # hash -> source -> ECs (non-UniProt sources)
br = collections.defaultdict(set)
for line in open(E / 'brenda_acc_ec.tsv'):
    if '\t' in line:
        acc, e = line.rstrip('\n').split('\t'); br[acc].update(ecs(e))
sec = collections.defaultdict(set)
for line in open(E / 'round3/sifts_acc_ec.tsv'):
    p = line.rstrip('\n').split('\t')
    if len(p) >= 2: sec[p[0]].update(ecs(p[1]))
b6 = collections.defaultdict(set)                      # BRENDA release 2026.1 (r5/brenda26_acc_ec.tsv)
for line in open(E / 'r5/brenda26_acc_ec.tsv'):
    acc, e = line.rstrip('\n').split('\t'); b6[acc].update(ecs(e))
need = set(br) | set(sec) | set(b6)
acc2h = collections.defaultdict(set)   # an accession can map to several sequence versions
# accession -> sequence fingerprint for BRENDA / SIFTS accessions: UniProtKB unreviewed (fetch_uniprot_hash.py output)
for p in sorted((E / 'uphash').rglob('*.tsv.gz')):
    with gzip.open(p, 'rt') as f:
        for line in f:
            acc, hx = line.split('\t', 2)[:2]
            if acc in need: acc2h[acc].add(int(hx, 16))
sp = pd.read_csv(E / 'sprot.tsv.gz', sep='\t'); sp.columns = ['acc', 'ec', 'seq']
sp_h = set()
for acc, e, s in zip(sp.acc, sp.ec, sp.seq):
    k = h(s)
    if ecs(e): sp_h.add(k)          # Swiss-Prot entries with a task EC are in the search reference already
    if acc in need: acc2h[acc].add(k)
for f in [E / 'trembl_ec.tsv.gz', E / 'trembl_ev2s.tsv.gz']:
    t = pd.read_csv(f, sep='\t'); t.columns = ['acc', 'ec', 'seq']
    for acc, s in zip(t.acc, t.seq):
        if acc in need and isinstance(s, str): acc2h[acc].add(h(s))
cur, buf = None, []
for line in list(open(E / 'brenda_missing.fasta')) + ['>end|end|\n']:
    if line.startswith('>'):
        if cur: acc2h[cur].add(h(''.join(buf)))
        cur = line[1:].split('|')[1] if '|' in line else line[1:].split()[0]; buf = []
    else:
        buf.append(line.strip())
bo = pd.read_csv(E / 'round3/brenda_uniparc.tsv', sep='\t')
for acc, s in zip(bo.accession, bo.sequence):
    acc2h[acc].add(h(s))
sn = pd.read_csv(E / 'round3/sifts_new.tsv', sep='\t')
for acc, s in zip(sn.iloc[:, 0], sn.iloc[:, -1]):
    if isinstance(s, str): acc2h[acc].add(h(s))
t6 = pd.read_csv(E / 'r5/brenda26_seqs.tsv', sep='\t')
for acc, s in zip(t6.accession, t6.sequence):
    if isinstance(s, str): acc2h[acc].add(h(s))
for acc, e in b6.items():
    for k in acc2h.get(acc, ()):
        if e: lab[k]['B6'].update(e)
print('BRENDA 2026.1 accessions mapped', sum(a in acc2h for a in b6), 'of', len(b6), flush=True)
for acc, e in br.items():
    for k in acc2h.get(acc, ()):
        if e: lab[k]['BR'].update(e)
for acc, e in sec.items():
    for k in acc2h.get(acc, ()):
        if e: lab[k]['PD'].update(e)
print('BRENDA accessions mapped', sum(a in acc2h for a in br), 'of', len(br), flush=True)
cp = pd.read_csv(E / 'catpred/catpred_seq_ec.tsv.gz', sep='\t')
for e, s in zip(cp.ec, cp.sequence):
    if isinstance(s, str) and ecs(e): lab[h(s)]['CP'].update(ecs(e))
for f in Path(a.catapro).glob('*fold.csv'):
    c = pd.read_csv(f)
    for e, s in zip(c.EC, c.Sequence):
        if isinstance(s, str) and ecs(e): lab[h(s)]['CT'].update(ecs(e))
k_ = pd.read_csv(E / 'kin_esp_seq_ec.tsv.gz', sep='\t')
for src, e, s in zip(k_.src, k_.ec, k_.sequence):
    if isinstance(s, str) and ecs(e): lab[h(s)]['DK' if src == 'dlk' else 'ES'].update(ecs(e))
en = pd.read_csv(E / 'round3/esp_noec_uniprot.tsv', sep='\t').dropna(subset=['EC number'])
for e, s in zip(en['EC number'], en.Sequence):
    if ecs(e): lab[h(s)]['E2'].update(ecs(e))
# current UniProtKB unreviewed EC from the fingerprint index (weights/uphash.npz)
z = np.load(Path(a.model) / 'weights' / 'uphash.npz'); UH, US = z['h'], z['s']
usets = json.load(open(Path(a.model) / 'weights' / 'uphash_sets.json'))
ex = set()
if a.exclude:
    cur = []
    for line in list(open(a.exclude)) + ['>']:
        if line.startswith('>'):
            if cur: ex.add(h(''.join(cur)))
            cur = []
        else:
            cur.append(line.strip())
set_id, sets = {}, []
def sid(t):
    t = tuple(sorted(t))
    if t not in set_id:
        set_id[t] = len(sets); sets.append(list(t))
    return set_id[t]
cnt = collections.Counter()
# 1. sequences in a non-UniProt source: priority order, UniProt (TR) in its place
H, S = [], []
for k, d in lab.items():
    if k in sp_h or k in ex:
        continue
    j = int(np.searchsorted(UH, np.uint64(k)))
    if j < len(UH) and UH[j] == k:
        d['TR'] = set(usets[int(US[j])])
    for src in ORDER:
        if d.get(src):
            H.append(k); S.append(sid(d[src])); cnt[src] += 1
            break
H = np.array(H, dtype=np.uint64); S = np.array(S, dtype=np.uint32)
# 2. all other UniProt unreviewed sequences keep their current EC
m = ~np.isin(UH, H) & ~np.isin(UH, np.array(sorted(sp_h | ex), dtype=np.uint64))
remap = np.array([sid(x) for x in usets], dtype=np.uint32)
cnt['TR'] += int(m.sum())
H = np.concatenate([H, UH[m]]); S = np.concatenate([S, remap[US[m]]])
o = np.argsort(H, kind='stable'); H, S = H[o], S[o]
w = Path(a.model) / 'weights'
np.savez(w / 'srchash.npz', h=H, s=S)
json.dump(sets, open(w / 'srchash_sets.json', 'w'))
print(f'{len(H)} sequences, label source: {dict(cnt)}, excluded {len(ex)}')