File size: 1,372 Bytes
2cb04e4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | """Build the reference database: one row per unique training sequence with its ECs (most frequent first)."""
import argparse
import json
from pathlib import Path
import pandas as pd
def build(train_path, output_path):
root = Path(__file__).parent
vocab = set(json.loads((root / 'ec_vocabulary.json').read_text()))
frame = pd.read_parquet(train_path)[['sequence', 'ec_number']].dropna()
bad = sorted(set(frame.ec_number) - vocab)
if bad:
raise ValueError(f'{len(bad)} training ECs not in vocabulary, e.g. {bad[:5]}')
counts = frame.ec_number.value_counts()
frame = frame.drop_duplicates()
frame['n'] = frame.ec_number.map(counts)
frame = frame.sort_values(['sequence', 'n', 'ec_number'], ascending=[True, False, True])
db = frame.groupby('sequence', sort=False).ec_number.agg(list).reset_index(name='ecs')
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
db.to_parquet(output_path, index=False, compression='zstd')
print(f'Wrote {len(db)} sequences, {frame.ec_number.nunique()} ECs to {output_path}')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--train', required=True)
parser.add_argument('--output', default=str(Path(__file__).parent / 'weights/db.parquet'))
args = parser.parse_args()
build(args.train, args.output)
|