File size: 3,625 Bytes
d35dd87 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | """Regression checks for lossless historical BPE reuse and label integrity."""
from pathlib import Path
import json
import sys
import pytest
from tokenizers import Tokenizer
sys.path.insert(0,str(Path(__file__).resolve().parent))
import laya_formal_experiment as core
from laya_direct_bpe import DirectBPE
from laya_direct_experiment import make_items
@pytest.fixture(scope='module')
def rep():
return DirectBPE()
def test_old_missing_n_regression_and_lossless_repair(rep):
old = Tokenizer.from_file(str(rep.root/'original_source_tokenizers/protein_bpe_8k.json'))
assert ''.join(old.encode('MNA').tokens) != 'MNA'
for kind,sequence in [('protein','MNA'),('protein','ANNAJXUOBZ'),('dna','ACGTNNN')]:
ids = rep.sequence_ids(sequence,kind)
assert ''.join(rep.inverse[i] for i in ids) == sequence
def test_original_merges_and_ids_unchanged(rep):
for name in ['dna_bpe_20k.json','protein_bpe_8k.json']:
old=json.loads((rep.root/'original_source_tokenizers'/name).read_text())['model']
new=json.loads((rep.root/'source_tokenizers'/name).read_text())['model']
assert old['merges']==new['merges']
assert all(new['vocab'][piece]==idx for piece,idx in old['vocab'].items())
assert rep.meta['alphabet_repair']=={'dna':['N'],'protein':['J','N']}
def test_initialization_uses_raw_piece_without_metadata_markers(rep):
for e in rep.entries:
assert e['base_ids']==rep.base(e['piece'],add_special_tokens=False)['input_ids']
expanded_vocab=rep.expanded.get_vocab()
assert all(expanded_vocab[s]==i for s,i in rep.base.get_vocab().items())
def test_raw_equivalence_and_candidate_label_permutation(rep):
_,_,builder=core.import_laya('vendor/laya')
for task in ('fold_class','promoter_detection'):
rows=core.load_split(core.ROOT/'artifacts/laya_formal_data',task,'train')[:3]
raw=make_items(rows,rep,'raw',builder,42,True)
full=make_items(rows,rep,'full_bpe',builder,42,True)
for row,x,y in zip(rows,raw,full):
assert x['label']==y['label']
assert x['markers']==y['markers']
for item in (x,y):
pos=item['markers'][item['label']]
answer=rep.base(' '+row['choices'][row['label']],add_special_tokens=False)['input_ids']
assert item['ids'][pos+1:pos+1+len(answer)]==answer
q={'t':'choice','ins':'Choose the correct biological label for the sequence.',
'crit':{c:None for c in row['choices']}}
ids,markers=rep.build(row,'raw',builder,list(range(len(row['choices']))))
expected=builder(rep.base,core.raw_state(row),q,max_len=1024,head_max_len=256)
assert (ids,markers)==expected
def test_natural_language_is_always_base_encoded_and_modalities_disjoint(rep):
_,_,builder=core.import_laya('vendor/laya')
row=core.load_split(core.ROOT/'artifacts/laya_formal_data','promoter_detection','train')[0]
row=dict(row,context='Natural text with <BIO_DNA:A> and protein ACGT.')
ids,markers=rep.build(row,'full_bpe',builder,[0,1])
seq=rep.sequence_ids(row['sequence'],'dna')
assert ids[-len(seq)-1:-1]==seq
assert all(i<len(rep.base) for i in ids[:-len(seq)-1])
assert set(rep.maps['dna'].values()).isdisjoint(rep.maps['protein'].values())
def test_overlength_input_is_rejected(rep):
_,_,builder=core.import_laya('vendor/laya')
row=core.load_split(core.ROOT/'artifacts/laya_formal_data','promoter_detection','train')[0]
with pytest.raises(ValueError,match='exceeds'):
rep.build(row,'full_bpe',builder,[0,1],max_len=20)
|