File size: 2,441 Bytes
7d004b5 99bfb2e b3d5b6b 7d004b5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | """Meaningful checks for scoring, alignment, ontology mapping and publication safety."""
from benchmark_schema import make_schema
from prepare_benchmarks import bio_document
from evaluate_model import chunks,counts,Adapter
from collections import defaultdict
import json
from pathlib import Path
def main():
d=bio_document(['New','York','and','York'],['B-place','I-place','O','B-place'],'test')
assert d['entities']==[[0,8,'place'],[13,17,'place']]
assert counts(d['entities'],[[0,8,'place'],[13,17,'wrong']])==[1,1,1]
assert counts([[0,5,'x']],[[0,4,'x']])==[0,1,1]
long=' '.join('t'+str(i) for i in range(180));parts=chunks(long)
assert all(long[o:o+len(t)]==t for o,t in parts)
covered=set(i for o,t in parts for i in range(o,o+len(t)))
assert all(i in covered for i,c in enumerate(long) if not c.isspace())
a=Adapter.__new__(Adapter);a.events=defaultdict(int)
assert a.parse_json('York York','{"place":["York","York"]}',{'place':'Named place'})==[[0,4,'place'],[5,9,'place']]
assert a.parse_json('York','{"place":["Paris"]}',{'place':'Named place'})[0][0]<0
assert a.parse_gner('New York is large','New(B-place), York(I-place), is(O), large(O)',{'place':'Named place'})==[[0,8,'place']]
assert a.parse_gner('New York is large','New(B-place) York(I-place) is(O) large(O)',{'place':'Named place'})==[[0,8,'place']]
assert a.parse_gner('New , York','New(B-place) ,(O) York(B-place)',{'place':'Named place'})==[[0,3,'place'],[6,10,'place']]
inv=json.loads((Path(__file__).resolve().parents[1]/'audit/labels.json').read_text())
s=make_schema(['Actor','Rating','Chemical','Year'],inv)
assert s['Actor']['status']=='new_specific_type_related'
assert s['Rating']['status']=='scope_mismatch_or_unresolved'
assert s['Chemical']['training_definition'] in [x['definition'] for x in inv['chemical_compound']['definitions']]
semantic=make_schema(['chemicalelement','theory'],inv)
assert semantic['chemicalelement']['training_definition']=='Chemical element from periodic table'
assert semantic['theory']['training_definition']=='Scientific theory or hypothesis'
from investigate_models import flat_predictions
flat=flat_predictions([[[0,5,'person',.7],[0,5,'scientist',.9],[6,10,'company',.8],[1,4,'other',.4]]])
assert flat==[[[0,5,'scientist',.9],[6,10,'company',.8]]]
print('Protocol checks passed',flush=True)
if __name__=='__main__':main()
|