| """Meaningful checks for scoring, alignment, ontology mapping and publication safety.""" |
| from benchmark_schema import make_schema |
| from prepare_benchmarks import bio_document |
| from evaluate_model import chunks,counts,Adapter |
| from collections import defaultdict |
| import json |
| from pathlib import Path |
|
|
| def main(): |
| d=bio_document(['New','York','and','York'],['B-place','I-place','O','B-place'],'test') |
| assert d['entities']==[[0,8,'place'],[13,17,'place']] |
| assert counts(d['entities'],[[0,8,'place'],[13,17,'wrong']])==[1,1,1] |
| assert counts([[0,5,'x']],[[0,4,'x']])==[0,1,1] |
| long=' '.join('t'+str(i) for i in range(180));parts=chunks(long) |
| assert all(long[o:o+len(t)]==t for o,t in parts) |
| covered=set(i for o,t in parts for i in range(o,o+len(t))) |
| assert all(i in covered for i,c in enumerate(long) if not c.isspace()) |
| a=Adapter.__new__(Adapter);a.events=defaultdict(int) |
| assert a.parse_json('York York','{"place":["York","York"]}',{'place':'Named place'})==[[0,4,'place'],[5,9,'place']] |
| assert a.parse_json('York','{"place":["Paris"]}',{'place':'Named place'})[0][0]<0 |
| assert a.parse_gner('New York is large','New(B-place), York(I-place), is(O), large(O)',{'place':'Named place'})==[[0,8,'place']] |
| assert a.parse_gner('New York is large','New(B-place) York(I-place) is(O) large(O)',{'place':'Named place'})==[[0,8,'place']] |
| assert a.parse_gner('New , York','New(B-place) ,(O) York(B-place)',{'place':'Named place'})==[[0,3,'place'],[6,10,'place']] |
| inv=json.loads((Path(__file__).resolve().parents[1]/'audit/labels.json').read_text()) |
| s=make_schema(['Actor','Rating','Chemical','Year'],inv) |
| assert s['Actor']['status']=='new_specific_type_related' |
| assert s['Rating']['status']=='scope_mismatch_or_unresolved' |
| assert s['Chemical']['training_definition'] in [x['definition'] for x in inv['chemical_compound']['definitions']] |
| semantic=make_schema(['chemicalelement','theory'],inv) |
| assert semantic['chemicalelement']['training_definition']=='Chemical element from periodic table' |
| assert semantic['theory']['training_definition']=='Scientific theory or hypothesis' |
| from investigate_models import flat_predictions |
| flat=flat_predictions([[[0,5,'person',.7],[0,5,'scientist',.9],[6,10,'company',.8],[1,4,'other',.4]]]) |
| assert flat==[[[0,5,'scientist',.9],[6,10,'company',.8]]] |
| print('Protocol checks passed',flush=True) |
|
|
| if __name__=='__main__':main() |
|
|
