File size: 2,415 Bytes
31f7037 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Named typed questions over the existing Julia scoring API."""
import math
from .data import validate_row
def predict_typed(engine, state, questions):
if not isinstance(questions,dict) or not questions:
raise ValueError('questions must be a nonempty mapping')
rows=[];metadata=[]
for qid,q in questions.items():
if not isinstance(qid,str) or not qid or not isinstance(q,dict):
raise ValueError('Questions require nonempty string IDs and question objects')
kind=q.get('type');criteria=q.get('criteria')
if kind=='choice':
if not isinstance(criteria,dict) or any(not isinstance(k,str) or not k for k in criteria):
raise ValueError('Choice criteria must map nonempty IDs to descriptions')
keys=list(criteria);labels=list(criteria.values())
elif kind=='score':
if not isinstance(criteria,list):raise ValueError('Score requires an ordered rubric')
labels=criteria;keys=[str(i) for i in range(len(labels))]
elif kind=='noul':
keys=['false','true']
if criteria is None:
labels=list(keys)
else:
if not isinstance(criteria,dict) or set(criteria)!=set(keys):
raise ValueError('Noul criteria must map false and true to descriptions')
labels=[criteria[key] for key in keys]
else:raise ValueError('Unsupported question type')
row=dict(state=state,question=q.get('instructions'),type=kind,options=labels)
validate_row(row,len(rows)+1)
rows.append(row);metadata.append((qid,kind,keys))
scores=engine.logits(rows)
if len(scores)!=len(rows):raise ValueError('Model returned an incorrect answer count')
answers={}
for (qid,kind,keys),z in zip(metadata,scores):
if len(z)!=len(keys) or not all(math.isfinite(x) for x in z):raise ValueError('Invalid model scores')
maximum=max(z);p=[math.exp(x-maximum) for x in z];total=sum(p);p=[x/total for x in p]
result=dict(type=kind,probabilities=dict(zip(keys,p)))
if kind=='choice':result['choice']=keys[max(range(len(p)),key=p.__getitem__)]
elif kind=='score':result['score']=sum(i*x for i,x in enumerate(p))
else:result['noul']=p[1]
if kind!='noul':result['max_probability']=max(p)
answers[qid]=result
return dict(answers=answers)
|