File size: 1,576 Bytes
fd74a4d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | """Frozen native scoring path; the sole alignment is empty-alpha identity."""
import re,types
from .common import VOCAB
from src.metrics import functional_correctness as native
from src.data_generation.primitives import ProgramBFCCNode
class PaperConstrainedNode(ProgramBFCCNode):
@classmethod
def from_string(cls,*args,**kwargs):
node=super().from_string(*args,**kwargs)
assert len(node.alpha)>0, 'paper forbids empty alpha'
return node
_globals=dict(native.eval_outputs.__globals__)
_globals['ProgramBFCCNode']=PaperConstrainedNode
eval_outputs=types.FunctionType(native.eval_outputs.__code__,_globals,'paper_eval_outputs',native.eval_outputs.__defaults__)
def extract(text):
return native.try_ast_literal_eval(native.extract_program_str_from_last_python_block(text))
def score(text,inputs,outputs):
cascade=extract(text)
predicted,valid,invalid=eval_outputs(inputs,outputs,cascade,VOCAB,5)
reasons=[]
for item in cascade[:5]:
if not isinstance(item,str): reasons.append('NON_STRING_RULE'); continue
try:
node=PaperConstrainedNode.from_string(item,vocabulary=native.ProgramVocabulary(VOCAB))
except (AssertionError,IndexError) as exc:
reasons.append(str(exc) or type(exc).__name__)
return {'pass_at_1':int(predicted==outputs),'predicted_outputs':predicted,'predicted_cascade':cascade,'valid_rules':valid,'invalid_rules':invalid,'invalid_rule_reasons':reasons,'complete_python_block_found':bool(re.search(r'```python\s(.*?)```',text,re.S)),'empty_cascade':not cascade}
|