File size: 1,576 Bytes
fd74a4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
"""Frozen native scoring path; the sole alignment is empty-alpha identity."""
import re,types
from .common import VOCAB
from src.metrics import functional_correctness as native
from src.data_generation.primitives import ProgramBFCCNode

class PaperConstrainedNode(ProgramBFCCNode):
    @classmethod
    def from_string(cls,*args,**kwargs):
        node=super().from_string(*args,**kwargs)
        assert len(node.alpha)>0, 'paper forbids empty alpha'
        return node

_globals=dict(native.eval_outputs.__globals__)
_globals['ProgramBFCCNode']=PaperConstrainedNode
eval_outputs=types.FunctionType(native.eval_outputs.__code__,_globals,'paper_eval_outputs',native.eval_outputs.__defaults__)
def extract(text):
    return native.try_ast_literal_eval(native.extract_program_str_from_last_python_block(text))
def score(text,inputs,outputs):
    cascade=extract(text)
    predicted,valid,invalid=eval_outputs(inputs,outputs,cascade,VOCAB,5)
    reasons=[]
    for item in cascade[:5]:
        if not isinstance(item,str): reasons.append('NON_STRING_RULE'); continue
        try:
            node=PaperConstrainedNode.from_string(item,vocabulary=native.ProgramVocabulary(VOCAB))
        except (AssertionError,IndexError) as exc:
            reasons.append(str(exc) or type(exc).__name__)
    return {'pass_at_1':int(predicted==outputs),'predicted_outputs':predicted,'predicted_cascade':cascade,'valid_rules':valid,'invalid_rules':invalid,'invalid_rule_reasons':reasons,'complete_python_block_found':bool(re.search(r'```python\s(.*?)```',text,re.S)),'empty_cascade':not cascade}