File size: 8,581 Bytes
fd74a4d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | """Portable native scorer: verbatim used definitions from the archived scientific sources.
The source files are retained in code/frozen. Only imports/namespace binding are adapted.
No generated program is executed as Python code.
"""
import ast, re, types
from typing import Union
VOCAB="abcdefghijkuvwxyz"
class ProgramVocabulary(set):
"""The set of characters used for creating predicate (input) and
transform (output) windows where a given program operates
"""
def __init__(self, *args, **kwargs):
if len(args) > 2: # just some syntax sugar.
super().__init__(list(args))
else: super().__init__(*args, **kwargs)
def __str__(self):
return "".join(list(self))
def __repr__(self):
return "".join(list(self))
def has_word(self, word: str):
word_vocab = set(list(word.strip()))
return word_vocab.issubset(self)
class ProgramBFCCNode:
"""Abstract representation of program in the BFCC interaction graph."""
def __init__(self, predicate: str, transform: str, vocabulary: ProgramVocabulary=ProgramVocabulary('a','b','c','d','e','f'), max_window_size: int=3):
self.predicate = predicate
self.transform = transform
self.vocabulary = vocabulary
self.max_window_size = max_window_size
@property
def alpha(self):
return self.predicate
@property
def beta(self):
return self.transform
@classmethod
def from_string(cls, string_repr: str, max_window_size: int=3, vocabulary: ProgramVocabulary=ProgramVocabulary('a','b','c','d','e','f')):
self = cls("","")
self.vocabulary = vocabulary
self.max_window_size = max_window_size
assert string_repr.strip() != '', "passed blank program input"
assert 'replace(' in string_repr, f"replace function not found in program string: {string_repr}"
# extract predicate and transform
predicate = string_repr.split("replace(")[-1].split(",")[0].strip()
transform = string_repr.split("replace(")[-1].split(",")[1].split(")")[0].strip()
# strip quotations off
predicate = predicate.replace("'","").replace('"','')
transform = transform.replace("'","").replace('"','')
# print("predicate:", predicate)
# print("transform:", transform)
# restrict the length of predicate and transform windows (no. of characters in them).
assert len(predicate) <= self.max_window_size, f"predicate has more than {self.max_window_size} characters: {predicate} in ({string_repr})"
assert len(transform) <= self.max_window_size, f"transform has more than {self.max_window_size} characters: {transform} in ({string_repr})"
# restrict the vocabulary of predicates and transforms.
assert self.vocabulary.has_word(predicate), f"the predicate {predicate} has some symbols not in the vocabulary: {self.vocabulary}"
assert self.vocabulary.has_word(transform), f"the transform {transform} has some symbols not in the vocabulary: {self.vocabulary}"
self.predicate = predicate
self.transform = transform
return self
def __call__(self, inputs: list[str]) -> list[str]:
outputs: list[str] = []
for input in inputs:
output = input.replace(self.predicate, self.transform)
outputs.append(output)
return outputs
def __eq__(self, other):
return self.predicate == other.predicate and self.transform == other.transform
def __repr__(self):
return f'replace("{self.predicate}", "{self.transform}")'
def __str__(self):
return self.__repr__()
def identity(x: list[str]) -> list[str]: return x
def eval_outputs(
inputs: list[str], outputs: list[str],
pred_cascade: list[Union[str, ProgramBFCCNode]],
vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
pred_outputs: list[str] = [i for i in inputs]
valid_programs = 0
invalid_programs = 0
for program in pred_cascade[:max_programs]: # we should only pick the first max_programs (m in the paper) programs predicted by the model in the cascade (in case the model disobeys the instruction and uses more programs than max_programs)
if isinstance(program, str):
try:
program = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
valid_programs += 1
except IndexError as e:
# print(f"\x1b[31;1m{e}\x1b[0m")
program = identity
invalid_programs += 1
except AssertionError as e:
# print(f"\x1b[31;1m{e}\x1b[0m")
program = identity # replace invalid programs with identity programs.
invalid_programs += 1
else:
# assert isinstance(program, ProgramBFCCNode), f"Invalid program type: {type(program)}"
# print(f"\x1b[31;1mInvalid program type: {type(program)}. Replacing with identity. Check if this isn't the result of WRONG FORMATTING!!!\x1b[0m")
program = identity
invalid_programs += 1
pred_outputs: list[str] = program(pred_outputs)
return pred_outputs, valid_programs, invalid_programs
def extract_last_python_block(markdown_text: str) -> str:
"""
Extracts first Python code block from a markdown string.
Args:
markdown_text (str): The markdown text containing code blocks.
Returns:
List[str]: A list of Python code blocks.
"""
code_block_pattern = r"```python\s(.*?)```"
matches = re.findall(code_block_pattern, markdown_text, re.DOTALL)
try: return [block.strip() for block in matches][-1]
except IndexError: return "[]"
def extract_program_str_from_last_python_block(model_response: str) -> str:
try: pred = extract_last_python_block(model_response)
except TypeError as e:
if model_response is None: pass
else: print(e)
pred = "replace('x','x')"
# wrap any replace(...) that isn’t already quoted in single quotes
# e.g. replace('kd','ka') → 'replace('kd','ka')'
pred = re.sub(
r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
r'"\1"',
pred
)
return pred
def try_ast_literal_eval(string): # do some minor fixes.
"""Takes program sequence string and returns the extracted sequence of actual replace programs."""
try:
try:
program_seq = ast.literal_eval(string)
if isinstance(program_seq, str):
program_seq = [program_seq]
# print(program_seq, type(program_seq))
try:
assert isinstance(program_seq, list), f"program sequence of list form couldn't be extracted or derived: {program_seq}"
return program_seq
except AssertionError: return []
except ValueError: return []
except SyntaxError: return []
native=types.SimpleNamespace(eval_outputs=eval_outputs, try_ast_literal_eval=try_ast_literal_eval, extract_program_str_from_last_python_block=extract_program_str_from_last_python_block, ProgramVocabulary=ProgramVocabulary)
class PaperConstrainedNode(ProgramBFCCNode):
@classmethod
def from_string(cls,*args,**kwargs):
node=super().from_string(*args,**kwargs)
assert len(node.alpha)>0, 'paper forbids empty alpha'
return node
_globals=dict(native.eval_outputs.__globals__)
_globals['ProgramBFCCNode']=PaperConstrainedNode
eval_outputs=types.FunctionType(native.eval_outputs.__code__,_globals,'paper_eval_outputs',native.eval_outputs.__defaults__)
def extract(text):
return native.try_ast_literal_eval(native.extract_program_str_from_last_python_block(text))
def score(text,inputs,outputs):
cascade=extract(text)
predicted,valid,invalid=eval_outputs(inputs,outputs,cascade,VOCAB,5)
reasons=[]
for item in cascade[:5]:
if not isinstance(item,str): reasons.append('NON_STRING_RULE'); continue
try:
node=PaperConstrainedNode.from_string(item,vocabulary=native.ProgramVocabulary(VOCAB))
except (AssertionError,IndexError) as exc:
reasons.append(str(exc) or type(exc).__name__)
return {'pass_at_1':int(predicted==outputs),'predicted_outputs':predicted,'predicted_cascade':cascade,'valid_rules':valid,'invalid_rules':invalid,'invalid_rule_reasons':reasons,'complete_python_block_found':bool(re.search(r'```python\s(.*?)```',text,re.S)),'empty_cascade':not cascade}
|