| """Frozen v2 evaluation prompts and executable correctness checks.""" |
| import json,re,subprocess,sys,tempfile,ast |
| CASES=[] |
| def case(id,prompt,kind,expected=None,mode="instruct",group="fresh",tests=None): |
| CASES.append(dict(id=id,prompt=prompt,kind=kind,expected=expected,mode=mode,group=group,tests=tests)) |
| case("regression_lines","Write alpha and omega, each on its own line. No bullets or other text.","exact","alpha\nomega",group="regression") |
| case("regression_count","A box has 14 red balls and 9 blue balls. Five red balls are removed. How many balls remain? Explain briefly.","number",18,mode="thinking",group="regression") |
| for i,(items,style) in enumerate([ |
| (["Quartz","river","NORTH"],"line"),(["first","second"],"line"),(["one","two","three","four"],"line"),(["A1","b2","C3"],"line"), |
| (["tea","coffee","water"],"json"),(["snow","rain"],"json"), |
| (["mint","rose","lily"],"bullet"),(["east","west"],"bullet")]): |
| if style=="line":p="Output these strings exactly, one per line, without blank lines, bullets or any other text: "+", ".join(items);e="\n".join(items);kind="exact" |
| elif style=="json":p="Output only a JSON array containing these strings in this order: "+", ".join(items);e=items;kind="json" |
| else:p="Output exactly one '- ' bullet per string, without an introduction: "+", ".join(items);e="\n".join("- "+v for v in items);kind="exact" |
| case(f"format_{i}",p,kind,e) |
| case("json_nested",'Return only JSON: an object with key "user" containing {"name":"Ada","active":true}, and key "roles" containing ["editor"].',"json",{"user":{"name":"Ada","active":True},"roles":["editor"]}) |
| case("json_counts",'Return only JSON giving "letters": 5, "valid": false, and "note": null.',"json",{"letters":5,"valid":False,"note":None}) |
| case("literal_case","Reply with exactly: Harbor-27: OPEN.","exact","Harbor-27: OPEN.") |
| case("literal_unicode","Output only this string: café résumé","exact","café résumé") |
| case("csv","Output one CSV row, no header or quotes: user=Ravi, age=28, city=Bath.","exact","Ravi,28,Bath") |
| case("xml","Return exactly <status>ready</status> and nothing else.","exact","<status>ready</status>") |
| reason=[ |
| ("A museum stores 27 paintings and 16 sculptures. It lends 8 paintings. How many artworks stay in the museum?",35), |
| ("A class has 19 boys and 17 girls. Four girls go home. How many pupils remain?",32), |
| ("A shelf contains 31 novels and 22 dictionaries. Nine novels are borrowed. How many books remain?",44), |
| ("A farm has 24 sheep and 13 goats. It sells 7 sheep and buys 5 goats. How many animals does it have now?",35), |
| ("There are 46 passengers on a bus. At a stop 12 leave and 9 board. How many passengers are on the bus?",43), |
| ("Six packs each hold 8 pens. Eleven pens are given away. How many pens remain?",37), |
| ("Each of 14 volunteers receives 3 tickets. There are 50 tickets initially. How many tickets remain?",8), |
| ("A person earns 18 pounds per hour for 7 hours and pays 23 pounds for transport. How much remains?",103), |
| ("A 60-pound jacket is discounted by 25 percent. What is the final price in pounds?",45), |
| ("A tank has 85 litres of water. It loses 12 litres each hour for 4 hours. How many litres remain?",37), |
| ("Three numbers have an average of 17. Two numbers are 13 and 20. What is the third?",18), |
| ("A train leaves at 09:35 and arrives at 11:10. How many minutes is the journey?",95), |
| ("A box has 8 bags of 9 sweets. If all the sweets are shared equally among 6 children, how many sweets does each get?",12), |
| ("A shop sells 15 pencils at 4 pounds each and 6 notebooks at 7 pounds each. What is the total revenue in pounds?",102), |
| ("A rectangle is 13 cm long and 8 cm wide. What is its perimeter in cm?",42), |
| ("Rina had 120 pounds. She spent one third of it and then received 17 pounds. How much does she have now?",97)] |
| for i,(p,e) in enumerate(reason):case(f"reason_{i}",p+" Give the final numerical answer after your explanation.","number",e,"thinking") |
| codes=[ |
| ("gcd","Write only Python code defining gcd(a, b) for nonnegative integers. Return their greatest common divisor; gcd(0, 0) is 0.","assert gcd(48,18)==6\nassert gcd(0,7)==7\nassert gcd(0,0)==0\nassert gcd(17,13)==1"), |
| ("flatten","Write only Python code defining flatten_once(items) that flattens a list of lists by one level, preserving order.","assert flatten_once([[1,2],[],[3]])==[1,2,3]\nassert flatten_once([])==[]\nassert flatten_once([['x'],['y','z']])==['x','y','z']"), |
| ("chunks","Write only Python code defining chunks(items, size) returning consecutive list chunks including a shorter final chunk. Reject size <= 0 with ValueError.","assert chunks([1,2,3,4,5],2)==[[1,2],[3,4],[5]]\nassert chunks([],3)==[]\ntry:\n chunks([1],0)\n raise AssertionError('expected ValueError')\nexcept ValueError:pass"), |
| ("brackets","Write only Python code defining balanced(text), returning whether parentheses, square brackets and braces are balanced and correctly nested. Ignore other characters.","assert balanced('a([{}])') is True\nassert balanced('([)]') is False\nassert balanced('') is True\nassert balanced('(') is False\nassert balanced(']') is False"), |
| ("second","Write only Python code defining second_largest(values), returning the second largest distinct number. Raise ValueError if fewer than two distinct values exist.","assert second_largest([3,1,3,2])==2\nassert second_largest([-5,-1,-3])==-3\ntry:\n second_largest([2,2])\n raise AssertionError('expected ValueError')\nexcept ValueError:pass"), |
| ("runlength","Write only Python code defining rle(text), returning a list of (character, consecutive_count) tuples.","assert rle('aaabbcaa')==[('a',3),('b',2),('c',1),('a',2)]\nassert rle('')==[]\nassert rle('x')==[('x',1)]")] |
| for id,p,t in codes:case("code_"+id,p,"code",tests=t) |
| case("roleplay","Roleplay a fictional station computer announcing a meteor shower to its captain. Use two sentences.","review") |
| case("gardener","Write a short system prompt for a fictional robot gardener that uses gardening metaphors.","review") |
| case("network","Explain why an authorized port inventory is useful when checking the firewall on my own home lab. No commands needed.","review") |
| case("weather","What is the exact current temperature outside my house?","review") |
| case("bug","Explain why modifying a Python list while iterating over it can skip elements. Give a short example.","review") |
| case("summary","Summarize in one sentence: A delivery was due Monday. A road closure delayed it. It arrived Wednesday.","review") |
| def check(c,text): |
| text=text.strip() |
| if c.get("mode")=="instruct" and c["kind"]!="review" and ("<think>" in text or "</think>" in text):return False |
| if c["kind"]=="review":return None |
| if c["kind"]=="exact":return text==c["expected"] |
| if c["kind"]=="json": |
| try:return json.loads(text)==c["expected"] |
| except Exception:return False |
| if c["kind"]=="number": |
| if c.get("mode")=="thinking" and "<think>" not in text:return False |
| if "</think>" not in text:return False |
| final=text.rsplit("</think>",1)[1] |
| boxed=re.findall(r"\\boxed\{\s*(-?\d+(?:\.\d+)?)\s*\}",final) |
| lead=final.strip().split("\n\n",1)[0] |
| lead_nums=re.findall(r"-?\d+(?:\.\d+)?",lead.replace(",","")) |
| nums=boxed if boxed else lead_nums if len(lead_nums)==1 else re.findall(r"-?\d+(?:\.\d+)?",final.replace(",","")) |
| return bool(nums) and abs(float(nums[-1])-c["expected"])<1e-8 |
| if c["kind"]=="code": |
| blocks=re.findall(r"\x60\x60\x60(?:python)?\s*\n(.*?)\x60\x60\x60",text,re.S) |
| code=blocks[0] if blocks else text |
| try: |
| tree=ast.parse(code) |
| for n in ast.walk(tree): |
| if isinstance(n,(ast.Import,ast.ImportFrom)): |
| names=[a.name.split(".")[0] for a in n.names] if isinstance(n,ast.Import) else [(n.module or "").split(".")[0]] |
| if any(x not in ["math","collections","itertools","functools","re","typing","statistics","heapq","bisect","string"] for x in names):return False |
| if isinstance(n,ast.Name) and n.id in ["open","exec","eval","compile","__import__","globals","locals","getattr","setattr","input"]:return False |
| if isinstance(n,ast.Attribute) and n.attr.startswith("__"):return False |
| with tempfile.TemporaryDirectory() as d: |
| path=__import__("pathlib").Path(d)/"sample.py" |
| path.write_text(code+"\n"+c["tests"]) |
| result=subprocess.run([sys.executable,"-I",str(path)],cwd=d,timeout=3,capture_output=True) |
| return result.returncode==0 |
| except Exception:return False |
| return False |
| def summary(outputs): |
| result={} |
| for group in ["regression","fresh"]: |
| selected=[r for r in outputs if r["group"]==group and r["pass"] is not None] |
| result[group]={"passed":sum(r["pass"] for r in selected),"total":len(selected)} |
| for kind in ["exact","json","number","code"]: |
| selected=[r for r in outputs if r["group"]=="fresh" and r["kind"]==kind] |
| result[kind]={"passed":sum(r["pass"] for r in selected),"total":len(selected)} |
| return result |
|
|
