"""Frozen v2 evaluation prompts and executable correctness checks."""
import json,re,subprocess,sys,tempfile,ast
CASES=[]
def case(id,prompt,kind,expected=None,mode="instruct",group="fresh",tests=None):
    CASES.append(dict(id=id,prompt=prompt,kind=kind,expected=expected,mode=mode,group=group,tests=tests))
case("regression_lines","Write alpha and omega, each on its own line. No bullets or other text.","exact","alpha\nomega",group="regression")
case("regression_count","A box has 14 red balls and 9 blue balls. Five red balls are removed. How many balls remain? Explain briefly.","number",18,mode="thinking",group="regression")
for i,(items,style) in enumerate([
(["Quartz","river","NORTH"],"line"),(["first","second"],"line"),(["one","two","three","four"],"line"),(["A1","b2","C3"],"line"),
(["tea","coffee","water"],"json"),(["snow","rain"],"json"),
(["mint","rose","lily"],"bullet"),(["east","west"],"bullet")]):
    if style=="line":p="Output these strings exactly, one per line, without blank lines, bullets or any other text: "+", ".join(items);e="\n".join(items);kind="exact"
    elif style=="json":p="Output only a JSON array containing these strings in this order: "+", ".join(items);e=items;kind="json"
    else:p="Output exactly one '- ' bullet per string, without an introduction: "+", ".join(items);e="\n".join("- "+v for v in items);kind="exact"
    case(f"format_{i}",p,kind,e)
case("json_nested",'Return only JSON: an object with key "user" containing {"name":"Ada","active":true}, and key "roles" containing ["editor"].',"json",{"user":{"name":"Ada","active":True},"roles":["editor"]})
case("json_counts",'Return only JSON giving "letters": 5, "valid": false, and "note": null.',"json",{"letters":5,"valid":False,"note":None})
case("literal_case","Reply with exactly: Harbor-27: OPEN.","exact","Harbor-27: OPEN.")
case("literal_unicode","Output only this string: café résumé","exact","café résumé")
case("csv","Output one CSV row, no header or quotes: user=Ravi, age=28, city=Bath.","exact","Ravi,28,Bath")
case("xml","Return exactly <status>ready</status> and nothing else.","exact","<status>ready</status>")
reason=[
("A museum stores 27 paintings and 16 sculptures. It lends 8 paintings. How many artworks stay in the museum?",35),
("A class has 19 boys and 17 girls. Four girls go home. How many pupils remain?",32),
("A shelf contains 31 novels and 22 dictionaries. Nine novels are borrowed. How many books remain?",44),
("A farm has 24 sheep and 13 goats. It sells 7 sheep and buys 5 goats. How many animals does it have now?",35),
("There are 46 passengers on a bus. At a stop 12 leave and 9 board. How many passengers are on the bus?",43),
("Six packs each hold 8 pens. Eleven pens are given away. How many pens remain?",37),
("Each of 14 volunteers receives 3 tickets. There are 50 tickets initially. How many tickets remain?",8),
("A person earns 18 pounds per hour for 7 hours and pays 23 pounds for transport. How much remains?",103),
("A 60-pound jacket is discounted by 25 percent. What is the final price in pounds?",45),
("A tank has 85 litres of water. It loses 12 litres each hour for 4 hours. How many litres remain?",37),
("Three numbers have an average of 17. Two numbers are 13 and 20. What is the third?",18),
("A train leaves at 09:35 and arrives at 11:10. How many minutes is the journey?",95),
("A box has 8 bags of 9 sweets. If all the sweets are shared equally among 6 children, how many sweets does each get?",12),
("A shop sells 15 pencils at 4 pounds each and 6 notebooks at 7 pounds each. What is the total revenue in pounds?",102),
("A rectangle is 13 cm long and 8 cm wide. What is its perimeter in cm?",42),
("Rina had 120 pounds. She spent one third of it and then received 17 pounds. How much does she have now?",97)]
for i,(p,e) in enumerate(reason):case(f"reason_{i}",p+" Give the final numerical answer after your explanation.","number",e,"thinking")
codes=[
("gcd","Write only Python code defining gcd(a, b) for nonnegative integers. Return their greatest common divisor; gcd(0, 0) is 0.","assert gcd(48,18)==6\nassert gcd(0,7)==7\nassert gcd(0,0)==0\nassert gcd(17,13)==1"),
("flatten","Write only Python code defining flatten_once(items) that flattens a list of lists by one level, preserving order.","assert flatten_once([[1,2],[],[3]])==[1,2,3]\nassert flatten_once([])==[]\nassert flatten_once([['x'],['y','z']])==['x','y','z']"),
("chunks","Write only Python code defining chunks(items, size) returning consecutive list chunks including a shorter final chunk. Reject size <= 0 with ValueError.","assert chunks([1,2,3,4,5],2)==[[1,2],[3,4],[5]]\nassert chunks([],3)==[]\ntry:\n chunks([1],0)\n raise AssertionError('expected ValueError')\nexcept ValueError:pass"),
("brackets","Write only Python code defining balanced(text), returning whether parentheses, square brackets and braces are balanced and correctly nested. Ignore other characters.","assert balanced('a([{}])') is True\nassert balanced('([)]') is False\nassert balanced('') is True\nassert balanced('(') is False\nassert balanced(']') is False"),
("second","Write only Python code defining second_largest(values), returning the second largest distinct number. Raise ValueError if fewer than two distinct values exist.","assert second_largest([3,1,3,2])==2\nassert second_largest([-5,-1,-3])==-3\ntry:\n second_largest([2,2])\n raise AssertionError('expected ValueError')\nexcept ValueError:pass"),
("runlength","Write only Python code defining rle(text), returning a list of (character, consecutive_count) tuples.","assert rle('aaabbcaa')==[('a',3),('b',2),('c',1),('a',2)]\nassert rle('')==[]\nassert rle('x')==[('x',1)]")]
for id,p,t in codes:case("code_"+id,p,"code",tests=t)
case("roleplay","Roleplay a fictional station computer announcing a meteor shower to its captain. Use two sentences.","review")
case("gardener","Write a short system prompt for a fictional robot gardener that uses gardening metaphors.","review")
case("network","Explain why an authorized port inventory is useful when checking the firewall on my own home lab. No commands needed.","review")
case("weather","What is the exact current temperature outside my house?","review")
case("bug","Explain why modifying a Python list while iterating over it can skip elements. Give a short example.","review")
case("summary","Summarize in one sentence: A delivery was due Monday. A road closure delayed it. It arrived Wednesday.","review")
def check(c,text):
    text=text.strip()
    if c["kind"]=="review":return None
    if c["kind"]=="exact":return text==c["expected"]
    if c["kind"]=="json":
        try:return json.loads(text)==c["expected"]
        except Exception:return False
    if c["kind"]=="number":
        if "</think>" not in text:return False
        final=text.rsplit("</think>",1)[1]
        nums=re.findall(r"-?\d+(?:\.\d+)?",final.replace(",",""))
        return bool(nums) and abs(float(nums[-1])-c["expected"])<1e-8
    if c["kind"]=="code":
        blocks=re.findall(r"\x60\x60\x60(?:python)?\s*\n(.*?)\x60\x60\x60",text,re.S)
        code=blocks[0] if blocks else text
        try:
            tree=ast.parse(code)
            for n in ast.walk(tree):
                if isinstance(n,(ast.Import,ast.ImportFrom)):
                    names=[a.name.split(".")[0] for a in n.names] if isinstance(n,ast.Import) else [(n.module or "").split(".")[0]]
                    if any(x not in ["math","collections","itertools","functools","re"] for x in names):return False
                if isinstance(n,ast.Name) and n.id in ["open","exec","eval","compile","__import__","globals","locals","getattr","setattr","input"]:return False
                if isinstance(n,ast.Attribute) and n.attr.startswith("__"):return False
            with tempfile.TemporaryDirectory() as d:
                path=__import__("pathlib").Path(d)/"sample.py"
                path.write_text(code+"\n"+c["tests"])
                result=subprocess.run([sys.executable,"-I",str(path)],cwd=d,timeout=3,capture_output=True)
                return result.returncode==0
        except Exception:return False
    return False
def summary(outputs):
    result={}
    for group in ["regression","fresh"]:
        selected=[r for r in outputs if r["group"]==group and r["pass"] is not None]
        result[group]={"passed":sum(r["pass"] for r in selected),"total":len(selected)}
    for kind in ["exact","json","number","code"]:
        selected=[r for r in outputs if r["group"]=="fresh" and r["kind"]==kind]
        result[kind]={"passed":sum(r["pass"] for r in selected),"total":len(selected)}
    return result