Download source.py from VALOR0316/koth-miner-99: direct link, hf CLI and curl.
- Browser
- Download file 70.9 kB
-
https://huggingface.co/VALOR0316/koth-miner-99/resolve/main/source.py
- Command line
-
hf download hf://VALOR0316/koth-miner-99/source.py
-
curl -L -o source.py https://huggingface.co/VALOR0316/koth-miner-99/resolve/main/source.py
70.9 kB
| """UID 101 Aegis clean v5.32 task-independent consensus candidate. | |
| Provider-diverse candidates are selected by literal statement samples, bounded execution, | |
| generated counterexamples, and runtime evidence. The policy has no task IDs, prompt | |
| fingerprints, lookup tables, stored solutions, or benchmark-trained answer selector. | |
| """ | |
| import ast | |
| import json | |
| import math | |
| import re | |
| import queue | |
| import threading | |
| import subprocess | |
| import sys | |
| import time | |
| _MODELS = ( | |
| "qwen/qwen3.7-flash", | |
| "deepseek/deepseek-v4-flash", | |
| "deepseek/deepseek-v4-pro", | |
| "z-ai/glm-5.2", | |
| "openai/gpt-5.6-luna", | |
| "google/gemini-3.6-flash", | |
| "moonshotai/kimi-k3", | |
| ) | |
| _KIND = "valor-uid101-aegis-clean-v5.32" | |
| _SAMPLE_MARK = re.compile(r"^Sample (Input|Output) (\d+)\s*$", re.M) | |
| _CODE_WORD = re.compile(r"\b(?:input|print|sys|def|import|from)\b") | |
| _NUMBER = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?\Z") | |
| _FLOAT_JUDGE = re.compile( | |
| r"(?:absolute\s+or\s+relative|relative\s+or\s+absolute)\s+(?:error|difference)|" | |
| r"(?:absolute|relative)\s+error|error[^\n]{0,80}(?:10\^|1e-)", re.I | |
| ) | |
| _AMBIGUOUS_JUDGE = re.compile( | |
| r"(?:print|output|return)\s+any\b|any\s+(?:valid\s+)?(?:answer|solution)\b|" | |
| r"multiple\s+(?:answers|solutions)[^\n]{0,100}(?:accepted|print any)", re.I | |
| ) | |
| _SEQUENTIAL_RISK = re.compile( | |
| r"(?:in|chronological)\s+order[\s\S]{0,300}(?:replace|overwrite|update)|" | |
| r"(?:replace|overwrite|update)[\s\S]{0,300}(?:in|chronological)\s+order|" | |
| r"for\s+each[\s\S]{0,180}(?:in\s+this\s+order|in\s+order)[\s\S]{0,250}" | |
| r"(?:operations?|append|delete|replace|update)", | |
| re.I, | |
| ) | |
| _MUTABLE_GRAPH_RISK = re.compile( | |
| r"(?:becomes?|turns?).{0,80}(?:into|from).{0,80}(?:road|open|available)|" | |
| r"(?:destroy|remove|unlock|open).{0,100}(?:wall|edge|cell|node)", re.I | re.S | |
| ) | |
| _LARGE_N = re.compile( | |
| r"\bN\s*(?:\\leq|<=|≤)\s*(?:[1-9]\s*(?:\\times|[x×*])\s*)?" | |
| r"10\^(?:[5-9]|[1-9][0-9]+)|" | |
| r"\bN\s*(?:\\leq|<=|≤)\s*[1-9][0-9]{5,}", re.I | |
| ) | |
| _MANY_OUTPUT_RISK = re.compile( | |
| r"(?:for\s+each|for\s+every).{0,100}(?:integer|value|number)|" | |
| r"(?:all|many).{0,80}(?:coefficients?|answers?|values?)", re.I | re.S | |
| ) | |
| _MODULAR_RISK = re.compile( | |
| r"(?:prime|modulo|modulus).{0,100}\bP\b|\bP\b.{0,100}(?:prime|modulo|modulus)", | |
| re.I | re.S, | |
| ) | |
| _COUNTING_RISK = re.compile( | |
| r"(?:number\s+of|count).{0,120}(?:graphs?|ways?|configurations?|sequences?|subsets?)", | |
| re.I | re.S, | |
| ) | |
| _NAMED_BLOCK = re.compile(r"```(candidate|generator|reference|benchmark)\s*\n(.*?)```", re.I | re.S) | |
| _FLOAT_TOL = 1e-9 | |
| _CASE_TIMEOUT = 3.0 | |
| _GEN_TIMEOUT = 1.0 | |
| _STRESS_TIMEOUT = 3.0 | |
| _PERFORMANCE_MARGIN_TIMEOUT = 8.0 | |
| _STRESS_SECONDS = 8.0 | |
| _MAX_CODE_CALLS = 4 | |
| _EVIDENCE_BUDGET_S = 54.0 | |
| _ORACLE_BUDGET_S = 12.0 | |
| _CANDIDATE_EVIDENCE_BUDGET_S = 18.0 | |
| _STRESS_SIZES = (1, 2, 3, 5, 8, 12, 24, 48) | |
| _RUN_BUDGET_S = 640.0 | |
| _EXPECTED_TASKS = 6 | |
| _EXPECTED_CODE_TASKS = 2 | |
| _RESERVE_PER_TASK_S = 20.0 | |
| _RESERVE_PER_CODE_TASK_S = 280.0 | |
| _MIN_CALL_WINDOW_S = 12.0 | |
| _DEFAULT_CALL_ESTIMATE_S = 20.0 | |
| _MODEL_CALL_FLOOR_S = (60.0, 90.0, 180.0, 90.0, 150.0, 45.0, 90.0) | |
| _HARD_CALL_TIMEOUT_S = 300.0 | |
| _RESOURCE_AUDIT = ( | |
| "Estimate maximum states, leaves, and retained objects before coding. A time-feasible " | |
| "enumeration can still exceed memory if every leaf is stored. When only distinct " | |
| "aggregates are required, update the final set or counter online. The largest legal input " | |
| "must retain a substantial margin below the execution limit, not merely finish at its edge." | |
| ) | |
| _INDEX_BOUND_AUDIT = ( | |
| "For every list or array, derive its allocated length and the maximum reachable subscript " | |
| "over every legal input. Reject code when an index can grow faster than the allocation " | |
| "expression or can otherwise exceed it; a sample-sized run is not boundary evidence." | |
| ) | |
| _PERFORMANCE_AUDIT = ( | |
| "Construct a largest-constraint legal input from the statement and estimate the exact dominant " | |
| "operation count. Reject boundary-time code. Prefer a candidate that preserves correctness " | |
| "evidence while completing every executable scale probe with the largest timing margin." | |
| ) | |
| _RECURSION_AUDIT = ( | |
| "Treat self-recursion inside an input-dependent loop as exponential until a strict maximum " | |
| "leaf count proves wide margin. Another recursive branch increases that bound; sample-sized " | |
| "completion is not largest-input performance evidence." | |
| ) | |
| _BATCH_OUTPUT_AUDIT = ( | |
| "When a counting problem requests all answers or coefficients over a range, reject rerunning " | |
| "the full combinatorial dynamic program independently at every interpolation point unless the " | |
| "proved total operation count has a wide margin. Prefer propagating coefficient vectors or " | |
| "batched transforms once through the state graph, while proving the coefficient conversion." | |
| ) | |
| _PERFORMANCE_REPAIR = ( | |
| "The program below has the strongest independent correctness evidence but missed a strict " | |
| "largest-input timing gate. Preserve its verified semantics and exact I/O while reducing the " | |
| "largest legal input below 4 seconds on one CPU. Re-derive the algorithm if " | |
| "micro-optimization cannot provide that margin. Return only complete raw Python 3 source." | |
| ) | |
| _SOLVE = ( | |
| "Derive the algorithm only from the statement and constraints. Check indexing, duplicates, " | |
| "boundaries, precision, and complexity. Never mutate outer-loop operands during pair iterations; " | |
| "use local aliases with attached metadata. Before coding a greedy or compressed-state solution, " | |
| "differential-check its invariant against exhaustive tiny cases." | |
| ) | |
| _INTEGER_SAFETY = ( | |
| "Python integers are arbitrary precision. Do not narrow values, partial sums, XOR states, or " | |
| "hash keys into a fixed-width array or mask unless every stated bound proves that no legal " | |
| "value can be truncated or collide. Prefer built-in set/dict for exact deduplication; " | |
| "use Python-level open addressing only with proved bounds and a faster executable benchmark." | |
| ) | |
| _PRIMARY_FORMAT = ( | |
| "Return two fenced blocks and no prose. First is the complete Python 3 solution; its first " | |
| "source line must be the literal comment # input so the parser recognizes sys.stdin programs. " | |
| "Reference is a literal exhaustive solver for small legal inputs and must not reuse the " | |
| "candidate invariant.\n" | |
| "```python\n<program>\n```\n```reference\n<program>\n```" | |
| ) | |
| _SEQUENTIAL_A = ( | |
| "For mandatory sequential overwrite operations, every incoming value is consumed exactly once, " | |
| "but an early incoming value may be deliberately erased by a later write to the same target. " | |
| "Separate values that survive in the final state from operations sacrificed by later overwrites. " | |
| ) | |
| _SEQUENTIAL_B = ( | |
| "Do not treat all incoming values as freely placeable or refundable. Prove any greedy choice by an " | |
| "exchange argument, preserve chronological last-write semantics, and exhaustively enumerate all " | |
| "target sequences for tiny state and operation counts before returning code." | |
| ) | |
| _MUTABLE_GRAPH = ( | |
| "During shortest-path search, never mutate a graph, grid, or availability table shared by all " | |
| "queued paths. A change discovered at higher cost must not leak into lower-cost states. Attach " | |
| "persistent changes to the search state, or prove an equivalent weighted transition on an " | |
| "immutable graph before using Dijkstra or 0-1 BFS." | |
| ) | |
| _REPAIR = ( | |
| "Re-derive independently; use the failure as evidence of a general defect, never a special " | |
| "case. Test greedy, compressed-state, and sequential-update invariants against a literal " | |
| "exhaustive solver on the smallest legal instances. Trace every statement sample through input " | |
| "parsing, state changes, and output length. Return only complete raw Python 3 source." | |
| ) | |
| _REVIEW = ( | |
| "Solve independently, then audit both candidates for logic, boundaries, complexity, and format. " | |
| "Test greedy or compressed-state invariants against a literal exhaustive solver on the smallest " | |
| "legal instances, including duplicates, ordering, and overwrites. Trace every statement sample " | |
| "through parsing and output length. Return one corrected raw Python 3 program without prose." | |
| ) | |
| _REVIEW = (_REVIEW + " " + _PERFORMANCE_AUDIT + " " + _BATCH_OUTPUT_AUDIT | |
| + " " + _RECURSION_AUDIT) | |
| _MUST_PASS = ( | |
| "Every statement sample is an executable acceptance condition. Re-derive the algorithm rather " | |
| "than fitting sample outputs, mentally execute the complete program on every stated sample, and " | |
| "return code only after all samples agree. If prior candidates share an invariant, actively seek " | |
| "a counterexample to that invariant before reusing it." | |
| ) | |
| _EQUIVALENCE_AUDIT = ( | |
| "Audit every claimed equivalence and every necessary-or-sufficient reduction in both " | |
| "directions. Construct the smallest legal counterexample to each unproved direction before " | |
| "accepting a recurrence, compressed state, interpolation formula, or graph characterization. " | |
| ) | |
| _INDEPENDENCE_AUDIT = ( | |
| "Do not reuse a shared invariant merely because several programs agree. Derive the invariant " | |
| "from the statement, compare it with a literal small-instance enumerator when tractable, and " | |
| "separately justify completeness, uniqueness, and the largest-input complexity bound." | |
| ) | |
| _QUANTIFIER_AUDIT = ( | |
| "Preserve the order and scope of every universal or existential condition. Before replacing a " | |
| "quantified condition by a familiar named property, prove both implications and enumerate the " | |
| "smallest legal structures where the proposed property may hold but the original condition may fail." | |
| ) | |
| _CHALLENGE_A = ( | |
| "Solve independently without copying the prior candidate. Return exactly three fenced blocks and " | |
| "no prose. The candidate block must contain a complete Python 3 solution. " | |
| ) | |
| _CHALLENGE_B = ( | |
| "The generator block must contain a Python program accepting integer seed and size command-line " | |
| "arguments, calling random.seed(seed), and printing one small legal input derived only from the " | |
| "statement. Vary ties, duplicates, boundaries, operation order, and branch combinations.\n" | |
| ) | |
| _CHALLENGE_FORMAT = ( | |
| "Reference is an independent literal small-input solver. Benchmark accepts seed and size and " | |
| "prints one largest legal input, never an answer. Label the first fence python and start it with " | |
| "the literal comment # input so parser and returned candidate match.\n" | |
| "```python\n<program>\n```\n```generator\n<program>\n```" | |
| "\n```reference\n<program>\n```\n```benchmark\n<program>\n```" | |
| ) | |
| _CHALLENGE = _CHALLENGE_A + _CHALLENGE_B + _CHALLENGE_FORMAT | |
| _GENERATOR_ONLY = _CHALLENGE | |
| _ARBITER_ONLY = ( | |
| "Derive a third small-input oracle independently from the statement, without reading or " | |
| "reusing any optimized candidate invariant. Return exactly one fenced reference block and no " | |
| "prose. The program must be a literal exhaustive solver for small legal inputs. It must cover " | |
| "operation order, ties, duplicates, and boundary states whenever those concepts occur in the " | |
| "statement.\n```reference\n<program>\n```" | |
| ) | |
| _FLOAT_FORMAT = "".join(( | |
| "The validator compares output tokens literally even when the statement describes a numeric " | |
| "tolerance. Match the fixed-decimal style and number of places shown consistently by all " | |
| "statement samples. ", | |
| "If samples show more than 15 significant decimal digits, do not use binary float for the final " | |
| "computation: use decimal.Decimal with sufficient guard precision, including Decimal.sqrt or an " | |
| "equivalent high-precision derivation. ", | |
| "Never paste sample values or special-case their inputs; the same derivation and formatting must " | |
| "cover every legal input.", | |
| )) | |
| _EXPLICIT_DECIMALS = re.compile( | |
| r"(?:exactly\s+)?\d+\s+(?:digits?\s+after\s+the\s+decimal|decimal\s+places?)", | |
| re.I, | |
| ) | |
| _FIXED_FORMAT = re.compile(r"\.(\d+)f") | |
| def _sample_decimal_places(statement): | |
| places = set() | |
| for _stdin, expected in _samples(statement, 4): | |
| for token in str(expected).split(): | |
| if _NUMBER.fullmatch(token) and "." in token.lower() and "e" not in token.lower(): | |
| places.add(len(token.rsplit(".", 1)[1])) | |
| return next(iter(places)) if len(places) == 1 and 1 <= next(iter(places)) <= 30 else None | |
| class _DecimalHypotTransformer(ast.NodeTransformer): | |
| """Promote generic Euclidean primitives when public output requires high precision.""" | |
| def __init__(self): | |
| self.changed = False | |
| def visit_Call(self, node): | |
| node = self.generic_visit(node) | |
| if (isinstance(node.func, ast.Attribute) | |
| and isinstance(node.func.value, ast.Name) | |
| and node.func.value.id == "math" and node.func.attr == "hypot" | |
| and len(node.args) == 2 and not node.keywords): | |
| left, right = node.args | |
| def square(value): | |
| converted = ast.Call( | |
| func=ast.Name(id="_RouterDecimal", ctx=ast.Load()), | |
| args=[value], keywords=[], | |
| ) | |
| return ast.BinOp(left=converted, op=ast.Mult(), right=converted) | |
| self.changed = True | |
| return ast.copy_location(ast.Call( | |
| func=ast.Attribute( | |
| value=ast.BinOp(left=square(left), op=ast.Add(), right=square(right)), | |
| attr="sqrt", ctx=ast.Load(), | |
| ), | |
| args=[], keywords=[], | |
| ), node) | |
| return node | |
| class _DecimalFsumTransformer(ast.NodeTransformer): | |
| def visit_Call(self, node): | |
| node = self.generic_visit(node) | |
| if (isinstance(node.func, ast.Attribute) | |
| and isinstance(node.func.value, ast.Name) | |
| and node.func.value.id == "math" and node.func.attr == "fsum" | |
| and len(node.args) == 1 and not node.keywords): | |
| return ast.copy_location(ast.Call( | |
| func=ast.Name(id="sum", ctx=ast.Load()), | |
| args=[node.args[0], ast.Call( | |
| func=ast.Name(id="_RouterDecimal", ctx=ast.Load()), | |
| args=[ast.Constant(value=0)], keywords=[], | |
| )], keywords=[], | |
| ), node) | |
| return node | |
| def _promote_decimal_hypot(value, sample_places): | |
| if sample_places <= 15 or "Decimal" in value or "decimal" in value: | |
| return value | |
| try: | |
| tree = ast.parse(value) | |
| transformer = _DecimalHypotTransformer() | |
| tree = transformer.visit(tree) | |
| if not transformer.changed: | |
| return value | |
| tree = _DecimalFsumTransformer().visit(tree) | |
| insertion = 1 if (tree.body and isinstance(tree.body[0], ast.Expr) | |
| and isinstance(tree.body[0].value, ast.Constant) | |
| and isinstance(tree.body[0].value.value, str)) else 0 | |
| while (insertion < len(tree.body) and isinstance(tree.body[insertion], ast.ImportFrom) | |
| and tree.body[insertion].module == "__future__"): | |
| insertion += 1 | |
| setup = ast.parse( | |
| "from decimal import Decimal as _RouterDecimal, getcontext as _router_decimal_context\n" | |
| + "_router_decimal_context().prec = " + str(sample_places + 20) + "\n" | |
| ).body | |
| tree.body[insertion:insertion] = setup | |
| ast.fix_missing_locations(tree) | |
| return ast.unparse(tree) + "\n" | |
| except (SyntaxError, ValueError, TypeError): | |
| return value | |
| def _normalize_float_program(code, prompt): | |
| """Apply the public sample's literal float style without task identity or stored outputs.""" | |
| value = str(code or "") | |
| statement = str(prompt or "") | |
| if not _FLOAT_JUDGE.search(statement) or _EXPLICIT_DECIMALS.search(statement): | |
| return value | |
| try: | |
| ast.parse(value) | |
| except (SyntaxError, ValueError, TypeError): | |
| return value | |
| sample_places = _sample_decimal_places(statement) | |
| if sample_places is None: | |
| return value | |
| updated = _FIXED_FORMAT.sub(lambda _match: "." + str(sample_places) + "f", value) | |
| updated = _promote_decimal_hypot(updated, sample_places) | |
| try: | |
| ast.parse(updated) | |
| except (SyntaxError, ValueError, TypeError): | |
| return value | |
| return updated | |
| def _finish_candidate(code, prompt): | |
| """Reject prose, truncation, and malformed programs before any expensive evidence stage.""" | |
| value = _normalize_float_program(code, prompt) | |
| if not str(value).strip(): | |
| return "" | |
| try: | |
| ast.parse(value) | |
| except (SyntaxError, ValueError, TypeError): | |
| return "" | |
| return value | |
| def _load(weights): | |
| try: | |
| data = json.loads(bytes(weights).decode("utf-8")) | |
| except Exception as exc: | |
| raise ValueError("UID 101 clean settings are not valid JSON") from exc | |
| required = { | |
| "schema", "kind", "non_code", "primary", "challenger", "generator", "reviewer", "rescue", | |
| "batch_primary", "batch_challenger", | |
| "primary_max_tokens", "challenge_max_tokens", "review_max_tokens", | |
| "batch_primary_max_tokens", "batch_challenge_max_tokens", | |
| "generator_max_tokens", "rescue_max_tokens", "sample_limit", "stress_cases", | |
| "min_consensus_cases", "critic", | |
| "temperature", "primary_reasoning", "challenge_reasoning", "review_reasoning", | |
| "batch_primary_reasoning", "batch_challenge_reasoning", | |
| "generator_reasoning", "rescue_reasoning", | |
| } | |
| if not isinstance(data, dict) or set(data) != required: | |
| raise ValueError("UID 101 clean settings have an invalid schema") | |
| if data["schema"] != 3 or data["kind"] != _KIND: | |
| raise ValueError("UID 101 clean settings do not match this source") | |
| for field in ("non_code", "primary", "challenger", "generator", "reviewer", "rescue", | |
| "batch_primary", "batch_challenger"): | |
| if type(data[field]) is not int or not 0 <= data[field] < len(_MODELS): | |
| raise ValueError("UID 101 clean model index is invalid") | |
| if (type(data["primary_max_tokens"]) is not int | |
| or not 4096 <= data["primary_max_tokens"] <= 32768): | |
| raise ValueError("UID 101 clean primary token limit is invalid") | |
| for field in ("challenge_max_tokens", "generator_max_tokens", "review_max_tokens", | |
| "rescue_max_tokens", "batch_primary_max_tokens", "batch_challenge_max_tokens"): | |
| if type(data[field]) is not int or not 2048 <= data[field] <= 32768: | |
| raise ValueError("UID 101 clean fallback token limit is invalid") | |
| if type(data["sample_limit"]) is not int or not 1 <= data["sample_limit"] <= 4: | |
| raise ValueError("UID 101 clean sample limit is invalid") | |
| if type(data["stress_cases"]) is not int or not 4 <= data["stress_cases"] <= 24: | |
| raise ValueError("UID 101 clean stress case limit is invalid") | |
| if (type(data["min_consensus_cases"]) is not int | |
| or not 2 <= data["min_consensus_cases"] <= data["stress_cases"]): | |
| raise ValueError("UID 101 clean consensus threshold is invalid") | |
| critic = data["critic"] | |
| if (not isinstance(critic, dict) | |
| or set(critic) != {"schema", "mean", "scale", "coef", "intercept"} | |
| or critic["schema"] != 1): | |
| raise ValueError("UID 101 critic schema is invalid") | |
| for field in ("mean", "scale", "coef"): | |
| if (not isinstance(critic[field], list) or len(critic[field]) != 18 | |
| or any(type(value) not in (int, float) or not math.isfinite(value) | |
| for value in critic[field])): | |
| raise ValueError("UID 101 critic vector is invalid") | |
| if (any(value <= 0 for value in critic["scale"]) | |
| or type(critic["intercept"]) not in (int, float) | |
| or not math.isfinite(critic["intercept"])): | |
| raise ValueError("UID 101 critic normalization is invalid") | |
| if type(data["temperature"]) not in (int, float) or data["temperature"] != 0: | |
| raise ValueError("UID 101 clean temperature must be deterministic") | |
| for field in ("primary_reasoning", "challenge_reasoning", "generator_reasoning", | |
| "review_reasoning", "rescue_reasoning", "batch_primary_reasoning", | |
| "batch_challenge_reasoning"): | |
| if data[field] not in ("none", "low", "medium", "high"): | |
| raise ValueError("UID 101 clean reasoning effort is invalid") | |
| return data | |
| def _is_code(prompt): | |
| text = str(prompt) | |
| return ("Write a complete Python 3 program" in text | |
| and "standard input" in text and "standard output" in text) | |
| def _is_mcq(prompt): | |
| text = "\n" + str(prompt) | |
| return all("\n" + option in text for option in ("A)", "B)", "C)", "D)")) | |
| def _non_code_request(prompt): | |
| text = str(prompt) | |
| if _is_mcq(text): | |
| return text | |
| return text + "\n\nDerive the answer strictly from the question and return only the answer." | |
| def _samples(prompt, limit): | |
| text = str(prompt).replace("\r\n", "\n").replace("\r", "\n") | |
| if _AMBIGUOUS_JUDGE.search(text): | |
| return [] | |
| marks = [(m.start(), m.end(), m.group(1), int(m.group(2))) | |
| for m in _SAMPLE_MARK.finditer(text)] | |
| blocks = {} | |
| for index, (_start, end, kind, number) in enumerate(marks): | |
| stop = marks[index + 1][0] if index + 1 < len(marks) else len(text) | |
| lines = text[end:stop].split("\n") | |
| while lines and not lines[0].strip(): | |
| lines.pop(0) | |
| kept = [] | |
| for line in lines: | |
| if not line.strip(): | |
| break | |
| kept.append(line) | |
| blocks[(kind, number)] = "\n".join(kept) | |
| pairs = [] | |
| for number in sorted({number for _kind, number in blocks}): | |
| stdin = blocks.get(("Input", number)) | |
| stdout = blocks.get(("Output", number)) | |
| if stdin and stdout: | |
| pairs.append((stdin, stdout)) | |
| return pairs[:limit] | |
| def _source(answer): | |
| """Match the validator's first-qualifying-block extraction exactly. | |
| Returning a later or longest fence is provenance-invalid even if that program also appeared in | |
| the response: the validator canonicalizes only the first block containing input/print. | |
| """ | |
| text = str(answer or "").strip() | |
| if not text: | |
| return "" | |
| if "```" in text: | |
| for block in (part for part in text.split("```") if part.strip()): | |
| block = block[len("python"):] if block.lstrip().lower().startswith("python") else block | |
| if "input" in block or "print" in block: | |
| return block.strip() + "\n" | |
| return text + "\n" | |
| def _returnable_candidate(answer, programs=None): | |
| """Return only the exact program token the validator attributes to this response.""" | |
| canonical = _source(answer) | |
| parsed = (programs or _named_programs(answer)).get("candidate") | |
| if parsed and " ".join(parsed.split()) == " ".join(canonical.split()): | |
| return parsed | |
| return canonical | |
| def _tokens_match(observed, expected, numeric): | |
| got = str(observed).split() | |
| want = str(expected).split() | |
| if got == want: | |
| return True | |
| if not numeric or len(got) != len(want) or not got: | |
| return False | |
| for left, right in zip(got, want): | |
| if _NUMBER.fullmatch(left) is None or _NUMBER.fullmatch(right) is None: | |
| return False | |
| try: | |
| a, b = float(left), float(right) | |
| except ValueError: | |
| return False | |
| if not math.isfinite(a) or not math.isfinite(b): | |
| return False | |
| if abs(a - b) > _FLOAT_TOL * max(1.0, abs(b)): | |
| return False | |
| return True | |
| def _run_program(code, stdin_text, timeout, argv=(), deadline=None): | |
| if deadline is not None: | |
| remaining = deadline - time.monotonic() | |
| if remaining <= 0: | |
| return None | |
| timeout = min(float(timeout), max(0.01, remaining)) | |
| try: | |
| run = subprocess.run( | |
| [sys.executable, "-I", "-c", code, *[str(value) for value in argv]], | |
| input=str(stdin_text), | |
| capture_output=True, text=True, timeout=timeout, | |
| ) | |
| except (subprocess.TimeoutExpired, OSError, ValueError): | |
| return None | |
| return run.stdout if run.returncode == 0 else None | |
| def _check(code, samples, numeric, deadline=None): | |
| if not code.strip(): | |
| return 0, max(1, len(samples)), ( | |
| samples[0][0] if samples else "", "<empty>", | |
| samples[0][1] if samples else "valid Python", | |
| ) | |
| try: | |
| compile(code, "<candidate>", "exec") | |
| except (SyntaxError, ValueError, TypeError) as exc: | |
| return 0, max(1, len(samples)), ( | |
| samples[0][0] if samples else "", | |
| type(exc).__name__ + ": " + str(exc)[:600], | |
| samples[0][1] if samples else "valid Python", | |
| ) | |
| passed = 0 | |
| first_bad = None | |
| for stdin, expected in samples: | |
| observed = _run_program(code, stdin, _CASE_TIMEOUT, deadline=deadline) | |
| # The validator's LCB grader is literal-token exact, including numeric tasks. | |
| good = observed is not None and str(observed).split() == str(expected).split() | |
| if good: | |
| passed += 1 | |
| elif first_bad is None: | |
| first_bad = (stdin, observed if observed is not None else "<no output>", expected) | |
| return passed, len(samples), first_bad | |
| def _failure_note(passed, total, failure): | |
| if failure is None: | |
| return "sample verification: " + str(passed) + "/" + str(total) | |
| stdin, observed, expected = failure | |
| return ( | |
| "sample verification: " + str(passed) + "/" + str(total) | |
| + "\nFailing sample input:\n" + str(stdin)[:500] | |
| + "\nCandidate output:\n" + str(observed)[:500] | |
| + "\nExpected output:\n" + str(expected)[:500] | |
| ) | |
| def _named_programs(answer): | |
| programs = {} | |
| python_blocks = [] | |
| text = str(answer or "") | |
| # Recover explicitly labelled tooling blocks first. A provider may emit an unlabelled closing | |
| # fence after raw candidate code; generic left-to-right fence pairing can otherwise consume the | |
| # next labelled opener as a closer and hide valid tooling. | |
| for label in ("generator", "reference", "benchmark"): | |
| match = re.search(r"```" + label + r"\s*\n(.*?)```", text, re.I | re.S) | |
| if match: | |
| code = match.group(1).strip() | |
| if code: | |
| programs[label] = code + "\n" | |
| for match in re.finditer(r"```([^\n`]*)\n(.*?)```", text, re.I | re.S): | |
| # Some providers add one extra fence marker around an otherwise correctly labelled block. | |
| # Normalize only fence punctuation; arbitrary prose labels are never accepted as programs. | |
| tag = match.group(1).strip().lower().strip("`") | |
| lines = [line for line in match.group(2).strip().splitlines() | |
| if line.strip().lower() not in ("<program>", "program")] | |
| code = "\n".join(lines).strip() | |
| if not code: | |
| continue | |
| if tag in ("python", "python3", "py"): | |
| python_blocks.append(code + "\n") | |
| if "candidate" not in programs: | |
| programs["candidate"] = code + "\n" | |
| elif tag in ("generator", "reference", "benchmark") and tag not in programs: | |
| programs[tag] = code + "\n" | |
| elif tag == "candidate" and "candidate" not in programs: | |
| # Backward-compatible parsing for stored development responses. New prompts never ask | |
| # for this tag because the validator would retain the word `candidate` in its token. | |
| programs["candidate"] = code + "\n" | |
| # Positional recovery is format-only and independent of task identity. It supports providers | |
| # that rewrite every requested fence tag to `python` while preserving fence order. | |
| if len(python_blocks) >= 2 and "generator" not in programs: | |
| programs["generator"] = python_blocks[1] | |
| if len(python_blocks) >= 3 and "reference" not in programs: | |
| programs["reference"] = python_blocks[2] | |
| return programs | |
| def _plain_program_blocks(answer): | |
| blocks = [] | |
| for raw in re.findall(r"```(?:python|python3|py)?\s*\n(.*?)```", str(answer or ""), | |
| re.I | re.S): | |
| code = raw.strip() | |
| if code: | |
| blocks.append(code + "\n") | |
| return blocks | |
| def _generated_inputs(generator, case_limit, require_scale=True, deadline=None): | |
| if not generator or not generator.strip(): | |
| return [] | |
| # A generator that ignores the supplied scale only exercises toy inputs and can hide | |
| # asymptotic failures. This is a task-independent interface requirement, not prompt dispatch. | |
| direct_size = re.search(r"(?:sys\.)?argv\s*\[\s*2\s*\]", generator) | |
| sliced_args = (re.search(r"(?:sys\.)?argv\s*\[\s*1\s*:\s*\]", generator) | |
| and re.search(r"\[\s*1\s*\]", generator)) | |
| if require_scale and direct_size is None and not sliced_args: | |
| return [] | |
| try: | |
| compile(generator, "<generator>", "exec") | |
| except (SyntaxError, ValueError, TypeError): | |
| return [] | |
| local_deadline = time.monotonic() + _STRESS_SECONDS | |
| deadline = min(local_deadline, deadline) if deadline is not None else local_deadline | |
| cases = [] | |
| observations = [] | |
| if case_limit <= 1: | |
| size_schedule = _STRESS_SIZES[:case_limit] | |
| elif case_limit < len(_STRESS_SIZES): | |
| last = len(_STRESS_SIZES) - 1 | |
| size_schedule = tuple( | |
| _STRESS_SIZES[round(index * last / (case_limit - 1))] | |
| for index in range(case_limit) | |
| ) | |
| else: | |
| size_schedule = tuple( | |
| _STRESS_SIZES[index % len(_STRESS_SIZES)] for index in range(case_limit) | |
| ) | |
| for index, size in enumerate(size_schedule): | |
| if time.monotonic() >= deadline: | |
| break | |
| stdin = _run_program(generator, "", _GEN_TIMEOUT, (index + 1, size), deadline) | |
| if not stdin or not stdin.strip() or len(stdin) > 20000: | |
| continue | |
| observations.append((size, stdin)) | |
| if stdin not in cases: | |
| cases.append(stdin) | |
| if require_scale: | |
| small = [value for size, value in observations if size <= 5] | |
| large = [value for size, value in observations if size >= 24] | |
| if not small or not large: | |
| return [] | |
| def shape(value): | |
| tokens = str(value).split() | |
| integers = [] | |
| for token in tokens[:32]: | |
| try: | |
| integers.append(abs(int(token))) | |
| except ValueError: | |
| integers.append(0) | |
| return len(str(value)), len(tokens), integers | |
| small_shapes = [shape(value) for value in small] | |
| large_shapes = [shape(value) for value in large] | |
| small_chars = max(row[0] for row in small_shapes) | |
| small_tokens = max(row[1] for row in small_shapes) | |
| grew = (max(row[0] for row in large_shapes) >= max(8, int(small_chars * 1.25)) | |
| or max(row[1] for row in large_shapes) >= max(3, int(small_tokens * 1.25))) | |
| width = max((len(row[2]) for row in small_shapes + large_shapes), default=0) | |
| for position in range(width): | |
| low = max((row[2][position] for row in small_shapes | |
| if position < len(row[2])), default=0) | |
| high = max((row[2][position] for row in large_shapes | |
| if position < len(row[2])), default=0) | |
| if high > low and high >= max(2, int(low * 1.25)): | |
| grew = True | |
| break | |
| if not grew: | |
| return [] | |
| return cases | |
| def _deep_code_risk(text): | |
| """Recognize broad derivation risk without prompt identity or stored-solution routing.""" | |
| value = str(text) | |
| signals = sum(bool(pattern.search(value)) for pattern in ( | |
| _MANY_OUTPUT_RISK, _MODULAR_RISK, _COUNTING_RISK, | |
| )) | |
| return signals >= 2 | |
| def _fixed_width_integer_risk(code): | |
| """Flag lossy Python integer storage; this is a language safety check, not task dispatch.""" | |
| try: | |
| tree = ast.parse(code) | |
| except (SyntaxError, ValueError, TypeError): | |
| return False | |
| has_fixed_array = False | |
| has_integer_narrowing = False | |
| for node in ast.walk(tree): | |
| if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) | |
| and node.func.id == "array" and node.args | |
| and isinstance(node.args[0], ast.Constant) | |
| and str(node.args[0].value) in {"b", "B", "h", "H", "i", "I", "l", "L", "q", "Q"}): | |
| has_fixed_array = True | |
| if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitAnd): | |
| for operand in (node.left, node.right): | |
| if (isinstance(operand, ast.Constant) and type(operand.value) is int | |
| and operand.value >= (1 << 31) - 1 | |
| and (operand.value + 1) & operand.value == 0): | |
| has_integer_narrowing = True | |
| return has_fixed_array or has_integer_narrowing | |
| def _expression_degree(node, degrees=None): | |
| """Conservative polynomial degree used only for generic allocation/index diagnostics.""" | |
| degrees = degrees or {} | |
| if node is None or isinstance(node, ast.Constant): | |
| return 0 | |
| if isinstance(node, ast.Name): | |
| return degrees.get(node.id, 1) | |
| if isinstance(node, ast.UnaryOp): | |
| return _expression_degree(node.operand, degrees) | |
| if isinstance(node, ast.BinOp): | |
| left = _expression_degree(node.left, degrees) | |
| right = _expression_degree(node.right, degrees) | |
| if isinstance(node.op, (ast.Add, ast.Sub, ast.Mod)): | |
| return max(left, right) | |
| if isinstance(node.op, ast.Mult): | |
| return left + right | |
| if isinstance(node.op, (ast.Div, ast.FloorDiv)): | |
| return left | |
| if (isinstance(node.op, ast.Pow) and isinstance(node.right, ast.Constant) | |
| and type(node.right.value) is int and 0 <= node.right.value <= 8): | |
| return left * node.right.value | |
| return max(left, right) | |
| if isinstance(node, ast.Call): | |
| return max((_expression_degree(arg, degrees) for arg in node.args), default=1) | |
| return max((_expression_degree(child, degrees) | |
| for child in ast.iter_child_nodes(node)), default=0) | |
| def _repeated_list_size(node): | |
| if not isinstance(node, ast.BinOp) or not isinstance(node.op, ast.Mult): | |
| return None | |
| if isinstance(node.left, (ast.List, ast.Tuple)): | |
| return node.right | |
| if isinstance(node.right, (ast.List, ast.Tuple)): | |
| return node.left | |
| return None | |
| def _allocation_extent(node): | |
| repeated = _repeated_list_size(node) | |
| if repeated is not None: | |
| return repeated | |
| if (isinstance(node, ast.ListComp) and len(node.generators) == 1 | |
| and isinstance(node.generators[0].iter, ast.Call) | |
| and isinstance(node.generators[0].iter.func, ast.Name) | |
| and node.generators[0].iter.func.id == "range"): | |
| args = node.generators[0].iter.args | |
| if len(args) == 1: | |
| return args[0] | |
| if len(args) >= 2: | |
| return args[1] | |
| return None | |
| def _allocation_index_risk(code): | |
| """Flag an index whose polynomial growth outruns its list allocation growth.""" | |
| try: | |
| tree = ast.parse(code) | |
| except (SyntaxError, ValueError, TypeError): | |
| return False | |
| allocations = {} | |
| degrees = {} | |
| for node in ast.walk(tree): | |
| if isinstance(node, (ast.Assign, ast.AnnAssign)): | |
| value = node.value | |
| targets = node.targets if isinstance(node, ast.Assign) else [node.target] | |
| size = _allocation_extent(value) | |
| for target in targets: | |
| if isinstance(target, ast.Name): | |
| if size is None: | |
| degrees[target.id] = _expression_degree(value, degrees) | |
| else: | |
| allocations[target.id] = _expression_degree(size, degrees) | |
| for node in ast.walk(tree): | |
| if not isinstance(node, ast.Subscript) or not isinstance(node.value, ast.Name): | |
| continue | |
| if node.value.id not in allocations: | |
| continue | |
| index = node.slice.value if isinstance(node.slice, ast.Index) else node.slice | |
| if _expression_degree(index, degrees) > allocations[node.value.id]: | |
| return True | |
| return False | |
| def _quality(code): | |
| """Return a small task-independent static safety score; never infer a task identity.""" | |
| try: | |
| tree = ast.parse(code) | |
| except (SyntaxError, ValueError, TypeError): | |
| return -100 | |
| score = 0 | |
| nodes = list(ast.walk(tree)) | |
| score += int(any(isinstance(node, (ast.For, ast.While)) for node in nodes)) | |
| score += int(any(isinstance(node, ast.FunctionDef) for node in nodes)) | |
| score += int(40 <= len(code) <= 12000) | |
| dangerous = {"eval", "exec", "compile", "__import__"} | |
| score -= 4 * sum( | |
| isinstance(node, ast.Call) and isinstance(node.func, ast.Name) | |
| and node.func.id in dangerous for node in nodes | |
| ) | |
| score -= int(len(code) > 20000) | |
| return score | |
| def _has_input_scaled_nested_loop(code): | |
| """Reject an obvious quadratic loop only when the statement also declares large N.""" | |
| try: | |
| tree = ast.parse(code) | |
| except (SyntaxError, ValueError, TypeError): | |
| return False | |
| loop_types = (ast.For, ast.AsyncFor, ast.While) | |
| for outer in (node for node in ast.walk(tree) if isinstance(node, loop_types)): | |
| for child in ast.walk(outer): | |
| if child is outer or not isinstance(child, loop_types): | |
| continue | |
| expression = child.iter if isinstance(child, (ast.For, ast.AsyncFor)) else child.test | |
| names = {node.id for node in ast.walk(expression) if isinstance(node, ast.Name)} | |
| if names: | |
| return True | |
| return False | |
| def _ast_depth(node): | |
| children = list(ast.iter_child_nodes(node)) | |
| return 1 + max((_ast_depth(child) for child in children), default=0) | |
| def _critic_features(code, sample_ratio, completion_ratio, consensus_ratio): | |
| try: | |
| tree = ast.parse(code) | |
| except (SyntaxError, ValueError, TypeError): | |
| return [0.0] * 15 + [float(sample_ratio), float(completion_ratio), | |
| float(consensus_ratio)] | |
| nodes = list(ast.walk(tree)) | |
| dangerous = {"eval", "exec", "compile", "__import__"} | |
| return [ | |
| math.log1p(len(code)), math.log1p(code.count("\n") + 1), math.log1p(len(nodes)), | |
| float(sum(isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) for node in nodes)), | |
| float(sum(isinstance(node, (ast.For, ast.AsyncFor, ast.While)) for node in nodes)), | |
| float(sum(isinstance(node, (ast.If, ast.IfExp, ast.Match)) for node in nodes)), | |
| float(sum(isinstance(node, ast.Call) for node in nodes)), | |
| float(sum(isinstance(node, ast.Subscript) for node in nodes)), | |
| float(sum(isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)) | |
| for node in nodes)), | |
| float(sum(isinstance(node, (ast.Try, ast.Raise)) for node in nodes)), | |
| float(sum(isinstance(node, (ast.Import, ast.ImportFrom)) for node in nodes)), | |
| float(sum(isinstance(node, ast.Return) for node in nodes)), | |
| float(sum(isinstance(node, ast.Constant) and type(node.value) is int for node in nodes)), | |
| float(_ast_depth(tree)), | |
| float(sum(isinstance(node, ast.Call) and isinstance(node.func, ast.Name) | |
| and node.func.id in dangerous for node in nodes)), | |
| float(sample_ratio), float(completion_ratio), float(consensus_ratio), | |
| ] | |
| def _critic_score(code, sample_ratio, completion_ratio, consensus_ratio, critic): | |
| if not critic: | |
| return 0.0 | |
| values = _critic_features(code, sample_ratio, completion_ratio, consensus_ratio) | |
| logit = float(critic["intercept"]) | |
| for value, mean, scale, coefficient in zip( | |
| values, critic["mean"], critic["scale"], critic["coef"]): | |
| logit += ((value - mean) / scale) * coefficient | |
| return 1.0 / (1.0 + math.exp(-max(-30.0, min(30.0, logit)))) | |
| def _reference_valid(reference, samples, numeric, deadline=None): | |
| if not reference or not reference.strip(): | |
| return False | |
| try: | |
| compile(reference, "<reference>", "exec") | |
| except (SyntaxError, ValueError, TypeError): | |
| return False | |
| if not samples: | |
| return True | |
| completed = 0 | |
| for stdin, expected in samples: | |
| observed = _run_program(reference, stdin, _STRESS_TIMEOUT, deadline=deadline) | |
| if observed is None: | |
| continue | |
| completed += 1 | |
| if not _tokens_match(observed, expected, numeric): | |
| return False | |
| return completed > 0 | |
| def _trusted_oracle(reference_a, reference_b, samples, generated, numeric, minimum, | |
| deadline=None): | |
| if (not _reference_valid(reference_a, samples, numeric, deadline) | |
| or not _reference_valid(reference_b, samples, numeric, deadline)): | |
| return [] | |
| agreed = [] | |
| for stdin in generated: | |
| left = _run_program(reference_a, stdin, _STRESS_TIMEOUT, deadline=deadline) | |
| right = _run_program(reference_b, stdin, _STRESS_TIMEOUT, deadline=deadline) | |
| if left is None or right is None: | |
| continue | |
| if not _tokens_match(left, right, numeric): | |
| return [] | |
| agreed.append((stdin, left)) | |
| return agreed if len(agreed) >= minimum else [] | |
| def _first_trusted_oracle(references, samples, generated, numeric, minimum, deadline=None): | |
| """Admit an oracle only when two independently returned references fully agree.""" | |
| local_deadline = time.monotonic() + _ORACLE_BUDGET_S | |
| deadline = min(local_deadline, deadline) if deadline is not None else local_deadline | |
| for left_index in range(len(references)): | |
| for right_index in range(left_index + 1, len(references)): | |
| agreed = _trusted_oracle( | |
| references[left_index], references[right_index], samples, generated, | |
| numeric, minimum, deadline, | |
| ) | |
| if agreed: | |
| return agreed | |
| return [] | |
| def _candidate_evidence(candidates, samples, generated, numeric, oracle_cases=(), critic=None, | |
| large_input=False, performance_inputs=(), deadline=None): | |
| rows = [] | |
| outputs = [] | |
| for index, code in enumerate(candidates): | |
| local_deadline = time.monotonic() + _CANDIDATE_EVIDENCE_BUDGET_S | |
| candidate_deadline = (min(local_deadline, deadline) | |
| if deadline is not None else local_deadline) | |
| passed, total, failure = _check(code, samples, numeric, candidate_deadline) | |
| sample_valid = bool(code.strip() and failure is None and passed == total) | |
| if sample_valid: | |
| values = [_run_program(code, stdin, _STRESS_TIMEOUT, deadline=candidate_deadline) | |
| for stdin in generated] | |
| performance_started = time.monotonic() | |
| performance_values = [ | |
| _run_program(code, stdin, _PERFORMANCE_MARGIN_TIMEOUT, deadline=candidate_deadline) | |
| for stdin in performance_inputs | |
| ] | |
| performance_latency = time.monotonic() - performance_started | |
| else: | |
| # A program that fails the statement samples can never be returned. Do not spend its | |
| # reviewer reserve on generated or scale probes. | |
| values = [None] * len(generated) | |
| performance_values = [None] * len(performance_inputs) | |
| performance_latency = 0.0 | |
| outputs.append(values) | |
| rows.append({ | |
| "index": index, | |
| "sample_passed": passed, | |
| "sample_total": total, | |
| "sample_valid": sample_valid, | |
| "completed": sum(value is not None for value in values), | |
| "consensus": 0, | |
| "oracle": 0, | |
| "oracle_total": len(oracle_cases), | |
| "feasible": not (large_input and _has_input_scaled_nested_loop(code)), | |
| "integer_safe": not _fixed_width_integer_risk(code), | |
| "boundary_safe": not _allocation_index_risk(code), | |
| "performance_completed": sum(value is not None for value in performance_values), | |
| "performance_total": len(performance_inputs), | |
| "performance_latency_s": performance_latency, | |
| "critic": 0.0, | |
| "quality": _quality(code), | |
| }) | |
| for case_index in range(len(generated)): | |
| groups = [] | |
| for candidate_index, values in enumerate(outputs): | |
| value = values[case_index] | |
| if value is None: | |
| continue | |
| placed = False | |
| for representative, members in groups: | |
| if _tokens_match(value, representative, numeric): | |
| members.append(candidate_index) | |
| placed = True | |
| break | |
| if not placed: | |
| groups.append((value, [candidate_index])) | |
| if not groups: | |
| continue | |
| largest = max(len(members) for _value, members in groups) | |
| if largest < 2: | |
| continue | |
| winners = [members for _value, members in groups if len(members) == largest] | |
| if len(winners) == 1: | |
| for candidate_index in winners[0]: | |
| rows[candidate_index]["consensus"] += 1 | |
| for row, code in zip(rows, candidates): | |
| for stdin, expected in oracle_cases: | |
| observed = _run_program(code, stdin, _STRESS_TIMEOUT, deadline=deadline) | |
| if observed is not None and _tokens_match(observed, expected, numeric): | |
| row["oracle"] += 1 | |
| row["critic"] = _critic_score( | |
| code, | |
| row["sample_passed"] / max(1, row["sample_total"]), | |
| row["completed"] / max(1, len(generated)), | |
| row["consensus"] / max(1, len(generated)), | |
| critic, | |
| ) | |
| return rows, outputs | |
| def _best_candidate(candidates, rows): | |
| viable = [row for row in rows if row["sample_valid"]] | |
| pool = viable if viable else rows | |
| # Executed largest-input evidence is stronger than a conservative static hint. If at least | |
| # one sample-valid candidate finishes every generated performance probe inside the published | |
| # ten-second case contract with a two-second reserve, exclude candidates that do not. | |
| performance_viable = [ | |
| row for row in pool | |
| if (not row.get("performance_total") | |
| or row.get("performance_completed") == row.get("performance_total")) | |
| ] | |
| if performance_viable: | |
| pool = performance_viable | |
| best = max(pool, key=lambda row: ( | |
| row["sample_valid"], row["sample_passed"], | |
| bool(row["oracle_total"] and row["oracle"] == row["oracle_total"]), | |
| row["oracle"], row["consensus"], row.get("integer_safe", True), | |
| row.get("boundary_safe", True), row.get("feasible", True), | |
| bool(not row.get("performance_total") | |
| or row.get("performance_completed") == row.get("performance_total")), | |
| -row.get("performance_latency_s", 0.0), row["critic"], | |
| row["completed"], row["quality"], -row["index"], | |
| )) | |
| return candidates[best["index"]], best | |
| def _disagreement_note(generated, outputs): | |
| notes = [] | |
| for case_index, stdin in enumerate(generated): | |
| values = [rows[case_index] for rows in outputs] | |
| completed = [value for value in values if value is not None] | |
| if len(completed) < 2 or all(value.split() == completed[0].split() | |
| for value in completed[1:]): | |
| continue | |
| notes.append( | |
| "Input:\n" + stdin[:800] + "\nCandidate outputs:\n" | |
| + "\n---\n".join((value if value is not None else "<no output>")[:800] | |
| for value in values) | |
| ) | |
| if len(notes) >= 2: | |
| break | |
| return "\n\n".join(notes) | |
| def _hard_bounded_call(callback, timeout=_HARD_CALL_TIMEOUT_S): | |
| """Return an empty response when a provider call exceeds a real wall-clock deadline.""" | |
| result = queue.Queue(maxsize=1) | |
| def run(): | |
| try: | |
| result.put((True, callback())) | |
| except BaseException as exc: | |
| result.put((False, exc)) | |
| worker = threading.Thread(target=run, daemon=True) | |
| worker.start() | |
| worker.join(timeout) | |
| if worker.is_alive(): | |
| return "" | |
| ok, value = result.get_nowait() | |
| if not ok: | |
| raise value | |
| return value | |
| def build_agent(weights): | |
| config = _load(weights) | |
| state = {"started": None, "done": 0, "code_done": 0, | |
| "reserved_code_s": 0.0, "latency": {}} | |
| def invoke(call_model, model_index, prompt, max_tokens, reasoning): | |
| params = { | |
| "max_tokens": max_tokens, | |
| "reasoning": {"effort": reasoning}, | |
| "temperature": config["temperature"], | |
| } | |
| started = time.monotonic() | |
| try: | |
| # The runtime/harness owns the single request deadline. Nesting another daemon-thread | |
| # timeout here leaves an uncancellable provider request alive across later tasks. | |
| return call_model( | |
| _MODELS[model_index], [{"role": "user", "content": prompt}], params | |
| ) | |
| except Exception: | |
| # A transient provider/transport failure is an invalid candidate, not a reason to abort | |
| # the whole six-task proof. The bounded next stage still observes the same statement. | |
| return "" | |
| finally: | |
| elapsed = time.monotonic() - started | |
| floor = _MODEL_CALL_FLOOR_S[model_index] | |
| prior = state["latency"].get(model_index, floor) | |
| state["latency"][model_index] = max(floor, 0.7 * prior + 0.3 * elapsed) | |
| def agent(prompt, call_model): | |
| if state["started"] is None: | |
| state["started"] = time.monotonic() | |
| state["done"] += 1 | |
| epoch_deadline = state["started"] + _RUN_BUDGET_S | |
| def room_for(model_index, multiplier=1.0, stage_cap=None): | |
| remaining = max(0, _EXPECTED_TASKS - state["done"]) | |
| estimate = state["latency"].get(model_index, _MODEL_CALL_FLOOR_S[model_index]) | |
| # stage_cap used to reduce the estimate without enforcing a matching wall timeout. | |
| # Never admit a long provider call on that optimistic fiction. | |
| reserve = max( | |
| _RESERVE_PER_TASK_S * remaining, | |
| state["reserved_code_s"], | |
| ) | |
| return (time.monotonic() + max(_MIN_CALL_WINDOW_S, estimate * multiplier) | |
| + reserve) < epoch_deadline | |
| original = str(prompt) | |
| if not _is_code(original): | |
| state["reserved_code_s"] = 0.0 | |
| return invoke( | |
| call_model, config["non_code"], _non_code_request(original), | |
| config["primary_max_tokens"], config["primary_reasoning"], | |
| ) | |
| state["code_done"] += 1 | |
| remaining_code = max(0, _EXPECTED_CODE_TASKS - state["code_done"]) | |
| state["reserved_code_s"] = _RESERVE_PER_CODE_TASK_S * remaining_code | |
| model_calls = 0 | |
| def ask(model_index, request_text, max_tokens, reasoning): | |
| nonlocal model_calls | |
| if model_calls >= _MAX_CODE_CALLS: | |
| return "" | |
| model_calls += 1 | |
| return invoke(call_model, model_index, request_text, max_tokens, reasoning) | |
| samples = _samples(original, config["sample_limit"]) | |
| numeric = bool(_FLOAT_JUDGE.search(original)) | |
| finish = lambda code: _finish_candidate(code, original) | |
| sequential_risk = bool(_SEQUENTIAL_RISK.search(original)) | |
| mutable_graph_risk = bool(_MUTABLE_GRAPH_RISK.search(original)) | |
| large_input = bool(_LARGE_N.search(original)) | |
| deep_risk = _deep_code_risk(original) | |
| batch_output_risk = bool( | |
| _MANY_OUTPUT_RISK.search(original) and _COUNTING_RISK.search(original) | |
| ) | |
| high_risk = sequential_risk or mutable_graph_risk or large_input or deep_risk | |
| request = (original + "\n\nGeneral reliability protocol: " + _SOLVE | |
| + "\n\nResource-bound audit: " + _RESOURCE_AUDIT | |
| + "\n\nAllocation/index audit: " + _INDEX_BOUND_AUDIT) | |
| if numeric: | |
| request += "\n\nNumeric-output protocol: " + _FLOAT_FORMAT | |
| if sequential_risk: | |
| request += "\n\nSequential-update protocol: " + _SEQUENTIAL_A + _SEQUENTIAL_B | |
| if mutable_graph_risk: | |
| request += "\n\nMutable-state shortest-path protocol: " + _MUTABLE_GRAPH | |
| if large_input: | |
| request += ("\n\nLarge-input protocol: Prove the asymptotic bound for the largest N. " | |
| "Reject any nested iteration whose inner bound grows with the input unless " | |
| "a strict amortized or logarithmic bound is established.") | |
| if deep_risk: | |
| request += ("\n\nLogical-equivalence audit: " + _EQUIVALENCE_AUDIT | |
| + _QUANTIFIER_AUDIT + _INDEPENDENCE_AUDIT) | |
| primary_model = config["batch_primary"] if batch_output_risk else config["primary"] | |
| challenger_model = config["batch_challenger"] if batch_output_risk else config["challenger"] | |
| primary_max_tokens = (config["batch_primary_max_tokens"] if batch_output_risk | |
| else config["primary_max_tokens"]) | |
| challenge_max_tokens = (config["batch_challenge_max_tokens"] if batch_output_risk | |
| else config["challenge_max_tokens"]) | |
| primary_reasoning = (config["batch_primary_reasoning"] if batch_output_risk | |
| else config["primary_reasoning"]) | |
| challenge_reasoning = (config["batch_challenge_reasoning"] if batch_output_risk | |
| else config["challenge_reasoning"]) | |
| if batch_output_risk: | |
| request += ( | |
| "\n\nBatched-output complexity audit: " + _BATCH_OUTPUT_AUDIT | |
| + "\n\nExecutable performance target: construct a largest-legal-input " | |
| "benchmark from the statement. The returned program must preserve the derived " | |
| "semantics and target at most 6 seconds per official case on one CPU, leaving " | |
| "headroom below the 10-second validator limit." | |
| ) | |
| primary_answer = ask( | |
| primary_model, request + "\n\nOutput protocol:\n" + _PRIMARY_FORMAT, | |
| primary_max_tokens, primary_reasoning, | |
| ) | |
| primary_programs = _named_programs(primary_answer) | |
| first = finish(_returnable_candidate(primary_answer, primary_programs)) | |
| references = [] | |
| if primary_programs.get("reference"): | |
| references.append(primary_programs["reference"]) | |
| if deep_risk: | |
| # Do not anchor a provider-diverse solver on the primary program. Agreement after | |
| # seeing the same compressed-state idea is correlated evidence, not independence. | |
| challenge_request = ( | |
| request + "\n\nLogical-equivalence audit: " + _EQUIVALENCE_AUDIT | |
| + _INDEPENDENCE_AUDIT | |
| + "\n\nIndependent challenge protocol:\n" + _CHALLENGE | |
| ) | |
| else: | |
| # Preserve provider independence on ordinary tasks too: the challenger receives the | |
| # statement and generic audits, never the primary program. | |
| challenge_request = ( | |
| request + "\n\nLogical-equivalence audit: " + _EQUIVALENCE_AUDIT | |
| + _INDEPENDENCE_AUDIT | |
| + "\n\nIndependent challenge protocol: derive from the statement before " | |
| "considering any other implementation.\n" + _CHALLENGE | |
| ) | |
| challenge_answer = (ask( | |
| challenger_model, challenge_request, | |
| challenge_max_tokens, challenge_reasoning, | |
| ) if room_for(challenger_model, 1.5) else "") | |
| evidence_deadline = min( | |
| epoch_deadline - 8.0, time.monotonic() + _EVIDENCE_BUDGET_S | |
| ) | |
| programs = _named_programs(challenge_answer) | |
| second = finish(_returnable_candidate(challenge_answer, programs)) | |
| generator = programs.get("generator") | |
| benchmark = programs.get("benchmark") | |
| if programs.get("reference"): | |
| references.append(programs["reference"]) | |
| candidates = [first] | |
| if second.strip(): | |
| candidates.append(second) | |
| generated = _generated_inputs( | |
| generator, config["stress_cases"], require_scale=not deep_risk, deadline=evidence_deadline | |
| ) | |
| generator_candidate_added = False | |
| # Preserve the final model call for an independent reviewer. A missing or malformed | |
| # challenge must not spend that reserve on generator-only tooling. | |
| if (not generated or (not benchmark and not deep_risk)) and model_calls < 2 and room_for( | |
| config["generator"], 1.0, 45.0): | |
| generator_request = original + "\n\nInput-generator protocol: " + _GENERATOR_ONLY | |
| generator_answer = ask( | |
| config["generator"], generator_request, | |
| config["generator_max_tokens"], config["generator_reasoning"], | |
| ) | |
| generated_programs = _named_programs(generator_answer) | |
| if not generated_programs: | |
| plain_blocks = _plain_program_blocks(generator_answer) | |
| if len(plain_blocks) in (2, 3): | |
| generated_programs = { | |
| "generator": plain_blocks[0], "reference": plain_blocks[1], | |
| } | |
| if len(plain_blocks) == 3: | |
| generated_programs["benchmark"] = plain_blocks[2] | |
| elif not plain_blocks: | |
| # Providers occasionally ignore a tooling-only format and return a complete | |
| # raw solution. It is not generator evidence, but it is still an independently | |
| # derived runtime candidate and must not be silently discarded. | |
| recovered = _source(generator_answer) | |
| if recovered.strip(): | |
| candidates.append(finish(recovered)) | |
| generator_candidate_added = True | |
| generated_candidate = generated_programs.get("candidate") | |
| if generated_candidate and generated_candidate.strip(): | |
| candidates.append(finish(generated_candidate)) | |
| generator_candidate_added = True | |
| generator = generated_programs.get("generator") or "" | |
| benchmark = generated_programs.get("benchmark") or benchmark | |
| if generated_programs.get("reference"): | |
| references.append(generated_programs["reference"]) | |
| generated = _generated_inputs( | |
| generator, config["stress_cases"], require_scale=not deep_risk | |
| ) | |
| # Every returned benchmark is only admission evidence and uses an eight-second | |
| # timeout, leaving margin under the validator's published ten-second case contract. | |
| performance_inputs = _generated_inputs( | |
| benchmark, 2, require_scale=False, deadline=evidence_deadline | |
| ) if benchmark else [] | |
| oracle_cases = _first_trusted_oracle( | |
| references, samples, generated, numeric, config["min_consensus_cases"], evidence_deadline | |
| ) | |
| # High-risk statements may have many sample-valid but semantically wrong compressed or | |
| # greedy programs. Ask a provider-diverse model for a third literal oracle when the first | |
| # two stages did not establish one. A single reference is never trusted by itself. | |
| if (generated and not oracle_cases and not deep_risk | |
| and model_calls < 2 and room_for(config["generator"], 1.5)): | |
| arbiter_answer = ask( | |
| config["generator"], original + "\n\nOracle protocol: " + _ARBITER_ONLY, | |
| config["generator_max_tokens"], config["generator_reasoning"], | |
| ) | |
| arbiter_programs = _named_programs(arbiter_answer) | |
| arbiter_reference = arbiter_programs.get("reference") | |
| if not arbiter_reference: | |
| plain_blocks = _plain_program_blocks(arbiter_answer) | |
| if len(plain_blocks) == 1: | |
| arbiter_reference = plain_blocks[0] | |
| if (arbiter_reference and arbiter_reference.strip() | |
| and arbiter_reference not in references): | |
| references.append(arbiter_reference) | |
| oracle_cases = _first_trusted_oracle( | |
| references, samples, generated, numeric, config["min_consensus_cases"], evidence_deadline | |
| ) | |
| rows, outputs = _candidate_evidence( | |
| candidates, samples, generated, numeric, oracle_cases, config["critic"], large_input, performance_inputs, evidence_deadline | |
| ) | |
| best, best_row = _best_candidate(candidates, rows) | |
| # Two independent generations are normally sufficient. A third call is admitted | |
| # only below, when execution/oracle evidence cannot safely accept either candidate. | |
| disagreement = _disagreement_note(generated, outputs) | |
| fixed_width_risk = any(_fixed_width_integer_risk(code) for code in candidates) | |
| boundary_risk = any(_allocation_index_risk(code) for code in candidates) | |
| review_required = high_risk or fixed_width_risk or boundary_risk | |
| performance_ok = ( | |
| not performance_inputs | |
| or best_row["performance_completed"] == best_row["performance_total"] | |
| ) | |
| # A generator-format violation may still yield a complete independent program. | |
| # On an ordinary statement, a sample-valid arbitrary-precision candidate is strictly safer | |
| # than a fixed-width candidate that needs collision/bound proofs. This is a generic language | |
| # safety fallback, not task routing or answer lookup. | |
| if (generator_candidate_added and not high_risk and fixed_width_risk | |
| and best_row["sample_valid"] and best_row.get("integer_safe", True) | |
| and performance_ok): | |
| return finish(best) | |
| if (best_row["sample_valid"] and performance_ok and oracle_cases | |
| and best_row["oracle"] == best_row["oracle_total"] | |
| and len(generated) >= config["min_consensus_cases"]): | |
| return finish(best) | |
| if (not review_required and best_row["sample_valid"] and performance_ok and ( | |
| (oracle_cases and best_row["oracle"] == best_row["oracle_total"]) | |
| or (not high_risk and not oracle_cases and len(candidates) >= 2 | |
| and len(generated) >= config["min_consensus_cases"] | |
| and all(row["sample_valid"] for row in rows) | |
| and best_row["consensus"] == len(generated) | |
| and best_row["completed"] == len(generated)))): | |
| return finish(best) | |
| candidate_text = "\n\n".join( | |
| "Candidate " + str(index + 1) + ":\n" + code | |
| for index, code in enumerate(candidates) | |
| ) | |
| if deep_risk: | |
| # Correctness evidence selects the repair seed; no task identity or stored answer is used. | |
| # The reviewer sees only the current statement, generic audit policy, and runtime candidate. | |
| review_request = ( | |
| request + "\n\nPerformance repair protocol: " + _PERFORMANCE_REPAIR | |
| + "\n\nLogical-equivalence audit: " + _EQUIVALENCE_AUDIT | |
| + _INDEPENDENCE_AUDIT | |
| + "\n\nStatement-sample acceptance protocol: " + _MUST_PASS | |
| + "\n\nEvidence-selected program to repair:\n" + best | |
| ) | |
| else: | |
| review_request = ( | |
| request + "\n\nIndependent review protocol: " + _REVIEW | |
| + "\n\nStatement-sample acceptance protocol: " + _MUST_PASS | |
| + "\n\nPrograms under review:\n" + candidate_text | |
| ) | |
| if disagreement: | |
| review_request += ( | |
| "\n\nThe programs disagreed on small generated inputs. These outputs are evidence " | |
| "of disagreement only; none is a trusted oracle.\n" + disagreement | |
| ) | |
| narrowed = [str(index + 1) for index, code in enumerate(candidates) | |
| if _fixed_width_integer_risk(code)] | |
| if narrowed: | |
| review_request += ( | |
| "\n\nStatic language-safety diagnostics: candidate(s) " + ", ".join(narrowed) | |
| + " use fixed-width integer storage or masking. " + _INTEGER_SAFETY | |
| ) | |
| boundary_candidates = [str(index + 1) for index, code in enumerate(candidates) | |
| if _allocation_index_risk(code)] | |
| if boundary_candidates: | |
| review_request += ( | |
| "\n\nStatic allocation/index diagnostics: candidate(s) " | |
| + ", ".join(boundary_candidates) + " contain a subscript expression whose " | |
| "growth outruns the tracked list allocation. " + _INDEX_BOUND_AUDIT | |
| ) | |
| if oracle_cases: | |
| counterexamples = [] | |
| for stdin, expected in oracle_cases: | |
| observed = _run_program(best, stdin, _STRESS_TIMEOUT) | |
| if observed is None or not _tokens_match(observed, expected, numeric): | |
| counterexamples.append( | |
| "Input:\n" + stdin[:800] + "\nTrusted dual-reference output:\n" | |
| + expected[:800] + "\nSelected-candidate output:\n" | |
| + (observed if observed is not None else "<no output>")[:800] | |
| ) | |
| if len(counterexamples) >= 2: | |
| break | |
| if counterexamples: | |
| review_request += ( | |
| "\n\nTwo independent small references agreed on these counterexamples:\n" | |
| + "\n\n".join(counterexamples) | |
| ) | |
| third = finish(_source(ask( | |
| config["reviewer"], review_request, config["review_max_tokens"], | |
| config["review_reasoning"], | |
| ))) if room_for(config["reviewer"], 1.0, 75.0) else "" | |
| reviewer_row = None | |
| if third.strip(): | |
| candidates.append(third) | |
| review_deadline = min( | |
| epoch_deadline - 8.0, | |
| time.monotonic() + _CANDIDATE_EVIDENCE_BUDGET_S, | |
| ) | |
| third_rows, third_outputs = _candidate_evidence( | |
| [third], samples, generated, numeric, oracle_cases, config["critic"], | |
| large_input, performance_inputs, review_deadline, | |
| ) | |
| if third_rows: | |
| third_rows[0]["index"] = len(candidates) - 1 | |
| rows.extend(third_rows) | |
| outputs.extend(third_outputs) | |
| reviewer_row = third_rows[0] | |
| best, best_row = _best_candidate(candidates, rows) | |
| if model_calls >= _MAX_CODE_CALLS: | |
| return finish(best) | |
| # A reviewer is another candidate, never an authority. The same execution, | |
| # oracle, scale, and learned-critic evidence selects among every generated program. | |
| performance_ok = ( | |
| not performance_inputs | |
| or best_row["performance_completed"] == best_row["performance_total"] | |
| ) | |
| if (best_row["sample_valid"] and performance_ok and ( | |
| (oracle_cases and best_row["oracle"] == best_row["oracle_total"]) | |
| or (not high_risk and not oracle_cases | |
| and best_row["consensus"] >= config["min_consensus_cases"] | |
| and best_row["completed"] >= config["min_consensus_cases"]) | |
| or (high_risk and not oracle_cases | |
| and len(candidates) >= 3 | |
| and len(generated) >= config["min_consensus_cases"] | |
| and sum(row["sample_valid"] for row in rows) >= 3 | |
| and best_row["consensus"] == len(generated) | |
| and best_row["completed"] == len(generated)))): | |
| return finish(best) | |
| # Preserve sample-valid answers on ordinary tasks when there is no contradictory evidence. | |
| # High-risk tasks may not use this sample-only escape hatch: they must establish a dual | |
| # oracle, unanimous three-way generated-case consensus, or continue to independent rescue. | |
| if (best_row["sample_valid"] and performance_ok | |
| and ((oracle_cases and best_row["oracle"] == best_row["oracle_total"]) | |
| or (not high_risk and not oracle_cases))): | |
| return finish(best) | |
| passed, total, failure = _check(best, samples, numeric) | |
| if (performance_inputs and best_row["sample_valid"] and not performance_ok): | |
| rescue_request = ( | |
| request + "\n\nSecond independent performance repair: " + _PERFORMANCE_REPAIR | |
| + "\n\nThe previous repair did not pass every executable largest-input " | |
| "probe. Preserve exact semantics; replace the algorithm when necessary." | |
| + "\n\nStatement-sample acceptance protocol: " + _MUST_PASS | |
| + "\n\nExecution-selected program to repair:\n" + best | |
| ) | |
| else: | |
| rescue_request = ( | |
| request + "\n\nEmergency independent derivation: " + _REPAIR | |
| + "\n\nStatement-sample acceptance protocol: " + _MUST_PASS | |
| + "\n\nThe deterministic selector found insufficient execution consensus. " | |
| "Do not vote between prior programs; derive and return one complete raw program." | |
| + "\n\nBest prior candidate diagnostics:\n" | |
| + _failure_note(passed, total, failure) | |
| + "\n\nBest prior candidate:\n" + best | |
| ) | |
| rescued = finish(_source(ask( | |
| config["rescue"], rescue_request, config["rescue_max_tokens"], | |
| config["rescue_reasoning"], | |
| ))) if room_for(config["rescue"], 1.0, 60.0) else "" | |
| if rescued.strip(): | |
| candidates.append(rescued) | |
| rescue_deadline = min( | |
| epoch_deadline - 8.0, | |
| time.monotonic() + _CANDIDATE_EVIDENCE_BUDGET_S, | |
| ) | |
| rescue_rows, _rescue_outputs = _candidate_evidence( | |
| [rescued], samples, generated, numeric, oracle_cases, config["critic"], | |
| large_input, performance_inputs, rescue_deadline, | |
| ) | |
| if rescue_rows: | |
| rescue_rows[0]["index"] = len(candidates) - 1 | |
| rows.extend(rescue_rows) | |
| return finish(_best_candidate(candidates, rows)[0]) | |
| return agent | |