Upload heldout_code_math_hf_results.json from Held-Out Code & Math A100-80GB evaluation
b31e2fb verified | { | |
| "title": "Held-Out Code & Math Test-Time Strategy Benchmark on Hugging Face A100-80GB", | |
| "hardware": { | |
| "platform": "Hugging Face Compute Jobs (a100-large)", | |
| "gpu_name": "NVIDIA A100-SXM4-80GB", | |
| "vram_total_gb": 79.25, | |
| "elapsed_sec": 2050.32 | |
| }, | |
| "track1_competition_math_tir": { | |
| "num_tasks": 35, | |
| "base_passed": 27, | |
| "base_pct": 77.14, | |
| "polystep_passed": 30, | |
| "polystep_pct": 85.71, | |
| "gain_pct": 8.57, | |
| "juncture_branches_switched": 10, | |
| "tasks": [ | |
| { | |
| "id": "AIME-HeldOut-01", | |
| "category": "Number Theory (Modular Exponentiation)", | |
| "gold_answer": "500", | |
| "base_answer": "500", | |
| "polystep_answer": "500", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t1", | |
| "elapsed_sec": 24.0 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-02", | |
| "category": "Combinatorics (Digit DP)", | |
| "gold_answer": "6923077", | |
| "base_answer": "", | |
| "polystep_answer": "6923265", | |
| "base_passed": false, | |
| "polystep_passed": false, | |
| "switched_branch": true, | |
| "selected_source": "polystep_pinned_self_repair", | |
| "elapsed_sec": 84.9 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-03", | |
| "category": "Number Theory (Chinese Remainder Theorem)", | |
| "gold_answer": "3667381", | |
| "base_answer": "", | |
| "polystep_answer": "", | |
| "base_passed": false, | |
| "polystep_passed": false, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 157.45 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-04", | |
| "category": "Algebra & Linear Recurrences", | |
| "gold_answer": "23240118", | |
| "base_answer": "23240118", | |
| "polystep_answer": "23240118", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 12.37 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-05", | |
| "category": "Counting & Probability (Dice Sum)", | |
| "gold_answer": "25090131", | |
| "base_answer": "25090131", | |
| "polystep_answer": "25090131", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 91.1 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-06", | |
| "category": "Combinatorics (Derangements)", | |
| "gold_answer": "176214841", | |
| "base_answer": "176214841", | |
| "polystep_answer": "176214841", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 19.66 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-07", | |
| "category": "Number Theory (Coprime Residues)", | |
| "gold_answer": "2285", | |
| "base_answer": "2285", | |
| "polystep_answer": "2285", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t2", | |
| "elapsed_sec": 51.34 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-08", | |
| "category": "Exact Rational Series (Telescoping Fraction)", | |
| "gold_answer": "1076", | |
| "base_answer": "1076", | |
| "polystep_answer": "1076", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t61", | |
| "elapsed_sec": 38.1 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-09", | |
| "category": "Number Theory (Divisor Sum)", | |
| "gold_answer": "10584", | |
| "base_answer": "10584", | |
| "polystep_answer": "10584", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 34.06 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-10", | |
| "category": "Combinatorics (Dyck Paths / Catalan)", | |
| "gold_answer": "2674440", | |
| "base_answer": "2674440", | |
| "polystep_answer": "2674440", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 18.06 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-11", | |
| "category": "Algebra (Polynomial Root Power Sums)", | |
| "gold_answer": "851", | |
| "base_answer": "851", | |
| "polystep_answer": "-903", | |
| "base_passed": true, | |
| "polystep_passed": false, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t60", | |
| "elapsed_sec": 50.29 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-12", | |
| "category": "Number Theory (p-adic Valuation)", | |
| "gold_answer": "335", | |
| "base_answer": "335", | |
| "polystep_answer": "335", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 33.49 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-13", | |
| "category": "Combinatorics (Pattern-Avoiding Binary Strings)", | |
| "gold_answer": "2555757", | |
| "base_answer": "2555757", | |
| "polystep_answer": "2555757", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 41.8 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-14", | |
| "category": "Graph Theory (Closed Walks on K_4)", | |
| "gold_answer": "10761681", | |
| "base_answer": "10761681", | |
| "polystep_answer": "10761681", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 38.79 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-15", | |
| "category": "Number Theory (Integer Square Root Divisibility)", | |
| "gold_answer": "352800", | |
| "base_answer": "352800", | |
| "polystep_answer": "352800", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 13.91 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-16", | |
| "category": "Exact Rational Series (Harmonic Alternating)", | |
| "gold_answer": "210631", | |
| "base_answer": "210631", | |
| "polystep_answer": "210631", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 20.8 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-17", | |
| "category": "Linear Algebra (GCD Matrix Determinant)", | |
| "gold_answer": "16", | |
| "base_answer": "16", | |
| "polystep_answer": "16", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 62.89 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-18", | |
| "category": "Combinatorics (Odd Partitions / Compositions)", | |
| "gold_answer": "2178309", | |
| "base_answer": "", | |
| "polystep_answer": "300540195", | |
| "base_passed": false, | |
| "polystep_passed": false, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t41", | |
| "elapsed_sec": 42.27 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-19", | |
| "category": "Number Theory (Euler Totient Sum)", | |
| "gold_answer": "12232", | |
| "base_answer": "", | |
| "polystep_answer": "12232", | |
| "base_passed": false, | |
| "polystep_passed": true, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t1", | |
| "elapsed_sec": 20.29 | |
| }, | |
| { | |
| "id": "AIME-HeldOut-20", | |
| "category": "Numerical Analysis (Barron Adaptive Robust Loss)", | |
| "gold_answer": "118418", | |
| "base_answer": "118418", | |
| "polystep_answer": "118418", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 21.68 | |
| }, | |
| { | |
| "id": "MATH500/test_prealgebra_1139.json", | |
| "category": "MATH-500 L5 (Prealgebra)", | |
| "gold_answer": "4", | |
| "base_answer": "2", | |
| "polystep_answer": "2", | |
| "base_passed": false, | |
| "polystep_passed": false, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 98.42 | |
| }, | |
| { | |
| "id": "MATH500/test_number_theory_737.json", | |
| "category": "MATH-500 L5 (Number Theory)", | |
| "gold_answer": "284", | |
| "base_answer": "284", | |
| "polystep_answer": "284", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 42.92 | |
| }, | |
| { | |
| "id": "MATH500/test_precalculus_1199.json", | |
| "category": "MATH-500 L4 (Precalculus)", | |
| "gold_answer": "3.1415927", | |
| "base_answer": "3.1415927", | |
| "polystep_answer": "3.1415927", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 19.79 | |
| }, | |
| { | |
| "id": "MATH500/test_algebra_1837.json", | |
| "category": "MATH-500 L5 (Algebra)", | |
| "gold_answer": "5", | |
| "base_answer": "5", | |
| "polystep_answer": "5", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 104.66 | |
| }, | |
| { | |
| "id": "MATH500/test_algebra_2427.json", | |
| "category": "MATH-500 L5 (Algebra)", | |
| "gold_answer": "10", | |
| "base_answer": "10", | |
| "polystep_answer": "10", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 104.61 | |
| }, | |
| { | |
| "id": "MATH500/test_counting_and_probability_525.json", | |
| "category": "MATH-500 L5 (Counting & Probability)", | |
| "gold_answer": "144", | |
| "base_answer": "", | |
| "polystep_answer": "144", | |
| "base_passed": false, | |
| "polystep_passed": true, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t26", | |
| "elapsed_sec": 146.59 | |
| }, | |
| { | |
| "id": "MATH500/test_counting_and_probability_134.json", | |
| "category": "MATH-500 L4 (Counting & Probability)", | |
| "gold_answer": "720", | |
| "base_answer": "720", | |
| "polystep_answer": "720", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 25.94 | |
| }, | |
| { | |
| "id": "MATH500/test_counting_and_probability_119.json", | |
| "category": "MATH-500 L4 (Counting & Probability)", | |
| "gold_answer": "-125", | |
| "base_answer": "-125", | |
| "polystep_answer": "-125", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 43.88 | |
| }, | |
| { | |
| "id": "MATH500/test_geometry_627.json", | |
| "category": "MATH-500 L4 (Geometry)", | |
| "gold_answer": "17", | |
| "base_answer": "", | |
| "polystep_answer": "17", | |
| "base_passed": false, | |
| "polystep_passed": true, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t509", | |
| "elapsed_sec": 114.16 | |
| }, | |
| { | |
| "id": "MATH500/test_prealgebra_1558.json", | |
| "category": "MATH-500 L5 (Prealgebra)", | |
| "gold_answer": "1.25", | |
| "base_answer": "1.25", | |
| "polystep_answer": "1.25", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 22.74 | |
| }, | |
| { | |
| "id": "MATH500/test_algebra_305.json", | |
| "category": "MATH-500 L5 (Algebra)", | |
| "gold_answer": "5", | |
| "base_answer": "5", | |
| "polystep_answer": "5", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 27.88 | |
| }, | |
| { | |
| "id": "MATH500/test_prealgebra_1388.json", | |
| "category": "MATH-500 L4 (Prealgebra)", | |
| "gold_answer": "83", | |
| "base_answer": "83", | |
| "polystep_answer": "83", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 16.11 | |
| }, | |
| { | |
| "id": "MATH500/test_number_theory_1055.json", | |
| "category": "MATH-500 L5 (Number Theory)", | |
| "gold_answer": "203", | |
| "base_answer": "203", | |
| "polystep_answer": "203", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 74.49 | |
| }, | |
| { | |
| "id": "MATH500/test_prealgebra_951.json", | |
| "category": "MATH-500 L5 (Prealgebra)", | |
| "gold_answer": "12", | |
| "base_answer": "12", | |
| "polystep_answer": "12", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "switched_branch": false, | |
| "selected_source": "base_greedy", | |
| "elapsed_sec": 80.43 | |
| }, | |
| { | |
| "id": "MATH500/test_algebra_187.json", | |
| "category": "MATH-500 L4 (Algebra)", | |
| "gold_answer": "3", | |
| "base_answer": "", | |
| "polystep_answer": "3", | |
| "base_passed": false, | |
| "polystep_passed": true, | |
| "switched_branch": true, | |
| "selected_source": "polystep_juncture_t100", | |
| "elapsed_sec": 130.81 | |
| } | |
| ] | |
| }, | |
| "track2_algorithmic_code_math_fuzz": { | |
| "num_tasks": 10, | |
| "base_passed": 9, | |
| "base_pct": 90.0, | |
| "polystep_passed": 10, | |
| "polystep_pct": 100.0, | |
| "gain_pct": 10.0, | |
| "tasks": [ | |
| { | |
| "task_id": "HeldOut-CodeMath/01_rolling_geometric_max", | |
| "entry_point": "rolling_max_product_pair", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": false, | |
| "base_completion": "if not nums:\n return []\n max_so_far = nums[0]\n result = [max_so_far]\n for num in nums[1:]:\n max_so_far = max(max_so_far, num)\n result.append(max_so_far)\n return result", | |
| "polystep_completion": "if not nums:\n return []\n max_so_far = nums[0]\n result = [max_so_far]\n for num in nums[1:]:\n max_so_far = max(max_so_far, num)\n result.append(max_so_far)\n return result", | |
| "elapsed_sec": 10.19 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/02_circular_digit_shift", | |
| "entry_point": "circular_shift_digits", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": false, | |
| "base_completion": "x = str(x)\n if shift > len(x):\n return x[::-1]\n else:\n return x[-shift:] + x[:-shift]", | |
| "polystep_completion": "x = str(x)\n if shift > len(x):\n return x[::-1]\n else:\n return x[-shift:] + x[:-shift]", | |
| "elapsed_sec": 5.71 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/03_caesar_double_two", | |
| "entry_point": "encrypt_shift_four", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": false, | |
| "base_completion": "result = []\n for char in s:\n if char.isalpha():\n # Determine the base ASCII value for the case of the character\n base = ord('a') if char.islower() else ord('A')\n # Shift by 4 (2 * 2) and wrap around within the 26-letter alphabet\n shifted = (ord(char) - base + 4) % 26 + base\n result.append(chr(shifted))\n else:\n # Non-alphabetic characters remain unchanged\n result.append(char)\n return ''.join(result)", | |
| "polystep_completion": "result = []\n for char in s:\n if char.isalpha():\n # Determine the base ASCII value for the case of the character\n base = ord('a') if char.islower() else ord('A')\n # Shift by 4 (2 * 2) and wrap around within the 26-letter alphabet\n shifted = (ord(char) - base + 4) % 26 + base\n result.append(chr(shifted))\n else:\n # Non-alphabetic characters remain unchanged\n result.append(char)\n return ''.join(result)", | |
| "elapsed_sec": 22.15 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/04_balanced_concat_parens", | |
| "entry_point": "match_parens_pair", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": false, | |
| "base_completion": "def is_good(s: str) -> bool:\n balance = 0\n for char in s:\n if char == '(':\n balance += 1\n else:\n balance -= 1\n if balance < 0:\n return False\n return balance == 0\n\n return 'Yes' if is_good(lst[0] + lst[1]) or is_good(lst[1] + lst[0]) else 'No'", | |
| "polystep_completion": "def is_good(s: str) -> bool:\n balance = 0\n for char in s:\n if char == '(':\n balance += 1\n else:\n balance -= 1\n if balance < 0:\n return False\n return balance == 0\n\n return 'Yes' if is_good(lst[0] + lst[1]) or is_good(lst[1] + lst[0]) else 'No'", | |
| "elapsed_sec": 12.94 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/05_sum_two_digit_prefix", | |
| "entry_point": "add_two_digit_elements", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": false, | |
| "base_completion": "return sum(x for x in arr[:k] if abs(x) < 100)", | |
| "polystep_completion": "return sum(x for x in arr[:k] if abs(x) < 100)", | |
| "elapsed_sec": 1.63 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/06_sorted_at_most_two_dups", | |
| "entry_point": "is_sorted_max_two_dups", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": false, | |
| "base_completion": "for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return False\n if i < len(lst) - 2 and lst[i] == lst[i + 1] == lst[i + 2]:\n return False\n return True", | |
| "polystep_completion": "for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return False\n if i < len(lst) - 2 and lst[i] == lst[i + 1] == lst[i + 2]:\n return False\n return True", | |
| "elapsed_sec": 10.32 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/07_sum_of_four_positive_evens", | |
| "entry_point": "is_sum_four_positive_even", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": false, | |
| "base_completion": "return n >= 8 and n % 2 == 0", | |
| "polystep_completion": "return n >= 8 and n % 2 == 0", | |
| "elapsed_sec": 1.45 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/08_sum_squares_cubes_mod3_mod4", | |
| "entry_point": "sum_squares_and_cubes", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": true, | |
| "base_completion": "return sum(x**2 if i % 3 == 0 else x**3 if i % 4 == 0 else x for i, x in enumerate(lst))", | |
| "polystep_completion": "return sum(x ** 3 if i % 4 == 0 and i % 3 != 0 else x ** 2 if i % 3 == 0 else x for i, x in enumerate(lst))", | |
| "elapsed_sec": 5.78 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/09_snake_to_upper_camel", | |
| "entry_point": "snake_to_camel_case", | |
| "base_passed": false, | |
| "polystep_passed": true, | |
| "base_probe_passed": false, | |
| "polystep_probe_passed": true, | |
| "switched_branch": true, | |
| "base_completion": "return ''.join([word[0].upper()] + [w[0].upper() + w[1:] for w in word.split('_')[1:]])", | |
| "polystep_completion": "return ''.join([w.capitalize() for w in word.split('_')])", | |
| "elapsed_sec": 4.12 | |
| }, | |
| { | |
| "task_id": "HeldOut-CodeMath/10_modular_power_two", | |
| "entry_point": "modp_fast", | |
| "base_passed": true, | |
| "polystep_passed": true, | |
| "base_probe_passed": true, | |
| "polystep_probe_passed": true, | |
| "switched_branch": false, | |
| "base_completion": "if n == 0:\n return 1\n if n % 2 == 0:\n return modp_fast(n // 2, p) ** 2 % p\n else:\n return 2 * modp_fast(n - 1, p) % p", | |
| "polystep_completion": "if n == 0:\n return 1\n if n % 2 == 0:\n return modp_fast(n // 2, p) ** 2 % p\n else:\n return 2 * modp_fast(n - 1, p) % p", | |
| "elapsed_sec": 7.9 | |
| } | |
| ] | |
| }, | |
| "track3_heldout_competitions": { | |
| "bigcodebench_hard": { | |
| "num_tasks": 12, | |
| "base_passed": 3, | |
| "base_pass_at_1_pct": 25.0, | |
| "polystep_passed": 12, | |
| "polystep_pass_at_1_pct": 100.0, | |
| "polystep_gain_pct": 75.0 | |
| }, | |
| "livecodebench": { | |
| "num_tasks": 15, | |
| "base_passed": 5, | |
| "base_pass_at_1_pct": 33.33, | |
| "polystep_passed": 15, | |
| "polystep_pass_at_1_pct": 100.0, | |
| "polystep_gain_pct": 66.67 | |
| }, | |
| "hf_private_competition": { | |
| "num_tasks": 25, | |
| "base_passed": 7, | |
| "base_pass_at_1_pct": 28.0, | |
| "polystep_passed": 22, | |
| "polystep_pass_at_1_pct": 88.0, | |
| "polystep_gain_pct": 60.0, | |
| "tasks": [ | |
| { | |
| "id": "HFComp/Blind-007_bisect_left_lower_bound_step", | |
| "base_prediction": 0, | |
| "polystep_prediction": 3, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-010_fenwick_1_indexed_build", | |
| "base_prediction": 3, | |
| "polystep_prediction": 2, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-016_levenshtein_dp_base_row", | |
| "base_prediction": 2, | |
| "polystep_prediction": 1, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-022_circular_kadane_all_negative_guard", | |
| "base_prediction": 0, | |
| "polystep_prediction": 0, | |
| "base_passed": true, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-027_kahn_toposort_cycle_guard", | |
| "base_prediction": 0, | |
| "polystep_prediction": 0, | |
| "base_passed": false, | |
| "polystep_passed": false | |
| }, | |
| { | |
| "id": "HFComp/Blind-033_welford_streaming_mean_update", | |
| "base_prediction": 2, | |
| "polystep_prediction": 2, | |
| "base_passed": true, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-035_welford_bessel_sample_variance", | |
| "base_prediction": 3, | |
| "polystep_prediction": 1, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-045_coin_change_dp_step_increment", | |
| "base_prediction": 1, | |
| "polystep_prediction": 0, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-047_coin_change_unreachable_sentinel", | |
| "base_prediction": 0, | |
| "polystep_prediction": 3, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-048_dijkstra_stale_heap_guard", | |
| "base_prediction": 1, | |
| "polystep_prediction": 2, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-051_dijkstra_unreachable_sentinel", | |
| "base_prediction": 2, | |
| "polystep_prediction": 1, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-097_barron_shifted_log1p_logsumexp", | |
| "base_prediction": 0, | |
| "polystep_prediction": 0, | |
| "base_passed": true, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-100_sinkhorn_entropic_dual_scaling", | |
| "base_prediction": 2, | |
| "polystep_prediction": 3, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-104_floyd_warshall_zero_diagonal", | |
| "base_prediction": 3, | |
| "polystep_prediction": 2, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-108_kruskal_mst_weight_sort", | |
| "base_prediction": 1, | |
| "polystep_prediction": 1, | |
| "base_passed": true, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-120_fwht_butterfly_xor_partner", | |
| "base_prediction": 0, | |
| "polystep_prediction": 0, | |
| "base_passed": true, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-124_automaton_suffix_prefix_overlap", | |
| "base_prediction": 3, | |
| "polystep_prediction": 3, | |
| "base_passed": true, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-135_balanced_brackets_unclosed_guard", | |
| "base_prediction": 3, | |
| "polystep_prediction": 3, | |
| "base_passed": false, | |
| "polystep_passed": false | |
| }, | |
| { | |
| "id": "HFComp/Blind-137_posix_canonical_path_noop_filter", | |
| "base_prediction": 2, | |
| "polystep_prediction": 1, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-148_temporal_dedup_chronological_sort", | |
| "base_prediction": 1, | |
| "polystep_prediction": 0, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-150_temporal_dedup_entity_sort", | |
| "base_prediction": 0, | |
| "polystep_prediction": 0, | |
| "base_passed": false, | |
| "polystep_passed": false | |
| }, | |
| { | |
| "id": "HFComp/Blind-168_pca_mean_centering_before_svd", | |
| "base_prediction": 0, | |
| "polystep_prediction": 2, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-175_vectorized_2d_barron_log1p", | |
| "base_prediction": 2, | |
| "polystep_prediction": 1, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-192_chronological_embargo_split", | |
| "base_prediction": 0, | |
| "polystep_prediction": 0, | |
| "base_passed": true, | |
| "polystep_passed": true | |
| }, | |
| { | |
| "id": "HFComp/Blind-196_causal_rolling_window_zscore", | |
| "base_prediction": 1, | |
| "polystep_prediction": 3, | |
| "base_passed": false, | |
| "polystep_passed": true | |
| } | |
| ] | |
| } | |
| } | |
| } |