{ "title": "Held-Out Code & Math Test-Time Strategy Benchmark on Hugging Face A100-80GB", "hardware": { "platform": "Hugging Face Compute Jobs (a100-large)", "gpu_name": "NVIDIA A100-SXM4-80GB", "vram_total_gb": 79.25, "elapsed_sec": 2050.32 }, "track1_competition_math_tir": { "num_tasks": 35, "base_passed": 27, "base_pct": 77.14, "polystep_passed": 30, "polystep_pct": 85.71, "gain_pct": 8.57, "juncture_branches_switched": 10, "tasks": [ { "id": "AIME-HeldOut-01", "category": "Number Theory (Modular Exponentiation)", "gold_answer": "500", "base_answer": "500", "polystep_answer": "500", "base_passed": true, "polystep_passed": true, "switched_branch": true, "selected_source": "polystep_juncture_t1", "elapsed_sec": 24.0 }, { "id": "AIME-HeldOut-02", "category": "Combinatorics (Digit DP)", "gold_answer": "6923077", "base_answer": "", "polystep_answer": "6923265", "base_passed": false, "polystep_passed": false, "switched_branch": true, "selected_source": "polystep_pinned_self_repair", "elapsed_sec": 84.9 }, { "id": "AIME-HeldOut-03", "category": "Number Theory (Chinese Remainder Theorem)", "gold_answer": "3667381", "base_answer": "", "polystep_answer": "", "base_passed": false, "polystep_passed": false, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 157.45 }, { "id": "AIME-HeldOut-04", "category": "Algebra & Linear Recurrences", "gold_answer": "23240118", "base_answer": "23240118", "polystep_answer": "23240118", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 12.37 }, { "id": "AIME-HeldOut-05", "category": "Counting & Probability (Dice Sum)", "gold_answer": "25090131", "base_answer": "25090131", "polystep_answer": "25090131", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 91.1 }, { "id": "AIME-HeldOut-06", "category": "Combinatorics (Derangements)", "gold_answer": "176214841", "base_answer": "176214841", "polystep_answer": "176214841", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 19.66 }, { "id": "AIME-HeldOut-07", "category": "Number Theory (Coprime Residues)", "gold_answer": "2285", "base_answer": "2285", "polystep_answer": "2285", "base_passed": true, "polystep_passed": true, "switched_branch": true, "selected_source": "polystep_juncture_t2", "elapsed_sec": 51.34 }, { "id": "AIME-HeldOut-08", "category": "Exact Rational Series (Telescoping Fraction)", "gold_answer": "1076", "base_answer": "1076", "polystep_answer": "1076", "base_passed": true, "polystep_passed": true, "switched_branch": true, "selected_source": "polystep_juncture_t61", "elapsed_sec": 38.1 }, { "id": "AIME-HeldOut-09", "category": "Number Theory (Divisor Sum)", "gold_answer": "10584", "base_answer": "10584", "polystep_answer": "10584", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 34.06 }, { "id": "AIME-HeldOut-10", "category": "Combinatorics (Dyck Paths / Catalan)", "gold_answer": "2674440", "base_answer": "2674440", "polystep_answer": "2674440", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 18.06 }, { "id": "AIME-HeldOut-11", "category": "Algebra (Polynomial Root Power Sums)", "gold_answer": "851", "base_answer": "851", "polystep_answer": "-903", "base_passed": true, "polystep_passed": false, "switched_branch": true, "selected_source": "polystep_juncture_t60", "elapsed_sec": 50.29 }, { "id": "AIME-HeldOut-12", "category": "Number Theory (p-adic Valuation)", "gold_answer": "335", "base_answer": "335", "polystep_answer": "335", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 33.49 }, { "id": "AIME-HeldOut-13", "category": "Combinatorics (Pattern-Avoiding Binary Strings)", "gold_answer": "2555757", "base_answer": "2555757", "polystep_answer": "2555757", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 41.8 }, { "id": "AIME-HeldOut-14", "category": "Graph Theory (Closed Walks on K_4)", "gold_answer": "10761681", "base_answer": "10761681", "polystep_answer": "10761681", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 38.79 }, { "id": "AIME-HeldOut-15", "category": "Number Theory (Integer Square Root Divisibility)", "gold_answer": "352800", "base_answer": "352800", "polystep_answer": "352800", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 13.91 }, { "id": "AIME-HeldOut-16", "category": "Exact Rational Series (Harmonic Alternating)", "gold_answer": "210631", "base_answer": "210631", "polystep_answer": "210631", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 20.8 }, { "id": "AIME-HeldOut-17", "category": "Linear Algebra (GCD Matrix Determinant)", "gold_answer": "16", "base_answer": "16", "polystep_answer": "16", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 62.89 }, { "id": "AIME-HeldOut-18", "category": "Combinatorics (Odd Partitions / Compositions)", "gold_answer": "2178309", "base_answer": "", "polystep_answer": "300540195", "base_passed": false, "polystep_passed": false, "switched_branch": true, "selected_source": "polystep_juncture_t41", "elapsed_sec": 42.27 }, { "id": "AIME-HeldOut-19", "category": "Number Theory (Euler Totient Sum)", "gold_answer": "12232", "base_answer": "", "polystep_answer": "12232", "base_passed": false, "polystep_passed": true, "switched_branch": true, "selected_source": "polystep_juncture_t1", "elapsed_sec": 20.29 }, { "id": "AIME-HeldOut-20", "category": "Numerical Analysis (Barron Adaptive Robust Loss)", "gold_answer": "118418", "base_answer": "118418", "polystep_answer": "118418", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 21.68 }, { "id": "MATH500/test_prealgebra_1139.json", "category": "MATH-500 L5 (Prealgebra)", "gold_answer": "4", "base_answer": "2", "polystep_answer": "2", "base_passed": false, "polystep_passed": false, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 98.42 }, { "id": "MATH500/test_number_theory_737.json", "category": "MATH-500 L5 (Number Theory)", "gold_answer": "284", "base_answer": "284", "polystep_answer": "284", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 42.92 }, { "id": "MATH500/test_precalculus_1199.json", "category": "MATH-500 L4 (Precalculus)", "gold_answer": "3.1415927", "base_answer": "3.1415927", "polystep_answer": "3.1415927", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 19.79 }, { "id": "MATH500/test_algebra_1837.json", "category": "MATH-500 L5 (Algebra)", "gold_answer": "5", "base_answer": "5", "polystep_answer": "5", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 104.66 }, { "id": "MATH500/test_algebra_2427.json", "category": "MATH-500 L5 (Algebra)", "gold_answer": "10", "base_answer": "10", "polystep_answer": "10", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 104.61 }, { "id": "MATH500/test_counting_and_probability_525.json", "category": "MATH-500 L5 (Counting & Probability)", "gold_answer": "144", "base_answer": "", "polystep_answer": "144", "base_passed": false, "polystep_passed": true, "switched_branch": true, "selected_source": "polystep_juncture_t26", "elapsed_sec": 146.59 }, { "id": "MATH500/test_counting_and_probability_134.json", "category": "MATH-500 L4 (Counting & Probability)", "gold_answer": "720", "base_answer": "720", "polystep_answer": "720", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 25.94 }, { "id": "MATH500/test_counting_and_probability_119.json", "category": "MATH-500 L4 (Counting & Probability)", "gold_answer": "-125", "base_answer": "-125", "polystep_answer": "-125", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 43.88 }, { "id": "MATH500/test_geometry_627.json", "category": "MATH-500 L4 (Geometry)", "gold_answer": "17", "base_answer": "", "polystep_answer": "17", "base_passed": false, "polystep_passed": true, "switched_branch": true, "selected_source": "polystep_juncture_t509", "elapsed_sec": 114.16 }, { "id": "MATH500/test_prealgebra_1558.json", "category": "MATH-500 L5 (Prealgebra)", "gold_answer": "1.25", "base_answer": "1.25", "polystep_answer": "1.25", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 22.74 }, { "id": "MATH500/test_algebra_305.json", "category": "MATH-500 L5 (Algebra)", "gold_answer": "5", "base_answer": "5", "polystep_answer": "5", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 27.88 }, { "id": "MATH500/test_prealgebra_1388.json", "category": "MATH-500 L4 (Prealgebra)", "gold_answer": "83", "base_answer": "83", "polystep_answer": "83", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 16.11 }, { "id": "MATH500/test_number_theory_1055.json", "category": "MATH-500 L5 (Number Theory)", "gold_answer": "203", "base_answer": "203", "polystep_answer": "203", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 74.49 }, { "id": "MATH500/test_prealgebra_951.json", "category": "MATH-500 L5 (Prealgebra)", "gold_answer": "12", "base_answer": "12", "polystep_answer": "12", "base_passed": true, "polystep_passed": true, "switched_branch": false, "selected_source": "base_greedy", "elapsed_sec": 80.43 }, { "id": "MATH500/test_algebra_187.json", "category": "MATH-500 L4 (Algebra)", "gold_answer": "3", "base_answer": "", "polystep_answer": "3", "base_passed": false, "polystep_passed": true, "switched_branch": true, "selected_source": "polystep_juncture_t100", "elapsed_sec": 130.81 } ] }, "track2_algorithmic_code_math_fuzz": { "num_tasks": 10, "base_passed": 9, "base_pct": 90.0, "polystep_passed": 10, "polystep_pct": 100.0, "gain_pct": 10.0, "tasks": [ { "task_id": "HeldOut-CodeMath/01_rolling_geometric_max", "entry_point": "rolling_max_product_pair", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": false, "base_completion": "if not nums:\n return []\n max_so_far = nums[0]\n result = [max_so_far]\n for num in nums[1:]:\n max_so_far = max(max_so_far, num)\n result.append(max_so_far)\n return result", "polystep_completion": "if not nums:\n return []\n max_so_far = nums[0]\n result = [max_so_far]\n for num in nums[1:]:\n max_so_far = max(max_so_far, num)\n result.append(max_so_far)\n return result", "elapsed_sec": 10.19 }, { "task_id": "HeldOut-CodeMath/02_circular_digit_shift", "entry_point": "circular_shift_digits", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": false, "base_completion": "x = str(x)\n if shift > len(x):\n return x[::-1]\n else:\n return x[-shift:] + x[:-shift]", "polystep_completion": "x = str(x)\n if shift > len(x):\n return x[::-1]\n else:\n return x[-shift:] + x[:-shift]", "elapsed_sec": 5.71 }, { "task_id": "HeldOut-CodeMath/03_caesar_double_two", "entry_point": "encrypt_shift_four", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": false, "base_completion": "result = []\n for char in s:\n if char.isalpha():\n # Determine the base ASCII value for the case of the character\n base = ord('a') if char.islower() else ord('A')\n # Shift by 4 (2 * 2) and wrap around within the 26-letter alphabet\n shifted = (ord(char) - base + 4) % 26 + base\n result.append(chr(shifted))\n else:\n # Non-alphabetic characters remain unchanged\n result.append(char)\n return ''.join(result)", "polystep_completion": "result = []\n for char in s:\n if char.isalpha():\n # Determine the base ASCII value for the case of the character\n base = ord('a') if char.islower() else ord('A')\n # Shift by 4 (2 * 2) and wrap around within the 26-letter alphabet\n shifted = (ord(char) - base + 4) % 26 + base\n result.append(chr(shifted))\n else:\n # Non-alphabetic characters remain unchanged\n result.append(char)\n return ''.join(result)", "elapsed_sec": 22.15 }, { "task_id": "HeldOut-CodeMath/04_balanced_concat_parens", "entry_point": "match_parens_pair", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": false, "base_completion": "def is_good(s: str) -> bool:\n balance = 0\n for char in s:\n if char == '(':\n balance += 1\n else:\n balance -= 1\n if balance < 0:\n return False\n return balance == 0\n\n return 'Yes' if is_good(lst[0] + lst[1]) or is_good(lst[1] + lst[0]) else 'No'", "polystep_completion": "def is_good(s: str) -> bool:\n balance = 0\n for char in s:\n if char == '(':\n balance += 1\n else:\n balance -= 1\n if balance < 0:\n return False\n return balance == 0\n\n return 'Yes' if is_good(lst[0] + lst[1]) or is_good(lst[1] + lst[0]) else 'No'", "elapsed_sec": 12.94 }, { "task_id": "HeldOut-CodeMath/05_sum_two_digit_prefix", "entry_point": "add_two_digit_elements", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": false, "base_completion": "return sum(x for x in arr[:k] if abs(x) < 100)", "polystep_completion": "return sum(x for x in arr[:k] if abs(x) < 100)", "elapsed_sec": 1.63 }, { "task_id": "HeldOut-CodeMath/06_sorted_at_most_two_dups", "entry_point": "is_sorted_max_two_dups", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": false, "base_completion": "for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return False\n if i < len(lst) - 2 and lst[i] == lst[i + 1] == lst[i + 2]:\n return False\n return True", "polystep_completion": "for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return False\n if i < len(lst) - 2 and lst[i] == lst[i + 1] == lst[i + 2]:\n return False\n return True", "elapsed_sec": 10.32 }, { "task_id": "HeldOut-CodeMath/07_sum_of_four_positive_evens", "entry_point": "is_sum_four_positive_even", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": false, "base_completion": "return n >= 8 and n % 2 == 0", "polystep_completion": "return n >= 8 and n % 2 == 0", "elapsed_sec": 1.45 }, { "task_id": "HeldOut-CodeMath/08_sum_squares_cubes_mod3_mod4", "entry_point": "sum_squares_and_cubes", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": true, "base_completion": "return sum(x**2 if i % 3 == 0 else x**3 if i % 4 == 0 else x for i, x in enumerate(lst))", "polystep_completion": "return sum(x ** 3 if i % 4 == 0 and i % 3 != 0 else x ** 2 if i % 3 == 0 else x for i, x in enumerate(lst))", "elapsed_sec": 5.78 }, { "task_id": "HeldOut-CodeMath/09_snake_to_upper_camel", "entry_point": "snake_to_camel_case", "base_passed": false, "polystep_passed": true, "base_probe_passed": false, "polystep_probe_passed": true, "switched_branch": true, "base_completion": "return ''.join([word[0].upper()] + [w[0].upper() + w[1:] for w in word.split('_')[1:]])", "polystep_completion": "return ''.join([w.capitalize() for w in word.split('_')])", "elapsed_sec": 4.12 }, { "task_id": "HeldOut-CodeMath/10_modular_power_two", "entry_point": "modp_fast", "base_passed": true, "polystep_passed": true, "base_probe_passed": true, "polystep_probe_passed": true, "switched_branch": false, "base_completion": "if n == 0:\n return 1\n if n % 2 == 0:\n return modp_fast(n // 2, p) ** 2 % p\n else:\n return 2 * modp_fast(n - 1, p) % p", "polystep_completion": "if n == 0:\n return 1\n if n % 2 == 0:\n return modp_fast(n // 2, p) ** 2 % p\n else:\n return 2 * modp_fast(n - 1, p) % p", "elapsed_sec": 7.9 } ] }, "track3_heldout_competitions": { "bigcodebench_hard": { "num_tasks": 12, "base_passed": 3, "base_pass_at_1_pct": 25.0, "polystep_passed": 12, "polystep_pass_at_1_pct": 100.0, "polystep_gain_pct": 75.0 }, "livecodebench": { "num_tasks": 15, "base_passed": 5, "base_pass_at_1_pct": 33.33, "polystep_passed": 15, "polystep_pass_at_1_pct": 100.0, "polystep_gain_pct": 66.67 }, "hf_private_competition": { "num_tasks": 25, "base_passed": 7, "base_pass_at_1_pct": 28.0, "polystep_passed": 22, "polystep_pass_at_1_pct": 88.0, "polystep_gain_pct": 60.0, "tasks": [ { "id": "HFComp/Blind-007_bisect_left_lower_bound_step", "base_prediction": 0, "polystep_prediction": 3, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-010_fenwick_1_indexed_build", "base_prediction": 3, "polystep_prediction": 2, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-016_levenshtein_dp_base_row", "base_prediction": 2, "polystep_prediction": 1, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-022_circular_kadane_all_negative_guard", "base_prediction": 0, "polystep_prediction": 0, "base_passed": true, "polystep_passed": true }, { "id": "HFComp/Blind-027_kahn_toposort_cycle_guard", "base_prediction": 0, "polystep_prediction": 0, "base_passed": false, "polystep_passed": false }, { "id": "HFComp/Blind-033_welford_streaming_mean_update", "base_prediction": 2, "polystep_prediction": 2, "base_passed": true, "polystep_passed": true }, { "id": "HFComp/Blind-035_welford_bessel_sample_variance", "base_prediction": 3, "polystep_prediction": 1, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-045_coin_change_dp_step_increment", "base_prediction": 1, "polystep_prediction": 0, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-047_coin_change_unreachable_sentinel", "base_prediction": 0, "polystep_prediction": 3, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-048_dijkstra_stale_heap_guard", "base_prediction": 1, "polystep_prediction": 2, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-051_dijkstra_unreachable_sentinel", "base_prediction": 2, "polystep_prediction": 1, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-097_barron_shifted_log1p_logsumexp", "base_prediction": 0, "polystep_prediction": 0, "base_passed": true, "polystep_passed": true }, { "id": "HFComp/Blind-100_sinkhorn_entropic_dual_scaling", "base_prediction": 2, "polystep_prediction": 3, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-104_floyd_warshall_zero_diagonal", "base_prediction": 3, "polystep_prediction": 2, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-108_kruskal_mst_weight_sort", "base_prediction": 1, "polystep_prediction": 1, "base_passed": true, "polystep_passed": true }, { "id": "HFComp/Blind-120_fwht_butterfly_xor_partner", "base_prediction": 0, "polystep_prediction": 0, "base_passed": true, "polystep_passed": true }, { "id": "HFComp/Blind-124_automaton_suffix_prefix_overlap", "base_prediction": 3, "polystep_prediction": 3, "base_passed": true, "polystep_passed": true }, { "id": "HFComp/Blind-135_balanced_brackets_unclosed_guard", "base_prediction": 3, "polystep_prediction": 3, "base_passed": false, "polystep_passed": false }, { "id": "HFComp/Blind-137_posix_canonical_path_noop_filter", "base_prediction": 2, "polystep_prediction": 1, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-148_temporal_dedup_chronological_sort", "base_prediction": 1, "polystep_prediction": 0, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-150_temporal_dedup_entity_sort", "base_prediction": 0, "polystep_prediction": 0, "base_passed": false, "polystep_passed": false }, { "id": "HFComp/Blind-168_pca_mean_centering_before_svd", "base_prediction": 0, "polystep_prediction": 2, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-175_vectorized_2d_barron_log1p", "base_prediction": 2, "polystep_prediction": 1, "base_passed": false, "polystep_passed": true }, { "id": "HFComp/Blind-192_chronological_embargo_split", "base_prediction": 0, "polystep_prediction": 0, "base_passed": true, "polystep_passed": true }, { "id": "HFComp/Blind-196_causal_rolling_window_zscore", "base_prediction": 1, "polystep_prediction": 3, "base_passed": false, "polystep_passed": true } ] } } }