| window.FRONTIER_SWE = {"source": "https://www.frontierswe.com/tasks", "snapshot_utc": "2026-09-07T08:22:50Z", "n_tasks": 34, "tasks": [{"slug": "autonomous-astrometry-toolkit", "number": 1, "title": "Astronomy Toolkit", "url": "https://www.frontierswe.com/tasks/autonomous-astrometry-toolkit", "categories": ["Visual Reasoning", "Implementation"], "summary": "Build an offline astrometry pipeline that blind-solves FITS images against Gaia DR3, registers overlapping observations, and writes sky mosaics.", "scoring": "Hidden tests", "toolchains": ["Python"], "compute": {"label": "CPU", "kind": "cpu", "compute": "16 CPUs · 128 GB RAM", "gpu": null, "base_image": "python:3.13-slim-trixie", "tools": "Python 3", "time_limit": "20h"}, "background": "Astrometry is the problem of figuring out exactly where on the sky a telescope image points. Blind plate solving, recovering a World Coordinate System (WCS) with no prior pointing hint, means extracting stars from noisy pixels, matching their geometric patterns against a full-sky reference catalog, and fitting a projection that maps pixels to celestial coordinates, all while handling rotation, unknown plate scale, vignetting, and survey-to-survey differences in depth and resolution.\nProduction solvers took years of engineering to make this robust. Here the agent must build one from scratch that generalizes across surveys: the development suite spans native and cropped SDSS frames, deliberately degraded observations, a coarse mixed optical/infrared campaign, and a synthetic field that can only be solved by genuinely blind search over the full-sky Gaia DR3 catalog.", "task": "Implement /app/astrometry/localize.py , invoked without network access as python /app/astrometry/localize.py --input-dir /path/to/campaign --output-dir /path/to/output . Each campaign supplies a campaign.json , FITS images, and a star catalog that may be a small field-local extract or the 1.35 GB full-sky Gaia catalog.\n- Write wcs/<image_id>.json for every image: a TAN celestial projection with ctype , crpix , crval , and a 2×2 cd matrix.\n- Write registrations.json with one 3×3 pixel-to-pixel transform per unordered overlapping image pair (no duplicate inverse entries).\n- Write mosaic.fits (a finite two-dimensional image with a celestial WCS header) plus a run_summary.json .\n- Each campaign, including catalog indexing, must complete within 15 minutes under bounded memory; the solver must not depend on fixed sky coordinates, image IDs, or catalog row order.", "task_bullets": ["Write wcs/<image_id>.json for every image: a TAN celestial projection with ctype , crpix , crval , and a 2×2 cd matrix.", "Write registrations.json with one 3×3 pixel-to-pixel transform per unordered overlapping image pair (no duplicate inverse entries).", "Write mosaic.fits (a finite two-dimensional image with a celestial WCS header) plus a run_summary.json .", "Each campaign, including catalog indexing, must complete within 15 minutes under bounded memory; the solver must not depend on fixed sky coordinates, image IDs, or catalog row order."], "agent_gets": "An example campaign at /app/example_campaign/ and five public development campaigns under /app/development_suite/campaigns/ , each with public truth (truth/truth.json ) containing reference WCS parameters and catalog samples with expected pixel coordinates: native SDSS frames, clean offset crops, degraded crops with noise and vignetting, a coarse mixed-survey campaign of rotated optical and infrared observations, and a synthetic star field wired to the full-sky catalog.\n- The Gaia DR3 global catalog (1.35 GB CSV) and a geometric index staged read-only at /data/astrometry/ .\n- A structural output checker, python /app/astrometry/validate_outputs.py , that verifies artifact schemas without estimating scientific quality.\n- Python 3.13 with NumPy, SciPy, and Astropy preinstalled; the machine is offline and the evaluation campaigns use different sky fields than the public suite.", "verification": "A root-only verifier runs the submitted solver as the unprivileged agent user on sealed real campaigns covering sky fields never shown to the agent, plus a metamorphic exact-invariance case: a transformed copy of a sealed campaign that must produce equivalent results, catching solvers that key on incidental identifiers. Solver syscalls are audited with strace for network, reward, or hidden-truth access.\n- Each campaign scores completeness (the fraction of images localized, raised to a per-campaign power) times quality, times a runtime multiplier. Quality is a weighted geometric mean of WCS accuracy (0.55, reprojection error against truth star positions in arcseconds), registration residuals (0.30, median pixel error of submitted transforms), and mosaic fidelity (0.15).\n- Runtime earns full credit at 15 s/image and falls to its floor by 150 s/image; a correct slow solver keeps partial credit. Each campaign has a 15-minute solver timeout.\n- Zeroed outright: no images localized, missing or schema-invalid artifacts, safeguard violations flagged by the syscall audit, or an incomplete sealed campaign set.", "verification_bullets": ["Each campaign scores completeness (the fraction of images localized, raised to a per-campaign power) times quality, times a runtime multiplier. Quality is a weighted geometric mean of WCS accuracy (0.55, reprojection error against truth star positions in arcseconds), registration residuals (0.30, median pixel error of submitted transforms), and mosaic fidelity (0.15).", "Runtime earns full credit at 15 s/image and falls to its floor by 150 s/image; a correct slow solver keeps partial credit. Each campaign has a 15-minute solver timeout.", "Zeroed outright: no images localized, missing or schema-invalid artifacts, safeguard violations flagged by the syscall audit, or an incomplete sealed campaign set."], "safeguards": ["Runtime earns full credit at 15 s/image and falls to its floor by 150 s/image; a correct slow solver keeps partial credit. Each campaign has a 15-minute solver timeout.", "Zeroed outright: no images localized, missing or schema-invalid artifacts, safeguard violations flagged by the syscall audit, or an incomplete sealed campaign set."], "environment": {"Base image": "python:3.13-slim-trixie", "Tools available": "Python 3", "Compute": "16 CPUs · 128 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "Gaia Data Release 3 (CC BY-NC 3.0 IGO): Catalog rows for the full-sky star catalog and geometric index. Credit: ESA/Gaia/DPAC.", "url": "https://www.cosmos.esa.int/web/gaia-users/license"}, {"title": "Sloan Digital Sky Survey DR17 (Public domain (data); CC BY (web imagery)): Frames used as transformed cutouts in the real-image campaigns.", "url": "https://www.sdss.org/collaboration/image-use-policy/"}, {"title": "Digitized Sky Survey II (Copyrighted (Caltech/Palomar; AAO/ROE)): Cutouts in the mixed-survey campaign, retrieved through the CDS hips2fits service; used for research and education, with commercial reuse requiring permission from the survey holders.", "url": "https://gsss.stsci.edu/zzzoldwebsite/dss/data_use.html"}, {"title": "Two Micron All Sky Survey (2MASS) : Infrared cutouts in the mixed-survey campaign, retrieved through the CDS hips2fits service. 2MASS is a joint project of the University of Massachusetts and IPAC/Caltech, funded by NASA and NSF.", "url": "https://irsa.ipac.caltech.edu/data/2MASS/docs/releases/second/doc/sec1_8b.html"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.8369, "std": 0.053, "steps": "253", "in_tok": "77.2M", "out_tok": "322k", "avg_cost": "$46.43", "avg_time": "4.6h"}, {"rank": 2, "model": "GLM-5.3", "score": 0.4371, "std": 0.154, "steps": "483", "in_tok": "217.2M", "out_tok": "576k", "avg_cost": "$69.06", "avg_time": "16.6h"}, {"rank": 3, "model": "Kimi K3", "score": 0.4198, "std": 0.122, "steps": "423", "in_tok": "148.4M", "out_tok": "427k", "avg_cost": "$59.05", "avg_time": "14.8h"}, {"rank": 4, "model": "Grok 4.6", "score": 0.2653, "std": 0.205, "steps": "163", "in_tok": "31.1M", "out_tok": "358k", "avg_cost": "$32.22", "avg_time": "3.2h"}, {"rank": 5, "model": "Gemini 3.7 Flash", "score": 0.2262, "std": 0.331, "steps": "179", "in_tok": "31.6M", "out_tok": "273k", "avg_cost": "$4.93", "avg_time": "1.4h"}, {"rank": 6, "model": "GPT-5.6", "score": 0.1334, "std": 0.212, "steps": "242", "in_tok": "45.3M", "out_tok": "182k", "avg_cost": "$36.56", "avg_time": "2.5h"}, {"rank": 7, "model": "Muse Spark 1.2", "score": 0.0515, "std": 0.05, "steps": "315", "in_tok": "99.6M", "out_tok": "492k", "avg_cost": "$22.74", "avg_time": "4.5h"}, {"rank": 8, "model": "Qwen3.8-Max", "score": 0.0, "std": 0.0, "steps": "488", "in_tok": "142.7M", "out_tok": "1.2M", "avg_cost": "$43.60", "avg_time": "18.3h"}, {"rank": 9, "model": "DeepSeek V4 Flash Exp", "score": 0.0, "std": 0.0, "steps": "474", "in_tok": "203.2M", "out_tok": "574k", "avg_cost": "$3.68", "avg_time": "15.3h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "126", "in_tok": "5.3M", "out_tok": "37k", "avg_cost": "$5.22", "avg_time": "0.9h"}], "best_score": 0.8369, "mean_score": 0.237, "n_models": 10}, {"slug": "cranelift-codegen-opt", "number": 2, "title": "Cranelift Codegen Optimization", "url": "https://www.frontierswe.com/tasks/cranelift-codegen-opt", "categories": ["Performance Optimisation"], "summary": "Make Cranelift, the compiler inside Wasmtime, emit faster machine code, without slowing down compilation or breaking a single spec test.", "scoring": "Perf speedup", "toolchains": ["Rust", "Python", "x86-64 asm", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "8 CPUs · 128 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Rust, CMake, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Cranelift is the optimizing compiler backend inside Wasmtime. Its design point is fast compilation, so the machine code it emits leaves measurable performance on the table compared to LLVM-class backends: in the mid-end rewrite rules (written in the ISLE DSL), the e-graph optimizer and its cost model, the x86-64 instruction selector, and the vendored regalloc2 register allocator. Improving generated code without materially slowing the compiler is exactly the trade-off production compiler engineers navigate.\nCorrectness makes this unforgiving: WebAssembly semantics pin down exact float results and NaN propagation, integer division/remainder edges, shift-count masking, trap conditions and their ordering; a rewrite rule that is almost right miscompiles real programs.", "task": "The agent must make the native code Cranelift generates run faster, without making the compiler itself do materially more work to compile. What counts is the machine code emitted for a WebAssembly module, not how fast Wasmtime itself runs. All changes are confined to the Wasmtime source tree at /app/wasmtime/ , written as .rs and .isle sources.\n- /app/perf-check models the full performance requirement: it rebuilds the compiler, measures the generated code for the visible workloads against the unmodified compiler, and separately reports compile work on the biggest modules as a ratio to the baseline.\n- Compile time is a constraint, not free: the guidance is to stay within about +10% of baseline compile work; a runtime win bought by a much slower compile does not count.\n- Every workload's reference output and every spec test that passes today must still pass; one regression is as bad as making no change.\n- Cargo is offline with a fully vendored dependency graph: Cargo.toml , Cargo.lock , cargo config, and build.rs files are fixed, so no new dependencies can be added.", "task_bullets": ["/app/perf-check models the full performance requirement: it rebuilds the compiler, measures the generated code for the visible workloads against the unmodified compiler, and separately reports compile work on the biggest modules as a ratio to the baseline.", "Compile time is a constraint, not free: the guidance is to stay within about +10% of baseline compile work; a runtime win bought by a much slower compile does not count.", "Every workload's reference output and every spec test that passes today must still pass; one regression is as bad as making no change.", "Cargo is offline with a fully vendored dependency graph: Cargo.toml , Cargo.lock , cargo config, and build.rs files are fixed, so no new dependencies can be added."], "agent_gets": "The full pinned Wasmtime source tree with a warm target/ for incremental rebuilds, regalloc2 vendored as an editable path dependency, and the Wasm spec and misc test suites. Ten visible benchmark workloads ship as pre-built .wasm with inputs and the unmodified compiler's recorded outputs: an interpreter, compression codecs, SQLite, crypto, and loop/recursion micro-kernels drawn largely from the Sightglass corpus.\n/app/perf-check is the self-check loop: rebuild, compile every workload, measure the generated code deterministically, and compare against the baseline (with --wast for the spec suites and wasmtime-baseline on PATH for manual A/B). The measurement stack in /app/performance/ (per-address attribution plus per-opcode pricing) and the baseline measurements are all visible. The machine is offline with a pinned Rust toolchain preinstalled.", "verification": "A clean-room verifier in a separate container reconstructs the scored tree from pristine Wasmtime sources plus only the agent's .rs /.isle edits, rebuilds it, and scores ten held-out workloads: same corpus families as the visible set (SIMD, compression, regex, crypto, jump-table and integer kernels) but never seen by the agent.\n- The metric is deterministic simulated work, not wall-clock: per-address execution counts are restricted to the code Cranelift generated (excluding Wasmtime startup), and each opcode is priced by a reciprocal-throughput cost table derived from uops.info, with codegen pinned to x86-64-v3.\n- Correctness gates run first: must-pass and must-fail canary suites, two passes over the full spec suites (nothing that passes on the reference may regress), edge-case binaries, and every workload's output compared against the reference compiler's build. Any failure zeroes the result.\n- The scored speedup is outlier-dampened (held down to the geometric mean that survives dropping the single best workload), then mapped on a convex curve with a 0.5% deadband reaching full credit at a 1.20x geomean. A one-sided compile-work penalty multiplies the result: free within +10% of baseline, linear to zero at +20%.\n- A single workload \"improving\" by more than 2.5x is treated as implausible and zeroed for review; modified sources that reach an external code generator are rejected; a compiler that fails to build or to run a workload the baseline handles scores zero.", "verification_bullets": ["The metric is deterministic simulated work, not wall-clock: per-address execution counts are restricted to the code Cranelift generated (excluding Wasmtime startup), and each opcode is priced by a reciprocal-throughput cost table derived from uops.info, with codegen pinned to x86-64-v3.", "Correctness gates run first: must-pass and must-fail canary suites, two passes over the full spec suites (nothing that passes on the reference may regress), edge-case binaries, and every workload's output compared against the reference compiler's build. Any failure zeroes the result.", "The scored speedup is outlier-dampened (held down to the geometric mean that survives dropping the single best workload), then mapped on a convex curve with a 0.5% deadband reaching full credit at a 1.20x geomean. A one-sided compile-work penalty multiplies the result: free within +10% of baseline, linear to zero at +20%.", "A single workload \"improving\" by more than 2.5x is treated as implausible and zeroed for review; modified sources that reach an external code generator are rejected; a compiler that fails to build or to run a workload the baseline handles scores zero."], "safeguards": ["The metric is deterministic simulated work, not wall-clock: per-address execution counts are restricted to the code Cranelift generated (excluding Wasmtime startup), and each opcode is priced by a reciprocal-throughput cost table derived from uops.info, with codegen pinned to x86-64-v3.", "Correctness gates run first: must-pass and must-fail canary suites, two passes over the full spec suites (nothing that passes on the reference may regress), edge-case binaries, and every workload's output compared against the reference compiler's build. Any failure zeroes the result.", "The scored speedup is outlier-dampened (held down to the geometric mean that survives dropping the single best workload), then mapped on a convex curve with a 0.5% deadband reaching full credit at a 1.20x geomean. A one-sided compile-work penalty multiplies the result: free within +10% of baseline, linear to zero at +20%.", "A single workload \"improving\" by more than 2.5x is treated as implausible and zeroed for review; modified sources that reach an external code generator are rejected; a compiler that fails to build or to run a workload the baseline handles scores zero."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Rust, CMake, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 128 GB RAM", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of Cranelift Codegen Optimization from FrontierSWE V1.\n- The metric moved from wall-clock timings in a benchmark runner to deterministic simulated work: per-address execution counts restricted to Cranelift-generated code, priced per opcode from a uops.info-derived cost table, with codegen pinned to x86-64-v3. One measurement per change replaces iteration counts and host noise.\n- Scoring moved from the visible tiered benchmark suite to ten held-out workloads from the same corpus families that the agent never sees, replacing the weighted-harmonic-mean formula with an outlier-dampened geometric mean (held to what survives dropping the single best workload) on a convex curve reaching full credit at 1.20x.\n- The compile-time constraint became an explicit modeled penalty: compile work on the biggest modules is measured against the baseline and multiplies the reward (free within +10%, linear to zero at +20%) instead of V1's unquantified \"compile time must not regress\".\n- Anti-cheat hardened: the verifier reconstructs the scored tree from pristine sources plus only the agent's .rs /.isle diffs, rejects modified sources that reach an external code generator, auto-zeroes implausible single-workload jumps above 2.5x, and adds must-pass/must-fail canary suites and edge-case binaries alongside the spec suites.\n- Verification became a separate clean-room container with its timeout cut from 24 hours to 2.5 hours, and the /app/.timer daemon files were replaced by the sandbox-timer CLI under the new harbor schema.", "references": [{"title": "Wasmtime / Cranelift (Apache-2.0 WITH LLVM-exception): The full source tree at a pinned commit, fetched at image build to /app/wasmtime/ as the codebase the agent optimizes; its WebAssembly spec and misc test suites are also baked into the verifier's pristine assets.", "url": "https://github.com/bytecodealliance/wasmtime"}, {"title": "regalloc2 (Apache-2.0 WITH LLVM-exception): v0.15.0, cloned at image build into /app/wasmtime/vendor/regalloc2/ as a modifiable local path dependency of the Cranelift build.", "url": "https://github.com/bytecodealliance/regalloc2"}, {"title": "uops.info instruction data (Cited per authors' request; format via XED-to-XML (Apache-2.0)): The per-instruction cost table is a derived aggregate of reciprocal-throughput measurements from Abel & Reineke's uops.info (ASPLOS 2019), generated once offline, not a copy of the dataset.", "url": "https://uops.info/"}, {"title": "Sightglass benchmark corpus (Apache-2.0 OR MIT (corpus; members under their own licenses)): Pre-compiled .wasm workloads and inputs derived from open-source projects including brotli, SQLite, SpiderMonkey, Lua, zstd, serde-json, bzip2, meshoptimizer, pulldown-cmark, rust regex, libsodium, BLAKE3, intgemm, LLVM test-suite and PolyBench/C kernels, and Computer Language Shootout micro-benchmarks, plus their bundled text input corpora (CommonMark spec, Learn X in Y Minutes, bzip2 documentation). Reference outputs are re-recorded from the task's own baseline compiler at image build.", "url": "https://github.com/bytecodealliance/sightglass"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.234, "std": 0.143, "steps": "935", "in_tok": "366.5M", "out_tok": "719k", "avg_cost": "$200.53", "avg_time": "16.3h"}, {"rank": 2, "model": "Kimi K3", "score": 0.0243, "std": 0.028, "steps": "1,043", "in_tok": "402.4M", "out_tok": "884k", "avg_cost": "$138.87", "avg_time": "18.9h"}, {"rank": 3, "model": "Gemini 3.7 Flash", "score": 0.022, "std": 0.033, "steps": "587", "in_tok": "221.0M", "out_tok": "245k", "avg_cost": "$31.50", "avg_time": "5.4h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.0079, "std": 0.007, "steps": "725", "in_tok": "298.7M", "out_tok": "452k", "avg_cost": "$88.78", "avg_time": "12.5h"}, {"rank": 5, "model": "DeepSeek V4 Flash Exp", "score": 0.0075, "std": 0.01, "steps": "835", "in_tok": "327.5M", "out_tok": "469k", "avg_cost": "$5.38", "avg_time": "8.1h"}, {"rank": 6, "model": "GPT-5.6", "score": 0.0035, "std": 0.008, "steps": "523", "in_tok": "253.5M", "out_tok": "231k", "avg_cost": "$264.94", "avg_time": "7.7h"}, {"rank": 7, "model": "Qwen3.8-Max", "score": 0.0023, "std": 0.005, "steps": "504", "in_tok": "116.7M", "out_tok": "2.1M", "avg_cost": "$42.60", "avg_time": "19.3h"}, {"rank": 8, "model": "Grok 4.6", "score": 0.0, "std": 0.0, "steps": "573", "in_tok": "140.2M", "out_tok": "1.0M", "avg_cost": "$147.90", "avg_time": "9.9h"}, {"rank": 9, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "120", "in_tok": "11.7M", "out_tok": "18k", "avg_cost": "$11.20", "avg_time": "1.0h"}, {"rank": 10, "model": "Muse Spark 1.2", "score": 0.0, "std": 0.0, "steps": "445", "in_tok": "202.3M", "out_tok": "460k", "avg_cost": "$53.21", "avg_time": "8.5h"}], "best_score": 0.234, "mean_score": 0.0302, "n_models": 10}, {"slug": "flash-fs", "number": 3, "title": "Crash-Proof Flash Filesystem", "url": "https://www.frontierswe.com/tasks/flash-fs", "categories": ["Implementation"], "summary": "Implement a crash-proof NOR flash filesystem in Zig, down to the exact on-disk byte layout.", "scoring": "Hidden tests", "toolchains": ["Zig", "Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 4 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Zig, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Filesystems for NOR flash live under constraints that desktop filesystems never face: writes only flip bits one way until a whole block is erased, blocks wear out after a bounded number of erase cycles, and embedded devices can lose power at any instant. A correct design must guarantee that power loss between any two operations never corrupts the filesystem, while spreading erases across the device and keeping RAM usage tiny.\nThe task workspace is derived from a production embedded filesystem, de-branded so the agent cannot simply recall the implementation: the test and bench suites define the behavior, and the agent has to reconstruct the design from what they pin down.", "task": "Implement the filesystem in Zig inside /app/flash-fs/ , exporting C-compatible symbols (export fn , callconv(.c) ) that match what the test suites call. Using @cImport to wrap existing C code is not allowed; the filesystem itself must be written in Zig.\n- The tests/ suites are white-box: they read raw on-disk metadata directly, so correctness means reproducing the exact on-disk block-device state they pin down, not merely passing at the API surface.\n- Survive simulated power loss at every point the suites inject it, across multiple block-device geometries.\n- The benches/ suites measure bytes read, bytes programmed, and blocks erased; lower is better.\n- Every suite runs under a fixed per-suite time cap that counts as failed if exceeded.", "task_bullets": ["The tests/ suites are white-box: they read raw on-disk metadata directly, so correctness means reproducing the exact on-disk block-device state they pin down, not merely passing at the API surface.", "Survive simulated power loss at every point the suites inject it, across multiple block-device geometries.", "The benches/ suites measure bytes read, bytes programmed, and blocks erased; lower is better.", "Every suite runs under a fixed per-suite time cap that counts as failed if exceeded."], "agent_gets": "A ready-to-run Zig project at /app/flash-fs/ : the correctness suites in tests/ , the flash-I/O efficiency suites in benches/ , an emulated block device, the test runner, and a build system driven by ./build_and_test.sh (all suites) or ./build_and_test.sh <suite> (one). The machine is offline; everything needed is preinstalled, including the Zig toolchain.", "verification": "A clean-room verifier, separate from the agent's environment, rebuilds the submission and runs the hidden versions of the suites. The scored answer key is the reference implementation's per-permutation block-device checksums across four flash geometries, so credit requires a genuinely working filesystem that produces the exact on-disk state; a recalled description of the original design is not enough.\n- Correctness comes from the hidden test suites; efficiency is scored on deterministic emulated-flash I/O counters (bytes read/programmed, blocks erased), not wall-clock time.\n- Build failures, missing symbols, and anti-cheat violations zero the result.", "verification_bullets": ["Correctness comes from the hidden test suites; efficiency is scored on deterministic emulated-flash I/O counters (bytes read/programmed, blocks erased), not wall-clock time.", "Build failures, missing symbols, and anti-cheat violations zero the result."], "safeguards": ["Correctness comes from the hidden test suites; efficiency is scored on deterministic emulated-flash I/O counters (bytes read/programmed, blocks erased), not wall-clock time.", "Build failures, missing symbols, and anti-cheat violations zero the result."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Zig, C/C++ toolchain, Python 3", "Compute": "4 CPUs · 4 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "littlefs (BSD-3-Clause): The test framework, test/bench suites, block-device emulation, build system, and the reference implementation used only by the oracle (v2.11, on-disk format 2.1). Agent-visible copies are de-branded as an anti-contamination measure; SPDX license headers are retained in the vendored files.", "url": "https://github.com/littlefs-project/littlefs"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.7779, "std": 0.428, "steps": "249", "in_tok": "95.4M", "out_tok": "448k", "avg_cost": "$58.57", "avg_time": "3.7h"}, {"rank": 2, "model": "Grok 4.6", "score": 0.0209, "std": 0.024, "steps": "1,452", "in_tok": "393.7M", "out_tok": "2.0M", "avg_cost": "$402.94", "avg_time": "14.4h"}, {"rank": 3, "model": "GPT-5.6", "score": 0.0204, "std": 0.003, "steps": "311", "in_tok": "92.1M", "out_tok": "204k", "avg_cost": "$94.52", "avg_time": "2.4h"}, {"rank": 4, "model": "Kimi K3", "score": 0.013, "std": 0.017, "steps": "1,170", "in_tok": "513.6M", "out_tok": "1.6M", "avg_cost": "$188.95", "avg_time": "19.9h"}, {"rank": 5, "model": "GLM-5.3", "score": 0.0109, "std": 0.006, "steps": "1,338", "in_tok": "596.2M", "out_tok": "1.2M", "avg_cost": "$163.96", "avg_time": "19.0h"}, {"rank": 6, "model": "Gemini 3.7 Flash", "score": 0.0082, "std": 0.008, "steps": "1,878", "in_tok": "756.6M", "out_tok": "1.9M", "avg_cost": "$87.06", "avg_time": "20.0h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0062, "std": 0.003, "steps": "2,894", "in_tok": "1331.7M", "out_tok": "2.3M", "avg_cost": "$22.22", "avg_time": "19.3h"}, {"rank": 8, "model": "Muse Spark 1.2", "score": 0.0033, "std": 0.002, "steps": "415", "in_tok": "184.7M", "out_tok": "495k", "avg_cost": "$31.67", "avg_time": "2.7h"}, {"rank": 9, "model": "Qwen3.8-Max", "score": 0.0003, "std": 0.001, "steps": "142", "in_tok": "16.9M", "out_tok": "1.4M", "avg_cost": "$12.66", "avg_time": "19.3h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "167", "in_tok": "13.6M", "out_tok": "26k", "avg_cost": "$13.03", "avg_time": "0.7h"}], "best_score": 0.7779, "mean_score": 0.0861, "n_models": 10}, {"slug": "dart-style-haskell", "number": 4, "title": "Dart Style in Haskell", "url": "https://www.frontierswe.com/tasks/dart-style-haskell", "categories": ["Implementation"], "summary": "Implement the Dart code formatter in Haskell, byte-exact against its test corpus in both short and tall styles.", "scoring": "Byte exact", "toolchains": ["Haskell", "Go", "Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 8 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Haskell (GHC), C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "A production code formatter is a full language frontend plus a constraint solver: it must scan and parse the entire language (every declaration form, expression, comment placement, and string interpolation) and then choose line splits that satisfy a page width while matching thousands of precedents byte for byte. Dart's formatter is harder still because it has two complete layout engines: the classic \"short\" style and the current \"tall\" style, with different splitting rules and trailing-comma handling.\nThe agent is handed tests, not source: the workspace contains no formatter implementation to port from, only the formatter's input/expected-output test corpus. The behaviour has to be reconstructed from what those cases pin down, in Haskell.", "task": "Implement the Dart code formatter as a Haskell executable named dart-style that reads Dart source on stdin and writes the formatted result to stdout, working in the provided cabal project at /app/dart-style/ .\n- Changes are confined to .hs files under src/ ; the build configuration (dart-style.cabal , cabal.project ) is fixed and language extensions go in per-file pragmas.\n- Both formatting styles must work, selected by --style short|tall , along with arbitrary --page-width and --indent values, --compilation-unit versus --statement parsing, and --trailing-commas automate|preserve .\n- Correctness is byte-exact: each corpus case feeds input on stdin and compares stdout against the expected output byte for byte.\n- cabal build all must succeed and cabal list-bin dart-style must locate the executable.", "task_bullets": ["Changes are confined to .hs files under src/ ; the build configuration (dart-style.cabal , cabal.project ) is fixed and language extensions go in per-file pragmas.", "Both formatting styles must work, selected by --style short|tall , along with arbitrary --page-width and --indent values, --compilation-unit versus --statement parsing, and --trailing-commas automate|preserve .", "Correctness is byte-exact: each corpus case feeds input on stdin and compares stdout against the expected output byte for byte.", "cabal build all must succeed and cabal list-bin dart-style must locate the executable."], "agent_gets": "A compiling cabal scaffold at /app/dart-style/ , an /app/README.md spelling out the build commands and the full CLI contract, and the formatter's test corpus at /app/tests/ : short/ and tall/ .stmt /.unit case files plus large real-world benchmark/ inputs. A provided /app/run-tests.sh builds the project and runs the corpus (all of it, a matching subset, or with per-case mismatch diffs), backed by a documented Python runner the agent can reuse for its own tooling. GHC 9.6.7 and cabal 3.12 are preinstalled with the scaffold's dependencies cached for offline use; the machine is offline and a sandbox-timer CLI reports the remaining budget. The real Dart SDK is baked into the image for verification only and is not reachable by the agent.", "verification": "A clean-room verifier, separate from the agent's environment, reconstructs the graded project from a pristine baked scaffold plus only the agent's .hs sources, strips the Dart runtime, clean-rebuilds with cabal as a non-root user, and runs a root-only scored corpus of roughly 5,000 cases. The scored corpus is a perturbed rendering: the same pinned dart format re-renders every case at a different page width and indent at image build, so memorising the visible corpus's expected outputs does not transfer.\n- The reward is byte-exact passes divided by scored cases, where scored cases are the changed-input cases the pinned reference formatter passed at image build. Unchanged-input cases (about 11% of the corpus) are excluded from both numerator and denominator, so copying stdin to stdout scores 0.\n- Hard gates: a failed build or missing executable, a reachable Dart runtime, or submitting the unmodified starting scaffold zeroes the result.\n- Each case runs under a fixed 30-second timeout and the whole suite under a fixed 4600-second deadline; tripping the deadline records an invalid run rather than a partial score.", "verification_bullets": ["The reward is byte-exact passes divided by scored cases, where scored cases are the changed-input cases the pinned reference formatter passed at image build. Unchanged-input cases (about 11% of the corpus) are excluded from both numerator and denominator, so copying stdin to stdout scores 0.", "Hard gates: a failed build or missing executable, a reachable Dart runtime, or submitting the unmodified starting scaffold zeroes the result.", "Each case runs under a fixed 30-second timeout and the whole suite under a fixed 4600-second deadline; tripping the deadline records an invalid run rather than a partial score."], "safeguards": ["Hard gates: a failed build or missing executable, a reachable Dart runtime, or submitting the unmodified starting scaffold zeroes the result.", "Each case runs under a fixed 30-second timeout and the whole suite under a fixed 4600-second deadline; tripping the deadline records an invalid run rather than a partial score."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Haskell (GHC), C/C++ toolchain, Python 3", "Compute": "4 CPUs · 8 GB RAM", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of Dart → Haskell from FrontierSWE V1.\n- The V1 workspace shipped dart_style's full Dart source at /app/reference/ (plus analyzer dependency sources) to port from; V2 removes the source entirely. The agent gets the test corpus as the executable specification, plus a fixed cabal scaffold and a documented corpus runner.\n- Verification moved from a bash script hardening the agent's own container in place (PATH locking, system-binary checksums, strace probes) to a clean-room verifier in a separate, hash-pinned image that reconstructs the project from a pristine scaffold plus only the agent's .hs sources; wrapper binaries, build-hook tampering, and cabal-file edits are discarded by construction.\n- The scored corpus is now perturbed at image build: the pinned dart format re-renders every case at a different page width and indent, so hardcoding the visible corpus's expected outputs no longer pays.\n- Scoring gates tightened: unchanged-input cases (about 11%) are excluded from both sides of the fraction so a copy-stdin-to-stdout submission scores exactly 0, the denominator is fixed to the cases the reference passed at bake, and submitting the unmodified scaffold is an explicit anti-cheat failure.\n- The CLI contract was reworked from version-based pipeline selection (--language-version ) to an explicit --style short|tall flag, with the full contract documented in the workspace README instead of the task prompt.\n- The verifier timeout dropped from 24 hours to 2 hours, with a fixed 30-second per-case timeout and a fixed 4600-second suite deadline that records an invalid run when tripped; the ad-hoc /app/.timer file interface was replaced by the standard sandbox-timer CLI.", "references": [{"title": "dart_style (BSD-3-Clause): The test corpus (test/short , test/tall , benchmark/case ), fetched at image build (not vendored) at the exact revision Dart SDK 3.11.3 vendors as its formatter (dart_style 3.1.4-wip), so the baked dart format reproduces the corpus by construction.", "url": "https://github.com/dart-lang/dart_style"}, {"title": "Dart SDK (BSD-3-Clause): The dart format implementation (SDK 3.11.3, pinned; fetched at image build, not vendored), baked at /opt/dart-sdk as build and verification infrastructure only; it is locked away from the agent and stripped before scoring.", "url": "https://github.com/dart-lang/sdk"}, {"title": "Flutter (BSD-3-Clause): A few benchmark and regression corpus cases derive from Flutter framework test code via dart_style's own benchmark suite, used solely as formatter inputs and expected formatted outputs.", "url": "https://github.com/flutter/flutter"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.9136, "std": 0.031, "steps": "1,902", "in_tok": "860.2M", "out_tok": "1.3M", "avg_cost": "$461.38", "avg_time": "18.5h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.6337, "std": 0.123, "steps": "810", "in_tok": "325.0M", "out_tok": "403k", "avg_cost": "$328.24", "avg_time": "6.2h"}, {"rank": 3, "model": "Gemini 3.7 Flash", "score": 0.4727, "std": 0.128, "steps": "2,036", "in_tok": "894.0M", "out_tok": "1.3M", "avg_cost": "$91.05", "avg_time": "12.8h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.4606, "std": 0.054, "steps": "1,131", "in_tok": "526.3M", "out_tok": "727k", "avg_cost": "$143.78", "avg_time": "19.6h"}, {"rank": 5, "model": "Kimi K3", "score": 0.4045, "std": 0.232, "steps": "1,062", "in_tok": "454.5M", "out_tok": "1.0M", "avg_cost": "$161.34", "avg_time": "19.9h"}, {"rank": 6, "model": "Qwen3.8-Max", "score": 0.2859, "std": 0.142, "steps": "975", "in_tok": "347.6M", "out_tok": "1.3M", "avg_cost": "$95.98", "avg_time": "18.4h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.2199, "std": 0.119, "steps": "2,120", "in_tok": "954.7M", "out_tok": "1.4M", "avg_cost": "$15.48", "avg_time": "15.9h"}, {"rank": 8, "model": "Grok 4.6", "score": 0.2133, "std": 0.245, "steps": "1,889", "in_tok": "467.9M", "out_tok": "2.3M", "avg_cost": "$467.46", "avg_time": "19.4h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.1948, "std": 0.242, "steps": "127", "in_tok": "15.4M", "out_tok": "112k", "avg_cost": "$3.19", "avg_time": "1.2h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "175", "in_tok": "11.6M", "out_tok": "27k", "avg_cost": "$11.15", "avg_time": "0.8h"}], "best_score": 0.9136, "mean_score": 0.3799, "n_models": 10}, {"slug": "ffmpeg-swscale-rewrite", "number": 5, "title": "FFmpeg libswscale Optimization", "url": "https://www.frontierswe.com/tasks/ffmpeg-swscale-rewrite", "categories": ["Performance Optimisation"], "summary": "Re-implement FFmpeg's libswscale in Zig, matching its output quality while beating its C scalar code with portable SIMD.", "scoring": "Perf speedup", "toolchains": ["Zig", "Python", "x86-64 asm", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "8 CPUs · 32 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Zig, FFmpeg n7.1 sources and compiled scalar baseline, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "libswscale is FFmpeg's image scaling and pixel-format conversion library: the code that turns YUV into RGB, packs and unpacks planes, and resamples frames, in some of the hottest loops in video processing. Re-implementing it means reproducing the numerics of decades-old scalar C (coefficient tables, rounding, chroma subsampling) closely enough to pass per-plane PSNR bars, across dozens of format pairs and three scaling algorithms, and then making it substantially faster.\nThe rewrite must be genuinely new work: it is written in Zig behind a small custom C ABI, FFmpeg's own code cannot be wrapped, linked, loaded, or embedded, and inline assembly is banned; the speed has to come from portable SIMD (Zig's @Vector ) and better algorithms, not from recalling or repackaging the original.", "task": "Build libswscale_candidate.so from the sources in /app/swscale-impl/ , exporting the three C-linkage functions in /app/swscale_api.h (swscale_create / swscale_process / swscale_destroy ), covering ten pixel formats and three scaling algorithms.\n- The correctness contract is 42 same-size conversions plus 16 scaling configurations (58 in all), checked plane by plane against the FFmpeg scalar reference: PSNR of at least 60 dB for same-size conversion and 40 dB for scaling. All 58 must pass; a library fast on 57 of 58 counts the same as an empty scaffold.\n- The contract fixes format pairs and algorithms, not frame sizes, so the implementation has to generalize rather than special-case the visible workloads.\n- Performance is single-threaded measured work, fully modeled into /app/perf-check ; the goal is beating the reference's per-workload numbers.\n- Zig 0.14.0 only, no inline assembly, no dlopen /dlsym , and no FFmpeg code in any form; target x86-64-v3 (AVX2, FMA, BMI2) and no higher, since the measurement simulator has no AVX-512.\n- Changes are confined to /app/swscale-impl/ .", "task_bullets": ["The correctness contract is 42 same-size conversions plus 16 scaling configurations (58 in all), checked plane by plane against the FFmpeg scalar reference: PSNR of at least 60 dB for same-size conversion and 40 dB for scaling. All 58 must pass; a library fast on 57 of 58 counts the same as an empty scaffold.", "The contract fixes format pairs and algorithms, not frame sizes, so the implementation has to generalize rather than special-case the visible workloads.", "Performance is single-threaded measured work, fully modeled into /app/perf-check ; the goal is beating the reference's per-workload numbers.", "Zig 0.14.0 only, no inline assembly, no dlopen /dlsym , and no FFmpeg code in any form; target x86-64-v3 (AVX2, FMA, BMI2) and no higher, since the measurement simulator has no AVX-512.", "Changes are confined to /app/swscale-impl/ ."], "agent_gets": "A Zig starter scaffold at /app/scaffold/zig/ that already exports the three functions, the ABI header, and /app/libswscale_public_baseline.so : FFmpeg's scalar swscale (built with --disable-asm ) behind the same ABI, the reference the output is compared against and whose work is being beaten. /app/perf-check rebuilds, compares output against the unmodified reference, and measures (--contract for the full 58-conversion sweep, --quick to skip measurement, or a named workload); reference numbers are baked in /app/baseline-work.json . The /app/driver C program that loads and exercises a library is usable under gdb /perf , the measurement stack is readable at /app/performance/ , and FFmpeg's libswscale /libavutil C sources are at /reference/ffmpeg-src/ for study. The driver synthesizes deterministic source pixels, so there are no image files and every run sees the same input. The machine is offline.", "verification": "A clean-room verifier, separate from the agent's environment, first strips every FFmpeg source and archive from the container, rebuilds the submission from source as a non-root user, and runs the full 58-conversion contract plus ten hidden held-out benchmark workloads: the same conversion families and size classes as the public set, at dimensions and format pairs the agent has not seen. Speed is not wall-clock: each workload's work is measured by callgrind-based per-opcode instruction pricing (a deterministic cost model derived from uops.info throughput data), so the score is independent of host speed.\n- Reward is gated on build success, provenance, and full correctness, then scored as 2**u - 1 where u scales the geometric-mean work reduction across held-out workloads, reaching full credit at 20x; matching the baseline or regressing scores zero.\n- Provenance checks reject FFmpeg delegation in any form: linked library names are restricted to a C-runtime allowlist, RPATH/RUNPATH and dynamic-loading imports are rejected, and the built objects are scanned for FFmpeg symbols, build strings, inline assembly, and large binary blobs.\n- The driver installs a seccomp filter that blocks fork/exec before the candidate loads, sources are perturbed between runs to defeat cached outputs, and iteration-identity and work-linearity gates catch skipped work.\n- A crash or hang on a measured workload counts as a correctness failure; if fewer than 75% of workloads could be measured the trial is marked invalid rather than scored.", "verification_bullets": ["Reward is gated on build success, provenance, and full correctness, then scored as 2**u - 1 where u scales the geometric-mean work reduction across held-out workloads, reaching full credit at 20x; matching the baseline or regressing scores zero.", "Provenance checks reject FFmpeg delegation in any form: linked library names are restricted to a C-runtime allowlist, RPATH/RUNPATH and dynamic-loading imports are rejected, and the built objects are scanned for FFmpeg symbols, build strings, inline assembly, and large binary blobs.", "The driver installs a seccomp filter that blocks fork/exec before the candidate loads, sources are perturbed between runs to defeat cached outputs, and iteration-identity and work-linearity gates catch skipped work.", "A crash or hang on a measured workload counts as a correctness failure; if fewer than 75% of workloads could be measured the trial is marked invalid rather than scored."], "safeguards": ["Reward is gated on build success, provenance, and full correctness, then scored as 2**u - 1 where u scales the geometric-mean work reduction across held-out workloads, reaching full credit at 20x; matching the baseline or regressing scores zero.", "Provenance checks reject FFmpeg delegation in any form: linked library names are restricted to a C-runtime allowlist, RPATH/RUNPATH and dynamic-loading imports are rejected, and the built objects are scanned for FFmpeg symbols, build strings, inline assembly, and large binary blobs.", "The driver installs a seccomp filter that blocks fork/exec before the candidate loads, sources are perturbed between runs to defeat cached outputs, and iteration-identity and work-linearity gates catch skipped work."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Zig, FFmpeg n7.1 sources and compiled scalar baseline, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 32 GB RAM", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of FFmpeg libswscale Re-implementation from FrontierSWE V1.\n- The language was narrowed from \"Zig or Rust\" to Zig 0.14.0 only, making @Vector the portable-SIMD surface under test, with an explicit x86-64-v3 ceiling.\n- The metric moved from wall-clock benchmark timing to deterministic per-opcode instruction pricing (callgrind-based, uops.info-derived), and the reward is now normalized to [0,1] behind the correctness gate; V1 reported the raw geometric-mean speedup, where 1.0 meant parity with the baseline.\n- The correctness contract became explicit and agent-checkable: 58 enumerated conversions via perf-check --contract , with held-out sizes still hidden; V1 checked hidden workloads against the same PSNR bars without a visible contract list.\n- Verification moved into a separate clean-room container with a pinned image. V1's verifier ran inside the agent's environment, deleting the reference material in place and importing helper code from /app ; the V2 scorer never imports agent code.\n- Anti-delegation was hardened: a seccomp filter blocks fork/exec before the candidate loads, linked libraries are allowlisted, RPATH is banned, binaries are scanned for FFmpeg symbols and blobs, and source perturbation plus work-linearity gates defeat cached or skipped runs.\n- The agent budget grew from 8 to 20 hours, and V1's full ASM-enabled FFmpeg binary (provided for generating golden outputs) is gone; only the C sources and the scalar baseline library remain.", "references": [{"title": "FFmpeg (LGPL-2.1-or-later (with GPL-2.0-or-later components as configured)): The libswscale /libavutil /compat source subtrees at tag n7.1, cloned at image build (nothing vendored) and baked read-only at /reference/ffmpeg-src/ , plus a --disable-asm scalar build wrapped by a task-authored shim into the baseline shared library. No FFmpeg code reaches the driver, measurement, scaffolds, or verifier.", "url": "https://github.com/FFmpeg/FFmpeg"}, {"title": "uops.info instruction data (Derived aggregate; format via andreas-abel/XED-to-XML (Apache-2.0)): The per-opcode cost table used by the measurement stack is a derived aggregate of reciprocal-throughput measurements from uops.info (Abel & Reineke, ASPLOS 2019), generated once offline; it is not a copy of the dataset.", "url": "https://uops.info/"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.4013, "std": 0.328, "steps": "857", "in_tok": "366.7M", "out_tok": "1.3M", "avg_cost": "$214.08", "avg_time": "9.9h"}, {"rank": 2, "model": "GLM-5.3", "score": 0.1309, "std": 0.035, "steps": "1,449", "in_tok": "618.7M", "out_tok": "1.7M", "avg_cost": "$173.60", "avg_time": "16.0h"}, {"rank": 3, "model": "Kimi K3", "score": 0.079, "std": 0.052, "steps": "1,044", "in_tok": "437.6M", "out_tok": "1.2M", "avg_cost": "$155.23", "avg_time": "19.0h"}, {"rank": 4, "model": "DeepSeek V4 Flash Exp", "score": 0.0478, "std": 0.028, "steps": "1,399", "in_tok": "618.9M", "out_tok": "1.2M", "avg_cost": "$10.62", "avg_time": "7.3h"}, {"rank": 5, "model": "Qwen3.8-Max", "score": 0.0122, "std": 0.016, "steps": "494", "in_tok": "165.8M", "out_tok": "2.3M", "avg_cost": "$56.27", "avg_time": "18.8h"}, {"rank": 6, "model": "Grok 4.6", "score": 0.0072, "std": 0.016, "steps": "1,482", "in_tok": "417.5M", "out_tok": "1.6M", "avg_cost": "$424.47", "avg_time": "11.9h"}, {"rank": 7, "model": "Gemini 3.7 Flash", "score": 0.0031, "std": 0.005, "steps": "580", "in_tok": "192.7M", "out_tok": "881k", "avg_cost": "$25.55", "avg_time": "3.5h"}, {"rank": 8, "model": "GPT-5.6", "score": 0.0, "std": 0.0, "steps": "374", "in_tok": "98.4M", "out_tok": "204k", "avg_cost": "$92.64", "avg_time": "2.6h"}, {"rank": 9, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "215", "in_tok": "19.4M", "out_tok": "41k", "avg_cost": "$18.61", "avg_time": "0.6h"}, {"rank": 10, "model": "Muse Spark 1.2", "score": 0.0, "std": 0.0, "steps": "312", "in_tok": "120.6M", "out_tok": "519k", "avg_cost": "$21.59", "avg_time": "1.7h"}], "best_score": 0.4013, "mean_score": 0.0681, "n_models": 10}, {"slug": "remotion-video-generation", "number": 6, "title": "Fitness-Recap Video in Remotion", "url": "https://www.frontierswe.com/tasks/remotion-video-generation", "categories": ["Visual Reasoning", "Implementation"], "summary": "Reverse-engineer a parametric motion-design system: reproduce an animated fitness-recap video (frames and soundtrack) exactly, for any input.", "scoring": "Pixel match", "toolchains": ["Python", "Remotion"], "compute": {"label": "CPU", "kind": "cpu", "compute": "8 CPUs · 16 GB RAM", "gpu": null, "base_image": "node:22-bookworm", "tools": "Node.js, Python 3", "time_limit": "20h"}, "background": "Parametric video generators (the engines behind year-in-review recaps) turn structured data into minutes of choreographed animation: scene sequencing, spring-based motion curves, particle effects, data-driven layouts, and a soundtrack whose stingers land on visual beats. Reproducing one exactly means recovering not just the look of every scene but the full mapping from input data to timing, layout, and audio mix.\nThe reference here is a re-themed derivative of a real production recap video, rebuilt around a wellness \"year in motion\" dataset. It is observable only as a black box: the agent can render it on any input it likes and inspect the resulting frames and audio, but never its source. Every visual is a deterministic function of the input data, so the behavior is fully inferable from probing.", "task": "The agent must build a parametric video generator that turns a JSON file (one person's year of fitness data) into a fully animated 1080×1080, 30 fps recap video with a soundtrack, matching the reference generator's output exactly in both video and audio.\n- Probe the reference with reference-generator <input.json> <out_dir> , which writes the video as a PNG frame sequence plus an audio.wav (with --mp4 for a watchable preview and --seconds for a quick trim), on any input the agent constructs.\n- Build a Remotion project at /app/generator (sources in src/ ) so that ./render.sh <input.json> <out_dir> reproduces the reference's frames and audio from the agent's source alone.\n- Handle the full input space described by samples/schema.json : video length and scene content depend on the data, so the duration logic is part of the task.\n- Compare against the reference with /app/samples/check.sh <sample> [seconds] , which renders both sides and reports per-frame and audio differences.", "task_bullets": ["Probe the reference with reference-generator <input.json> <out_dir> , which writes the video as a PNG frame sequence plus an audio.wav (with --mp4 for a watchable preview and --seconds for a quick trim), on any input the agent constructs.", "Build a Remotion project at /app/generator (sources in src/ ) so that ./render.sh <input.json> <out_dir> reproduces the reference's frames and audio from the agent's source alone.", "Handle the full input space described by samples/schema.json : video length and scene content depend on the data, so the duration logic is part of the task.", "Compare against the reference with /app/samples/check.sh <sample> [seconds] , which renders both sides and reports per-frame and audio differences."], "agent_gets": "A scaffolded Remotion project at /app/generator with render.sh wired up and the complete static asset pack already in public/ (illustration art, mascot planets, digit sprites, fonts, and the CC0 soundtrack and SFX), plus three example inputs in /app/samples/ and samples/schema.json documenting the input format and its conventions. The runnable-but-not-readable reference generator is on the PATH. The image is built on Node 22 with Remotion 4.0.501 and its renderer preinstalled from a pinned offline dependency store, along with the headless browser runtime it renders through. The machine is offline.", "verification": "A clean-room verifier in a separate container removes the reference probe tool, then renders six hidden JSON inputs with the agent's ./render.sh as the non-root agent user (each render capped at 1200 s), kills every agent process, and renders the reference bundle on the same inputs into root-only directories. Per input, the score is 0.85 × visual + 0.15 × audio , and the reward is the mean over the six inputs.\n- Visual closeness is an exponential of per-pixel MSE computed on every reference frame: a pixel-exact render scores 1.0, a near-miss rebuild lands orders of magnitude lower, and a blank render scores ~0. The mean is divided by whichever side rendered more frames, so missing frames count zero and extra frames dilute the score.\n- Audio closeness is an exponential of the reference-normalized waveform MSE times a length ratio: identical audio scores 1, while silence, missing sound effects, or a wrong mix score ~0.\n- Anti-cheat: frames and audio are rejected if they are symlinks or resolve outside the agent's own output directory, the scorer only reads files and never executes agent code, and the reward directory is locked before any agent code runs. A failed reference render marks the trial invalid as an infrastructure fault rather than zeroing the agent.", "verification_bullets": ["Visual closeness is an exponential of per-pixel MSE computed on every reference frame: a pixel-exact render scores 1.0, a near-miss rebuild lands orders of magnitude lower, and a blank render scores ~0. The mean is divided by whichever side rendered more frames, so missing frames count zero and extra frames dilute the score.", "Audio closeness is an exponential of the reference-normalized waveform MSE times a length ratio: identical audio scores 1, while silence, missing sound effects, or a wrong mix score ~0.", "Anti-cheat: frames and audio are rejected if they are symlinks or resolve outside the agent's own output directory, the scorer only reads files and never executes agent code, and the reward directory is locked before any agent code runs. A failed reference render marks the trial invalid as an infrastructure fault rather than zeroing the agent."], "safeguards": ["Visual closeness is an exponential of per-pixel MSE computed on every reference frame: a pixel-exact render scores 1.0, a near-miss rebuild lands orders of magnitude lower, and a blank render scores ~0. The mean is divided by whichever side rendered more frames, so missing frames count zero and extra frames dilute the score.", "Anti-cheat: frames and audio are rejected if they are symlinks or resolve outside the agent's own output directory, the scorer only reads files and never executes agent code, and the reward directory is locked before any agent code runs. A failed reference render marks the trial invalid as an infrastructure fault rather than zeroing the agent."], "environment": {"Base image": "node:22-bookworm", "Tools available": "Node.js, Python 3", "Compute": "8 CPUs · 16 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "GitHub Unwrapped (remotion-dev/github-unwrapped) (MIT): The reference generator is a re-themed derivative of this year-in-review video, along with its MIT-licensed illustration assets (cockpit, rockets, planets, UFO, star sprites, gradients, backgrounds). All GitHub semantics were replaced with a wellness theme, all GitHub/Octocat trademarked art was removed and replaced with original AI-generated mascots, and input-hashed randomness was replaced with deterministic formulas so every visual is inferable from the input data.", "url": "https://github.com/remotion-dev/github-unwrapped"}, {"title": "Remotion (Remotion License (source-available)): Remotion 4.0.501 and the @remotion/* packages are consumed as unmodified npm dependencies baked into the image; nothing in the task forks or redistributes modified Remotion source.", "url": "https://github.com/remotion-dev/remotion"}, {"title": "DSEG14 Classic (by keshikan) (SIL OFL 1.1): Replaces the upstream \"Seven Segment\" font (free for personal use only); the pre-rendered digit sprites were regenerated from DSEG14 Classic so no Krafti Lab material remains.", "url": "https://github.com/keshikan/DSEG"}, {"title": "Mona Sans (GitHub) (SIL OFL 1.1): Typeface shipped in the task's asset pack.", "url": "https://github.com/github/mona-sans"}, {"title": "Freesound audio (various authors) (CC0): The soundtrack and all sound effects (theme by Sergmusic; whooshes by qubodup; impacts and riser by AudioPapkin and Rizzard; tick by malle99), replacing the upstream's non-redistributable SmartSound-licensed audio entirely.", "url": "https://freesound.org"}, {"title": "Lucide (ISC): The \"dumbbell\" workout icon, shipped as an SVG asset and vendored as SVG paths in the reference source.", "url": "https://lucide.dev"}], "results": [{"rank": 1, "model": "GPT-5.6", "score": 0.1853, "std": 0.029, "steps": "1,154", "in_tok": "465.8M", "out_tok": "490k", "avg_cost": "$475.97", "avg_time": "11.3h"}, {"rank": 2, "model": "Claude Fable 5.1", "score": 0.1685, "std": 0.057, "steps": "1,128", "in_tok": "497.1M", "out_tok": "776k", "avg_cost": "$266.22", "avg_time": "10.4h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.1408, "std": 0.046, "steps": "786", "in_tok": "339.3M", "out_tok": "621k", "avg_cost": "$102.31", "avg_time": "17.6h"}, {"rank": 4, "model": "Kimi K3", "score": 0.128, "std": 0.063, "steps": "1,468", "in_tok": "638.6M", "out_tok": "1.1M", "avg_cost": "$215.12", "avg_time": "19.9h"}, {"rank": 5, "model": "DeepSeek V4 Flash Exp", "score": 0.0657, "std": 0.058, "steps": "1,242", "in_tok": "497.0M", "out_tok": "722k", "avg_cost": "$8.08", "avg_time": "17.0h"}, {"rank": 6, "model": "Qwen3.8-Max", "score": 0.0479, "std": 0.063, "steps": "796", "in_tok": "280.1M", "out_tok": "1.1M", "avg_cost": "$100.48", "avg_time": "13.3h"}, {"rank": 7, "model": "Grok 4.6", "score": 0.0264, "std": 0.005, "steps": "2,615", "in_tok": "715.1M", "out_tok": "1.8M", "avg_cost": "$972.20", "avg_time": "19.9h"}, {"rank": 8, "model": "Gemini 3.7 Flash", "score": 0.009, "std": 0.003, "steps": "282", "in_tok": "59.1M", "out_tok": "144k", "avg_cost": "$7.94", "avg_time": "2.6h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.004, "std": 0.003, "steps": "289", "in_tok": "77.8M", "out_tok": "451k", "avg_cost": "$33.88", "avg_time": "5.0h"}, {"rank": 10, "model": "Inkling", "score": 0.0001, "std": 0.0, "steps": "170", "in_tok": "12.0M", "out_tok": "37k", "avg_cost": "$11.51", "avg_time": "0.9h"}], "best_score": 0.1853, "mean_score": 0.0776, "n_models": 10}, {"slug": "opengl-scene-engine", "number": 7, "title": "Flight-Sim Renderer in OpenGL", "url": "https://www.frontierswe.com/tasks/opengl-scene-engine", "categories": ["Visual Reasoning", "Implementation"], "summary": "Rebuild a deterministic flight-sim renderer from scratch (physics, cameras, lighting, and effects) until its frames match the reference pixel for pixel.", "scoring": "Pixel match", "toolchains": ["Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 8 GB RAM", "gpu": null, "base_image": "debian:bookworm-slim", "tools": "FFmpeg, OSMesa/llvmpipe, CMake, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "A real-time 3D engine is a tower of interlocking systems: mesh and texture loading, camera projection, physically-based shading, shadows, reflections, particles, and, in a flight simulator, a physics integrator whose state feeds every frame. Reproducing one pixel for pixel means getting every layer exactly right at once: a slightly wrong lift coefficient or gamma curve diverges within frames, and the error compounds over a 40-second sortie.\nThe task fixes one world (a hillside airfield) and defines the target purely by behavior: a reference renderer turns keystroke input logs into videos, and the agent must build an engine whose output is indistinguishable from it. The reference is runnable on any log the agent writes and dumps its per-tick state, so the physics coefficients and shading math are identifiable by experiment, not given.", "task": "The agent must build its own renderer plus flight simulator from scratch in /app/src so that make -C /app produces /app/render , and its frames match the reference's output pixel for pixel on any input log over the declared event set.\n- The world is fixed and fully declared in /app/world.json , with all content in /app/assets/ ; a scene in /app/scenes/ is a plain-text input log where keys are pressed and released at 240 Hz ticks and every 8th tick is a video frame.\n- A deterministic flight model integrates throttle, ailerons, elevator, rudder, brakes, and flaps from a named spawn pose; the simulation must be reproduced along with the rendering.\n- Instant events switch cameras, drop payloads, fade daylight, trail smoke, and toggle a ray-traced pipeline; all of them are in scope.\n- The reference is runnable as reference-renderer world.json <scene.txt> <out_dir> , and /app/run_tests.py [name ...] renders both sides scene by scene and reports the difference.", "task_bullets": ["The world is fixed and fully declared in /app/world.json , with all content in /app/assets/ ; a scene in /app/scenes/ is a plain-text input log where keys are pressed and released at 240 Hz ticks and every 8th tick is a video frame.", "A deterministic flight model integrates throttle, ailerons, elevator, rudder, brakes, and flaps from a named spawn pose; the simulation must be reproduced along with the rendering.", "Instant events switch cameras, drop payloads, fade daylight, trail smoke, and toggle a ray-traced pipeline; all of them are in scope.", "The reference is runnable as reference-renderer world.json <scene.txt> <out_dir> , and /app/run_tests.py [name ...] renders both sides scene by scene and reports the difference."], "agent_gets": "A workspace at /app with the fixed world blueprint (world.json ), the full asset pack (meshes, PBR texture sets, an HDR environment), sample scene logs, a Makefile , the run_tests.py comparison harness with an MSE tool, and /app/README.md , the contract documenting the file formats, CLI, input-log grammar, and the event/camera/world vocabulary, but not the physics coefficients or shading math. The runnable reference renderer dumps its per-tick state for any log the agent writes. The image preinstalls a C++ toolchain (g++ , make , cmake ), the software OpenGL stack (OSMesa/llvmpipe), ffmpeg , and NumPy. The machine is offline.", "verification": "A clean-room verifier in a separate container rebuilds the submission with make -C /app as the non-root agent user, restores a pristine copy of the assets and world blueprint from a root-only mirror, and renders a hidden scored corpus of 12 seeded scenes spanning 10 weighted families (missions, aerobatics, landings, night takeoffs, payload drops, a ray-traced pond pass, camera tours, stills, and effects). Each scene's score is exp(-0.004 × MSE) over the per-pixel error across every frame; the reward averages within each family, then weights across families.\n- The reference renders live alongside the agent's binary in the same container, into root-only directories; no expected values are hardcoded anywhere.\n- Anti-cheat: a frame is rejected if it is a symlink, resolves outside the agent's own output directory, or has the wrong size; agent processes are killed before any reference frame exists; and submissions are capped at 16 MB of source-like bytes, so baked frame dumps cannot masquerade as code.\n- A failed build zeroes the result, and a missing or short frame zeroes that scene. A failed reference render is treated as a harness defect and skipped, never blamed on the agent.", "verification_bullets": ["The reference renders live alongside the agent's binary in the same container, into root-only directories; no expected values are hardcoded anywhere.", "Anti-cheat: a frame is rejected if it is a symlink, resolves outside the agent's own output directory, or has the wrong size; agent processes are killed before any reference frame exists; and submissions are capped at 16 MB of source-like bytes, so baked frame dumps cannot masquerade as code.", "A failed build zeroes the result, and a missing or short frame zeroes that scene. A failed reference render is treated as a harness defect and skipped, never blamed on the agent."], "safeguards": ["Anti-cheat: a frame is rejected if it is a symlink, resolves outside the agent's own output directory, or has the wrong size; agent processes are killed before any reference frame exists; and submissions are capped at 16 MB of source-like bytes, so baked frame dumps cannot masquerade as code.", "A failed build zeroes the result, and a missing or short frame zeroes that scene. A failed reference render is treated as a harness defect and skipped, never blamed on the agent."], "environment": {"Base image": "debian:bookworm-slim", "Tools available": "FFmpeg, OSMesa/llvmpipe, CMake, C/C++ toolchain, Python 3", "Compute": "4 CPUs · 8 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "Light Aircraft (by weirdybeardyman) (CC0 1.0): The player aircraft mesh and livery textures. Exported to OBJ, rescaled and reoriented, control surfaces and prop split into hinged submeshes, converted to the task's PMESH format, and liveries recolored and repacked per part.", "url": "https://opengameart.org/content/light-aircraft"}, {"title": "Cesium Milk Truck (CC BY 4.0 (with trademark limitations)): Re-themed as the airfield fuel truck. Rescaled and converted to PMESH; the Cesium wordmark band and every roundel in the vendored texture are repainted flat, so no Cesium trademark ships with the task.", "url": "https://github.com/KhronosGroup/glTF-Sample-Assets/tree/main/Models/CesiumMilkTruck"}, {"title": "Kenney City Kit (Industrial) (CC0 1.0): Industrial buildings, tank, and chimney meshes with their colormap texture, converted to PMESH and scaled per instance in the world blueprint.", "url": "https://kenney.nl/assets/city-kit-industrial"}, {"title": "ambientCG PBR texture sets (CC0 1.0): Seven 1K PBR material sets (grass, dirt, asphalt, concrete, steel, wood, metal), renamed to per-set albedo/normal/rough/ao/metal maps in the NormalGL convention.", "url": "https://ambientcg.com"}, {"title": "Kloofendal 48d Partly Cloudy (Pure Sky), Poly Haven (CC0 1.0): The HDR sky environment by Greg Zaal (capture) and Jarod Guest (sky-only edit); the cubemap and IBL products are derived deterministically at image build.", "url": "https://polyhaven.com/a/kloofendal_48d_partly_cloudy_puresky"}, {"title": "stb_image.h (Public domain / MIT (dual)): Image decoding (PNG/JPG/HDR) shipped to both the reference engine and the agent workspace.", "url": "https://github.com/nothings/stb"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.3423, "std": 0.097, "steps": "601", "in_tok": "263.8M", "out_tok": "554k", "avg_cost": "$144.63", "avg_time": "11.1h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.1419, "std": 0.046, "steps": "844", "in_tok": "318.0M", "out_tok": "342k", "avg_cost": "$318.24", "avg_time": "10.7h"}, {"rank": 3, "model": "Grok 4.6", "score": 0.0319, "std": 0.014, "steps": "1,232", "in_tok": "332.0M", "out_tok": "1.8M", "avg_cost": "$375.13", "avg_time": "19.9h"}, {"rank": 4, "model": "Kimi K3", "score": 0.0129, "std": 0.01, "steps": "1,046", "in_tok": "460.4M", "out_tok": "1.3M", "avg_cost": "$165.73", "avg_time": "19.9h"}, {"rank": 5, "model": "GLM-5.3", "score": 0.0053, "std": 0.007, "steps": "763", "in_tok": "330.5M", "out_tok": "671k", "avg_cost": "$99.46", "avg_time": "19.4h"}, {"rank": 6, "model": "Gemini 3.7 Flash", "score": 0.0052, "std": 0.004, "steps": "379", "in_tok": "99.7M", "out_tok": "270k", "avg_cost": "$16.38", "avg_time": "5.2h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0038, "std": 0.003, "steps": "1,424", "in_tok": "627.2M", "out_tok": "1.0M", "avg_cost": "$10.35", "avg_time": "18.4h"}, {"rank": 8, "model": "Qwen3.8-Max", "score": 0.0009, "std": 0.001, "steps": "1,258", "in_tok": "500.9M", "out_tok": "1.6M", "avg_cost": "$169.09", "avg_time": "19.8h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0, "std": 0.0, "steps": "146", "in_tok": "24.9M", "out_tok": "183k", "avg_cost": "$5.30", "avg_time": "2.1h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "146", "in_tok": "6.8M", "out_tok": "29k", "avg_cost": "$6.60", "avg_time": "1.1h"}], "best_score": 0.3423, "mean_score": 0.0544, "n_models": 10}, {"slug": "frogsgame-rl", "number": 8, "title": "FrogsGame Post-Training", "url": "https://www.frontierswe.com/tasks/frogsgame-rl", "categories": ["AI Research"], "summary": "Post-train Qwen3-8B on a local A100 to solve unseen constraint-puzzle boards through iterative tool calls.", "scoring": "Hidden tests", "toolchains": ["CUDA", "Python", "C/C++"], "compute": {"label": "1× A100-80GB", "kind": "gpu", "compute": "8 CPUs · 64 GB RAM", "gpu": "1× A100-80GB", "base_image": "nvidia/cuda:12.5.1-devel-ubuntu24.04", "tools": "CUDA, PyTorch, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "The Frog Placement Game is a Queens-style constraint puzzle: an N×N grid of N colors, on which exactly one frog must be placed in every row, every column, and every color, with no two frogs adjacent — including diagonally. Boards range from N=6 (easy) to N=13 (expert), and the game is played through a tool-call interface: the board is not included in the prompt, so the model must call get_state to discover the layout, then place, check, and revise frogs one call at a time within a 200-call budget.\nPost-training a small model to do this is hard because nothing is given: no training boards are provided, so the agent must generate its own solvable boards across every difficulty tier, build a multi-turn tool-use dataset in exactly the format the verifier replays, and execute the entire fine-tuning pipeline on a single local GPU within the time budget — while producing a policy that generalizes to board sizes it may have undertrained on.", "task": "Post-train Qwen/Qwen3-8B to solve unseen Frog Placement Game boards through iterative tool use, generalizing across board sizes N=6 through N=13. The agent implements the training pipeline in /app/train.py , executes it on the local GPU, and leaves a trained adapter in place.\n- The deliverable is an unmerged PEFT LoRA adapter at /app/checkpoint/adapter/ (an adapter_config.json plus adapter_model.safetensors ) that targets Qwen/Qwen3-8B and has rank at most 256.\n- /app/prepare.py — the immutable game engine, tool schemas, and shared prompt builder — must not be modified; training must use its build_system_prompt() and USER_MESSAGE so the format matches the verifier exactly.\n- No training boards are provided: the agent must generate its own boards and verify each one is solvable before using it.\n- The trained model must operate without solver or solution access at inference time, and the LoRA must not be merged into the base model.\n- The pipeline must actually be executed — the adapter files must exist in the workspace before the sandbox times out.", "task_bullets": ["The deliverable is an unmerged PEFT LoRA adapter at /app/checkpoint/adapter/ (an adapter_config.json plus adapter_model.safetensors ) that targets Qwen/Qwen3-8B and has rank at most 256.", "/app/prepare.py — the immutable game engine, tool schemas, and shared prompt builder — must not be modified; training must use its build_system_prompt() and USER_MESSAGE so the format matches the verifier exactly.", "No training boards are provided: the agent must generate its own boards and verify each one is solvable before using it.", "The trained model must operate without solver or solution access at inference time, and the LoRA must not be merged into the base model.", "The pipeline must actually be executed — the adapter files must exist in the workspace before the sandbox times out."], "agent_gets": "A self-contained workspace at /app : prepare.py (game engine, tool interface, and eval harness), train.py (the editable training entry point), infer.py (a vLLM smoke test that verifies an adapter loads and generates), and a local copy of the Qwen3-8B tokenizer. The Qwen3-8B base weights are baked into the image's Hugging Face cache, and the pinned stack — Axolotl 0.18.0 for LoRA fine-tuning, vLLM 0.23.0 for local inference, PyTorch with CUDA, and PEFT — is preinstalled. Training and inference run on a dedicated A100-80GB GPU. The machine is offline; a sandbox-timer CLI reports the remaining time budget.", "verification": "A clean-room verifier, running as root in a separate container, generates its own 500 hidden test boards (125 per difficulty tier), loads a pristine Qwen3-8B plus the submitted LoRA adapter in local vLLM, and replays every board through the same prompt and tool interface defined in prepare.py , with the same 200-tool-call budget. The reward is the solve rate over the fixed 500-board denominator; boards left unevaluated when the scoring deadline expires count as unsolved.\n- The visible /app/prepare.py is checked against a SHA-256 hash pin of the trusted baked copy — any modification is a contract violation that zeroes the result.\n- The adapter is strictly audited before evaluation: LoRA-only tensors with matching A/B pairs, safetensors format (no adapter_model.bin ), no symlinks, no DoRA, modules_to_save , or per-module rank/alpha patterns, at most 1000 files and 8 GiB total.\n- A declared base model other than Qwen/Qwen3-8B or a LoRA rank outside 1–256 zeroes the result; a missing or invalid adapter cannot be evaluated and scores zero.\n- Pipeline-quality checks on train.py , the generated boards, and the checkpoint are reported as diagnostics but are excluded from the reward.", "verification_bullets": ["The visible /app/prepare.py is checked against a SHA-256 hash pin of the trusted baked copy — any modification is a contract violation that zeroes the result.", "The adapter is strictly audited before evaluation: LoRA-only tensors with matching A/B pairs, safetensors format (no adapter_model.bin ), no symlinks, no DoRA, modules_to_save , or per-module rank/alpha patterns, at most 1000 files and 8 GiB total.", "A declared base model other than Qwen/Qwen3-8B or a LoRA rank outside 1–256 zeroes the result; a missing or invalid adapter cannot be evaluated and scores zero.", "Pipeline-quality checks on train.py , the generated boards, and the checkpoint are reported as diagnostics but are excluded from the reward."], "safeguards": ["The visible /app/prepare.py is checked against a SHA-256 hash pin of the trusted baked copy — any modification is a contract violation that zeroes the result.", "A declared base model other than Qwen/Qwen3-8B or a LoRA rank outside 1–256 zeroes the result; a missing or invalid adapter cannot be evaluated and scores zero."], "environment": {"Base image": "nvidia/cuda:12.5.1-devel-ubuntu24.04", "Tools available": "CUDA, PyTorch, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 64 GB RAM", "GPU": "1× A100-80GB", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of FrogsGame Post-Training from FrontierSWE V1.\n- Training moved from the remote Tinker API (internet access and a TINKER_API_KEY , no local GPU) to a fully offline local pipeline: Axolotl on a dedicated A100-80GB with the Qwen3-8B weights baked into the image.\n- The deliverable changed from a Tinker checkpoint path plus a self-reported results.json to a strictly validated unmerged PEFT LoRA adapter at /app/checkpoint/adapter/ (safetensors only, rank at most 256, LoRA-only tensors, no DoRA or modules_to_save , at most 8 GiB).\n- Scoring is now clean-room and hardware-fixed: a separate no-network verifier container with a pinned A100-80GB loads clean Qwen3-8B plus the adapter in local vLLM, instead of evaluating through the Tinker API.\n- The reward is an absolute solve rate over a fixed 500-board denominator (unevaluated boards count as unsolved), replacing V1's raw solved-count reward.\n- Anti-cheat hardened: prepare.py is SHA-256 hash-pinned against a trusted baked copy, verifier assets are root-locked before scoring, and the adapter is audited for symlinks, non-LoRA tensors, and size caps.\n- The agent budget grew from 8 to 20 hours, the timer moved from /app/.timer/ files to the sandbox-timer CLI, and both agent and verifier images are pinned by digest under the new harbor schema.", "references": [{"title": "Frog Placement Game : The board generator, solver, and game engine are first-party, authored for this evaluation. The puzzle is a Queens-style constraint game (row/column/color/adjacency uniqueness), a family of folk logic puzzles; no third-party puzzle code or board data is vendored.", "url": null}, {"title": "Axolotl (Apache-2.0): Version 0.18.0, installed into the image; the framework the agent uses for LoRA fine-tuning on the local GPU.", "url": "https://github.com/axolotl-ai-cloud/axolotl"}, {"title": "vLLM (Apache-2.0): Version 0.23.0; serves the verifier's base model plus the submitted LoRA during scoring, and backs the agent-visible infer.py smoke test.", "url": "https://github.com/vllm-project/vllm"}, {"title": "Qwen3-8B (Apache-2.0): Weights and tokenizer baked into the image's Hugging Face cache at build time so training and inference run fully offline.", "url": "https://huggingface.co/Qwen/Qwen3-8B"}, {"title": "uv (Apache-2.0 OR MIT): Version 0.11.7; its binaries install the hash-pinned Python environment at image build.", "url": "https://github.com/astral-sh/uv"}, {"title": "NVIDIA CUDA base image (NVIDIA Deep Learning Container License / CUDA EULA): nvidia/cuda:12.5.1-devel-ubuntu24.04 supplies the CUDA toolkit for building GPU extensions; the CUDA image Dockerfiles are BSD-3-Clause and the OS layer is Ubuntu 24.04 under Canonical terms.", "url": "https://gitlab.com/nvidia/container-images/cuda"}, {"title": "Supporting Python libraries : PyTorch (BSD-3-Clause, CUDA wheels), Hugging Face transformers /tokenizers /huggingface_hub (Apache-2.0), peft (Apache-2.0), the openai client (Apache-2.0), and NumPy (BSD), installed from PyPI at image build.", "url": null}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.7344, "std": 0.226, "steps": "154", "in_tok": "21.9M", "out_tok": "141k", "avg_cost": "$14.36", "avg_time": "16.4h"}, {"rank": 2, "model": "GLM-5.3", "score": 0.4388, "std": 0.32, "steps": "257", "in_tok": "49.8M", "out_tok": "206k", "avg_cost": "$17.73", "avg_time": "19.4h"}, {"rank": 3, "model": "Kimi K3", "score": 0.2244, "std": 0.16, "steps": "179", "in_tok": "26.6M", "out_tok": "169k", "avg_cost": "$12.32", "avg_time": "19.7h"}, {"rank": 4, "model": "Grok 4.6", "score": 0.1428, "std": 0.098, "steps": "202", "in_tok": "33.5M", "out_tok": "318k", "avg_cost": "$34.65", "avg_time": "18.7h"}, {"rank": 5, "model": "Qwen3.8-Max", "score": 0.0936, "std": 0.075, "steps": "329", "in_tok": "49.2M", "out_tok": "357k", "avg_cost": "$15.24", "avg_time": "19.4h"}, {"rank": 6, "model": "GPT-5.6", "score": 0.046, "std": 0.029, "steps": "237", "in_tok": "42.3M", "out_tok": "110k", "avg_cost": "$42.99", "avg_time": "16.2h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0456, "std": 0.049, "steps": "342", "in_tok": "71.3M", "out_tok": "237k", "avg_cost": "$1.36", "avg_time": "19.8h"}, {"rank": 8, "model": "Gemini 3.7 Flash", "score": 0.0356, "std": 0.017, "steps": "260", "in_tok": "69.3M", "out_tok": "124k", "avg_cost": "$24.39", "avg_time": "12.4h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0136, "std": 0.012, "steps": "650", "in_tok": "302.1M", "out_tok": "357k", "avg_cost": "$53.70", "avg_time": "4.7h"}, {"rank": 10, "model": "Inkling", "score": 0.0016, "std": 0.004, "steps": "107", "in_tok": "3.8M", "out_tok": "21k", "avg_cost": "$3.73", "avg_time": "0.5h"}], "best_score": 0.7344, "mean_score": 0.1776, "n_models": 10}, {"slug": "git-to-zig", "number": 9, "title": "Git to Zig", "url": "https://www.frontierswe.com/tasks/git-to-zig", "categories": ["Implementation"], "summary": "Reimplement git in Zig from its own behavioural test suite: no C source, no real git to lean on.", "scoring": "Hidden tests", "toolchains": ["Zig", "Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 16 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Zig, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "git is one of the most widely deployed systems programs in existence: an object database, a content-addressed storage format, an index, refs, diff and merge machinery, and dozens of porcelain commands, all with two decades of precisely specified behaviour down to exact output strings and exit codes. Reimplementing it is a marathon of format compatibility (SHA-1 object encoding, pack files, the index format) layered under a very large CLI surface.\nIn this task the agent is handed tests, not source. git's C code is not in the workspace and the real git binary is locked root-only, so the behaviour must be reconstructed from git's own behavioural test scripts, which serve as the executable specification.", "task": "Implement git's version control functionality in Zig inside the pre-scaffolded project at /app/zig-git/ . The build scaffold is fixed: zig build compiles src/ into zig-out/bin/git , and the agent's sources may only be .zig files under src/ .\n- Match real git's behaviour (its subcommands, output formats, and exit codes) as pinned down by the behavioural test scripts in /app/tests/t/ .\n- Run the tests with /app/run_tests.py (the full set), /app/run_tests.py --sample (a quick representative slice), or by naming individual scripts.\n- The implementation must be Zig itself: delegating to an existing git binary or linking libgit2 zeroes the result.\n- All changes must stay inside .zig sources under /app/zig-git/src/ ; the verifier discards everything else.", "task_bullets": ["Match real git's behaviour (its subcommands, output formats, and exit codes) as pinned down by the behavioural test scripts in /app/tests/t/ .", "Run the tests with /app/run_tests.py (the full set), /app/run_tests.py --sample (a quick representative slice), or by naming individual scripts.", "The implementation must be Zig itself: delegating to an existing git binary or linking libgit2 zeroes the result.", "All changes must stay inside .zig sources under /app/zig-git/src/ ; the verifier discards everything else."], "agent_gets": "A ready-to-build Zig scaffold at /app/zig-git/ (build.zig , build.zig.zon , src/ ) and an agent-readable copy of git v2.47.0's own behavioural test suite at /app/tests/ : the test-lib.sh harness, prebuilt test-helper binaries, and a visible subset of the t/ scripts, with git's C source and git's own binaries removed. A provided /app/run_tests.py runner drives the scripts against the agent's binary. Zig 0.14.0 is preinstalled, the machine is offline, and a sandbox-timer CLI reports the remaining time budget. The real git is installed but root-only: unreadable and unrunnable by the agent.", "verification": "A clean-room verifier, separate from the agent's environment, first reconstructs the scored project from a pristine baked scaffold plus only the agent's .zig sources (symlinks and caches excluded), so tampering with the build configuration or stashing binaries in the workspace cannot survive. It then builds the candidate as a non-root user and runs 743 scored scripts from a trusted, root-only copy of git's test suite against the resulting binary.\n- Scoring is a weighted mean of per-script completion fractions: each script's TAP passes are capped at the reference implementation's count, a per-script floor subtracts everything the empty scaffold already passes (a no-op submission scores exactly 0), and a contribution cap keeps any single script from outweighing roughly 1/133 of the total.\n- TAP parsing is plan-bounded and per-script results are captured by the root harness into a nonce-named root-only directory, so forged \"ok\" lines cannot inflate the score.\n- Build failure, a missing or non-ELF binary, linking libgit2, or a failed workspace reset zeroes the result; per-script time caps scaled from baked reference durations and a global suite deadline make hangs score ~0 instead of erroring.", "verification_bullets": ["Scoring is a weighted mean of per-script completion fractions: each script's TAP passes are capped at the reference implementation's count, a per-script floor subtracts everything the empty scaffold already passes (a no-op submission scores exactly 0), and a contribution cap keeps any single script from outweighing roughly 1/133 of the total.", "TAP parsing is plan-bounded and per-script results are captured by the root harness into a nonce-named root-only directory, so forged \"ok\" lines cannot inflate the score.", "Build failure, a missing or non-ELF binary, linking libgit2, or a failed workspace reset zeroes the result; per-script time caps scaled from baked reference durations and a global suite deadline make hangs score ~0 instead of erroring."], "safeguards": ["Build failure, a missing or non-ELF binary, linking libgit2, or a failed workspace reset zeroes the result; per-script time caps scaled from baked reference durations and a global suite deadline make hangs score ~0 instead of erroring."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Zig, C/C++ toolchain, Python 3", "Compute": "4 CPUs · 16 GB RAM", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of Git to Zig from FrontierSWE V1.\n- The V1 workspace shipped git's full C source at /app/git-src/ and a working system git to test against; V2 removes both. The agent now gets only git's behavioural test suite as the specification, and the real git binary is locked root-only.\n- Verification moved from a bash script running inside the agent's own container to a clean-room verifier in a separate, hash-pinned image (environment_mode = \"separate\" ).\n- Anti-cheat was rebuilt from detection to construction: V1 relied on grep and strace heuristics to catch wrappers and embedded C; V2 reconstructs the scored project from a pristine baked scaffold plus only the agent's .zig sources, so build-config tampering, stashed binaries, and wrapper tricks are discarded before scoring.\n- Scoring gates hardened: V1 divided raw TAP passes by a fixed oracle total across the whole suite; V2 scores 743 curated scripts against a baked per-script reference, subtracts an empty-scaffold floor so a no-op scores exactly 0, caps credit at the reference ceiling, and bounds any single script's contribution, with plan-bounded TAP parsing against forged output.\n- The verifier now runs the agent's build and tests as a non-root user with the reward directory locked before any agent code executes; its timeout dropped from 24 hours to 2.5 hours via per-script time caps scaled from baked reference durations.\n- The brief was rewritten around the runnable test suite (with a --sample mode for quick iteration), and the ad-hoc /app/.timer file interface was replaced by the standard sandbox-timer CLI.", "references": [{"title": "git (GPL-2.0): git v2.47.0's own behavioural test suite (the t/ scripts and test-lib.sh harness) and its build outputs, cloned and built once at image build from the pinned tag. The full built tree is baked root-only as the verifier's trusted corpus; the agent-readable copy keeps the harness, prebuilt test-helpers, and a visible subset of the scripts, with git's C source and git's own binaries removed; the agent is handed tests, not source.", "url": "https://github.com/git/git"}, {"title": "sha1collisiondetection (MIT): git's bundled SHA-1 collision-detection library (pinned by git v2.47.0's submodule config), needed to compile the suite's test-helpers at image build. Its C source is stripped from the agent-readable /app/tests .", "url": "https://github.com/cr-marcstevens/sha1collisiondetection"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.5105, "std": 0.03, "steps": "1,997", "in_tok": "859.9M", "out_tok": "2.2M", "avg_cost": "$483.24", "avg_time": "19.1h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.2658, "std": 0.058, "steps": "2,244", "in_tok": "1037.2M", "out_tok": "806k", "avg_cost": "$1044.53", "avg_time": "19.3h"}, {"rank": 3, "model": "Kimi K3", "score": 0.1633, "std": 0.012, "steps": "1,609", "in_tok": "711.5M", "out_tok": "1.1M", "avg_cost": "$237.57", "avg_time": "19.8h"}, {"rank": 4, "model": "Grok 4.6", "score": 0.149, "std": 0.018, "steps": "2,237", "in_tok": "595.3M", "out_tok": "2.1M", "avg_cost": "$593.06", "avg_time": "17.9h"}, {"rank": 5, "model": "GLM-5.3", "score": 0.1415, "std": 0.014, "steps": "1,737", "in_tok": "777.5M", "out_tok": "807k", "avg_cost": "$214.26", "avg_time": "19.8h"}, {"rank": 6, "model": "Gemini 3.7 Flash", "score": 0.1195, "std": 0.07, "steps": "2,189", "in_tok": "998.3M", "out_tok": "1.6M", "avg_cost": "$97.22", "avg_time": "19.9h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.1156, "std": 0.012, "steps": "3,369", "in_tok": "1556.6M", "out_tok": "2.0M", "avg_cost": "$24.95", "avg_time": "19.1h"}, {"rank": 8, "model": "Qwen3.8-Max", "score": 0.1038, "std": 0.011, "steps": "1,752", "in_tok": "702.5M", "out_tok": "1.4M", "avg_cost": "$186.05", "avg_time": "19.6h"}, {"rank": 9, "model": "Inkling", "score": 0.0324, "std": 0.066, "steps": "191", "in_tok": "12.6M", "out_tok": "24k", "avg_cost": "$12.07", "avg_time": "1.1h"}, {"rank": 10, "model": "Muse Spark 1.2", "score": 0.0037, "std": 0.008, "steps": "344", "in_tok": "130.4M", "out_tok": "382k", "avg_cost": "$22.49", "avg_time": "2.3h"}], "best_score": 0.5105, "mean_score": 0.1605, "n_models": 10}, {"slug": "granite-mamba2-inference-optimization", "number": 10, "title": "Granite Mamba2 Inference Optimization", "url": "https://www.frontierswe.com/tasks/granite-mamba2-inference-optimization", "categories": ["AI Research"], "summary": "Make the real Granite hybrid Mamba2 layer's CUDA inference path faster on a B200, without changing what it computes.", "scoring": "Perf speedup", "toolchains": ["Go", "CUDA", "Python", "C/C++"], "compute": {"label": "1× B200", "kind": "gpu", "compute": "8 CPUs · 64 GB RAM", "gpu": "1× B200", "base_image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", "tools": "CUDA, PyTorch, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "State-space models like Mamba2 replace attention with a recurrent selective-scan, which makes their inference profile very different from a transformer's: long prefill is a chunked parallel scan, while decode is a per-token recurrence over convolution and SSM cache state. Production inference engines squeeze this path with hand-tuned Triton kernels, and the gap between a straightforward eager PyTorch implementation and an optimized one is large.\nThe workspace is a standalone port of the real Hugging Face GraniteMoeHybridMambaLayer , with weights extracted from the pinned checkpoint ibm-granite/granite-4.0-h-1b-base (layer-0 Mamba weights, final norm, and tied embedding head). Everything runs in bfloat16 on CUDA, which constrains optimization choices such as Triton intrinsics and accumulation precision.", "task": "The agent must implement CandidateBlock in /app/src/candidate_impl.py (a stub subclassing the reference is provided) and make the layer's inference path faster without changing its semantics. The performance bar is an optimized implementation built on the same production Triton kernels vendored in /app/vllm_ops/ : match it, then beat it.\n- Keep the fixed constructor and forward(hidden_states, cache=None, attention_mask=None) signature; the internal cache layout may change as long as the returned cache still exposes conv_state , ssm_state , has_previous_state , and decode-position semantics.\n- Match the reference on hidden states, convolution and SSM cache states, and last-token readout logits, across prefill, cached decode, and variable-length padded batches. Speed only counts once parity holds.\n- Gains are measured on long prefill, batched variable-length prefill, and per-token decode latency; consistent gains across shapes beat a single outlier.\n- torch.compile , Triton, custom CUDA kernels, CUDA streams, and calls into transformers are all allowed. /app/reference_impl.py , /app/task_fixtures.py , and /app/vllm_ops/ must stay untouched; all changes go in the single candidate file.", "task_bullets": ["Keep the fixed constructor and forward(hidden_states, cache=None, attention_mask=None) signature; the internal cache layout may change as long as the returned cache still exposes conv_state , ssm_state , has_previous_state , and decode-position semantics.", "Match the reference on hidden states, convolution and SSM cache states, and last-token readout logits, across prefill, cached decode, and variable-length padded batches. Speed only counts once parity holds.", "Gains are measured on long prefill, batched variable-length prefill, and per-token decode latency; consistent gains across shapes beat a single outlier.", "torch.compile , Triton, custom CUDA kernels, CUDA streams, and calls into transformers are all allowed. /app/reference_impl.py , /app/task_fixtures.py , and /app/vllm_ops/ must stay untouched; all changes go in the single candidate file."], "agent_gets": "A ready-to-run workspace at /app : the fixed reference port (reference_impl.py , a clean port of the HF torch_forward path that does not call transformers inside the forward), the extracted checkpoint slice in assets/ , the vendored vLLM Triton kernels in vllm_ops/ as building blocks, and fixed utilities in task_fixtures.py including the visible correctness and benchmark workloads.\nLocal loops are provided: verify_api.py checks parity against both the reference and the pinned transformers implementation, and run_dev_bench.py runs a local latency comparison. The hidden optimized baseline itself is not exposed to candidate code. There is no internet access at run time; the image bakes a ready .venv , so uv run --no-sync python ... works offline, and a sandbox-timer CLI reports the remaining budget.", "verification": "A clean-room verifier runs in a fresh container on the captured /app : it restages pristine copies of the fixed files (task_fixtures.py , reference_impl.py , vllm_ops/ ) so only the candidate file carries over, then runs a correctness gauntlet followed by a paired GPU benchmark. Candidate code executes only inside an unprivileged worker process; the trusted parent owns workload generation, comparisons, and timing.\n- Correctness is checked against both the eager reference and the pinned transformers implementation on hidden prefill and decode workloads: hidden states, conv and SSM cache states, readout logits, and readout KL divergence. Any failure zeroes the result before performance is considered.\n- The timed path is additionally validated at the large benchmark shapes, with tolerances calibrated from the trusted baseline's own drift, and an output digest is compared on every timed call, so returning garbage-fast output only on the timed shapes is a hard correctness failure.\n- Speed is a paired ABBA benchmark against a hidden optimized baseline built on the same Triton kernel family, timed exclusively on the trusted parent's wall clock (worker self-reported timings are never scored), with median-of-pairs aggregation and a frozen Triton autotune cache.\n- The reward is the fraction of starter runtime eliminated: with G the geometric-mean speedup over the unchanged starter (measured against a live no-op floor), a correct candidate scores 0 when G ≤ 1 and 1 − 1/G otherwise; 2x maps to 0.5, 4x to 0.75.\n- Source scans reject references to verifier internals and any delegation to the trusted baseline; build failures and a missing candidate file zero the result.", "verification_bullets": ["Correctness is checked against both the eager reference and the pinned transformers implementation on hidden prefill and decode workloads: hidden states, conv and SSM cache states, readout logits, and readout KL divergence. Any failure zeroes the result before performance is considered.", "The timed path is additionally validated at the large benchmark shapes, with tolerances calibrated from the trusted baseline's own drift, and an output digest is compared on every timed call, so returning garbage-fast output only on the timed shapes is a hard correctness failure.", "Speed is a paired ABBA benchmark against a hidden optimized baseline built on the same Triton kernel family, timed exclusively on the trusted parent's wall clock (worker self-reported timings are never scored), with median-of-pairs aggregation and a frozen Triton autotune cache.", "The reward is the fraction of starter runtime eliminated: with G the geometric-mean speedup over the unchanged starter (measured against a live no-op floor), a correct candidate scores 0 when G ≤ 1 and 1 − 1/G otherwise; 2x maps to 0.5, 4x to 0.75.", "Source scans reject references to verifier internals and any delegation to the trusted baseline; build failures and a missing candidate file zero the result."], "safeguards": ["Correctness is checked against both the eager reference and the pinned transformers implementation on hidden prefill and decode workloads: hidden states, conv and SSM cache states, readout logits, and readout KL divergence. Any failure zeroes the result before performance is considered.", "Source scans reject references to verifier internals and any delegation to the trusted baseline; build failures and a missing candidate file zero the result."], "environment": {"Base image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", "Tools available": "CUDA, PyTorch, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 64 GB RAM", "GPU": "1× B200", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of Granite Mamba2 Inference Optimization from FrontierSWE V1.\n- The score changed from an uncapped geometric-mean speedup over the hidden optimized baseline (1.0 = parity) to the fraction of starter runtime eliminated: 0 when the aggregate speedup over the unchanged starter is ≤ 1, otherwise 1 − 1/G, anchored by a live no-op floor measured in the same run.\n- Timing moved from worker-reported CUDA-event measurements to the trusted parent's wall clock as the sole scored source, so candidate code can no longer influence its own timings; worker CUDA numbers are retained as diagnostics only.\n- New benchmark-shape correctness gates: the timed path is validated at the large benchmark shapes against the eager reference with baseline-calibrated tolerances, and an output digest is checked on every timed call, closing the gap between small checked shapes and large timed shapes.\n- Verification is now clean-room: the captured /app is rescored in a separate pinned container that restages pristine fixed files, with the candidate worker running as an unprivileged user in per-user private IPC directories and a frozen Triton autotune cache, replacing V1's in-place hash checks.\n- The submission contracted from a /app/submission/ tree (helper files allowed) to the single file /app/src/candidate_impl.py , with source scans rejecting references to verifier internals or delegation to the trusted baseline.\n- Infrastructure moved to the new harbor schema with artifact capture, and the /app/.timer daemon files were replaced by the sandbox-timer CLI.", "references": [{"title": "IBM Granite 4.0-H 1B base (Apache-2.0): A single-layer slice (layer-0 Mamba weights, final norm, and tied embedding head) is extracted from the pinned checkpoint revision at image build; no weights are vendored in the task repository.", "url": "https://huggingface.co/ibm-granite/granite-4.0-h-1b-base"}, {"title": "vLLM Mamba2 Triton kernels (Apache-2.0): Extracted SSM scan / chunk / state-passing Triton kernels (main branch, March 2026), themselves adapted from state-spaces/mamba by Tri Dao and Albert Gu, vendored as agent-visible building blocks in /app/vllm_ops/ ; the hidden optimized baseline calls the same kernels.", "url": "https://github.com/vllm-project/vllm"}, {"title": "Hugging Face Transformers (Apache-2.0): /app/reference_impl.py is a standalone port of the GraniteMoeHybridMambaLayer torch_forward path; the pinned package (4.57.6) is also installed in the image and used as a parity target.", "url": "https://github.com/huggingface/transformers"}, {"title": "mamba-ssm / causal-conv1d (Apache-2.0 / BSD-3-Clause): Pinned pip packages (mamba-ssm==2.3.1 , causal-conv1d==1.6.1 ) compiled into the image as CUDA extensions the optimized baseline relies on; not vendored.", "url": "https://github.com/state-spaces/mamba"}], "results": [{"rank": 1, "model": "GLM-5.3", "score": 0.8196, "std": 0.069, "steps": "473", "in_tok": "213.6M", "out_tok": "614k", "avg_cost": "$61.42", "avg_time": "7.6h"}, {"rank": 2, "model": "Claude Fable 5.1", "score": 0.8187, "std": 0.024, "steps": "670", "in_tok": "287.5M", "out_tok": "1.2M", "avg_cost": "$171.77", "avg_time": "8.3h"}, {"rank": 3, "model": "Kimi K3", "score": 0.7379, "std": 0.07, "steps": "627", "in_tok": "277.1M", "out_tok": "826k", "avg_cost": "$100.99", "avg_time": "15.7h"}, {"rank": 4, "model": "Qwen3.8-Max", "score": 0.7366, "std": 0.06, "steps": "497", "in_tok": "163.7M", "out_tok": "1.8M", "avg_cost": "$52.96", "avg_time": "18.0h"}, {"rank": 5, "model": "DeepSeek V4 Flash Exp", "score": 0.7262, "std": 0.152, "steps": "749", "in_tok": "295.7M", "out_tok": "756k", "avg_cost": "$5.25", "avg_time": "13.5h"}, {"rank": 6, "model": "Grok 4.6", "score": 0.6077, "std": 0.188, "steps": "183", "in_tok": "42.8M", "out_tok": "659k", "avg_cost": "$49.33", "avg_time": "4.0h"}, {"rank": 7, "model": "GPT-5.6", "score": 0.5617, "std": 0.164, "steps": "370", "in_tok": "104.3M", "out_tok": "241k", "avg_cost": "$101.96", "avg_time": "4.5h"}, {"rank": 8, "model": "Inkling", "score": 0.471, "std": 0.43, "steps": "99", "in_tok": "5.3M", "out_tok": "28k", "avg_cost": "$5.18", "avg_time": "1.3h"}, {"rank": 9, "model": "Gemini 3.7 Flash", "score": 0.3885, "std": 0.232, "steps": "318", "in_tok": "99.7M", "out_tok": "538k", "avg_cost": "$10.96", "avg_time": "1.5h"}, {"rank": 10, "model": "Muse Spark 1.2", "score": 0.3055, "std": 0.42, "steps": "294", "in_tok": "83.5M", "out_tok": "532k", "avg_cost": "$15.81", "avg_time": "2.2h"}], "best_score": 0.8196, "mean_score": 0.6173, "n_models": 10}, {"slug": "higgs-uncertainty-inference", "number": 11, "title": "Higgs Uncertainty Inference", "url": "https://www.frontierswe.com/tasks/higgs-uncertainty-inference", "categories": ["Scientific Computing"], "summary": "Estimate the Higgs signal strength from LHC pseudo-experiments with calibrated confidence intervals that survive hidden systematic shifts.", "scoring": "Skill vs baseline", "toolchains": ["Python"], "compute": {"label": "CPU", "kind": "cpu", "compute": "8 CPUs · 32 GB RAM", "gpu": null, "base_image": "python:3.11-slim-bookworm", "tools": "PyTorch, Python 3", "time_limit": "20h"}, "background": "Measuring the Higgs signal strength mu (how many signal events appear relative to the Standard Model expectation) is a canonical LHC analysis. The hard part is not the point estimate but the uncertainty: detector effects and modeling assumptions (systematics) shift the event distributions in ways that are only partly known, and an interval that is honest under nominal conditions can silently under-cover once the true nuisance parameters drift from the training distribution.\nThe physics data derives from the FAIR Universe HiggsML Uncertainty Challenge. The task turns it into a stress test of uncertainty quantification: hidden pseudo-experiments mix a nominal regime with one whose systematic shifts fall outside the documented training configuration, and the two are indistinguishable per experiment by design.", "task": "Build a pipeline that estimates mu for each unlabelled pseudo-experiment and reports a calibrated central 68.27% confidence interval, packaged as a self-contained deliverable at /app/higgs_model/ (checkpoint, model.py , predict.py , run_summary.json ).\n- Inference runs as python /app/higgs_model/predict.py --data-dir … --checkpoint … --output-path … and writes one JSONL row per experiment: {\"experiment_id\":…,\"mu\":…,\"mu_lo\":…,\"mu_hi\":…} with finite values satisfying mu_lo < mu <= mu_hi .\n- Inference must be deterministic, read only its supplied input, and use no external services or hardcoded per-experiment lookups.\n- The checkpoint may occupy at most 2.5 GB, and a few hundred experiments must finish within 20 minutes.\n- Intervals must remain useful when hidden nuisance parameters differ from the nominal training distribution.", "task_bullets": ["Inference runs as python /app/higgs_model/predict.py --data-dir … --checkpoint … --output-path … and writes one JSONL row per experiment: {\"experiment_id\":…,\"mu\":…,\"mu_lo\":…,\"mu_hi\":…} with finite values satisfying mu_lo < mu <= mu_hi .", "Inference must be deterministic, read only its supplied input, and use no external services or hardcoded per-experiment lookups.", "The checkpoint may occupy at most 2.5 GB, and a few hundred experiments must finish within 20 minutes.", "Intervals must remain useful when hidden nuisance parameters differ from the nominal training distribution."], "agent_gets": "Visible data staged read-only under /data : labeled training events (train/events.parquet with features, labels, weights, and detailed process labels), unlabelled validation experiments, calibration experiments with separate truth in calibration/labels.parquet , plus systematics.json and metadata.json describing the feature columns and documented nuisance model.\n- A runnable starter model under /app/higgs_model/ exercising the full contract.\n- A pinned CPU scientific Python stack: NumPy, pandas, pyarrow, scikit-learn, SciPy, XGBoost, LightGBM, iminuit, Optuna, PyTorch Lightning, and statsmodels.\n- The machine is offline; everything needed is already present.", "verification": "A root-only verifier runs the submitted model as the unprivileged agent user, without network access, on hidden pseudo-experiments spanning two regimes: known systematics and shifted systematics. Per-regime and pooled metrics are normalized between an event-blind baseline anchor and a reference-solution anchor, then combined as a weighted geometric mean (known 0.3, shifted 0.3, overall 0.4). The per-regime slices blend point RMSE on mu with the mean Winkler interval score; the pooled slice adds a challenge-style quantiles calibration score. The Winkler score punishes every missed interval by its distance, so covering the aggregate while sacrificing individual experiments cannot profit.\n- Hard gates zero the reward: pooled empirical coverage more than 0.15 from 68.27% (with a soft penalty beyond 0.05), mean interval width of 1.60 or more on either regime (the trivial-width anchor), or failing to beat the event-blind baseline on any scored term.\n- Safeguard probes also zero it: repeated runs must agree within 1e-6, and a response probe that resamples each experiment's events must change at least half the predictions; a lookup table or constant answer fails.\n- Contract failures (malformed rows, non-finite values, inverted intervals, an oversized checkpoint, or a timed-out run) also zero the result.", "verification_bullets": ["Hard gates zero the reward: pooled empirical coverage more than 0.15 from 68.27% (with a soft penalty beyond 0.05), mean interval width of 1.60 or more on either regime (the trivial-width anchor), or failing to beat the event-blind baseline on any scored term.", "Safeguard probes also zero it: repeated runs must agree within 1e-6, and a response probe that resamples each experiment's events must change at least half the predictions; a lookup table or constant answer fails.", "Contract failures (malformed rows, non-finite values, inverted intervals, an oversized checkpoint, or a timed-out run) also zero the result."], "safeguards": ["Hard gates zero the reward: pooled empirical coverage more than 0.15 from 68.27% (with a soft penalty beyond 0.05), mean interval width of 1.60 or more on either regime (the trivial-width anchor), or failing to beat the event-blind baseline on any scored term.", "Safeguard probes also zero it: repeated runs must agree within 1e-6, and a response probe that resamples each experiment's events must change at least half the predictions; a lookup table or constant answer fails.", "Contract failures (malformed rows, non-finite values, inverted intervals, an oversized checkpoint, or a timed-out run) also zero the result."], "environment": {"Base image": "python:3.11-slim-bookworm", "Tools available": "PyTorch, Python 3", "Compute": "8 CPUs · 32 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "FAIR Universe HiggsML Uncertainty Challenge dataset (CC BY 4.0): The simulated LHC physics data. The quantiles metric in the verifier is an independent reimplementation of the published challenge scoring formula; no upstream utility code is redistributed, and the pseudo-experiment generation, packaging, and verification code are original to this benchmark.", "url": "https://doi.org/10.5281/zenodo.15131565"}], "results": [{"rank": 1, "model": "Grok 4.6", "score": 0.7451, "std": 0.301, "steps": "418", "in_tok": "97.7M", "out_tok": "833k", "avg_cost": "$102.30", "avg_time": "8.0h"}, {"rank": 2, "model": "Qwen3.8-Max", "score": 0.5656, "std": 0.294, "steps": "333", "in_tok": "57.0M", "out_tok": "560k", "avg_cost": "$18.16", "avg_time": "10.1h"}, {"rank": 3, "model": "Claude Fable 5.1", "score": 0.5193, "std": 0.44, "steps": "205", "in_tok": "51.0M", "out_tok": "309k", "avg_cost": "$33.00", "avg_time": "6.1h"}, {"rank": 4, "model": "Muse Spark 1.2", "score": 0.507, "std": 0.381, "steps": "149", "in_tok": "29.0M", "out_tok": "328k", "avg_cost": "$6.56", "avg_time": "1.8h"}, {"rank": 5, "model": "DeepSeek V4 Flash Exp", "score": 0.3776, "std": 0.435, "steps": "316", "in_tok": "83.5M", "out_tok": "365k", "avg_cost": "$1.70", "avg_time": "7.0h"}, {"rank": 6, "model": "Kimi K3", "score": 0.3133, "std": 0.394, "steps": "343", "in_tok": "99.2M", "out_tok": "407k", "avg_cost": "$43.65", "avg_time": "15.4h"}, {"rank": 7, "model": "GPT-5.6", "score": 0.2171, "std": 0.329, "steps": "253", "in_tok": "66.0M", "out_tok": "166k", "avg_cost": "$64.71", "avg_time": "3.5h"}, {"rank": 8, "model": "Inkling", "score": 0.1769, "std": 0.391, "steps": "118", "in_tok": "3.9M", "out_tok": "52k", "avg_cost": "$3.95", "avg_time": "1.5h"}, {"rank": 9, "model": "Gemini 3.7 Flash", "score": 0.1099, "std": 0.041, "steps": "114", "in_tok": "11.9M", "out_tok": "156k", "avg_cost": "$2.33", "avg_time": "1.2h"}, {"rank": 10, "model": "GLM-5.3", "score": 0.0756, "std": 0.057, "steps": "248", "in_tok": "59.0M", "out_tok": "329k", "avg_cost": "$18.22", "avg_time": "7.1h"}], "best_score": 0.7451, "mean_score": 0.3607, "n_models": 10}, {"slug": "audio-compression-csound", "number": 12, "title": "Kolmogorov Audio Compression", "url": "https://www.frontierswe.com/tasks/audio-compression-csound", "categories": ["Performance Optimisation"], "summary": "Compress a synthetic WAV corpus into the smallest self-extracting archive that reproduces every file byte-for-byte.", "scoring": "Byte exact", "toolchains": ["Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 16 GB RAM", "gpu": null, "base_image": "ubuntu:22.04", "tools": "Audio analysis: Python 3 with numpy, scipy, librosa, soundfile, pandas Compression: flac, xz, zstd, brotli, lz4, gzip, bzip2, plus a C/C++ toolchain for custom codecs", "time_limit": "20h"}, "background": "Lossless compression is a practical probe of Kolmogorov complexity : the size of the smallest self-extracting archive is an upper bound on the length of the shortest program that reproduces the data, so compressing further means recovering more of the program that generated it. This corpus makes that gap enormous by construction: every WAV is synthetic, rendered offline from procedural scores by the csound synthesis engine, so hundreds of megabytes of PCM are ultimately described by a tiny amount of generative information. Only the rendered audio ships; the generator does not.\nThe scoring anchors follow directly: a whole-corpus flac -8 (strong generic modeling) earns nothing, and full credit sits an order of magnitude beyond it, in territory reachable only by rediscovering the corpus's generative structure.", "task": "Compress the audio files in /app/audio/ losslessly, shipping the result as a self-contained directory /app/dist/ . Running python3 /app/dist/decompress.py <out_dir> must recreate every file byte-for-byte using only what is under /app/dist/ , and the total size of /app/dist/ is the compressed size being scored.\n- The round trip is a hard prerequisite: every WAV must come back byte-exact, or the result is zero regardless of size.\n- Size accounting counts every file's contents plus every relative path name under /app/dist/ (only __pycache__ is excluded); the decoder itself is part of the payload.\n- The decoder may call tools already on the machine without those counting toward size, but anything the agent writes must live inside /app/dist/ .\n- Decompression must finish within a 1800-second budget.\n- Changes are confined to /app/dist/ .\n- A starter decoder ships at /app/dist/decompress.py , and /app/check.py self-checks the round trip (--sample for fast iteration) against the flac -8 baseline in /app/anchors.json .\n- The machine is offline; no lossless audio codec stronger than FLAC is installed, and csound is absent from every agent-visible path.", "task_bullets": ["The round trip is a hard prerequisite: every WAV must come back byte-exact, or the result is zero regardless of size.", "Size accounting counts every file's contents plus every relative path name under /app/dist/ (only __pycache__ is excluded); the decoder itself is part of the payload.", "The decoder may call tools already on the machine without those counting toward size, but anything the agent writes must live inside /app/dist/ .", "Decompression must finish within a 1800-second budget.", "Changes are confined to /app/dist/ .", "A starter decoder ships at /app/dist/decompress.py , and /app/check.py self-checks the round trip (--sample for fast iteration) against the flac -8 baseline in /app/anchors.json .", "The machine is offline; no lossless audio codec stronger than FLAC is installed, and csound is absent from every agent-visible path."], "agent_gets": "", "verification": "A clean-room verifier, separate from the agent's environment, stages the submitted /app/dist/ and runs decompress.py under the 1800-second cap. The output is byte-compared against a held-out copy of the corpus; only after every file matches is size scored.\n- Once lossless, the reward maps the size ratio onto a curve: zero at the measured flac -8 baseline ratio, full credit at a ratio of 0.01, with a mild convex bend so each further halving of size is worth more than the last.\n- Any byte mismatch, missing file, decoder crash, or timeout zeroes the result; so does a ratio at or above the FLAC baseline.", "verification_bullets": ["Once lossless, the reward maps the size ratio onto a curve: zero at the measured flac -8 baseline ratio, full credit at a ratio of 0.01, with a mild convex bend so each further halving of size is worth more than the last.", "Any byte mismatch, missing file, decoder crash, or timeout zeroes the result; so does a ratio at or above the FLAC baseline."], "safeguards": ["Once lossless, the reward maps the size ratio onto a curve: zero at the measured flac -8 baseline ratio, full credit at a ratio of 0.01, with a mild convex bend so each further halving of size is worth more than the last.", "Any byte mismatch, missing file, decoder crash, or timeout zeroes the result; so does a ratio at or above the FLAC baseline."], "environment": {"Base image": "ubuntu:22.04", "Tools available": "Audio analysis: Python 3 with numpy, scipy, librosa, soundfile, pandas Compression: flac, xz, zstd, brotli, lz4, gzip, bzip2, plus a C/C++ toolchain for custom codecs", "Compute": "4 CPUs · 16 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "csound (LGPL-2.1): The synthesis engine that rendered the corpus offline from procedural scores.", "url": "https://csound.com/"}, {"title": "music21 corpus (BSD-3-Clause; underlying works public domain): Public-domain melodic and bass patterns that seed the score generator; no upstream audio is redistributed.", "url": "https://web.mit.edu/music21/"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.0723, "std": 0.004, "steps": "355", "in_tok": "93.1M", "out_tok": "322k", "avg_cost": "$54.37", "avg_time": "16.2h"}, {"rank": 2, "model": "GLM-5.3", "score": 0.0305, "std": 0.005, "steps": "354", "in_tok": "94.6M", "out_tok": "330k", "avg_cost": "$34.25", "avg_time": "19.2h"}, {"rank": 3, "model": "GPT-5.6", "score": 0.0274, "std": 0.013, "steps": "362", "in_tok": "81.4M", "out_tok": "183k", "avg_cost": "$73.02", "avg_time": "18.8h"}, {"rank": 4, "model": "Kimi K3", "score": 0.024, "std": 0.007, "steps": "398", "in_tok": "109.7M", "out_tok": "415k", "avg_cost": "$42.47", "avg_time": "19.8h"}, {"rank": 5, "model": "Gemini 3.7 Flash", "score": 0.0173, "std": 0.007, "steps": "280", "in_tok": "69.7M", "out_tok": "379k", "avg_cost": "$14.33", "avg_time": "7.5h"}, {"rank": 6, "model": "Grok 4.6", "score": 0.0163, "std": 0.005, "steps": "697", "in_tok": "156.4M", "out_tok": "603k", "avg_cost": "$155.68", "avg_time": "19.8h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0146, "std": 0.01, "steps": "599", "in_tok": "238.8M", "out_tok": "519k", "avg_cost": "$4.11", "avg_time": "11.1h"}, {"rank": 8, "model": "Qwen3.8-Max", "score": 0.0126, "std": 0.004, "steps": "399", "in_tok": "63.2M", "out_tok": "654k", "avg_cost": "$20.21", "avg_time": "19.1h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0096, "std": 0.002, "steps": "398", "in_tok": "123.8M", "out_tok": "342k", "avg_cost": "$36.20", "avg_time": "7.4h"}, {"rank": 10, "model": "Inkling", "score": 0.0053, "std": 0.002, "steps": "154", "in_tok": "7.7M", "out_tok": "31k", "avg_cost": "$7.45", "avg_time": "1.7h"}], "best_score": 0.0723, "mean_score": 0.023, "n_models": 10}, {"slug": "lean-checker-pascal", "number": 13, "title": "Lean 4 Kernel Type Checker in Pascal", "url": "https://www.frontierswe.com/tasks/lean-checker-pascal", "categories": ["Implementation"], "summary": "Implement the Lean 4 kernel's type checker in Free Pascal, deciding real Mathlib-scale exports without ever certifying a proof of False.", "scoring": "Hidden tests", "toolchains": ["Pascal", "Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 8 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Free Pascal, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "The Lean 4 kernel is the small trusted core of a major proof assistant: a dependent type theory with universes, inductive families, recursors, quotient types, and definitional equality (everything Mathlib's formalized mathematics ultimately rests on). Independent kernel re-implementations exist precisely so the kernel can be cross-checked, and this task asks for one: a standalone checker that reads exported Lean environments and decides, declaration by declaration, whether they are admissible.\nThe difficulty is twofold. Semantically, the checker must reproduce subtle kernel behavior: reduction, universe handling, the Nat and String literal extensions, and the admissibility rules for (possibly mutual and nested) inductives. Operationally, real inputs are large: exports range from a few hundred bytes to about 90 MB with over 1.5 million objects, so throughput is part of what is measured.", "task": "Implement the Lean 4 kernel's type checker in the Free Pascal project at /app/checker/ , with sources only as Pascal files under src/ and the program entry in src/checker.pas . The documented fpc build line produces build/checker , invoked as checker <path-to-export-file> .\n- Exit 0 if and only if every declaration in the lean4export NDJSON file is well-formed, well-typed and admissible, and the file is structurally valid; exit non-zero otherwise. The exit code is the result.\n- Never exit 0 on an input the checker failed to understand (an unparsed construct or an internal error is not evidence of admissibility), and never reject a file merely because it is expensive.\n- Soundness is paramount: wrongly certifying an environment that derives a closed proof of False is the most serious error the checker can make.\n- Decide everything in Pascal itself, the way the Lean kernel would: the checker must be a self-contained static binary.", "task_bullets": ["Exit 0 if and only if every declaration in the lean4export NDJSON file is well-formed, well-typed and admissible, and the file is structurally valid; exit non-zero otherwise. The exit code is the result.", "Never exit 0 on an input the checker failed to understand (an unparsed construct or an internal error is not evidence of admissibility), and never reject a file merely because it is expensive.", "Soundness is paramount: wrongly certifying an environment that derives a closed proof of False is the most serious error the checker can make.", "Decide everything in Pascal itself, the way the Lean kernel would: the checker must be a self-contained static binary."], "agent_gets": "/app/README.md carries the CLI contract, the complete NDJSON wire-format specification (names, levels, expressions, and all six declaration kinds, adapted from lean4export 's format document), and the build and test commands. /app/exports/ holds worked examples of both verdicts (every file under accept/ must exit 0 and every file under reject/ must not, with expected.tsv listing them all), and /app/run-tests.sh builds the checker and runs any subset. The Free Pascal compiler is preinstalled and the machine is offline.", "verification": "A clean-room verifier, separate from the agent's environment, stages only the Pascal sources under src/ , rebuilds them offline with fpc -O2 as a non-root user, and runs a hidden corpus of 607 export files (~2.4 GB): genuine Lean, Mathlib, Batteries and Aesop environments to accept, plus 131 deliberately corrupted or False -grafted exports to reject, each labeled offline by an independent reference kernel (nanoda_lib ). Cases are staged under content-hash names with uniform timestamps, so neither path nor order reveals a verdict; the checker's exit status on each file is the verdict.\n- Reward is the weighted accept rate times the weighted reject rate, with soundness-tier cases weighted 10×. Exiting 0 on any case containing a closed proof of False is a hard zero.\n- A static anti-delegation scan (process-spawning APIs, dynamic loading, spawn-shaped FFI declarations) plus a runtime exec tripwire zero the result if the checker hands the decision to another program.\n- Each case gets a size-scaled budget (15 s plus ~1.2 MB per second, about 10× the reference kernel's throughput) inside a 2400-second suite budget; a crash, hang, timeout or unreached case counts as a wrong verdict, never a skip.\n- Build failures (600-second cap) and a missing src/checker.pas entry point zero the result.", "verification_bullets": ["Reward is the weighted accept rate times the weighted reject rate, with soundness-tier cases weighted 10×. Exiting 0 on any case containing a closed proof of False is a hard zero.", "A static anti-delegation scan (process-spawning APIs, dynamic loading, spawn-shaped FFI declarations) plus a runtime exec tripwire zero the result if the checker hands the decision to another program.", "Each case gets a size-scaled budget (15 s plus ~1.2 MB per second, about 10× the reference kernel's throughput) inside a 2400-second suite budget; a crash, hang, timeout or unreached case counts as a wrong verdict, never a skip.", "Build failures (600-second cap) and a missing src/checker.pas entry point zero the result."], "safeguards": ["Reward is the weighted accept rate times the weighted reject rate, with soundness-tier cases weighted 10×. Exiting 0 on any case containing a closed proof of False is a hard zero.", "A static anti-delegation scan (process-spawning APIs, dynamic loading, spawn-shaped FFI declarations) plus a runtime exec tripwire zero the result if the checker hands the decision to another program.", "Each case gets a size-scaled budget (15 s plus ~1.2 MB per second, about 10× the reference kernel's throughput) inside a 2400-second suite budget; a crash, hang, timeout or unreached case counts as a wrong verdict, never a skip.", "Build failures (600-second cap) and a missing src/checker.pas entry point zero the result."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Free Pascal, C/C++ toolchain, Python 3", "Compute": "4 CPUs · 8 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "Lean 4 (Apache-2.0): The v4.33.0-rc1 toolchain compiled the modules whose environments were exported; the corpus also exports declarations from its own Init , Lean and Std libraries. Not shipped in the image.", "url": "https://github.com/leanprover/lean4"}, {"title": "lean4export (Apache-2.0): Produced every .ndjson case (format version 3.1.0); its format document is the source of the wire-format section of /app/README.md , adapted under the same license. Not shipped in the image.", "url": "https://github.com/leanprover/lean4export"}, {"title": "Mathlib 4 (Apache-2.0): Source of the mathematical declarations in the majority of the corpus (419 of the 607 hidden cases). The reject cases are modified exports (deliberately corrupted, or grafted with derivations of False ) and must not be mistaken for valid Mathlib output. Not shipped in the image.", "url": "https://github.com/leanprover-community/mathlib4"}, {"title": "Batteries (Apache-2.0): Compiled as a Mathlib dependency; 10 hidden cases export declarations from its modules.", "url": "https://github.com/leanprover-community/batteries"}, {"title": "Aesop (Apache-2.0): Compiled as a Mathlib dependency; 6 hidden cases export declarations from its modules.", "url": "https://github.com/leanprover-community/aesop"}, {"title": "ProofWidgets4 (Apache-2.0): Compiled as a Mathlib dependency; 1 hidden case exports declarations from its modules.", "url": "https://github.com/leanprover-community/ProofWidgets4"}, {"title": "nanoda_lib (Apache-2.0): The independent reference Lean 4 kernel that labeled every case offline; redistributed unmodified inside the image, root-only in the verifier tree, never reachable from /app .", "url": "https://github.com/ammkrn/nanoda_lib"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 1.0, "std": 0.0, "steps": "194", "in_tok": "64.4M", "out_tok": "384k", "avg_cost": "$41.54", "avg_time": "3.2h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.9314, "std": 0.083, "steps": "728", "in_tok": "249.9M", "out_tok": "372k", "avg_cost": "$248.76", "avg_time": "7.1h"}, {"rank": 3, "model": "Kimi K3", "score": 0.593, "std": 0.276, "steps": "1,025", "in_tok": "446.5M", "out_tok": "1.3M", "avg_cost": "$163.81", "avg_time": "20.0h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.4429, "std": 0.402, "steps": "905", "in_tok": "428.9M", "out_tok": "827k", "avg_cost": "$126.63", "avg_time": "19.0h"}, {"rank": 5, "model": "Gemini 3.7 Flash", "score": 0.4029, "std": 0.493, "steps": "1,471", "in_tok": "586.8M", "out_tok": "1.3M", "avg_cost": "$75.03", "avg_time": "16.0h"}, {"rank": 6, "model": "DeepSeek V4 Flash Exp", "score": 0.1884, "std": 0.17, "steps": "1,533", "in_tok": "700.2M", "out_tok": "1.4M", "avg_cost": "$11.96", "avg_time": "11.3h"}, {"rank": 7, "model": "Grok 4.6", "score": 0.0765, "std": 0.171, "steps": "888", "in_tok": "216.0M", "out_tok": "1.7M", "avg_cost": "$222.15", "avg_time": "13.4h"}, {"rank": 8, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "119", "in_tok": "7.7M", "out_tok": "32k", "avg_cost": "$7.42", "avg_time": "0.8h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0, "std": 0.0, "steps": "292", "in_tok": "92.4M", "out_tok": "660k", "avg_cost": "$21.30", "avg_time": "3.9h"}, {"rank": 10, "model": "Qwen3.8-Max", "score": 0.0, "std": 0.0, "steps": "156", "in_tok": "24.7M", "out_tok": "1.1M", "avg_cost": "$13.28", "avg_time": "20.0h"}], "best_score": 1.0, "mean_score": 0.3635, "n_models": 10}, {"slug": "libexpat-to-x86asm", "number": 14, "title": "libexpat Optimization", "url": "https://www.frontierswe.com/tasks/libexpat-to-x86asm", "categories": ["Performance Optimisation"], "summary": "Hand-write an XML parser in x86-64 assembly that matches libexpat's exact parse behaviour, and beats its C on measured work.", "scoring": "Hidden tests", "toolchains": ["Python", "x86-64 asm", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 8 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "libexpat is a widely used stream-oriented XML parser: callers register handlers, feed a document whole or in arbitrary chunks, and receive parse events in order. Reproducing it in hand-written assembly means matching not just well-formed parsing but libexpat's exact observable behaviour: the same events in the same order, the same coalescing of character data, and on malformed input the same specific error code, not merely an error. Chunked feeding is where from-scratch parsers first diverge: a construct can split across calls at any byte, so the parser must carry partial state and still emit exactly what a one-shot parse would.\nThe parser's C implementation is withheld: the agent gets the public API headers as the specification, example documents with their expected event traces, and nothing else to copy from. The performance bar makes assembly the point rather than a gimmick: the deliverable must get through a document doing measurably less work than libexpat's own compiled C.", "task": "Ship /app/asm-port/libexpat.so , assembled from the *.s /*.S /*.asm sources in /app/asm-port/ : a System V AMD64 C-ABI shared library implementing the interface in /app/tests/expat.h .\n- Thirteen entry points are resolved by name, from XML_ParserCreate and the handler setters through XML_Parse , XML_GetErrorCode , and XML_ParserFree ; a partial library still loads and runs.\n- Every document is parsed four ways (namespace processing off and on, one-shot and chunked), and all four must agree with libexpat: same events, same order, same final status and error code.\n- Correctness is all-or-nothing: one wrong document takes the result to zero however fast the rest are.\n- libc may be called from assembly (malloc , memcpy , memchr , …) and is the expected way to get memory, but its work is priced like the agent's own.\n- Target x86-64-v3 and no higher: the measurement simulator has no AVX-512, and an unmeasurable parse is no better than a wrong one.\n- Changes are confined to /app/asm-port/ .", "task_bullets": ["Thirteen entry points are resolved by name, from XML_ParserCreate and the handler setters through XML_Parse , XML_GetErrorCode , and XML_ParserFree ; a partial library still loads and runs.", "Every document is parsed four ways (namespace processing off and on, one-shot and chunked), and all four must agree with libexpat: same events, same order, same final status and error code.", "Correctness is all-or-nothing: one wrong document takes the result to zero however fast the rest are.", "libc may be called from assembly (malloc , memcpy , memchr , …) and is the expected way to get memory, but its work is priced like the agent's own.", "Target x86-64-v3 and no higher: the measurement simulator has no AVX-512, and an unmeasurable parse is no better than a wrong one.", "Changes are confined to /app/asm-port/ ."], "agent_gets": "The ABI headers (/app/tests/expat.h , expat_external.h ), example documents at /app/tests/corpus/ with the expected event traces for all four modes at /app/tests/expected/ , and the C source of the workers that produce and measure those traces. /app/build-lib.sh assembles and links the deliverable (nasm /as /ld , no C-compilation step anywhere); /app/run-tests.sh diffs the parser's events against libexpat's on every example document; /app/perf-check measures what one parse costs against the reference numbers baked in /app/baseline-work.json , over the benchmark documents in /app/bench/ . The measurement stack itself is readable at /app/performance/ , and /app/workloads.py defines the workload space: nine document shapes, any size in the 64–192 KB band, any of the four modes. The machine is offline, with nasm , as , ld , gdb , and valgrind preinstalled.", "verification": "A clean-room verifier, separate from the agent's environment, assembles the submission from its assembly sources alone and replays it over a root-only scored corpus drawn deterministically from the W3C XML Conformance Test Suite, including mutated twin documents, so a table of memorized traces is not an implementation. Every scored unit (a document in one of the four parse modes) must hash-match the trace of the reference libexpat 2.6.4 build; one wrong unit zeroes the result, with the unit pass rate kept only as a diagnostic.\n- Once fully correct, the reward is 2**u - 1 where u scales the geometric mean of reference-to-candidate work ratios, reaching full credit at 7x; doing no better than libexpat scores zero.\n- Work is whole-process instruction-priced measurement (callgrind-based per-opcode throughput pricing plus a capped branch-mispredict term, no cache term). It is deterministic, so one run per change is enough and host speed does not matter.\n- Anti-cheat gates: no withheld expat C files in the deliverable, hard-failing source scans for dlopen /dlsym , Python/pyexpat delegation, and .incbin ; an ELF import check restricted to libc; a preloaded guard that refuses loading a foreign XML library; and a no-exec seccomp filter during measured parses. Large binary files under /app are read as a parser carried in as data.\n- A work-linearity gate and a native wall-time audit catch submissions that skip priced iterations; if fewer than 75% of documents could be measured, the trial is marked invalid rather than scored.", "verification_bullets": ["Once fully correct, the reward is 2**u - 1 where u scales the geometric mean of reference-to-candidate work ratios, reaching full credit at 7x; doing no better than libexpat scores zero.", "Work is whole-process instruction-priced measurement (callgrind-based per-opcode throughput pricing plus a capped branch-mispredict term, no cache term). It is deterministic, so one run per change is enough and host speed does not matter.", "Anti-cheat gates: no withheld expat C files in the deliverable, hard-failing source scans for dlopen /dlsym , Python/pyexpat delegation, and .incbin ; an ELF import check restricted to libc; a preloaded guard that refuses loading a foreign XML library; and a no-exec seccomp filter during measured parses. Large binary files under /app are read as a parser carried in as data.", "A work-linearity gate and a native wall-time audit catch submissions that skip priced iterations; if fewer than 75% of documents could be measured, the trial is marked invalid rather than scored."], "safeguards": ["Once fully correct, the reward is 2**u - 1 where u scales the geometric mean of reference-to-candidate work ratios, reaching full credit at 7x; doing no better than libexpat scores zero.", "Work is whole-process instruction-priced measurement (callgrind-based per-opcode throughput pricing plus a capped branch-mispredict term, no cache term). It is deterministic, so one run per change is enough and host speed does not matter.", "Anti-cheat gates: no withheld expat C files in the deliverable, hard-failing source scans for dlopen /dlsym , Python/pyexpat delegation, and .incbin ; an ELF import check restricted to libc; a preloaded guard that refuses loading a foreign XML library; and a no-exec seccomp filter during measured parses. Large binary files under /app are read as a parser carried in as data."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "C/C++ toolchain, Python 3", "Compute": "4 CPUs · 8 GB RAM", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of libexpat to x86-64 Assembly from FrontierSWE V1.\n- V1 shipped libexpat's complete C source in the workspace as the specification; V2 withholds the implementation entirely: the agent gets only the public headers, example documents, and expected traces, and shipping any withheld C file is a hard anti-cheat failure.\n- The correctness target changed from compiling C test programs against the full ~60-function ABI to exact event-trace and error-code equivalence with a reference expat 2.6.4 build over a W3C conformance-suite corpus, in four parse modes including chunked feeding, narrowed to 13 dlsym-resolved entry points and made all-or-nothing.\n- Performance became the graded axis: after the correctness gate, the score is the geometric mean of deterministic instruction-priced work ratios against the reference, with full credit at 7x; V1 shipped expat's upstream wall-clock benchmark alongside weighted test modules.\n- Anti-cheat moved out of the agent's container: V1 hid gcc, python3, and the system libexpat behind encrypted bundles inside the same image; V2 verifies in a separate clean-room container with pinned images, ELF import checks, source scans, a preloaded dlopen guard, and a no-exec seccomp filter during measurement.\n- Infrastructure was rebuilt on the current harbor schema: the verifier timeout dropped from 86400 s to 9000 s, the /app/.timer file tree was replaced by the sandbox-timer CLI, and both agent and verifier images are pinned by content hash.", "references": [{"title": "libexpat (Expat XML parser) (MIT): The pinned 2.6.4 C library, cloned at image build (nothing vendored), builds the verifier-only reference libexpat.so whose parse-event traces are the answer key: real expat is the oracle. The agent receives only the public API headers and reference-produced traces; the C implementation is withheld.", "url": "https://github.com/libexpat/libexpat"}, {"title": "W3C XML Conformance Test Suite (xmlts) (W3C 3-clause BSD test-suite license option): A deterministic, self-contained XML 1.0 slice of the suite's test documents becomes the scored corpus (root-only) and a small disjoint visible slice of worked examples; the BSD election permits the derived mutated twins. James Clark's xmltest collection is excluded to honour its redistribute-only-intact terms. Fetched at a pinned sha256 at image build; no conformance claim is made.", "url": "https://www.w3.org/XML/Test/xmlts20130923.tar.gz"}, {"title": "uops.info instruction data (Derived aggregate; format via andreas-abel/XED-to-XML (Apache-2.0)): The per-opcode cost table used by the measurement stack is a derived aggregate of reciprocal-throughput measurements from uops.info (Abel & Reineke, ASPLOS 2019), generated once offline; it is not a copy of the dataset.", "url": "https://uops.info/"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.4661, "std": 0.11, "steps": "1,238", "in_tok": "534.3M", "out_tok": "1.4M", "avg_cost": "$301.57", "avg_time": "13.7h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.3143, "std": 0.043, "steps": "301", "in_tok": "80.1M", "out_tok": "220k", "avg_cost": "$78.14", "avg_time": "1.8h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.1616, "std": 0.096, "steps": "1,873", "in_tok": "838.6M", "out_tok": "1.4M", "avg_cost": "$229.18", "avg_time": "14.3h"}, {"rank": 4, "model": "Gemini 3.7 Flash", "score": 0.1105, "std": 0.106, "steps": "838", "in_tok": "324.7M", "out_tok": "836k", "avg_cost": "$34.17", "avg_time": "3.7h"}, {"rank": 5, "model": "Kimi K3", "score": 0.0916, "std": 0.085, "steps": "1,390", "in_tok": "599.3M", "out_tok": "1.5M", "avg_cost": "$210.38", "avg_time": "18.2h"}, {"rank": 6, "model": "Grok 4.6", "score": 0.0779, "std": 0.073, "steps": "403", "in_tok": "100.5M", "out_tok": "626k", "avg_cost": "$100.76", "avg_time": "3.3h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0287, "std": 0.041, "steps": "2,690", "in_tok": "1228.5M", "out_tok": "2.1M", "avg_cost": "$20.46", "avg_time": "9.5h"}, {"rank": 8, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "172", "in_tok": "10.8M", "out_tok": "43k", "avg_cost": "$10.44", "avg_time": "0.4h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0, "std": 0.0, "steps": "417", "in_tok": "140.3M", "out_tok": "675k", "avg_cost": "$34.19", "avg_time": "3.1h"}, {"rank": 10, "model": "Qwen3.8-Max", "score": 0.0, "std": 0.0, "steps": "232", "in_tok": "30.8M", "out_tok": "1.8M", "avg_cost": "$18.97", "avg_time": "19.9h"}], "best_score": 0.4661, "mean_score": 0.1251, "n_models": 10}, {"slug": "lua-native-compiler", "number": 15, "title": "Lua Native Compiler", "url": "https://www.frontierswe.com/tasks/lua-native-compiler", "categories": ["Implementation"], "summary": "Build an ahead-of-time Lua 5.4 compiler in Go that emits standalone native ELF binaries for three CPU architectures.", "scoring": "Hidden tests", "toolchains": ["Go", "Python", "x86-64 asm", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "8 CPUs · 32 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Go, QEMU, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Compiling a dynamic language ahead of time means replacing its interpreter loop with generated machine code: every arithmetic operation needs type-tag dispatch with metamethod fallbacks, and closures, upvalues, coroutines, and the garbage collector all have to keep working through native call frames. This task provides Lua's parser and full runtime as static libraries; the code generator, the native replacement for the bytecode dispatch loop, is the deliverable.\nThe compiler must also be genuinely retargetable: it emits code for three instruction sets (x86-64, aarch64, and riscv64), and the cross-architecture binaries are exercised under QEMU user-mode emulation. Correctness is defined differentially: each emitted binary must reproduce stock Lua 5.4's stdout byte for byte.", "task": "Build luanatc , an ahead-of-time compiler written in Go in the project at /app/lua-native-compiler/ (go build -o luanatc . ), invoked as luanatc program.lua -o out --target <x86_64|aarch64|riscv64> .\n- Each emitted binary must be a standalone native ELF for the requested target that reproduces stock Lua 5.4's stdout byte for byte and exits 0, not an interpreter that reads embedded bytecode at runtime.\n- Emit textual assembly and assemble/link it (as /ld and their aarch64/riscv64 cross variants) or construct the ELF directly. A C compiler may be used only as a link driver over object files; handing it generated C source is a disclosed boundary violation.\n- Emitted binaries link the provided per-target runtime archive /reference/lua-src/<arch>/liblua-runtime.a (GC, tables, strings, metamethods, coroutines, stdlib, VM helpers); the compiler itself can link liblua-compile.a to parse Lua source into bytecode.\n- The project must build purely from committed Go source; embedding prebuilt object files, archives, or binaries via //go:embed is not allowed.", "task_bullets": ["Each emitted binary must be a standalone native ELF for the requested target that reproduces stock Lua 5.4's stdout byte for byte and exits 0, not an interpreter that reads embedded bytecode at runtime.", "Emit textual assembly and assemble/link it (as /ld and their aarch64/riscv64 cross variants) or construct the ELF directly. A C compiler may be used only as a link driver over object files; handing it generated C source is a disclosed boundary violation.", "Emitted binaries link the provided per-target runtime archive /reference/lua-src/<arch>/liblua-runtime.a (GC, tables, strings, metamethods, coroutines, stdlib, VM helpers); the compiler itself can link liblua-compile.a to parse Lua source into bytecode.", "The project must build purely from committed Go source; embedding prebuilt object files, archives, or binaries via //go:embed is not allowed."], "agent_gets": "A Go scaffold at /app/lua-native-compiler/ and an /app/README.md documenting the CLI, the linking contract, and the per-target toolchain table. Example Lua programs live under /app/tests/programs/ with their exact expected stdout in /app/tests/expected/ , and /app/run-tests.sh compiles and checks every program on every target (cross targets under qemu-<arch>-static ). The image provides Lua 5.4 headers, the host parser archive, the three per-target runtime archives, native and cross binutils, and QEMU user-mode emulators. No Lua interpreter ships in the image: expected outputs were baked at image build and the interpreter deleted. The machine is offline and a sandbox-timer CLI reports the remaining budget.", "verification": "A clean-room verifier, separate from the agent's environment, scans the sources for disclosed-boundary violations (references to verifier internals, or invoking a C compiler on C source), reconstructs the project while dropping any compiled objects, archives, or ELF files, rebuilds it from Go source as a non-root user, and then scores a hidden corpus: perturbed twins of the public test chunks, each mutated and its expected output re-baked with the real reference interpreter at image build, so memorising the visible corpus does not transfer.\n- The reward is passed over attempted across (program × target) cells: every scored chunk is compiled and run for each of the three architectures, so a compiler that only handles one tops out near a third of the score. Per-invocation compile and run caps are 60 seconds each under a 9000-second suite deadline.\n- Each emitted binary is snapshotted to a root-owned immutable copy before checking and running, so the inspected bytes are exactly the executed bytes; runs go through a hidden ptrace no-exec launcher that kills any binary attempting to exec a second program.\n- Structural gates reject smuggled interpreters: embedded precompiled Lua chunks, real parser/code-generator/loader symbols beyond the provided error stubs, an oversized luaV_execute , or use of the C embedding API beyond one-time init. Build failures, a missing go.mod , and reset or scan violations zero the result.", "verification_bullets": ["The reward is passed over attempted across (program × target) cells: every scored chunk is compiled and run for each of the three architectures, so a compiler that only handles one tops out near a third of the score. Per-invocation compile and run caps are 60 seconds each under a 9000-second suite deadline.", "Each emitted binary is snapshotted to a root-owned immutable copy before checking and running, so the inspected bytes are exactly the executed bytes; runs go through a hidden ptrace no-exec launcher that kills any binary attempting to exec a second program.", "Structural gates reject smuggled interpreters: embedded precompiled Lua chunks, real parser/code-generator/loader symbols beyond the provided error stubs, an oversized luaV_execute , or use of the C embedding API beyond one-time init. Build failures, a missing go.mod , and reset or scan violations zero the result."], "safeguards": ["Structural gates reject smuggled interpreters: embedded precompiled Lua chunks, real parser/code-generator/loader symbols beyond the provided error stubs, an oversized luaV_execute , or use of the C embedding API beyond one-time init. Build failures, a missing go.mod , and reset or scan violations zero the result."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Go, QEMU, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 32 GB RAM", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of Lua Native Compiler from FrontierSWE V1.\n- The compiler is now multi-target: V1 scored x86-64 only, while this QEMU-based revision compiles and runs every program for x86-64, aarch64, and riscv64 (cross targets under QEMU user-mode emulation) and scores (program × target) cells; a single-architecture compiler tops out near a third of the reward.\n- The test corpus changed from a bespoke hidden suite of authored programs to the official Lua 5.4.7 test suite, split into self-contained chunks and bake-filtered against the reference. The full corpus is now agent-visible with expected outputs, and grading uses root-only perturbed twins whose expected outputs are re-baked by the real interpreter: anti-memorisation by mutation instead of hiding the tests.\n- The reference interpreter no longer ships: V1 gave the agent /reference/lua and luac during the run and deleted them before verification; V2 bakes expected outputs at image build and removes the interpreter from the image entirely.\n- The build contract tightened from five accepted build systems (or even a committed pre-built luanatc binary) to a single Go module rebuilt from committed source after a reset that drops all compiled objects, archives, and ELF files.\n- Anti-cheat moved from static heuristics to runtime enforcement: a hidden ptrace no-exec launcher kills emitted binaries that exec a second program, every binary is snapshotted to a root-owned immutable copy so the checked and executed bytes cannot diverge, and symbol-size gates verify the parser/VM-engine stubs stayed stubs. The C-compiler scan was also refined to reject only compiling C source, no longer false-flagging gcc used as a link driver.\n- Infrastructure standardized: a clean-room separate verifier with hash-pinned agent and verifier images, the agent budget grown from 8 to 20 hours, the verifier budget from 1 hour to 12000 seconds to cover three targets under emulation, and the standard sandbox-timer CLI replacing /app/.timer files.", "references": [{"title": "Lua (MIT): Lua 5.4.7, fetched at image build from the official distribution (nothing vendored). The image ships the public and internal headers and two derived static libraries (liblua-compile.a , liblua-runtime.a ) compiled from upstream sources with task-authored stubs replacing the dispatch loop, parser, and bytecode loader. The built interpreter is used only at image build to bake expected outputs, then deleted; no interpreter binary ships.", "url": "https://www.lua.org"}, {"title": "Lua 5.4 test suite (MIT): The official Lua 5.4 behavioural suite (version-matched to the runtime), fetched at image build. Its files are adapted to run standalone, split into small self-contained chunks, and bake-filtered against the real reference; the whole corpus ships to the agent un-mutated, while a perturbed twin of every chunk (the graded material) is held root-only.", "url": "https://www.lua.org/tests/"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.9026, "std": 0.148, "steps": "237", "in_tok": "80.4M", "out_tok": "420k", "avg_cost": "$50.45", "avg_time": "3.8h"}, {"rank": 2, "model": "GLM-5.3", "score": 0.7811, "std": 0.437, "steps": "1,778", "in_tok": "808.0M", "out_tok": "1.3M", "avg_cost": "$221.13", "avg_time": "15.6h"}, {"rank": 3, "model": "Gemini 3.7 Flash", "score": 0.5943, "std": 0.151, "steps": "984", "in_tok": "396.0M", "out_tok": "1.0M", "avg_cost": "$43.73", "avg_time": "4.8h"}, {"rank": 4, "model": "Kimi K3", "score": 0.525, "std": 0.401, "steps": "1,262", "in_tok": "560.6M", "out_tok": "1.3M", "avg_cost": "$195.62", "avg_time": "19.7h"}, {"rank": 5, "model": "DeepSeek V4 Flash Exp", "score": 0.2358, "std": 0.424, "steps": "2,475", "in_tok": "1151.0M", "out_tok": "2.1M", "avg_cost": "$19.30", "avg_time": "13.0h"}, {"rank": 6, "model": "Grok 4.6", "score": 0.1277, "std": 0.285, "steps": "1,113", "in_tok": "288.4M", "out_tok": "2.1M", "avg_cost": "$297.13", "avg_time": "10.7h"}, {"rank": 7, "model": "GPT-5.6", "score": 0.0, "std": 0.0, "steps": "224", "in_tok": "50.9M", "out_tok": "157k", "avg_cost": "$43.12", "avg_time": "1.5h"}, {"rank": 8, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "257", "in_tok": "24.1M", "out_tok": "74k", "avg_cost": "$23.20", "avg_time": "0.8h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0, "std": 0.0, "steps": "237", "in_tok": "71.5M", "out_tok": "370k", "avg_cost": "$18.96", "avg_time": "2.5h"}, {"rank": 10, "model": "Qwen3.8-Max", "score": 0.0, "std": 0.0, "steps": "740", "in_tok": "278.2M", "out_tok": "2.0M", "avg_cost": "$82.46", "avg_time": "18.3h"}], "best_score": 0.9026, "mean_score": 0.3166, "n_models": 10}, {"slug": "materials-ml-potential", "number": 16, "title": "Machine-Learned Interatomic Potential", "url": "https://www.frontierswe.com/tasks/materials-ml-potential", "categories": ["Scientific Computing"], "summary": "Train one interatomic potential that predicts DFT energies and forces for periodic structures, scored on hidden out-of-distribution compositions.", "scoring": "Skill vs baseline", "toolchains": ["CUDA", "Python"], "compute": {"label": "1× GPU (L4, A10G, L40S, or H100)", "kind": "gpu", "compute": "8 CPUs · 32 GB RAM", "gpu": "1× GPU (L4, A10G, L40S, or H100)", "base_image": "nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04", "tools": "CUDA, PyTorch, Python 3", "time_limit": "20h"}, "background": "Density functional theory (DFT) gives accurate energies and forces for atomic structures but costs hours per calculation. Machine-learned interatomic potentials replace it with a model that is orders of magnitude faster, which is what makes large-scale materials screening and molecular dynamics tractable. Building one is genuinely hard: the model must respect periodic boundary conditions, handle variable numbers of atoms across dozens of chemical elements, keep forces consistent with the energy surface, and (the real test) extrapolate to compositions and structure types it never trained on.\nThe structures and DFT labels derive from Meta FAIR Chemistry's Open Materials 2024 (OMat24) dataset of inorganic materials, with split preparation original to the benchmark so the scored structures cannot be recalled from public data.", "task": "Build one deployable model that predicts the total energy (eV) and per-atom forces (eV/Å) of periodic atomic structures at DFT accuracy, and keep everything required for replay under /app/materials_model/ (checkpoint, model.py , predict.py , run_summary.json ).\n- Inference runs as python /app/materials_model/predict.py --data-dir … --checkpoint … --output-path … and writes one JSONL object per structure with structure_id , total energy , and forces of exactly n_atoms rows of three finite numbers in input atom order.\n- The model must respect cell and pbc , and predictions must respond to changes in atomic geometry: no external services or hardcoded per-structure lookups.\n- One checkpoint of at most 2.5 GB; batch inference within 30 minutes; repeated inference must agree within an absolute tolerance of 1e-4.", "task_bullets": ["Inference runs as python /app/materials_model/predict.py --data-dir … --checkpoint … --output-path … and writes one JSONL object per structure with structure_id , total energy , and forces of exactly n_atoms rows of three finite numbers in input atom order.", "The model must respect cell and pbc , and predictions must respond to changes in atomic geometry: no external services or hardcoded per-structure lookups.", "One checkpoint of at most 2.5 GB; batch inference within 30 minutes; repeated inference must agree within an absolute tolerance of 1e-4."], "agent_gets": "Labeled training data at /data/train/structures.parquet (each row carries structure_id , n_atoms , atomic_numbers , positions , cell , and pbc , plus DFT energy and per-atom forces ), along with unlabeled validation inputs at /data/validation/structures.parquet and /data/metadata.json , all staged read-only from digest-pinned dataset images.\n- A runnable starter submission under /app/materials_model/ exercising the full inference contract.\n- A CUDA 12.4 stack with PyTorch 2.6 (cu124), ASE, NumPy, pandas, pyarrow, scikit-learn, and SciPy preinstalled.\n- The machine is offline; everything needed is already present.", "verification": "A root-only verifier reruns the submitted predict.py on hidden structures the agent never sees, grouped into workloads: the scored ones are out-of-distribution compositions and out-of-distribution bulk structures, weighted equally. For each workload the energy MAE (meV/atom) and force MAE (meV/Å) are converted to skill scores (0 at a non-trivial baseline anchor, 0.9 at the measured reference-solution anchor, 1 at zero error), and the reward is the weighted geometric mean of all skills.\n- A beat-the-baseline gate: if either energy or force MAE fails to improve on the baseline anchor in either primary workload, the reward is zero.\n- Safeguard probes: a determinism re-run must agree within 1e-4, and a geometry-perturbation probe must change predictions for at least half the perturbed structures, rejecting constant or lookup-table submissions. Network attempts during inference are audited via strace, and label columns are scrubbed from the inputs the model receives.\n- Contract failures (malformed rows, wrong force shapes, non-finite values, an oversized checkpoint, or exceeding the 30-minute inference timeout) zero the result.", "verification_bullets": ["A beat-the-baseline gate: if either energy or force MAE fails to improve on the baseline anchor in either primary workload, the reward is zero.", "Safeguard probes: a determinism re-run must agree within 1e-4, and a geometry-perturbation probe must change predictions for at least half the perturbed structures, rejecting constant or lookup-table submissions. Network attempts during inference are audited via strace, and label columns are scrubbed from the inputs the model receives.", "Contract failures (malformed rows, wrong force shapes, non-finite values, an oversized checkpoint, or exceeding the 30-minute inference timeout) zero the result."], "safeguards": ["A beat-the-baseline gate: if either energy or force MAE fails to improve on the baseline anchor in either primary workload, the reward is zero.", "Safeguard probes: a determinism re-run must agree within 1e-4, and a geometry-perturbation probe must change predictions for at least half the perturbed structures, rejecting constant or lookup-table submissions. Network attempts during inference are audited via strace, and label columns are scrubbed from the inputs the model receives.", "Contract failures (malformed rows, wrong force shapes, non-finite values, an oversized checkpoint, or exceeding the 30-minute inference timeout) zero the result."], "environment": {"Base image": "nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04", "Tools available": "CUDA, PyTorch, Python 3", "Compute": "8 CPUs · 32 GB RAM", "GPU": "1× GPU (L4, A10G, L40S, or H100)", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "Open Materials 2024 (OMat24) (CC BY 4.0): Structures and DFT energy/force labels, from Meta FAIR Chemistry's fairchem/OMAT24 dataset (Barroso-Luque et al., 2024). Split preparation, task scaffolding, reference models, and verification code are original to this benchmark.", "url": "https://arxiv.org/abs/2410.12771"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.9023, "std": 0.053, "steps": "173", "in_tok": "20.2M", "out_tok": "132k", "avg_cost": "$13.33", "avg_time": "17.3h"}, {"rank": 2, "model": "GLM-5.3", "score": 0.8944, "std": 0.09, "steps": "320", "in_tok": "51.1M", "out_tok": "193k", "avg_cost": "$22.78", "avg_time": "19.1h"}, {"rank": 3, "model": "Kimi K3", "score": 0.8557, "std": 0.082, "steps": "157", "in_tok": "17.1M", "out_tok": "149k", "avg_cost": "$11.61", "avg_time": "19.1h"}, {"rank": 4, "model": "Muse Spark 1.2", "score": 0.811, "std": 0.07, "steps": "207", "in_tok": "44.9M", "out_tok": "215k", "avg_cost": "$9.21", "avg_time": "2.0h"}, {"rank": 5, "model": "Grok 4.6", "score": 0.7812, "std": 0.136, "steps": "228", "in_tok": "39.9M", "out_tok": "257k", "avg_cost": "$54.01", "avg_time": "19.5h"}, {"rank": 6, "model": "GPT-5.6", "score": 0.5873, "std": 0.533, "steps": "313", "in_tok": "74.3M", "out_tok": "128k", "avg_cost": "$65.94", "avg_time": "12.6h"}, {"rank": 7, "model": "Qwen3.8-Max", "score": 0.5825, "std": 0.447, "steps": "206", "in_tok": "15.3M", "out_tok": "222k", "avg_cost": "$5.36", "avg_time": "18.7h"}, {"rank": 8, "model": "Gemini 3.7 Flash", "score": 0.5007, "std": 0.326, "steps": "189", "in_tok": "26.0M", "out_tok": "156k", "avg_cost": "$8.38", "avg_time": "6.2h"}, {"rank": 9, "model": "DeepSeek V4 Flash Exp", "score": 0.4758, "std": 0.343, "steps": "423", "in_tok": "106.5M", "out_tok": "316k", "avg_cost": "$1.96", "avg_time": "18.7h"}, {"rank": 10, "model": "Inkling", "score": 0.034, "std": 0.018, "steps": "144", "in_tok": "5.2M", "out_tok": "47k", "avg_cost": "$5.11", "avg_time": "1.6h"}], "best_score": 0.9023, "mean_score": 0.6425, "n_models": 10}, {"slug": "medium-range-weather-forecast", "number": 17, "title": "Medium-Range Weather Forecast", "url": "https://www.frontierswe.com/tasks/medium-range-weather-forecast", "categories": ["Scientific Computing"], "summary": "Train a deterministic global weather model that forecasts nine atmospheric channels out to ten days from a single gridded initial state.", "scoring": "Skill vs baseline", "toolchains": ["CUDA", "Python"], "compute": {"label": "1× L4", "kind": "gpu", "compute": "8 CPUs · 32 GB RAM", "gpu": "1× L4", "base_image": "nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04", "tools": "CUDA, PyTorch, Python 3", "time_limit": "20h"}, "background": "Medium-range weather forecasting — predicting the global atmosphere out to ten days — was long the exclusive domain of physics-based numerical models running on supercomputers. ML-based forecasting has recently become competitive, but building such a model from scratch is genuinely hard: the atmosphere is chaotic, so skill decays rapidly with lead time; the model must stay physically coherent across a full latitude–longitude grid; and small errors compound when a model is rolled forward through many autoregressive steps.\nThis task distills the problem into the WeatherBench-2 evaluation conventions on ERA5 reanalysis data: from one gridded initial state, forecast nine standard channels — geopotential and temperature at pressure levels, surface temperature and winds, sea-level pressure, and humidity — at twenty lead times from 12 to 240 hours, on a single mid-range GPU and a fixed time budget.", "task": "Build a deterministic global weather model that forecasts all required atmospheric channels and lead times from a gridded initial state, and package it as a replayable deliverable at /app/weather_model/ containing predict.py , model.py , run_summary.json , and exactly one checkpoint file under checkpoint/ .\n- predict.py is invoked with --data-dir , --checkpoint , and --output-path , and must write one NPZ archive with exactly four arrays: init_times , lead_hours , channels , and finite float32 predictions of shape (N, 20, 9, lat, lon).\n- The nine channels (z500 , z850 , t500 , t850 , t2m , u10 , v10 , msl , q700 ) and the twenty 12-hourly lead times from 12 to 240 hours are fixed by the contract.\n- Forecasts must be deterministic and use no internet or external services; a full batch must complete within 30 minutes and a small batch within 10 minutes.\n- The complete weather_model directory must be smaller than 2.5 GB and contain no symbolic links.\n- Validation RMSE on the provided splits is the development metric; /app/validate_forecast.py checks an archive against the output contract locally.", "task_bullets": ["predict.py is invoked with --data-dir , --checkpoint , and --output-path , and must write one NPZ archive with exactly four arrays: init_times , lead_hours , channels , and finite float32 predictions of shape (N, 20, 9, lat, lon).", "The nine channels (z500 , z850 , t500 , t850 , t2m , u10 , v10 , msl , q700 ) and the twenty 12-hourly lead times from 12 to 240 hours are fixed by the contract.", "Forecasts must be deterministic and use no internet or external services; a full batch must complete within 30 minutes and a small batch within 10 minutes.", "The complete weather_model directory must be smaller than 2.5 GB and contain no symbolic links.", "Validation RMSE on the provided splits is the development metric; /app/validate_forecast.py checks an archive against the output contract locally."], "agent_gets": "ERA5-derived reanalysis data under /data in training and validation splits: each split has an init_index.parquet of initialization times, an init_states.zarr of gridded atmospheric states with axis order (channel, lat, lon), and targets.npz with the matching future states; metadata.json defines the grid and channel metadata. A starter scaffold lives at /app/weather_model/ alongside the contract documentation and the local forecast validator. The preinstalled pinned stack includes torch 2.6.0 (CUDA), numpy , pandas , zarr , xarray , scipy , and pyarrow , with one L4 GPU for training. The machine is offline; a sandbox-timer CLI reports the remaining budget.", "verification": "A root-only clean-room verifier evaluates the submission on sealed, held-out ERA5 campaigns — multiple later temporal blocks never seen by the agent, integrity-checked file-by-file against a signed manifest. It freezes the deliverable read-only, revokes the agent user's access to the visible labeled data, and runs predict.py as the unprivileged agent user three times: a primary full batch (30-minute cap), a deterministic replay of a sealed probe subset (10-minute cap) that must match the primary forecasts within a strict float32 tolerance, and a physically perturbed probe whose forecasts must actually change — defeating lookup tables and input-ignoring models.\n- Skill is measured per channel and lead cohort (early 12–72 h, middle 84–168 h, late 180–240 h) using latitude-weighted anomaly correlation (ACC) against climatology and latitude-weighted RMSE.\n- Each metric is normalized against sealed anchors — a baseline maps to 0, a calibrated reference forecast to 0.5, and the ideal to 1 — then combined as a weighted mean across fields and campaigns.\n- Contract and safeguard gates are binary: any output-format violation, nondeterministic replay, insensitivity to perturbations, oversized deliverable, or failed run zeroes the result.", "verification_bullets": ["Skill is measured per channel and lead cohort (early 12–72 h, middle 84–168 h, late 180–240 h) using latitude-weighted anomaly correlation (ACC) against climatology and latitude-weighted RMSE.", "Each metric is normalized against sealed anchors — a baseline maps to 0, a calibrated reference forecast to 0.5, and the ideal to 1 — then combined as a weighted mean across fields and campaigns.", "Contract and safeguard gates are binary: any output-format violation, nondeterministic replay, insensitivity to perturbations, oversized deliverable, or failed run zeroes the result."], "safeguards": ["Contract and safeguard gates are binary: any output-format violation, nondeterministic replay, insensitivity to perturbations, oversized deliverable, or failed run zeroes the result."], "environment": {"Base image": "nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04", "Tools available": "CUDA, PyTorch, Python 3", "Compute": "8 CPUs · 32 GB RAM", "GPU": "1× L4", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "ERA5 (Copernicus Climate Data Store) (CC-BY): The reanalysis fields are derived from the CDS ERA5 pressure-level and single-level datasets (Copernicus Climate Change Service, 2017: ERA5, fifth generation of ECMWF atmospheric reanalyses of the global climate).", "url": "https://cds.climate.copernicus.eu/datasets/reanalysis-era5-pressure-levels"}, {"title": "WeatherBench 2 (Apache-2.0): The data layout and evaluation conventions follow WeatherBench 2; the task reimplements its task-specific metrics and copies no WeatherBench 2 source code. Temporal split preparation, task models, and verification code are original to this benchmark.", "url": "https://github.com/google-research/weatherbench2"}], "results": [{"rank": 1, "model": "Gemini 3.7 Flash", "score": 0.5473, "std": 0.018, "steps": "243", "in_tok": "42.3M", "out_tok": "152k", "avg_cost": "$15.28", "avg_time": "8.5h"}, {"rank": 2, "model": "Grok 4.6", "score": 0.5101, "std": 0.072, "steps": "539", "in_tok": "124.2M", "out_tok": "498k", "avg_cost": "$140.51", "avg_time": "19.6h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.4968, "std": 0.051, "steps": "323", "in_tok": "68.3M", "out_tok": "243k", "avg_cost": "$23.54", "avg_time": "18.8h"}, {"rank": 4, "model": "Claude Fable 5.1", "score": 0.4873, "std": 0.273, "steps": "195", "in_tok": "24.2M", "out_tok": "122k", "avg_cost": "$15.05", "avg_time": "16.4h"}, {"rank": 5, "model": "Qwen3.8-Max", "score": 0.3674, "std": 0.031, "steps": "292", "in_tok": "32.6M", "out_tok": "254k", "avg_cost": "$9.98", "avg_time": "19.1h"}, {"rank": 6, "model": "Muse Spark 1.2", "score": 0.3081, "std": 0.027, "steps": "237", "in_tok": "71.8M", "out_tok": "242k", "avg_cost": "$13.53", "avg_time": "1.5h"}, {"rank": 7, "model": "Inkling", "score": 0.1566, "std": 0.099, "steps": "119", "in_tok": "4.5M", "out_tok": "43k", "avg_cost": "$4.47", "avg_time": "1.5h"}, {"rank": 8, "model": "GPT-5.6", "score": 0.1187, "std": 0.265, "steps": "413", "in_tok": "90.0M", "out_tok": "156k", "avg_cost": "$75.26", "avg_time": "16.3h"}, {"rank": 9, "model": "DeepSeek V4 Flash Exp", "score": 0.0928, "std": 0.207, "steps": "428", "in_tok": "92.3M", "out_tok": "257k", "avg_cost": "$1.68", "avg_time": "19.0h"}, {"rank": 10, "model": "Kimi K3", "score": 0.0, "std": 0.0, "steps": "284", "in_tok": "40.2M", "out_tok": "191k", "avg_cost": "$15.94", "avg_time": "19.4h"}], "best_score": 0.5473, "mean_score": 0.3085, "n_models": 10}, {"slug": "meg-speech-decoding", "number": 18, "title": "MEG Speech Decoding", "url": "https://www.frontierswe.com/tasks/meg-speech-decoding", "categories": ["Scientific Computing"], "summary": "Build a neural decoder that predicts which word a listener heard from raw MEG brain recordings.", "scoring": "Hidden tests", "toolchains": ["CUDA", "Python", "C/C++"], "compute": {"label": "1× GPU (L4, A10G, L40S, or H100)", "kind": "gpu", "compute": "8 CPUs · 64 GB RAM", "gpu": "1× GPU (L4, A10G, L40S, or H100)", "base_image": "nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04", "tools": "CUDA, PyTorch, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Magnetoencephalography (MEG) records the magnetic fields produced by neural activity with millisecond resolution, and decoding heard speech from those recordings is a frontier problem in neuro-AI. The signal is faint and noisy, hundreds of sensor channels mix overlapping neural sources, and word-evoked responses vary across recordings — so a decoder that memorizes session quirks collapses on held-out data.\nThis task uses continuous LibriBrain recordings aligned to audiobook speech: given a recording and the sample index where a word was heard, the decoder must rank the vocabulary by how likely each word is to be the one the listener heard. The design space is deliberately open — from classical feature pipelines to end-to-end networks or adapting pretrained speech and MEG foundation models — but the result must survive sealed evaluation slices that stress rare words and long-duration generalization.", "task": "Build a word decoder and keep the complete replayable deliverable under /app/meg_decoder/ : predict.py , model.py , run_summary.json , and a trained checkpoint/ .\n- predict.py is invoked with --data-dir , --checkpoint , and --output-path , reads MEG recordings, event metadata, sensor metadata, and the vocabulary, and writes one JSONL row per input example_id .\n- Each row must contain a deterministic ranked list of at least ten unique vocabulary IDs (word_ids ); duplicate or out-of-vocabulary IDs invalidate the row.\n- A full inference batch must finish within 30 minutes without internet or external services.\n- Rankings must materially respond to the supplied MEG signals rather than relying on event identifiers or fixed vocabulary priors — a counterfactual probe enforces this.\n- The complete /app/meg_decoder/ directory must be no larger than 2.5 GB.", "task_bullets": ["predict.py is invoked with --data-dir , --checkpoint , and --output-path , reads MEG recordings, event metadata, sensor metadata, and the vocabulary, and writes one JSONL row per input example_id .", "Each row must contain a deterministic ranked list of at least ten unique vocabulary IDs (word_ids ); duplicate or out-of-vocabulary IDs invalidate the row.", "A full inference batch must finish within 30 minutes without internet or external services.", "Rankings must materially respond to the supplied MEG signals rather than relying on event identifiers or fixed vocabulary priors — a counterfactual probe enforces this.", "The complete /app/meg_decoder/ directory must be no larger than 2.5 GB."], "agent_gets": "Labeled LibriBrain development data under /data : training and validation splits with continuous MEG arrays in recordings.zarr , word-onset events with labels in events.parquet , sensor metadata, the complete vocabulary.json , and — for the training split only — the aligned stimulus audio. Optional pretrained checkpoints are staged offline under /models : the facebook/w2v-bert-2.0 speech encoder, the pnpl/MEG-XL MEG foundation model, and google-t5/t5-large . The pinned Python stack includes torch and torchaudio 2.6.0 (CUDA), transformers , mne , librosa , zarr , pandas , and scikit-learn , with one GPU available. A fail-closed scaffold sits at /app/meg_decoder/ and /app/validate_predictions.py checks outputs locally. The machine is offline; a sandbox-timer CLI reports the remaining budget.", "verification": "A clean-room verifier evaluates a read-only snapshot of /app/meg_decoder/ on sealed hidden data, stripping every label-bearing column from the events it hands to the decoder and locking /data , /models , and the workspace to root during replay. Inference runs as the unprivileged agent user with a 30-minute cap on the full batch.\n- Quality is scored per workload — held-out recordings, recording shift, rare words, and long duration (weighted 0.4/0.2/0.2/0.2) — using a chance-adjusted composite of macro top-10 accuracy (60%), macro MRR (25%), and macro top-1 accuracy (15%), combined across workloads by a smoothed geometric mean and calibrated onto the final reward scale.\n- A determinism probe reruns the full batch and requires identical top-10 rankings.\n- A signal-counterfactual probe time-shifts the real MEG while preserving event IDs and onsets; at least 50% of each top-10 ranking must change, defeating decoders that ignore the signal.\n- Contract and safeguard gates are binary: format violations, missing or duplicate predictions, a failed or nondeterministic replay, insufficient signal turnover, or an oversized deliverable zero the result.", "verification_bullets": ["Quality is scored per workload — held-out recordings, recording shift, rare words, and long duration (weighted 0.4/0.2/0.2/0.2) — using a chance-adjusted composite of macro top-10 accuracy (60%), macro MRR (25%), and macro top-1 accuracy (15%), combined across workloads by a smoothed geometric mean and calibrated onto the final reward scale.", "A determinism probe reruns the full batch and requires identical top-10 rankings.", "A signal-counterfactual probe time-shifts the real MEG while preserving event IDs and onsets; at least 50% of each top-10 ranking must change, defeating decoders that ignore the signal.", "Contract and safeguard gates are binary: format violations, missing or duplicate predictions, a failed or nondeterministic replay, insufficient signal turnover, or an oversized deliverable zero the result."], "safeguards": ["Quality is scored per workload — held-out recordings, recording shift, rare words, and long duration (weighted 0.4/0.2/0.2/0.2) — using a chance-adjusted composite of macro top-10 accuracy (60%), macro MRR (25%), and macro top-1 accuracy (15%), combined across workloads by a smoothed geometric mean and calibrated onto the final reward scale.", "A determinism probe reruns the full batch and requires identical top-10 rankings.", "Contract and safeguard gates are binary: format violations, missing or duplicate predictions, a failed or nondeterministic replay, insufficient signal turnover, or an oversized deliverable zero the result."], "environment": {"Base image": "nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04", "Tools available": "CUDA, PyTorch, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 64 GB RAM", "GPU": "1× GPU (L4, A10G, L40S, or H100)", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "LibriBrain (CC BY-NC 4.0): Human MEG recordings, event annotations, sensor metadata, and aligned stimulus audio are derived from this dataset; usage retains its participant-data and citation requirements. Split preparation, decoding scaffolding, and verification code are original to this benchmark.", "url": "https://huggingface.co/datasets/pnpl/LibriBrain"}, {"title": "w2v-bert-2.0 (MIT): Optional offline speech encoder staged under /models .", "url": "https://huggingface.co/facebook/w2v-bert-2.0"}, {"title": "MEG-XL (MIT): Optional MEG foundation-model checkpoint staged under /models .", "url": "https://huggingface.co/pnpl/MEG-XL"}, {"title": "T5-Large (Apache-2.0): Optional text-decoder checkpoint staged under /models .", "url": "https://huggingface.co/google-t5/t5-large"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.2924, "std": 0.043, "steps": "219", "in_tok": "32.5M", "out_tok": "174k", "avg_cost": "$20.47", "avg_time": "15.8h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.2447, "std": 0.06, "steps": "249", "in_tok": "71.4M", "out_tok": "145k", "avg_cost": "$75.64", "avg_time": "6.7h"}, {"rank": 3, "model": "Grok 4.6", "score": 0.21, "std": 0.065, "steps": "535", "in_tok": "121.4M", "out_tok": "713k", "avg_cost": "$129.11", "avg_time": "16.6h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.1361, "std": 0.09, "steps": "378", "in_tok": "85.6M", "out_tok": "270k", "avg_cost": "$33.74", "avg_time": "17.4h"}, {"rank": 5, "model": "DeepSeek V4 Flash Exp", "score": 0.1288, "std": 0.065, "steps": "410", "in_tok": "81.1M", "out_tok": "256k", "avg_cost": "$1.53", "avg_time": "15.5h"}, {"rank": 6, "model": "Kimi K3", "score": 0.1102, "std": null, "steps": "256", "in_tok": "32.2M", "out_tok": "173k", "avg_cost": "$14.90", "avg_time": "18.4h"}, {"rank": 7, "model": "Gemini 3.7 Flash", "score": 0.0811, "std": 0.047, "steps": "280", "in_tok": "41.5M", "out_tok": "130k", "avg_cost": "$9.85", "avg_time": "6.9h"}, {"rank": 8, "model": "Muse Spark 1.2", "score": 0.0769, "std": 0.074, "steps": "202", "in_tok": "59.3M", "out_tok": "260k", "avg_cost": "$13.61", "avg_time": "3.8h"}, {"rank": 9, "model": "Qwen3.8-Max", "score": 0.0489, "std": 0.069, "steps": "385", "in_tok": "57.1M", "out_tok": "348k", "avg_cost": "$16.80", "avg_time": "19.5h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "136", "in_tok": "5.6M", "out_tok": "31k", "avg_cost": "$5.45", "avg_time": "1.7h"}], "best_score": 0.2924, "mean_score": 0.1329, "n_models": 10}, {"slug": "msms-denovo-generation", "number": 19, "title": "MS/MS De Novo Generation", "url": "https://www.frontierswe.com/tasks/msms-denovo-generation", "categories": ["Scientific Computing"], "summary": "Train a model that proposes molecular structures de novo from tandem mass spectra, scored on hidden spectra with unseen scaffolds.", "scoring": "Hidden tests", "toolchains": ["CUDA", "Python"], "compute": {"label": "1× L4", "kind": "gpu", "compute": "8 CPUs · 32 GB RAM", "gpu": "1× L4", "base_image": "nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04", "tools": "CUDA, PyTorch, Python 3", "time_limit": "20h"}, "background": "Tandem mass spectrometry (MS/MS) fragments an ionized molecule and records the mass-to-charge ratios and intensities of the pieces. Reading the molecular structure back out of that fragment pattern (de novo structure elucidation) is a long-standing open problem in computational chemistry: the mapping from structure to spectrum is lossy and instrument-dependent, many distinct molecules share a molecular formula, and fragmentation chemistry is only partially understood. Models that work on molecules seen during training routinely fail on new chemical scaffolds.\nThe task data derives from the MassSpecGym benchmark, prepared with a scaffold-disjoint split so the hidden evaluation spectra come from molecular structures the agent's model has never seen. The verifier additionally runs a permutation test to confirm the model actually uses the spectrum rather than just the molecular formula.", "task": "Build a deployable model that predicts best-first candidate SMILES from each tandem mass spectrum and its supplied metadata, and leave the finished deployment under /app/msms_model/ . The entrypoint must run as python /app/msms_model/predict.py --data-dir … --checkpoint … --output-path … and write one JSONL row per input spectrum.\n- Each row carries 1–10 candidate SMILES ordered best-first. Invalid or formula-incompatible candidates score zero at their rank and never promote later candidates.\n- Candidates should be chemically valid and compatible with the supplied molecular formula (heavy-atom counts must match; hydrogen is ignored to avoid adduct artifacts).\n- Output must be deterministic for the same input and checkpoint, and predictions should use the measured mzs and intensities together with the metadata, not the formula alone.\n- The deployment must be self-contained and replayable offline: the checkpoint may occupy at most 2.5 GB and a prediction batch must finish within 30 minutes.", "task_bullets": ["Each row carries 1–10 candidate SMILES ordered best-first. Invalid or formula-incompatible candidates score zero at their rank and never promote later candidates.", "Candidates should be chemically valid and compatible with the supplied molecular formula (heavy-atom counts must match; hydrogen is ignored to avoid adduct artifacts).", "Output must be deterministic for the same input and checkpoint, and predictions should use the measured mzs and intensities together with the metadata, not the formula alone.", "The deployment must be self-contained and replayable offline: the checkpoint may occupy at most 2.5 GB and a prediction batch must finish within 30 minutes."], "agent_gets": "Labeled training and validation data staged read-only under /data/train/ and /data/validation/ as parquet pairs (spectra.parquet , labels.parquet ). Each spectrum row carries an opaque spectrum_id , precursor_mz , adduct , instrument , collision_energy , the molecular formula , and parallel fragment arrays; labels map IDs to canonical SMILES.\n- A starter workspace at /app with neutral data readers and contract validators in /app/msms_io and a runnable skeleton deployment in /app/msms_model .\n- A CUDA 12.4 Python stack preinstalled from a hash-pinned lock: PyTorch 2.6, RDKit, matchms, scikit-learn, SciPy, pandas, and pyarrow.\n- The machine is offline (Hugging Face and Transformers caches are forced offline); everything needed is already present.", "verification": "A root-only verifier reruns the submitted predict.py against hidden spectra that were never visible to the agent, then scores the candidates against hidden labels with RDKit. The reward is a weighted blend of structural metrics macro-averaged over connectivity classes (first InChIKey block): top-1 Tanimoto similarity (0.35), rank-discounted Tanimoto (0.25), top-1 exact-structure accuracy (0.20), top-10 accuracy (0.10), and top-10 mean reciprocal rank (0.10). Only formula-compatible candidates earn similarity credit.\n- Five prediction runs are captured: the primary hidden set, two determinism runs on a fixed subset that must agree exactly, and noise/swap probes with corrupted or shuffled fragment peaks.\n- A spectral-advantage permutation test (999 permutations) conditionally permutes predictions between structures sharing the same formula; if the model shows no statistically significant association with the actual spectra (p ≤ 0.10 with a minimum effect size), the reward is zero.\n- Zeroed outright: a missing or oversized (>2.5 GB) checkpoint, candidate-run failures or timeouts, fewer than 50% chemically valid candidates, nondeterministic output, missing or duplicate predictions, and submission code that references scoring internals.", "verification_bullets": ["Five prediction runs are captured: the primary hidden set, two determinism runs on a fixed subset that must agree exactly, and noise/swap probes with corrupted or shuffled fragment peaks.", "A spectral-advantage permutation test (999 permutations) conditionally permutes predictions between structures sharing the same formula; if the model shows no statistically significant association with the actual spectra (p ≤ 0.10 with a minimum effect size), the reward is zero.", "Zeroed outright: a missing or oversized (>2.5 GB) checkpoint, candidate-run failures or timeouts, fewer than 50% chemically valid candidates, nondeterministic output, missing or duplicate predictions, and submission code that references scoring internals."], "safeguards": ["Five prediction runs are captured: the primary hidden set, two determinism runs on a fixed subset that must agree exactly, and noise/swap probes with corrupted or shuffled fragment peaks.", "A spectral-advantage permutation test (999 permutations) conditionally permutes predictions between structures sharing the same formula; if the model shows no statistically significant association with the actual spectra (p ≤ 0.10 with a minimum effect size), the reward is zero.", "Zeroed outright: a missing or oversized (>2.5 GB) checkpoint, candidate-run failures or timeouts, fewer than 50% chemically valid candidates, nondeterministic output, missing or duplicate predictions, and submission code that references scoring internals."], "environment": {"Base image": "nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04", "Tools available": "CUDA, PyTorch, Python 3", "Compute": "8 CPUs · 32 GB RAM", "GPU": "1× L4", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "MassSpecGym (MIT): The spectra and molecular labels (dataset release on Polaris Hub; paper: NeurIPS 2024, arXiv:2410.23326). MassSpecGym aggregates spectra from the MoNA, MassBank, and GNPS community libraries, whose attribution metadata is retained in the prepared data manifest. The scaffold-disjoint split preparation, task scaffolding, and verification code are original to this task.", "url": "https://github.com/pluskal-lab/MassSpecGym"}], "results": [{"rank": 1, "model": "Claude Fable 5.1*", "score": 0.0808, "std": 0.074, "steps": "243", "in_tok": "66.0M", "out_tok": "370k", "avg_cost": "$46.47", "avg_time": "17.8h"}, {"rank": 2, "model": "Kimi K3", "score": 0.0658, "std": 0.06, "steps": "290", "in_tok": "53.0M", "out_tok": "248k", "avg_cost": "$23.42", "avg_time": "19.0h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.0605, "std": 0.055, "steps": "390", "in_tok": "102.2M", "out_tok": "353k", "avg_cost": "$37.48", "avg_time": "18.6h"}, {"rank": 4, "model": "Gemini 3.7 Flash", "score": 0.0555, "std": 0.051, "steps": "240", "in_tok": "40.1M", "out_tok": "191k", "avg_cost": "$9.38", "avg_time": "6.5h"}, {"rank": 5, "model": "GPT-5.6", "score": 0.0526, "std": 0.072, "steps": "389", "in_tok": "78.9M", "out_tok": "190k", "avg_cost": "$64.30", "avg_time": "11.5h"}, {"rank": 6, "model": "Grok 4.6", "score": 0.0346, "std": 0.049, "steps": "558", "in_tok": "132.0M", "out_tok": "762k", "avg_cost": "$141.37", "avg_time": "15.5h"}, {"rank": 7, "model": "Qwen3.8-Max", "score": 0.0306, "std": 0.043, "steps": "419", "in_tok": "71.5M", "out_tok": "533k", "avg_cost": "$21.57", "avg_time": "18.9h"}, {"rank": 8, "model": "Muse Spark 1.2", "score": 0.0213, "std": 0.029, "steps": "284", "in_tok": "81.4M", "out_tok": "279k", "avg_cost": "$15.14", "avg_time": "3.7h"}, {"rank": 9, "model": "DeepSeek V4 Flash Exp", "score": 0.0, "std": 0.0, "steps": "481", "in_tok": "138.4M", "out_tok": "393k", "avg_cost": "$2.52", "avg_time": "16.6h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "163", "in_tok": "6.9M", "out_tok": "56k", "avg_cost": "$6.80", "avg_time": "1.3h * Content-filtered by the provider. Scores shown are from Claude Opus 5 runs used as fallback."}], "best_score": 0.0808, "mean_score": 0.0402, "n_models": 10}, {"slug": "multi-gpu-efficient-finetuning", "number": 20, "title": "Multi-GPU Efficient Finetuning", "url": "https://www.frontierswe.com/tasks/multi-gpu-efficient-finetuning", "categories": ["AI Research"], "summary": "Squeeze a math-reasoning PEFT adapter for a frozen Qwen3-14B out of two T4 GPUs and a 20-hour budget.", "scoring": "Hidden tests", "toolchains": ["CUDA", "Python"], "compute": {"label": "2× T4", "kind": "gpu", "compute": "8 CPUs · 64 GB RAM", "gpu": "2× T4", "base_image": "nvidia/cuda:12.4.1-devel-ubuntu22.04", "tools": "CUDA, PyTorch, Python 3", "time_limit": "20h"}, "background": "Finetuning a 14-billion-parameter model on two T4 GPUs is a memory-bound exercise: the frozen base weights alone dwarf a single card's memory, so any training run has to combine parameter-efficient methods, sharding or offloading, and careful precision choices just to fit, and then still make efficient use of both devices inside a fixed wall-clock budget.\nThe target capability is exact-answer mathematical reasoning: competition-style problems where only a correctly boxed final answer counts. The agent must decide how to spend a hard 20-hour budget across data selection, training configuration, and validation, and leave behind a loadable adapter no matter when the clock runs out.", "task": "The agent must improve the exact-answer mathematical reasoning of the provided frozen Qwen3-14B model by producing an offline-trained PEFT adapter. All work happens under /app/math_adapter , and /app/math_adapter/train.sh must be an executable entrypoint that trains and exports the adapter.\n- The exported adapter/adapter_model.safetensors and its native PEFT configuration must load offline against the read-only base model at /models/qwen3-14b .\n- Only native methods supported by PEFT 0.9 are allowed; rank-based methods may use rank at most 64, and the serialized adapter is capped at 300,000,000 tensor elements and 1.25 GiB of safetensors weights.\n- Frozen-base tensors, saved embeddings or other full modules, modules_to_save , custom auto-mapping, pickle-capable weights, and symlinks are all forbidden in the adapter.\n- A run_summary.json with method_name , devices , total_elapsed_seconds , and checkpoint_path must accompany the adapter, and a usable checkpoint must remain if training ends early.\n- Hard-coding problem-specific answers, caching canned responses, and modifying /models or /app/data are prohibited.", "task_bullets": ["The exported adapter/adapter_model.safetensors and its native PEFT configuration must load offline against the read-only base model at /models/qwen3-14b .", "Only native methods supported by PEFT 0.9 are allowed; rank-based methods may use rank at most 64, and the serialized adapter is capped at 300,000,000 tensor elements and 1.25 GiB of safetensors weights.", "Frozen-base tensors, saved embeddings or other full modules, modules_to_save , custom auto-mapping, pickle-capable weights, and symlinks are all forbidden in the adapter.", "A run_summary.json with method_name , devices , total_elapsed_seconds , and checkpoint_path must accompany the adapter, and a usable checkpoint must remain if training ends early.", "Hard-coding problem-specific answers, caching canned responses, and modifying /models or /app/data are prohibited."], "agent_gets": "The frozen Qwen3-14B base model at /models/qwen3-14b (materialized from a pinned Hugging Face revision) and visible training data at /app/data/train.jsonl (math reasoning traces from the DAPO-MATH-17k-oss-reasoning dataset), both read-only. The workspace ships a placeholder train.sh and a README documenting the exact submission contract.\nThe machine is offline (Hugging Face hub access is disabled at the image level); everything needed is preinstalled, including PyTorch with CUDA 12.4, transformers 4.57.6, PEFT 0.9.0, accelerate, bitsandbytes, datasets, and lm-eval. A sandbox-timer CLI reports the remaining wall-clock budget.", "verification": "A separate clean-room verifier, running on different hardware than the agent ever sees, first validates the submission against the bounded adapter contract, then loads the adapter onto the integrity-checked frozen base and evaluates it on a sealed, root-only panel of 60 held-out AIME problems from 2025 and 2026, both past the base model's training cutoff.\n- Each problem gets exactly three sampled generations under a frozen contract (bfloat16, temperature 0.7, top-p 0.95, fixed seed, 16,384 max tokens) and is scored best-of-3 on exact-answer match, using boxed-answer normalization adapted from the lm-evaluation-harness.\n- The score is count-calibrated: the measured frozen-base performance (18 of 60 correct) maps to 0 and a perfect 60 of 60 maps to 1.0, with a constant increment per additional correct answer.\n- Model execution happens in unprivileged evaluation workers sharded across GPUs; only the root parent sees the sealed labels, and evidence files are schema-checked before scoring.\n- Contract violations zero the result: an oversized or non-native adapter, smuggled base tensors or full modules, pickle-capable weights, a missing run_summary.json , or an adapter that fails to load offline.", "verification_bullets": ["Each problem gets exactly three sampled generations under a frozen contract (bfloat16, temperature 0.7, top-p 0.95, fixed seed, 16,384 max tokens) and is scored best-of-3 on exact-answer match, using boxed-answer normalization adapted from the lm-evaluation-harness.", "The score is count-calibrated: the measured frozen-base performance (18 of 60 correct) maps to 0 and a perfect 60 of 60 maps to 1.0, with a constant increment per additional correct answer.", "Model execution happens in unprivileged evaluation workers sharded across GPUs; only the root parent sees the sealed labels, and evidence files are schema-checked before scoring.", "Contract violations zero the result: an oversized or non-native adapter, smuggled base tensors or full modules, pickle-capable weights, a missing run_summary.json , or an adapter that fails to load offline."], "safeguards": ["Contract violations zero the result: an oversized or non-native adapter, smuggled base tensors or full modules, pickle-capable weights, a missing run_summary.json , or an adapter that fails to load offline."], "environment": {"Base image": "nvidia/cuda:12.4.1-devel-ubuntu22.04", "Tools available": "CUDA, PyTorch, Python 3", "Compute": "8 CPUs · 64 GB RAM", "GPU": "2× T4", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "DAPO-MATH-17k-oss-reasoning (MIT): Visible reasoning traces provided as training data, built over BytedTsinghua-SIA/DAPO-Math-17k (Apache-2.0).", "url": "https://huggingface.co/datasets/thuzhizhi/DAPO-MATH-17k-oss-reasoning"}, {"title": "Qwen3-14B (Apache-2.0): The frozen base model, materialized from a pinned revision with a recorded file inventory and a deterministic tokenizer-template setting applied at image build.", "url": "https://huggingface.co/Qwen/Qwen3-14B"}, {"title": "lm-evaluation-harness (MIT): Exact-answer normalization in the verifier is adapted from the Hendrycks-MATH utilities.", "url": "https://github.com/EleutherAI/lm-evaluation-harness"}, {"title": "math-ai/aime25 and math-ai/aime26 (Apache-2.0): The held-out scored records are adapted from pinned revisions of both datasets, with fields and formatting normalized for the verifier; original AIME problem authorship is credited to the Mathematical Association of America.", "url": "https://huggingface.co/datasets/math-ai/aime26"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.2429, "std": 0.052, "steps": "106", "in_tok": "8.2M", "out_tok": "74k", "avg_cost": "$5.86", "avg_time": "17.5h"}, {"rank": 2, "model": "Muse Spark 1.2", "score": 0.2381, "std": 0.127, "steps": "1,422", "in_tok": "478.4M", "out_tok": "739k", "avg_cost": "$94.67", "avg_time": "11.7h"}, {"rank": 3, "model": "Kimi K3", "score": 0.1762, "std": 0.15, "steps": "157", "in_tok": "17.8M", "out_tok": "148k", "avg_cost": "$17.58", "avg_time": "19.1h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.1286, "std": 0.146, "steps": "211", "in_tok": "29.2M", "out_tok": "180k", "avg_cost": "$15.77", "avg_time": "19.2h"}, {"rank": 5, "model": "Inkling", "score": 0.119, "std": 0.227, "steps": "154", "in_tok": "8.7M", "out_tok": "38k", "avg_cost": "$8.43", "avg_time": "1.6h"}, {"rank": 6, "model": "Qwen3.8-Max", "score": 0.1095, "std": 0.15, "steps": "206", "in_tok": "15.2M", "out_tok": "271k", "avg_cost": "$5.67", "avg_time": "19.0h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.1, "std": 0.143, "steps": "265", "in_tok": "41.0M", "out_tok": "200k", "avg_cost": "$0.88", "avg_time": "19.9h"}, {"rank": 8, "model": "Gemini 3.7 Flash", "score": 0.0762, "std": 0.106, "steps": "238", "in_tok": "41.7M", "out_tok": "103k", "avg_cost": "$16.58", "avg_time": "15.3h"}, {"rank": 9, "model": "GPT-5.6", "score": 0.0524, "std": 0.104, "steps": "174", "in_tok": "29.6M", "out_tok": "89k", "avg_cost": "$22.00", "avg_time": "15.2h"}, {"rank": 10, "model": "Grok 4.6", "score": 0.0, "std": 0.0, "steps": "192", "in_tok": "28.0M", "out_tok": "287k", "avg_cost": "$28.49", "avg_time": "18.5h"}], "best_score": 0.2429, "mean_score": 0.1243, "n_models": 10}, {"slug": "notebook-compression", "number": 21, "title": "Notebook Compression", "url": "https://www.frontierswe.com/tasks/notebook-compression", "categories": ["Performance Optimisation"], "summary": "Build the smallest self-extracting archive of a ~650 MB corpus of real Jupyter notebooks: decoder included, byte-for-byte lossless.", "scoring": "Byte exact", "toolchains": ["Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 8 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "CMake, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Jupyter notebooks are a compressor's puzzle box: JSON structure, source code, markdown prose, and, dominating the bytes, base64-encoded cell outputs. In this corpus roughly 64% of all bytes are base64 image/png payloads, so beating a general-purpose compressor requires exploiting the format itself: undoing the base64 layer, modeling the JSON envelope, and finding redundancy across hundreds of related notebooks, all while guaranteeing byte-exact reconstruction.\nThe corpus is a fixed selection of real, permissively-licensed notebooks from twelve pinned open-source repositories (TensorFlow docs, Hugging Face, OpenAI cookbook, scikit-style ML tutorials, cloud-vendor samples, and more), shipped byte-for-byte as committed upstream (no re-serialization, no output stripping), with per-file provenance and hashes in a manifest.", "task": "The agent must compress the Jupyter notebooks in /app/corpus/ as small as possible, losslessly, and ship the result as a self-extracting archive under /app/dist/ .\n- python3 /app/dist/decompress.py <out_dir> must recreate every notebook byte-for-byte into <out_dir> , using only what is under /app/dist/ .\n- The total size of /app/dist/ (file contents plus every entry's path name, with only __pycache__ excluded) counts as the compressed size, so the decoder's own code is part of the price.\n- The decoder may shell out to tools already installed on the machine (those do not count toward the size); anything custom must live inside /app/dist/ .\n- Changes are confined to /app/dist , and the workspace must stay in working condition in case the sandbox times out mid-experiment.", "task_bullets": ["python3 /app/dist/decompress.py <out_dir> must recreate every notebook byte-for-byte into <out_dir> , using only what is under /app/dist/ .", "The total size of /app/dist/ (file contents plus every entry's path name, with only __pycache__ excluded) counts as the compressed size, so the decoder's own code is part of the price.", "The decoder may shell out to tools already installed on the machine (those do not count toward the size); anything custom must live inside /app/dist/ .", "Changes are confined to /app/dist , and the workspace must stay in working condition in case the sandbox times out mid-experiment."], "agent_gets": "The full ~650 MB notebook corpus at /app/corpus/ (the exact bytes that will be scored, not a proxy), plus python3 /app/check.py , which runs the decoder into a temporary directory, verifies the byte-exact round trip (with a --sample flag for fast iteration), and reports the size of /app/dist against the recorded xz -9 baseline.\nThe machine is offline (Ubuntu 24.04) with a compression-oriented toolchain preinstalled: standard compression CLIs and headers, a build toolchain, and a Python stack including zstandard , brotli , lz4 , NumPy, pandas, SciPy, and PyArrow. A sandbox-timer CLI reports the remaining budget.", "verification": "A clean-room verifier in a separate container scores the captured submission against root-only ground truth. It first wipes any harness logs restored from the agent's trial so a decoder cannot read an out-of-band stash, rejects any symlink under the submission, copies /app/dist into a scratch directory, and empties /app entirely before decoding; the decoder runs unprivileged with only its own files.\n- The decode stage is capped at 30 minutes; every recovered notebook is then compared byte-for-byte against the root-only corpus copy, with symlinked or non-regular outputs rejected. Any mismatch means the submission is not lossless and scores 0.\n- The metric is the submission ratio: total /app/dist bytes (contents plus path names) divided by corpus bytes; lower is better.\n- Reward is anchored to two fixed points: matching a whole-corpus xz -9 self-extracting archive scores 0, and reaching 15% of the corpus size scores 1.0. The curve is measured in compression factor and bent exponentially (2u − 1), so each further halving of the archive is worth more than the last.\n- A decoder that fails, times out, or depends on anything outside /app/dist zeroes the result.", "verification_bullets": ["The decode stage is capped at 30 minutes; every recovered notebook is then compared byte-for-byte against the root-only corpus copy, with symlinked or non-regular outputs rejected. Any mismatch means the submission is not lossless and scores 0.", "The metric is the submission ratio: total /app/dist bytes (contents plus path names) divided by corpus bytes; lower is better.", "Reward is anchored to two fixed points: matching a whole-corpus xz -9 self-extracting archive scores 0, and reaching 15% of the corpus size scores 1.0. The curve is measured in compression factor and bent exponentially (2u − 1), so each further halving of the archive is worth more than the last.", "A decoder that fails, times out, or depends on anything outside /app/dist zeroes the result."], "safeguards": ["The decode stage is capped at 30 minutes; every recovered notebook is then compared byte-for-byte against the root-only corpus copy, with symlinked or non-regular outputs rejected. Any mismatch means the submission is not lossless and scores 0.", "A decoder that fails, times out, or depends on anything outside /app/dist zeroes the result."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "CMake, C/C++ toolchain, Python 3", "Compute": "4 CPUs · 8 GB RAM", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of Notebook Compression from FrontierSWE V1.\n- The task was reframed from a three-stage fit / compress / decompress codec scored per-notebook on a hidden holdout set to a single self-extracting archive of the visible corpus itself: the agent now compresses exactly the bytes it can see, and the whole /app/dist directory (decoder included) is the scored size.\n- The corpus was rebuilt from scratch: V1 shipped pre-canonicalized notebooks on a data volume, while V2 ships ~650 MB of real, unmodified notebooks from twelve pinned permissively-licensed repositories with per-file provenance and hashes, deliberately retaining real cell outputs (about 64% of bytes are base64 PNG images).\n- Scoring changed from raw byte ratios to a two-anchor curve: a whole-corpus xz -9 archive marks zero reward and 15% of corpus size marks full credit, with an exponential bend that rewards escaping a compression plateau more than polishing inside one.\n- Verification hardened into a separate clean-room container: restored harness logs are wiped so decoders cannot read out-of-band stashes, symlinks are rejected, /app is emptied before the unprivileged capped decode, and the round trip is checked byte-for-byte against root-only ground truth.\n- The agent budget grew from 8 to 20 hours while the machine shrank from 16 CPUs / 32 GB RAM to 4 CPUs / 8 GB, and V1's per-stage wall-time limits and artifact caps were replaced by a single decode cap; the /app/.timer daemon files gave way to the sandbox-timer CLI under the new harbor schema.", "references": [{"title": "tensorflow/docs-l10n (Apache-2.0): TensorFlow tutorial/guide notebooks and their translations, at a pinned commit.", "url": "https://github.com/tensorflow/docs-l10n"}, {"title": "GoogleCloudPlatform/training-data-analyst (Apache-2.0): GCP ML/data tutorial notebooks with executed outputs, at a pinned commit.", "url": "https://github.com/GoogleCloudPlatform/training-data-analyst"}, {"title": "huggingface/notebooks (Apache-2.0): Executed Transformers/Diffusers/course notebooks, at a pinned commit.", "url": "https://github.com/huggingface/notebooks"}, {"title": "openai/openai-cookbook (MIT): Executed OpenAI API cookbook notebooks, at a pinned commit.", "url": "https://github.com/openai/openai-cookbook"}, {"title": "microsoft/computervision-recipes (MIT): Computer-vision example notebooks with image outputs, at a pinned commit.", "url": "https://github.com/microsoft/computervision-recipes"}, {"title": "ageron/handson-ml3 (Apache-2.0): \"Hands-on ML\" chapter notebooks with plots, at a pinned commit.", "url": "https://github.com/ageron/handson-ml3"}, {"title": "GoogleCloudPlatform/generative-ai (Apache-2.0): Vertex/GenAI sample notebooks, at a pinned commit.", "url": "https://github.com/GoogleCloudPlatform/generative-ai"}, {"title": "keras-team/keras-io (Apache-2.0): Keras example notebooks, at a pinned commit.", "url": "https://github.com/keras-team/keras-io"}, {"title": "aws/amazon-sagemaker-examples (Apache-2.0): SageMaker example notebooks, at a pinned commit; the repository's NOTICE file is preserved verbatim alongside the vendored licenses.", "url": "https://github.com/aws/amazon-sagemaker-examples"}, {"title": "GoogleCloudPlatform/vertex-ai-samples (Apache-2.0): Vertex AI sample notebooks, at a pinned commit.", "url": "https://github.com/GoogleCloudPlatform/vertex-ai-samples"}, {"title": "jax-ml/jax (Apache-2.0): JAX documentation notebooks, at a pinned commit.", "url": "https://github.com/jax-ml/jax"}, {"title": "tensorflow/docs (Apache-2.0): TensorFlow documentation notebooks, at a pinned commit.", "url": "https://github.com/tensorflow/docs"}, {"title": "Build/runtime Python tooling (BSD-3-Clause / MIT / Apache-2.0): Installed from PyPI, not vendored: zstandard (BSD-3-Clause), brotli (MIT), lz4 (BSD-3-Clause), NumPy/pandas/SciPy (BSD-3-Clause), and PyArrow (Apache-2.0). All task code and the oracle's xz -9 baseline archive are original, using the Python standard library.", "url": "https://pypi.org/"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.5459, "std": 0.045, "steps": "423", "in_tok": "118.0M", "out_tok": "369k", "avg_cost": "$68.00", "avg_time": "14.9h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.228, "std": 0.05, "steps": "497", "in_tok": "110.0M", "out_tok": "230k", "avg_cost": "$86.59", "avg_time": "13.2h"}, {"rank": 3, "model": "Kimi K3", "score": 0.2268, "std": 0.025, "steps": "337", "in_tok": "75.6M", "out_tok": "330k", "avg_cost": "$31.14", "avg_time": "16.3h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.2184, "std": 0.062, "steps": "412", "in_tok": "117.7M", "out_tok": "411k", "avg_cost": "$37.73", "avg_time": "17.5h"}, {"rank": 5, "model": "Grok 4.6", "score": 0.2122, "std": 0.01, "steps": "336", "in_tok": "73.2M", "out_tok": "481k", "avg_cost": "$75.63", "avg_time": "9.6h"}, {"rank": 6, "model": "Qwen3.8-Max", "score": 0.2061, "std": 0.015, "steps": "399", "in_tok": "52.1M", "out_tok": "724k", "avg_cost": "$17.78", "avg_time": "18.8h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.1974, "std": 0.012, "steps": "487", "in_tok": "141.2M", "out_tok": "412k", "avg_cost": "$2.58", "avg_time": "13.9h"}, {"rank": 8, "model": "Gemini 3.7 Flash", "score": 0.0944, "std": 0.073, "steps": "176", "in_tok": "19.0M", "out_tok": "131k", "avg_cost": "$4.09", "avg_time": "4.1h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0912, "std": 0.058, "steps": "807", "in_tok": "281.7M", "out_tok": "956k", "avg_cost": "$75.06", "avg_time": "15.3h"}, {"rank": 10, "model": "Inkling", "score": 0.0202, "std": 0.001, "steps": "172", "in_tok": "6.5M", "out_tok": "34k", "avg_cost": "$6.30", "avg_time": "2.6h"}], "best_score": 0.5459, "mean_score": 0.2041, "n_models": 10}, {"slug": "optimizer-design-v2", "number": 22, "title": "Optimizer Design", "url": "https://www.frontierswe.com/tasks/optimizer-design-v2", "categories": ["AI Research"], "summary": "Design a novel torch.optim.Optimizer that out-converges a strong reference portfolio across ten diverse workloads with a single fixed config.", "scoring": "Perf speedup", "toolchains": ["CUDA", "Python", "C/C++"], "compute": {"label": "1× H100", "kind": "gpu", "compute": "8 CPUs · 128 GB RAM", "gpu": "1× H100", "base_image": "nvidia/cuda:12.4.1-devel-ubuntu22.04", "tools": "CUDA, PyTorch, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Most published optimizers win on a narrow slice of architectures and lose elsewhere; a per-workload-tuned baseline is easy to beat on one model and very hard to beat on ten at once. This task asks for genuine optimization research: one optimizer class and one hyperparameter configuration that converge quickly across transformers, convolutional networks, plain MLPs, embedding-heavy recommenders, and contrastive encoders, and that generalize to architecture families it was never tuned on.\nThe bar is not vanilla AdamW. The frozen per-workload reference step counts were calibrated offline from a portfolio of strong controls (Nesterov SGD, normalized and shape-routed Muon hybrids, adaptive schedules, table-aware Muon/Adam variants, bounded SOAP and Shampoo, and Schedule-Free AdamW), with each denominator accepted only from repeated confirmation runs on the exact task hardware.", "task": "The agent must create /app/custom_optimizer.py defining CustomOptimizer(torch.optim.Optimizer) and /app/optimizer_config.json , whose object is passed to the constructor as **kwargs . The frozen training loop drives it through the ordinary construct / zero_grad / step contract.\n- Seven visible workloads for development: nano-GPT on WikiText-103, ResNet-18 on CIFAR-100, a masked atom-set transformer for QM9 dipole-moment regression, next-item prediction on MovieLens-1M, a ViT and a 12-layer plain MLP on CIFAR-10, and a SimCSE contrastive encoder on AG News. Three held-out workloads from different architecture families are scored at verification.\n- One class and one config for all ten workloads. Branching on workload or model class names is forbidden; adapting behavior based on parameter shape is explicitly allowed; that is legitimate optimizer design.\n- The optimizer must be self-contained: imports limited to torch , numpy , scipy , and a fixed set of standard-library modules that deliberately excludes filesystem, network, and OS access.\n- The training infrastructure (train_workload.py , run_visible.py , workloads/ ) is read-only; runs against modified copies are rejected.", "task_bullets": ["Seven visible workloads for development: nano-GPT on WikiText-103, ResNet-18 on CIFAR-100, a masked atom-set transformer for QM9 dipole-moment regression, next-item prediction on MovieLens-1M, a ViT and a 12-layer plain MLP on CIFAR-10, and a SimCSE contrastive encoder on AG News. Three held-out workloads from different architecture families are scored at verification.", "One class and one config for all ten workloads. Branching on workload or model class names is forbidden; adapting behavior based on parameter shape is explicitly allowed; that is legitimate optimizer design.", "The optimizer must be self-contained: imports limited to torch , numpy , scipy , and a fixed set of standard-library modules that deliberately excludes filesystem, network, and OS access.", "The training infrastructure (train_workload.py , run_visible.py , workloads/ ) is read-only; runs against modified copies are rejected."], "agent_gets": "A complete experiment loop on an H100: the seven visible workload definitions in /app/workloads/ (exact architectures, datasets, loss functions, and per-workload target losses), the frozen training loop, and python3 /app/run_visible.py to run one workload or all seven, saving per-step loss curves, speedups, and timing to /app/runs/ for comparison across experiments.\nThe starter custom_optimizer.py is a simple AdamW implementation provided only to demonstrate the interface; it is not the reference portfolio and has no promised speedup. Datasets are baked into the image from hash-pinned sources; the machine is offline for task purposes, and a sandbox-timer CLI reports the remaining budget.", "verification": "A clean-room verifier runs in a fresh container on the captured /app , first checking every frozen training file against a SHA-256 manifest, then training all ten workloads (seven visible plus three held-out) with the submitted optimizer. The scorer never imports candidate code: submitted optimizer code runs alone in a chrooted, privilege-dropped, seccomp-confined process that sees only mirrored parameters and gradients, while trusted root-owned processes own the models, data, forward/backward passes, step counting, checkpoints, and all scored quantities.\n- Per workload, the verifier records the first validation checkpoint at which the EMA validation loss crosses the unchanged target: a crossing earns baseline_steps / candidate_steps against the frozen reference denominator; a miss earns capped loss-ratio partial credit (at most 1.0). Reference optimizers are not re-run; only the frozen scalars are read.\n- The ten speedups combine by geometric mean G, mapped asymptotically to reward: 0 at or below the 1.0x reference baseline, 1 − 1/G above it. G = 2 (half the aggregate optimizer updates) maps to 0.5, with no finite full-credit target.\n- A crashed or diverged workload is floored near zero before the geometric mean, dragging the aggregate down hard: uniform gains beat one dramatic outlier.\n- Disallowed imports, tampered frozen files, and branching on workload identity are rejected outright; candidate stdout is diagnostic only and cannot influence the score.", "verification_bullets": ["Per workload, the verifier records the first validation checkpoint at which the EMA validation loss crosses the unchanged target: a crossing earns baseline_steps / candidate_steps against the frozen reference denominator; a miss earns capped loss-ratio partial credit (at most 1.0). Reference optimizers are not re-run; only the frozen scalars are read.", "The ten speedups combine by geometric mean G, mapped asymptotically to reward: 0 at or below the 1.0x reference baseline, 1 − 1/G above it. G = 2 (half the aggregate optimizer updates) maps to 0.5, with no finite full-credit target.", "A crashed or diverged workload is floored near zero before the geometric mean, dragging the aggregate down hard: uniform gains beat one dramatic outlier.", "Disallowed imports, tampered frozen files, and branching on workload identity are rejected outright; candidate stdout is diagnostic only and cannot influence the score."], "safeguards": ["A crashed or diverged workload is floored near zero before the geometric mean, dragging the aggregate down hard: uniform gains beat one dramatic outlier.", "Disallowed imports, tampered frozen files, and branching on workload identity are rejected outright; candidate stdout is diagnostic only and cannot influence the score."], "environment": {"Base image": "nvidia/cuda:12.4.1-devel-ubuntu22.04", "Tools available": "CUDA, PyTorch, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 128 GB RAM", "GPU": "1× H100", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of Optimizer Design from FrontierSWE V1.\n- The reward changed from the raw geometric-mean speedup G to an asymptotic map: 0 at or below the 1.0x reference baseline and 1 − 1/G above it, so matching the references earns nothing and there is no finite full-credit speedup.\n- The reference denominators were recalibrated from per-workload-tuned AdamW to a much stronger frozen control portfolio (Nesterov SGD, normalized and table-aware Muon hybrids, bounded SOAP and Shampoo, Schedule-Free AdamW), each accepted only from three-repeat confirmation on the exact task hardware; references are no longer re-run at verification; only frozen scalar step counts are read.\n- Candidate isolation was rebuilt: V1 imported custom_optimizer.py directly into the verifier's training process, while V2 runs it in a chrooted, privilege-dropped, seccomp-confined worker per workload that sees only mirrored parameters and gradients, with the import allowlist now enforced rather than stated.\n- Verification moved to a separate clean-room container on the captured /app , with frozen training files checked against a SHA-256 manifest and datasets pinned by digest in an OCI lock.\n- The first-EMA-crossing scoring rule is now applied identically to submissions and to the offline baseline calibration, and the /app/.timer daemon files were replaced by the sandbox-timer CLI under the new harbor schema.", "references": [{"title": "WikiText-103 / WikiText-2 (CC BY-SA 3.0): Training data for the nano_gpt workload and a held-out character-level language model (Merity et al. 2016).", "url": "https://huggingface.co/datasets/wikitext"}, {"title": "CIFAR-10 / CIFAR-100 (Research use): Image-classification data for the vit , deep_mlp , and resnet workloads, via torchvision (Krizhevsky 2009).", "url": "https://www.cs.toronto.edu/~kriz/cifar.html"}, {"title": "QM9 (CC0 / public domain): Dipole-moment regression data for the graph_transformer workload (Ramakrishnan et al. 2014, via PyTorch Geometric); atomic numbers only, unnormalized target index 0, source-order 80/20 split.", "url": "https://pytorch-geometric.readthedocs.io/"}, {"title": "MovieLens 1M (GroupLens research-use terms): Next-item prediction data for the next_item workload; only a derived tensor of remapped integer IDs and next-item targets is stored in the private evaluation registry.", "url": "https://grouplens.org/datasets/movielens/1m/"}, {"title": "AG News (Academic / research use): Text data for the contrastive workload and a held-out classifier.", "url": "https://huggingface.co/datasets/ag_news"}, {"title": "Multi30k (CC BY-NC-SA 4.0): Data for a held-out sequence-to-sequence workload.", "url": "https://huggingface.co/datasets/bentrevett/multi30k"}, {"title": "PyTorch + torchvision (BSD-3-Clause): PyTorch 2.5.1 and torchvision 0.20.1, installed from pinned package locks together with NumPy and SciPy (BSD-3-Clause).", "url": "https://github.com/pytorch/pytorch"}, {"title": "Muon (MIT): Primary source informing the offline calibration controls (not vendored; first-party ports and hybrids only).", "url": "https://github.com/KellerJordan/Muon"}, {"title": "SOAP (MIT): Primary source for the bounded SOAP calibration control (not vendored, and not a claim of full upstream equivalence).", "url": "https://github.com/nikhilvyas/SOAP"}, {"title": "Distributed Shampoo (BSD-3-Clause): Primary source for the bounded Shampoo calibration control (not vendored).", "url": "https://github.com/facebookresearch/optimizers"}, {"title": "Schedule-Free AdamW (Apache-2.0): Primary source for the Schedule-Free AdamW calibration control (not vendored).", "url": "https://github.com/facebookresearch/schedule_free"}], "results": [{"rank": 1, "model": "GPT-5.6", "score": 0.2914, "std": 0.047, "steps": "488", "in_tok": "100.6M", "out_tok": "149k", "avg_cost": "$84.09", "avg_time": "13.1h"}, {"rank": 2, "model": "GLM-5.3", "score": 0.1715, "std": 0.068, "steps": "217", "in_tok": "35.5M", "out_tok": "208k", "avg_cost": "$16.86", "avg_time": "18.8h"}, {"rank": 3, "model": "Grok 4.6", "score": 0.1658, "std": 0.064, "steps": "322", "in_tok": "69.4M", "out_tok": "443k", "avg_cost": "$69.50", "avg_time": "19.4h"}, {"rank": 4, "model": "Claude Fable 5.1", "score": 0.141, "std": 0.075, "steps": "160", "in_tok": "21.6M", "out_tok": "167k", "avg_cost": "$14.86", "avg_time": "13.3h"}, {"rank": 5, "model": "Kimi K3", "score": 0.1378, "std": 0.11, "steps": "264", "in_tok": "53.2M", "out_tok": "302k", "avg_cost": "$22.85", "avg_time": "18.3h"}, {"rank": 6, "model": "Qwen3.8-Max", "score": 0.0473, "std": 0.038, "steps": "244", "in_tok": "20.7M", "out_tok": "493k", "avg_cost": "$8.38", "avg_time": "19.8h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0448, "std": 0.049, "steps": "334", "in_tok": "67.4M", "out_tok": "251k", "avg_cost": "$1.32", "avg_time": "19.2h"}, {"rank": 8, "model": "Gemini 3.7 Flash", "score": 0.0427, "std": 0.049, "steps": "646", "in_tok": "170.1M", "out_tok": "153k", "avg_cost": "$19.83", "avg_time": "13.4h"}, {"rank": 9, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "85", "in_tok": "2.1M", "out_tok": "22k", "avg_cost": "$2.12", "avg_time": "2.3h"}, {"rank": 10, "model": "Muse Spark 1.2", "score": 0.0, "std": 0.0, "steps": "332", "in_tok": "86.7M", "out_tok": "398k", "avg_cost": "$36.96", "avg_time": "13.9h"}], "best_score": 0.2914, "mean_score": 0.1042, "n_models": 10}, {"slug": "postgres-sqlite-wire-adapter", "number": 23, "title": "PostgreSQL 18 on SQLite", "url": "https://www.frontierswe.com/tasks/postgres-sqlite-wire-adapter", "categories": ["Implementation"], "summary": "Build a PostgreSQL 18 wire-compatible server in Zig on SQLite storage, indistinguishable to real PostgreSQL clients.", "scoring": "Hidden tests", "toolchains": ["Zig", "Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "8 CPUs · 32 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Zig, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Standing in for a PostgreSQL server means implementing far more than SQL: the frontend/backend wire protocol with its startup packets, authentication handshakes, prepared statements and portals; the system catalogs that clients introspect; and the cluster lifecycle utilities (initdb , pg_ctl ) that tools use to create and manage database instances. Real clients probe all of these surfaces, and PostgreSQL's own regression suite pins the expected behaviour down to exact query output.\nThis task asks for that entire server-side surface from scratch in Zig, with SQLite as the only storage engine underneath: a frontier-difficulty exercise in protocol implementation, SQL semantics translation, and systems engineering, developed fully offline against the bundled PostgreSQL documentation.", "task": "Implement a PostgreSQL 18 wire-compatible server in Zig in the project at /app/postgres-sqlite : a single multi-call executable that stands in for PostgreSQL 18.3's server-side entrypoints well enough that PostgreSQL 18 clients and tools cannot tell it apart from a real server.\n- The binary dispatches on argv[0] and is invoked as postgres , initdb , and pg_ctl .\n- All program logic lives in .zig sources reached from src/main.zig , built with bash ./build.sh ; SQLite is the storage engine (linked via -lsqlite3 ).\n- Behaviour is exercised by PostgreSQL's own regression tests: /app/run-tests.sh runs everything, or named tests such as run-tests.sh boolean int4 .\n- build.sh compiles only .zig sources, so any data the program needs must live inside them as Zig literals rather than in separate files.", "task_bullets": ["The binary dispatches on argv[0] and is invoked as postgres , initdb , and pg_ctl .", "All program logic lives in .zig sources reached from src/main.zig , built with bash ./build.sh ; SQLite is the storage engine (linked via -lsqlite3 ).", "Behaviour is exercised by PostgreSQL's own regression tests: /app/run-tests.sh runs everything, or named tests such as run-tests.sh boolean int4 .", "build.sh compiles only .zig sources, so any data the program needs must live inside them as Zig literals rather than in separate files."], "agent_gets": "A Zig scaffold at /app/postgres-sqlite with a fixed build.sh , a visible slice of PostgreSQL's core regression suite at /app/tests as the behavioural spec with the /app/run-tests.sh runner, the complete offline PostgreSQL 18 documentation at /reference/postgresql-docs/ (readable with w3m ), and the packaged psql client and admin tools for local testing. Zig 0.15.2 and SQLite are preinstalled; the machine is offline and a sandbox-timer CLI reports the remaining budget. The real PostgreSQL 18.3 server binaries exist in the image but are locked root-only; the agent cannot run or read them.", "verification": "A clean-room verifier, separate from the agent's environment, reconstructs the scored project from the pristine build.sh plus only the agent's .zig sources, then runs PostgreSQL's core regression schedule through pg_regress against the candidate server. It uses a three-uid choreography: the orchestrator runs as root, the candidate server builds and runs as the non-root agent user, and pg_regress and psql run as a third uid, so candidate code can neither kill the test driver nor tamper with its results. The trusted per-test signal is pg_regress 's exit code, captured by root into a nonce-named root-only directory.\n- The scored suite is a semantic-preserving mutation of the public one: identifiers are consistently renamed (extracted with the PostgreSQL parser at image build) and expected outputs regenerated by the real server, so hardcoding upstream outputs fails while a general implementation passes.\n- The reward is the fraction of scored tests passed, against a fixed denominator of what real PostgreSQL 18.3 passes under the same harness (measured at image build, so the ceiling is exactly 1.0). Skipped or timed-out tests count as failures, and a no-op stub scores approximately 0.\n- Binary-provenance checks zero the result: a candidate that is byte-identical to a real server binary, links PostgreSQL libraries, is a C-built ELF with no Zig provenance, or has a real server binary vendored anywhere in the workspace fails hard, as do build failures and reset violations.", "verification_bullets": ["The scored suite is a semantic-preserving mutation of the public one: identifiers are consistently renamed (extracted with the PostgreSQL parser at image build) and expected outputs regenerated by the real server, so hardcoding upstream outputs fails while a general implementation passes.", "The reward is the fraction of scored tests passed, against a fixed denominator of what real PostgreSQL 18.3 passes under the same harness (measured at image build, so the ceiling is exactly 1.0). Skipped or timed-out tests count as failures, and a no-op stub scores approximately 0.", "Binary-provenance checks zero the result: a candidate that is byte-identical to a real server binary, links PostgreSQL libraries, is a C-built ELF with no Zig provenance, or has a real server binary vendored anywhere in the workspace fails hard, as do build failures and reset violations."], "safeguards": ["Binary-provenance checks zero the result: a candidate that is byte-identical to a real server binary, links PostgreSQL libraries, is a C-built ELF with no Zig provenance, or has a real server binary vendored anywhere in the workspace fails hard, as do build failures and reset violations."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Zig, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 32 GB RAM", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of PostgreSQL 18 on SQLite from FrontierSWE V1.\n- The regression tests are no longer hidden: V1 kept the whole suite secret until verification, while V2 ships a visible slice of PostgreSQL's core regression suite in /app/tests with a runner, and defends against memorisation by scoring a root-only mutated variant: identifiers renamed via the PostgreSQL parser and expected outputs regenerated by the real server.\n- Verification moved from a 600-line bash script inside the agent's container to a clean-room verifier in a separate, hash-pinned image with a three-uid design: root orchestrates, the candidate server runs as the agent user, and pg_regress /psql run as a third uid whose results only root can read.\n- Anti-cheat is now by construction plus binary provenance: the scored project is reconstructed from the pristine build.sh and only the agent's .zig sources, and static checks catch a copied real server (sha256 against baked hashes), PostgreSQL library linkage, C-built binaries with no Zig provenance, and real server binaries vendored anywhere in the workspace. The real PostgreSQL server entrypoints are locked root-only in the image.\n- Scoring was consolidated and given a real ceiling: V1 summed regression, TAP, and a bespoke compat script against a hardcoded expected total; V2 scores a single all-public slice of the core regression schedule (188 tests) as a pass fraction against what real PostgreSQL 18.3 passes under the same harness, measured at image build, so the oracle scores ~1.0 by construction.\n- The agent budget grew from 8 hours to 20 hours, and the long prescriptive V1 brief (scope guidance, strategy hints, smoke contract) was replaced by a compact three-paragraph instruction with the behavioural spec externalized into the runnable test suite.\n- Infrastructure standardized: hash-pinned agent and verifier images replace a mutable :v4 tag, and the /app/.timer file interface was replaced by the standard sandbox-timer CLI.", "references": [{"title": "PostgreSQL (PostgreSQL License): The official 18.3 source release (sha256-pinned, fetched at image build) and pinned PGDG packages: the packaged client/admin tools and offline HTML documentation the agent works with, and the core regression suite, pg_regress driver, regress.so , and libpq built once at image build. A visible slice of the suite ships to /app/tests as the agent-facing spec; the full suite plus a perturbed scoring slice is baked root-only. The real server binaries are locked root-only after the build-time reference measurement.", "url": "https://www.postgresql.org"}, {"title": "SQLite (Public domain): The storage engine the candidate adapter links against (-lsqlite3 ) and the sqlite3 CLI, from Ubuntu packages.", "url": "https://sqlite.org"}, {"title": "Zig (MIT): The pinned 0.15.2 toolchain the scaffold and candidate build with.", "url": "https://ziglang.org/download"}, {"title": "pglast (bundles libpg_query) (GPL-3.0-or-later (pglast); BSD-3-Clause (libpg_query)): Used at image-build time only: the suite-perturbation tool imports pglast to parse each scored regression script and extract the objects it creates, so their identifiers can be consistently renamed as the anti-memorisation measure. Nothing of pglast is needed at agent or verifier runtime.", "url": "https://github.com/lelit/pglast"}, {"title": "Ubuntu base image and system packages (Various open-source licenses): ubuntu:24.04 plus the standard apt toolchain and utilities (C toolchain, bison /flex /perl for building pg_regress , w3m for the offline docs), installed from Ubuntu's archives.", "url": "https://ubuntu.com"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.3418, "std": 0.068, "steps": "2,138", "in_tok": "920.8M", "out_tok": "2.2M", "avg_cost": "$512.57", "avg_time": "18.4h"}, {"rank": 2, "model": "Gemini 3.7 Flash", "score": 0.1901, "std": 0.084, "steps": "3,071", "in_tok": "1382.2M", "out_tok": "2.3M", "avg_cost": "$128.21", "avg_time": "14.6h"}, {"rank": 3, "model": "GPT-5.6", "score": 0.1846, "std": 0.255, "steps": "738", "in_tok": "281.7M", "out_tok": "355k", "avg_cost": "$287.35", "avg_time": "7.3h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.1363, "std": 0.023, "steps": "1,685", "in_tok": "711.2M", "out_tok": "893k", "avg_cost": "$194.30", "avg_time": "19.8h"}, {"rank": 5, "model": "Grok 4.6", "score": 0.1066, "std": 0.049, "steps": "1,310", "in_tok": "348.5M", "out_tok": "1.7M", "avg_cost": "$355.20", "avg_time": "14.6h"}, {"rank": 6, "model": "Qwen3.8-Max", "score": 0.0934, "std": 0.038, "steps": "1,584", "in_tok": "648.7M", "out_tok": "1.6M", "avg_cost": "$173.75", "avg_time": "19.6h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0923, "std": 0.052, "steps": "2,736", "in_tok": "1179.0M", "out_tok": "1.7M", "avg_cost": "$19.24", "avg_time": "19.8h"}, {"rank": 8, "model": "Kimi K3", "score": 0.0618, "std": 0.009, "steps": "1,414", "in_tok": "677.7M", "out_tok": "1.3M", "avg_cost": "$232.16", "avg_time": "20.0h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0022, "std": 0.003, "steps": "427", "in_tok": "157.4M", "out_tok": "571k", "avg_cost": "$31.64", "avg_time": "2.6h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "281", "in_tok": "19.5M", "out_tok": "49k", "avg_cost": "$18.73", "avg_time": "1.0h"}], "best_score": 0.3418, "mean_score": 0.1209, "n_models": 10}, {"slug": "qe-pwx-rust", "number": 24, "title": "Quantum ESPRESSO pw.x in Rust", "url": "https://www.frontierswe.com/tasks/qe-pwx-rust", "categories": ["Scientific Computing", "Implementation"], "summary": "Reimplement Quantum ESPRESSO's plane-wave DFT engine (the pw.x SCF core) from scratch in Rust, numerically faithful to the real thing.", "scoring": "Hidden tests", "toolchains": ["Rust", "Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 8 GB RAM", "gpu": null, "base_image": "debian:bookworm", "tools": "Rust, CMake, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Quantum ESPRESSO's pw.x is one of the workhorse codes of computational materials science: a plane-wave density-functional-theory engine that self-consistently solves the Kohn–Sham equations for a periodic crystal, then reports total energies, band eigenvalues, forces on atoms, and stress on the cell. Behind that sits decades of accumulated numerical machinery: FFT grids, pseudopotentials of several families (norm-conserving, ultrasoft, PAW), k-point sampling with symmetry reduction, iterative diagonalization, and charge-density mixing, all of it written in legacy Fortran.\nThe task is a clean-room port: no QE source is provided (the pinned QE tree baked into the image is root-only and locked away from the agent). The behavior to reproduce is defined entirely by example inputs from QE's own test suite, their reference outputs, and an authoritative comparator that encodes QE's shipping regression tolerances.", "task": "Implement the pw.x self-consistent-field core in Rust inside the Cargo project at /app/qe-pwx/ : serial, CPU-only, and numerically faithful. The binary is built with cargo build --release and run as target/release/qe-pwx <input.in> <outdir> --pseudo-dir <dir> , and must write <outdir>/results.json .\n- Parse QE PW input files and compute the quantities the reference reports: energies in Ry, eigenvalues/Fermi/HOMO in eV, forces in Ry/bohr, pressure and stress in kbar, cell volume in bohr³.\n- Every reported quantity must match the reference output at QE's own pw regression tolerances; the exact field contract and comparison semantics are defined by /app/tools/compare.py .\n- The 121 example cases span SCF variants, gamma-point and k-sampled runs, spin polarization, metals with smearing, norm-conserving/ultrasoft/PAW pseudopotentials, structural relaxation, and all lattice types.\n- Only the pre-vendored numerical crates (rustfft , nalgebra , num-complex , serde , serde_json ) are available; the build is fully offline.", "task_bullets": ["Parse QE PW input files and compute the quantities the reference reports: energies in Ry, eigenvalues/Fermi/HOMO in eV, forces in Ry/bohr, pressure and stress in kbar, cell volume in bohr³.", "Every reported quantity must match the reference output at QE's own pw regression tolerances; the exact field contract and comparison semantics are defined by /app/tools/compare.py .", "The 121 example cases span SCF variants, gamma-point and k-sampled runs, spin polarization, metals with smearing, norm-conserving/ultrasoft/PAW pseudopotentials, structural relaxation, and all lattice types.", "Only the pre-vendored numerical crates (rustfft , nalgebra , num-complex , serde , serde_json ) are available; the build is fully offline."], "agent_gets": "A ready-to-build Cargo skeleton at /app/qe-pwx/ , 121 QE PW input cases under /app/cases/ (each shipped with its expected output gold.out baked by the real pinned pw.x ), and the pseudopotential files under /app/pseudo/ (with ESPRESSO_PSEUDO already set). /app/tools/compare.py is the authoritative comparator, and /app/run-tests.sh builds the port and self-checks it against every public case. The Rust toolchain and the vendored crates are preinstalled; the machine has no usable network access for the agent.", "verification": "A clean-room verifier, separate from the agent's environment, rebuilds the port offline and scores it as a non-root user. The scored set is hidden: one numerically perturbed twin of each of the 121 public cases (plane-wave cutoff and lattice scale changed, topology preserved), re-baked with the same pinned pw.x , so replaying memorized public gold.out values scores approximately zero. Two oracle-free physical checks (finite-difference force consistency and symmetry-invariance of the total energy) add two more points, but their credit is gated behind at least one twin passing.\n- Each hidden twin is all-or-nothing: every reference-reported quantity must match at QE's pw tolerances. Reward is points earned over the 123-point total, clamped to [0, 1].\n- A QE-free scan zeroes the result on Fortran sources, smuggled QE binaries, or scripts invoking pw.x ; the in-image QE tree and all Fortran compilers are additionally made unusable before any agent code runs.\n- Reference outputs never leave the root-only verifier tree; the port only sees a reference-free staged copy of the inputs.\n- Build failures and provenance/scoring failures zero the result; each case runs under a 1500-second cap within a 5400-second scoring budget, and cases that never run count as failed.", "verification_bullets": ["Each hidden twin is all-or-nothing: every reference-reported quantity must match at QE's pw tolerances. Reward is points earned over the 123-point total, clamped to [0, 1].", "A QE-free scan zeroes the result on Fortran sources, smuggled QE binaries, or scripts invoking pw.x ; the in-image QE tree and all Fortran compilers are additionally made unusable before any agent code runs.", "Reference outputs never leave the root-only verifier tree; the port only sees a reference-free staged copy of the inputs.", "Build failures and provenance/scoring failures zero the result; each case runs under a 1500-second cap within a 5400-second scoring budget, and cases that never run count as failed."], "safeguards": ["A QE-free scan zeroes the result on Fortran sources, smuggled QE binaries, or scripts invoking pw.x ; the in-image QE tree and all Fortran compilers are additionally made unusable before any agent code runs.", "Build failures and provenance/scoring failures zero the result; each case runs under a 1500-second cap within a 5400-second scoring budget, and cases that never run count as failed."], "environment": {"Base image": "debian:bookworm", "Tools available": "Rust, CMake, C/C++ toolchain, Python 3", "Compute": "4 CPUs · 8 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "Quantum ESPRESSO (GPL-2.0-or-later): A pinned checkout (v7.6-dev) is built at image-build time into a root-only pw.x that bakes every grading reference and serves as the oracle candidate; its bundled test-suite/ supplies the 121 graded case inputs verbatim, and the harness's tolerance tables and output parser are derived from QE's own published test-suite configuration. The agent never gets QE source access; the tree is locked before any agent code runs, as an anti-contamination measure.", "url": "https://github.com/QEF/q-e"}, {"title": "Quantum ESPRESSO pseudopotential library (QE PP library terms (GPL-compatible notices in file headers)): The 25 pseudopotential files referenced by the selected test-suite inputs; some come from the pinned QE tree's own pseudo/ directory, the rest from the QE pseudopotential library that QE's test suite itself downloads for these inputs.", "url": "https://pseudopotentials.quantum-espresso.org"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.9268, "std": 0.013, "steps": "272", "in_tok": "119.8M", "out_tok": "652k", "avg_cost": "$75.78", "avg_time": "4.0h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.3691, "std": 0.21, "steps": "453", "in_tok": "157.8M", "out_tok": "223k", "avg_cost": "$159.58", "avg_time": "4.6h"}, {"rank": 3, "model": "Grok 4.6", "score": 0.2341, "std": 0.13, "steps": "1,207", "in_tok": "369.1M", "out_tok": "1.6M", "avg_cost": "$382.00", "avg_time": "13.1h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.1008, "std": 0.068, "steps": "936", "in_tok": "423.8M", "out_tok": "1.3M", "avg_cost": "$122.85", "avg_time": "19.5h"}, {"rank": 5, "model": "Kimi K3", "score": 0.0683, "std": 0.109, "steps": "644", "in_tok": "257.5M", "out_tok": "1.5M", "avg_cost": "$108.04", "avg_time": "20.0h"}, {"rank": 6, "model": "Gemini 3.7 Flash", "score": 0.0634, "std": 0.066, "steps": "795", "in_tok": "331.6M", "out_tok": "746k", "avg_cost": "$46.19", "avg_time": "9.8h"}, {"rank": 7, "model": "Muse Spark 1.2", "score": 0.0049, "std": 0.011, "steps": "275", "in_tok": "94.0M", "out_tok": "525k", "avg_cost": "$18.82", "avg_time": "1.5h"}, {"rank": 8, "model": "DeepSeek V4 Flash Exp", "score": 0.0, "std": 0.0, "steps": "800", "in_tok": "346.9M", "out_tok": "962k", "avg_cost": "$6.31", "avg_time": "13.9h"}, {"rank": 9, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "179", "in_tok": "14.2M", "out_tok": "48k", "avg_cost": "$13.67", "avg_time": "0.3h"}, {"rank": 10, "model": "Qwen3.8-Max", "score": 0.0, "std": 0.0, "steps": "321", "in_tok": "100.3M", "out_tok": "1.8M", "avg_cost": "$36.66", "avg_time": "19.9h"}], "best_score": 0.9268, "mean_score": 0.1767, "n_models": 10}, {"slug": "qubit-routing", "number": 25, "title": "Qubit Routing", "url": "https://www.frontierswe.com/tasks/qubit-routing", "categories": ["Performance Optimisation"], "summary": "Write a qubit router that schedules SWAPs so every two-qubit gate runs on adjacent hardware qubits in as few timesteps as possible.", "scoring": "Perf speedup", "toolchains": ["Python"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 16 GB RAM", "gpu": null, "base_image": "python:3.13-slim", "tools": "Python 3", "time_limit": "20h"}, "background": "A quantum processor can run a two-qubit gate only between physically adjacent qubits, so a compiler's router inserts SWAP operations to walk logical qubits along hardware edges until each pending gate's operands sit next to each other. Every SWAP costs time and occupies an edge, so routing is a combinatorial scheduling problem: pick which SWAPs to run in parallel each timestep so the whole circuit finishes as early as possible.\nThe task's environment models real hardware coupling graphs (grid topologies plus the connectivity of IBM Q 16 and Q 20, Rigetti Acorn, and Google Sycamore) and four timing models in which SWAPs and CNOTs lock edges for different durations. Circuits come from synthetic generator families and from real OpenQASM benchmark suites, and the routing engine ships with no router in it: the algorithm is written from scratch.", "task": "Implement route_instance(instance) -> list[list[int]] in /app/router.py : the outer list is timesteps, each inner list the indices into instance[\"edges\"] to SWAP in parallel that step. After each timestep the simulator runs any gate whose operands are adjacent on an unlocked edge.\n- The initial logical-to-physical layout is fixed, and every gate must finish within instance[\"max_steps\"] ; a schedule that leaves a gate unfinished, or that the simulator rejects, counts for nothing.\n- Four timing models must be handled: uniform , instant_cnot , slow_cnot_3x , and slow_swap_2x .\n- Completing every circuit is the entry ticket; past that, fewer timesteps is strictly better on a smooth curve.\n- Each route_instance call gets 10 seconds (an overrun abandons that circuit), and the whole scored run shares a 5400-second budget.\n- The router must be ordinary, self-contained Python: the standard library plus the provided package, everything inside /app .", "task_bullets": ["The initial logical-to-physical layout is fixed, and every gate must finish within instance[\"max_steps\"] ; a schedule that leaves a gate unfinished, or that the simulator rejects, counts for nothing.", "Four timing models must be handled: uniform , instant_cnot , slow_cnot_3x , and slow_swap_2x .", "Completing every circuit is the entry ticket; past that, fewer timesteps is strictly better on a smooth curve.", "Each route_instance call gets 10 seconds (an overrun abandons that circuit), and the whole scored run shares a 5400-second budget.", "The router must be ordinary, self-contained Python: the standard library plus the provided package, everything inside /app ."], "agent_gets": "The routing engine at /app/qubit_routing/ (the timestep/edge-lock simulator, the device set, circuit generators, and the instance format, but no router), plus a training split of normalized OpenQASM benchmark circuits at /app/qubit_routing/qasm_training/ . Running python3 -m qubit_routing.run (with --limit 50 for a quick pass) routes the local circuits with the agent's router.py and writes routing_results.json with circuits finished, timesteps used, and per-call time. /app/README.md documents the contract in full. The machine is offline, and the environment is deliberately dependency-light: Python 3.13 with the standard library only.", "verification": "A clean-room verifier, separate from the agent's environment, rebuilds the scored package from a pristine baked copy of the engine plus only the agent's own .py files (anything shadowing the engine is dropped), then generates a hidden, deterministically seeded instance pool spanning the same devices, timing models, and circuit families, but on held-out OpenQASM circuits and fresh synthetic seeds. The candidate driver runs de-rooted with the 10-second per-call cap, and every returned schedule is independently re-simulated on the pristine engine before it can score.\n- Per instance: zero unless the re-simulated schedule is valid and completes every gate; then zero at a trusted greedy baseline, full credit at a precomputed CP-SAT reference target, with 2**u - 1 in speedup space between the two.\n- The aggregate is a weighted mean in which synthetic and OpenQASM instances contribute 50/50; unrouted instances contribute zero.\n- The greedy baseline and CP-SAT targets live root-only in the verifier image, the scorer never imports code from /app , the reward directory is locked before any agent code runs, and stray agent processes are killed before scoring.\n- A missing router.py or a package-boundary violation zeroes the result.", "verification_bullets": ["Per instance: zero unless the re-simulated schedule is valid and completes every gate; then zero at a trusted greedy baseline, full credit at a precomputed CP-SAT reference target, with 2**u - 1 in speedup space between the two.", "The aggregate is a weighted mean in which synthetic and OpenQASM instances contribute 50/50; unrouted instances contribute zero.", "The greedy baseline and CP-SAT targets live root-only in the verifier image, the scorer never imports code from /app , the reward directory is locked before any agent code runs, and stray agent processes are killed before scoring.", "A missing router.py or a package-boundary violation zeroes the result."], "safeguards": ["Per instance: zero unless the re-simulated schedule is valid and completes every gate; then zero at a trusted greedy baseline, full credit at a precomputed CP-SAT reference target, with 2**u - 1 in speedup space between the two.", "The aggregate is a weighted mean in which synthetic and OpenQASM instances contribute 50/50; unrouted instances contribute zero.", "A missing router.py or a package-boundary violation zeroes the result."], "environment": {"Base image": "python:3.13-slim", "Tools available": "Python 3", "Compute": "4 CPUs · 16 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "iic-jku/ibm_qx_mapping (JKU OpenQASM examples) (MIT): RevLib-derived OpenQASM benchmark circuits, fetched from a pinned commit at image build, normalized to single-qreg CX-only circuits, and split deterministically into the agent-visible training pool and the hidden verifier test pool.", "url": "https://github.com/iic-jku/ibm_qx_mapping"}, {"title": "pnnl/QASMBench (BSD-3-Clause-style permissive (Battelle)): The small/medium/large OpenQASM benchmark circuits, fetched from a pinned commit at image build and normalized and split identically to the JKU set.", "url": "https://github.com/pnnl/QASMBench"}, {"title": "Hardware coupling topologies : Public factual device connectivity for IBM Q 16 (QX5 / Melbourne), IBM Q 20 (QX20 / Tokyo), Rigetti Acorn, and Google Sycamore, re-expressed as static edge lists; no upstream code or data files are copied.", "url": null}, {"title": "RL qubit-routing lineage (Pozzi et al. / qroute-router) (GPL-3.0 / MIT (design lineage only)): The timestep/edge-lock interaction model, device set, and synthetic circuit families follow the reinforcement-learning qubit-routing line of work, but the environment was reimplemented from scratch as stdlib-only Python; no upstream code or data is copied.", "url": "https://github.com/Macro206/qubit-routing-with-rl"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.8686, "std": 0.012, "steps": "282", "in_tok": "72.9M", "out_tok": "310k", "avg_cost": "$43.99", "avg_time": "9.6h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.7471, "std": 0.037, "steps": "515", "in_tok": "111.7M", "out_tok": "220k", "avg_cost": "$94.28", "avg_time": "8.2h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.6845, "std": 0.062, "steps": "339", "in_tok": "90.3M", "out_tok": "313k", "avg_cost": "$32.83", "avg_time": "16.4h"}, {"rank": 4, "model": "Kimi K3", "score": 0.6838, "std": 0.056, "steps": "467", "in_tok": "162.6M", "out_tok": "550k", "avg_cost": "$60.62", "avg_time": "19.9h"}, {"rank": 5, "model": "Grok 4.6", "score": 0.6693, "std": 0.052, "steps": "313", "in_tok": "76.8M", "out_tok": "490k", "avg_cost": "$80.91", "avg_time": "5.1h"}, {"rank": 6, "model": "Gemini 3.7 Flash", "score": 0.6539, "std": 0.066, "steps": "190", "in_tok": "31.8M", "out_tok": "219k", "avg_cost": "$6.14", "avg_time": "3.6h"}, {"rank": 7, "model": "Muse Spark 1.2", "score": 0.5861, "std": 0.021, "steps": "255", "in_tok": "46.1M", "out_tok": "340k", "avg_cost": "$9.23", "avg_time": "3.3h"}, {"rank": 8, "model": "DeepSeek V4 Flash Exp", "score": 0.5146, "std": 0.074, "steps": "409", "in_tok": "137.0M", "out_tok": "424k", "avg_cost": "$2.55", "avg_time": "10.2h"}, {"rank": 9, "model": "Qwen3.8-Max", "score": 0.478, "std": 0.158, "steps": "414", "in_tok": "83.8M", "out_tok": "780k", "avg_cost": "$26.21", "avg_time": "18.9h"}, {"rank": 10, "model": "Inkling", "score": 0.2102, "std": 0.156, "steps": "110", "in_tok": "4.9M", "out_tok": "42k", "avg_cost": "$4.81", "avg_time": "1.5h"}], "best_score": 0.8686, "mean_score": 0.6096, "n_models": 10}, {"slug": "rbc-chess-recovery", "number": 26, "title": "Reconnaissance Blind Chess Recovery", "url": "https://www.frontierswe.com/tasks/rbc-chess-recovery", "categories": ["AI Research"], "summary": "Build a blind chess bot that recovers Stockfish's strength against opponents who can see the full board.", "scoring": "Hidden tests", "toolchains": ["Python"], "compute": {"label": "CPU", "kind": "cpu", "compute": "6 CPUs · 24 GB RAM", "gpu": null, "base_image": "ubuntu:22.04", "tools": "Python 3", "time_limit": "15h"}, "background": "Reconnaissance Blind Chess (RBC) is an imperfect-information variant of chess: a player never sees the opponent's moves or the true board, only a private 3×3 sense result each turn and the outcomes of its own moves. Playing well requires maintaining a belief distribution over thousands of possible board states, choosing sense actions that maximize information, and moving under genuine uncertainty — all classic hard problems in game AI.\nThis task is deliberately RBC-adjacent rather than standard RBC: the submitted bot is blind, but its opponents are sighted chess engines that receive the true board every turn. The question is how much of Stockfish's playing strength can be recovered through the limited RBC observation and action channels — under a fast 3+3 Fischer clock that rules out brute-force belief enumeration.", "task": "Implement make_bot(game_id: str) in /app/blind_bot.py , returning a reconchess.Player . The bot receives the ordinary RBC callbacks — sense results and its own move outcomes — but never the true board or the opponent's hidden move, while its engine opponents see everything.\n- Games use a 3+3 Fischer clock; every callback must stay below three seconds, and a callback failure or timeout attributed to the bot counts as a loss.\n- game_id is opaque: the bot must not depend on an encoded opponent, color, schedule position, or random seed.\n- Matches are offline, the captured /app workspace is read-only during a match, and writable scratch is fresh for every game — no cross-game state is available.\n- Additional modules may be added under /app and imported from the factory.", "task_bullets": ["Games use a 3+3 Fischer clock; every callback must stay below three seconds, and a callback failure or timeout attributed to the bot counts as a loss.", "game_id is opaque: the bot must not depend on an encoded opponent, color, schedule position, or random seed.", "Matches are offline, the captured /app workspace is read-only during a match, and writable scratch is fresh for every game — no cross-game state is available.", "Additional modules may be added under /app and imported from the factory."], "agent_gets": "A CPU-only workspace with the bot stub at /app/blind_bot.py , interface documentation in /app/README.md , and a development match runner at /app/run_dev_matches.py that plays seeded games against sighted Stockfish at three strengths (--level easy targeting roughly 1320 Elo, medium roughly 1700, hard roughly 2200) and reports wins, losses, callback failures, timing, and replay paths. Preinstalled: Stockfish 15.1 at /usr/games/stockfish , Python 3.11 with reconchess 1.6.9, python-chess , and NumPy. The sealed evaluation policies and their configurations are root-only and never visible. The machine is offline; a sandbox-timer CLI reports the remaining budget.", "verification": "A root-only verifier in a separate no-network container plays an official 140-game evaluation: 10 color-balanced games against each of 14 sealed policies — two sighted-Stockfish policy families across seven strength bands from 800 to 2000. The reward is the mean game score (win 1, draw 0.5, loss 0) over all 140 games. The mixture score is uncalibrated and is not a human or classical Elo rating.\n- The submission runs isolated as the unprivileged agent user inside a required cgroup-v2 / PID-namespace containment with a read-only root, private tmpfs scratch, seccomp restrictions, and trusted turn timing enforced outside the bot's process.\n- The scorer accepts only a complete, security-marked official schedule and attests every harness source file, the policy manifest, and the Stockfish binary against pinned SHA-256 digests.\n- Entrant failures count as losses, and games missing when the 4.5-hour suite deadline hits score zero — but trusted-opponent, engine, or harness failures invalidate the evaluation rather than becoming free wins.\n- Opponent randomness is drawn from a private per-run entropy key, so policy behavior cannot be replayed or predicted from public seeds.", "verification_bullets": ["The submission runs isolated as the unprivileged agent user inside a required cgroup-v2 / PID-namespace containment with a read-only root, private tmpfs scratch, seccomp restrictions, and trusted turn timing enforced outside the bot's process.", "The scorer accepts only a complete, security-marked official schedule and attests every harness source file, the policy manifest, and the Stockfish binary against pinned SHA-256 digests.", "Entrant failures count as losses, and games missing when the 4.5-hour suite deadline hits score zero — but trusted-opponent, engine, or harness failures invalidate the evaluation rather than becoming free wins.", "Opponent randomness is drawn from a private per-run entropy key, so policy behavior cannot be replayed or predicted from public seeds."], "safeguards": ["Entrant failures count as losses, and games missing when the 4.5-hour suite deadline hits score zero — but trusted-opponent, engine, or harness failures invalidate the evaluation rather than becoming free wins."], "environment": {"Base image": "ubuntu:22.04", "Tools available": "Python 3", "Compute": "6 CPUs · 24 GB RAM", "Time limit": "15h"}, "changes_from_v1": "", "references": [{"title": "Stockfish (GPL-3.0): Version 15.1 (retained Debian package), baked at /usr/games/stockfish . Used sighted — granted the true board each turn — in the two sealed policy families; the submission never receives the true board or policy source.", "url": "https://stockfishchess.org"}, {"title": "reconchess (BSD-3-Clause): The Reconnaissance Blind Chess game framework by Johns Hopkins University Applied Physics Laboratory: the Player interface the submission implements and the game loop the harness drives (version 1.6.9).", "url": "https://github.com/reconnaissanceblindchess/reconchess"}, {"title": "python-chess (GPL-3.0): Board representation, move generation, and the UCI bridge to Stockfish (version 1.999, by Niklas Fiekas).", "url": "https://github.com/niklasf/python-chess"}, {"title": "uv (Apache-2.0 OR MIT): Version 0.11.7; installs the retained CPython 3.11.15 and the hash-locked Python dependency set during image construction. Supporting libraries (NumPy, tqdm, click, loguru, and others) come from PyPI at image build. The task harness, scoring logic, and instructions are first-party; Scorca was considered as a reference opponent but excluded because of its CC-BY-ND 4.0 license.", "url": "https://github.com/astral-sh/uv"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.8957, "std": 0.076, "steps": "266", "in_tok": "72.8M", "out_tok": "265k", "avg_cost": "$42.84", "avg_time": "9.9h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.8186, "std": 0.059, "steps": "240", "in_tok": "37.3M", "out_tok": "123k", "avg_cost": "$24.22", "avg_time": "7.6h"}, {"rank": 3, "model": "Kimi K3", "score": 0.43, "std": 0.228, "steps": "308", "in_tok": "81.9M", "out_tok": "325k", "avg_cost": "$32.14", "avg_time": "14.3h"}, {"rank": 4, "model": "Grok 4.6", "score": 0.3936, "std": 0.215, "steps": "417", "in_tok": "98.7M", "out_tok": "866k", "avg_cost": "$110.04", "avg_time": "12.3h"}, {"rank": 5, "model": "GLM-5.3", "score": 0.3729, "std": 0.111, "steps": "311", "in_tok": "89.0M", "out_tok": "330k", "avg_cost": "$33.18", "avg_time": "14.4h"}, {"rank": 6, "model": "Muse Spark 1.2", "score": 0.3464, "std": 0.082, "steps": "176", "in_tok": "38.5M", "out_tok": "470k", "avg_cost": "$8.40", "avg_time": "5.6h"}, {"rank": 7, "model": "Gemini 3.7 Flash", "score": 0.2121, "std": 0.164, "steps": "430", "in_tok": "151.1M", "out_tok": "658k", "avg_cost": "$23.43", "avg_time": "5.8h"}, {"rank": 8, "model": "DeepSeek V4 Flash Exp", "score": 0.0707, "std": 0.076, "steps": "382", "in_tok": "138.8M", "out_tok": "427k", "avg_cost": "$2.59", "avg_time": "11.6h"}, {"rank": 9, "model": "Inkling", "score": 0.0664, "std": 0.08, "steps": "85", "in_tok": "2.8M", "out_tok": "26k", "avg_cost": "$2.78", "avg_time": "1.4h"}, {"rank": 10, "model": "Qwen3.8-Max", "score": 0.065, "std": 0.047, "steps": "356", "in_tok": "59.5M", "out_tok": "728k", "avg_cost": "$19.73", "avg_time": "13.5h"}], "best_score": 0.8957, "mean_score": 0.3671, "n_models": 10}, {"slug": "inference-system-optimization-v2", "number": 27, "title": "SGLang Inference System Optimization", "url": "https://www.frontierswe.com/tasks/inference-system-optimization-v2", "categories": ["AI Research"], "summary": "Make an already well-tuned SGLang server serving Qwen3.5-4B on a B200 faster, without changing a token of its greedy outputs.", "scoring": "Perf speedup", "toolchains": ["CUDA", "Python", "C/C++"], "compute": {"label": "1× B200", "kind": "gpu", "compute": "8 CPUs · 128 GB RAM", "gpu": "1× B200", "base_image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", "tools": "CUDA, PyTorch, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "LLM serving systems are full-stack performance problems: request scheduling, batching, KV-cache memory management, speculative decoding, CUDA-graph capture, and attention kernels all interact, and the fastest configuration for single sequential requests is rarely the fastest under concurrent load. Optimizing one without regressing the other (while keeping the model's outputs bit-for-bit faithful under greedy decoding) is what production inference teams spend months on.\nThe starting point here is deliberately not a strawman: the provided SGLang launch configuration already uses an FP8 KV cache, NEXTN/MTP speculative decoding, the extra-buffer mamba scheduler, CUDA graphs, and tuned page and memory settings. Flag-tuning alone is unlikely to yield much; real gains require custom kernels, SGLang or FlashInfer source modifications, or model surgery that preserves output equivalence.", "task": "The agent must make an SGLang serving instance with Qwen3.5-4B on a B200 GPU serve requests as fast as possible. Single-request latency across input/output shapes and batched requests at several concurrency levels matter comparably.\n- The server is started through /app/server/launch_server.sh , which receives PORT and MODEL_PATH environment variables and must expose the OpenAI-compatible SGLang HTTP API.\n- Correctness is a hard requirement: with temperature-0 greedy decoding, the optimized server must keep producing the outputs of the starting configuration, with an average whitespace-token prefix match of at least 0.95.\n- Everything the server needs must live inside /app and be reapplied or imported by launch_server.sh : the server is later launched from a fresh copy of the workspace in which only /app carries over, so direct edits to site-packages or in-place edits to /app/model do not survive on their own.\n- The design space is wide: scheduler and batching configuration, tuning or replacing the speculative pipeline, custom Triton or CUDA kernels, direct SGLang and FlashInfer source modifications, and model modifications (quantise, prune, fuse), all under the output-equivalence requirement.", "task_bullets": ["The server is started through /app/server/launch_server.sh , which receives PORT and MODEL_PATH environment variables and must expose the OpenAI-compatible SGLang HTTP API.", "Correctness is a hard requirement: with temperature-0 greedy decoding, the optimized server must keep producing the outputs of the starting configuration, with an average whitespace-token prefix match of at least 0.95.", "Everything the server needs must live inside /app and be reapplied or imported by launch_server.sh : the server is later launched from a fresh copy of the workspace in which only /app carries over, so direct edits to site-packages or in-place edits to /app/model do not survive on their own.", "The design space is wide: scheduler and batching configuration, tuning or replacing the speculative pipeline, custom Triton or CUDA kernels, direct SGLang and FlashInfer source modifications, and model modifications (quantise, prune, fuse), all under the output-equivalence requirement."], "agent_gets": "A running serving workspace: Qwen3.5-4B weights pre-downloaded at /app/model , the tuned launch_server.sh starting configuration, and a local measurement loop. compare_outputs.py snapshots the pristine server's greedy outputs on the visible 174-prompt dev set (dev_prompts.jsonl , generated from the same distribution as the held-back set) and diffs after every change; verify_serving.py sanity-checks the server and run_dev_bench.py measures latency across input-length × output-length quadrants plus a concurrent batch.\nPreinstalled tooling includes the CUDA 12.8 dev toolkit (nvcc ), PyTorch with Triton and torch.compile , the pinned SGLang + sgl_kernel + FlashInfer serving stack in site-packages, and gcc /ninja for native builds. There is no internet access at runtime (no package installs, no model downloads), and a sandbox-timer CLI reports the remaining budget.", "verification": "Verification runs in a separate clean-room environment on the captured /app . It launches the baseline and candidate servers under the same unprivileged user, collects greedy outputs on a held-back prompt set, and measures an A/B/A benchmark: baseline session, candidate session, then a second baseline session whose samples are pooled with the first into the scored median.\n- The correctness gate compares candidate outputs against the baseline's on hidden prompts (normal text, code, math, long-context, degenerate and adversarial inputs); an average whitespace-token prefix-match below 0.95 zeroes the result.\n- Speed is measured on five sequential workloads covering the input-length × output-length quadrants plus a reasoning shape, and two concurrent workloads (4-way and 8-way mixed). Per-class geometric-mean speedups are mapped linearly to credit between a live-zero anchor at 1.02x (parity and no-op relaunches score exactly 0) and a 2.0x cap.\n- The aggregate is capped at 0.30 + 0.70 × the weaker class's credit, so the top of the range is unreachable without genuine improvement in both sequential and concurrent serving; continuous regression gates on each class and on the weakest single workload collapse the score for real regressions while tolerating measured replay noise.\n- The timed path itself is integrity-checked: every benchmark response is compared against the baseline's response to the same salted request, with mean and median prefix-match bars plus a completion-length ratio that catches servers truncating output under load. Failing any of these gates zeroes the result, as does a server that fails to launch.", "verification_bullets": ["The correctness gate compares candidate outputs against the baseline's on hidden prompts (normal text, code, math, long-context, degenerate and adversarial inputs); an average whitespace-token prefix-match below 0.95 zeroes the result.", "Speed is measured on five sequential workloads covering the input-length × output-length quadrants plus a reasoning shape, and two concurrent workloads (4-way and 8-way mixed). Per-class geometric-mean speedups are mapped linearly to credit between a live-zero anchor at 1.02x (parity and no-op relaunches score exactly 0) and a 2.0x cap.", "The aggregate is capped at 0.30 + 0.70 × the weaker class's credit, so the top of the range is unreachable without genuine improvement in both sequential and concurrent serving; continuous regression gates on each class and on the weakest single workload collapse the score for real regressions while tolerating measured replay noise.", "The timed path itself is integrity-checked: every benchmark response is compared against the baseline's response to the same salted request, with mean and median prefix-match bars plus a completion-length ratio that catches servers truncating output under load. Failing any of these gates zeroes the result, as does a server that fails to launch."], "safeguards": ["The correctness gate compares candidate outputs against the baseline's on hidden prompts (normal text, code, math, long-context, degenerate and adversarial inputs); an average whitespace-token prefix-match below 0.95 zeroes the result.", "Speed is measured on five sequential workloads covering the input-length × output-length quadrants plus a reasoning shape, and two concurrent workloads (4-way and 8-way mixed). Per-class geometric-mean speedups are mapped linearly to credit between a live-zero anchor at 1.02x (parity and no-op relaunches score exactly 0) and a 2.0x cap.", "The timed path itself is integrity-checked: every benchmark response is compared against the baseline's response to the same salted request, with mean and median prefix-match bars plus a completion-length ratio that catches servers truncating output under load. Failing any of these gates zeroes the result, as does a server that fails to launch."], "environment": {"Base image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", "Tools available": "CUDA, PyTorch, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 128 GB RAM", "GPU": "1× B200", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of SGLang Inference System Optimization from FrontierSWE V1.\n- The agent budget grew from 4 hours to 20 hours, and the verifier window from 1 hour to 3 hours to cover three full server launches with A/B/A baseline bracketing (both baseline sessions now pool into the scored median instead of serving only as an anomaly check).\n- Scoring changed from a raw geometric-mean speedup to a calibrated credit map: sequential and concurrent classes are scored separately against a 2.0x cap with a live-zero anchor at 1.02x so parity and no-op relaunches score exactly 0, a weak-class cap requires genuine improvement in both classes, and continuous regression gates on each class and the weakest workload replace cliff-edge failures.\n- New timed-path integrity gates: every benchmark response is compared against the baseline's response to the same salted request, with mean and median prefix-match bars and a completion-length ratio that catches truncation under load: a server can no longer buy latency by degrading outputs only while being timed.\n- Correctness became locally measurable: the workspace now ships a visible 174-prompt dev set drawn from the same generator as the held-back set plus a compare_outputs.py snapshot/diff loop, with the 0.95 prefix-match bar and the ~0.97 relaunch-noise budget documented up front.\n- Verification moved to a separate clean-room container with an explicit persistence contract: the server relaunches from a fresh workspace copy in which only /app carries over, and the submission root moved from /app/submission/ to /app/server/ .\n- Infrastructure moved to the new harbor schema with pinned agent and verifier images, and the /app/.timer daemon files were replaced by the sandbox-timer CLI.", "references": [{"title": "SGLang (+ sgl_kernel) (Apache-2.0): Installed into the task image via pip (sglang[srt] ) as the serving runtime the agent optimizes and may patch in site-packages; not vendored.", "url": "https://github.com/sgl-project/sglang"}, {"title": "FlashInfer (Apache-2.0): Installed via pip as SGLang's attention kernel backend.", "url": "https://github.com/flashinfer-ai/flashinfer"}, {"title": "PyTorch (BSD-3-Clause): The tensor/CUDA runtime (bundling Triton), pulled in by the pinned sglang[srt] .", "url": "https://github.com/pytorch/pytorch"}, {"title": "FlashAttention 4 (BSD-3-Clause): Installed as the pinned flash-attn-4 package for Blackwell attention kernels.", "url": "https://github.com/Dao-AILab/flash-attention"}, {"title": "Qwen3.5-4B (Apache-2.0): The served model weights, downloaded into the image at build time and exposed to the server through /app/model .", "url": "https://huggingface.co/Qwen/Qwen3.5-4B"}, {"title": "NVIDIA CUDA base image (NVIDIA Deep Learning Container License / CUDA EULA): nvidia/cuda:12.8.1-devel-ubuntu22.04 provides the CUDA 12.8 dev toolkit for Blackwell.", "url": "https://hub.docker.com/r/nvidia/cuda"}, {"title": "python-build-standalone (CPython 3.11) (MPL-2.0 (tooling); interpreter under PSF-2.0): The standalone CPython 3.11 interpreter installed in the image, avoiding a Triton segfault seen with the distro Python on Blackwell.", "url": "https://github.com/astral-sh/python-build-standalone"}, {"title": "uv (Apache-2.0 OR MIT): Pinned binaries used to create the environment and install its hash-locked Python dependencies.", "url": "https://github.com/astral-sh/uv"}, {"title": "pip (MIT): Installed as a pinned dependency so standard pip commands work inside the agent's environment.", "url": "https://github.com/pypa/pip"}, {"title": "NumPy (BSD-3-Clause): Pinned direct dependency for numerical operations.", "url": "https://github.com/numpy/numpy"}, {"title": "Pillow (HPND): Pinned direct dependency used by model-processing libraries.", "url": "https://github.com/python-pillow/Pillow"}, {"title": "Hugging Face Hub, Safetensors, and Transformers (Apache-2.0): Pinned direct dependencies for downloading the model at image-build time, reading its tensor files, and loading its configuration and tokenizer.", "url": "https://github.com/huggingface/transformers"}, {"title": "Requests (Apache-2.0): Pinned direct dependency for HTTP client support.", "url": "https://github.com/psf/requests"}], "results": [{"rank": 1, "model": "GPT-5.6", "score": 0.0387, "std": 0.048, "steps": "609", "in_tok": "258.0M", "out_tok": "221k", "avg_cost": "$261.91", "avg_time": "6.1h"}, {"rank": 2, "model": "Muse Spark 1.2", "score": 0.0152, "std": 0.022, "steps": "739", "in_tok": "245.0M", "out_tok": "810k", "avg_cost": "$69.69", "avg_time": "9.5h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.015, "std": 0.023, "steps": "626", "in_tok": "208.7M", "out_tok": "395k", "avg_cost": "$60.48", "avg_time": "13.5h"}, {"rank": 4, "model": "Grok 4.6", "score": 0.0073, "std": 0.014, "steps": "1,085", "in_tok": "281.2M", "out_tok": "1.6M", "avg_cost": "$295.86", "avg_time": "14.3h"}, {"rank": 5, "model": "Qwen3.8-Max", "score": 0.006, "std": 0.009, "steps": "639", "in_tok": "139.7M", "out_tok": "1.3M", "avg_cost": "$43.20", "avg_time": "16.8h"}, {"rank": 6, "model": "Gemini 3.7 Flash", "score": 0.0034, "std": 0.008, "steps": "350", "in_tok": "104.9M", "out_tok": "171k", "avg_cost": "$11.36", "avg_time": "2.7h"}, {"rank": 7, "model": "Kimi K3", "score": 0.0021, "std": 0.005, "steps": "1,032", "in_tok": "407.1M", "out_tok": "1.0M", "avg_cost": "$143.36", "avg_time": "16.6h"}, {"rank": 8, "model": "DeepSeek V4 Flash Exp", "score": 0.0, "std": 0.0, "steps": "622", "in_tok": "217.1M", "out_tok": "392k", "avg_cost": "$3.69", "avg_time": "9.4h"}, {"rank": 9, "model": "Claude Fable 5.1", "score": 0.0, "std": 0.0, "steps": "852", "in_tok": "357.3M", "out_tok": "878k", "avg_cost": "$199.85", "avg_time": "10.0h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "121", "in_tok": "8.1M", "out_tok": "17k", "avg_cost": "$7.77", "avg_time": "1.3h"}], "best_score": 0.0387, "mean_score": 0.0088, "n_models": 10}, {"slug": "snooker-prediction", "number": 28, "title": "Snooker Prediction", "url": "https://www.frontierswe.com/tasks/snooker-prediction", "categories": ["Visual Reasoning", "AI Research"], "summary": "Watch three seconds of a snooker shot and predict where every ball will be up to four seconds later.", "scoring": "Custom reward", "toolchains": ["Python"], "compute": {"label": "CPU", "kind": "cpu", "compute": "8 CPUs · 32 GB RAM", "gpu": null, "base_image": "python:3.11-slim-bookworm", "tools": "FFmpeg, Python 3", "time_limit": "20h"}, "background": "Predicting where snooker balls end up from a short video clip chains together most of classical computer vision and physical reasoning: detecting and tracking balls through occlusions and collisions, recovering the table's coordinate frame from pixels, estimating velocities from a handful of frames, and then rolling the dynamics forward through ball-ball impacts, cushion bounces, and friction, seconds past the last observed frame, where small velocity errors compound quickly.\nThe clips are first-party and procedurally generated: tables, balls, and trajectories are rendered from a snooker-physics simulation, and ground-truth coordinates come from the same simulation rather than human annotation. No broadcast or scraped footage is involved, so the answers cannot be recalled from training data.", "task": "Build a program that predicts the future position of every snooker ball for each clip and timestamp pair listed in /app/data/target_times.csv , writing the results to /app/predictions.csv . Each observation video covers 0.0–3.0 seconds; target timestamps range from 3.5 to 7.0 seconds, up to four seconds beyond the last observed frame.\n- Predictions use the schema clip_id,time,ball_id,color,x,y , with coordinates in meters in the semantic table frame (origin at the center of the playable surface, roughly ±1.70 m by ±0.80 m).\n- Non-red balls are matched by color; red-ball IDs need not be stable, since reds are matched by position.\n- Quality depends on positional accuracy and complete coverage of the requested balls and timestamps; omitted balls are penalized at the worst physically meaningful distance.\n- No external data or services; changes stay confined to /app .", "task_bullets": ["Predictions use the schema clip_id,time,ball_id,color,x,y , with coordinates in meters in the semantic table frame (origin at the center of the playable surface, roughly ±1.70 m by ±0.80 m).", "Non-red balls are matched by color; red-ball IDs need not be stable, since reds are matched by position.", "Quality depends on positional accuracy and complete coverage of the requested balls and timestamps; omitted balls are penalized at the worst physically meaningful distance.", "No external data or services; changes stay confined to /app ."], "agent_gets": "29 rendered MP4 observation clips under /app/data/videos/ , each ending at 3.0 seconds. Nine clips come with provided future-position examples (example_annotations.csv and example_target_times.csv ) for local development; the remaining 20 clips are the ones requiring predictions, with 160 clip/timestamp pairs in target_times.csv .\n- Starter code in /app/predict.py and a local diagnostic, python3 /app/evaluate_predictions.py , that checks predictions against the provided examples.\n- A data contract and coordinate-frame reference in /app/data/README.md .\n- Python 3.11 with NumPy, pandas, SciPy, OpenCV, Pillow, Matplotlib, tqdm, and FFmpeg preinstalled for frame extraction and analysis. The machine is effectively offline.", "verification": "A root-only verifier scores /app/predictions.csv against private annotations taken from the same simulation that rendered the clips. For each clip/timestamp group, predictions are matched to truth within each color class (reds via minimum-cost Hungarian assignment), and per-ball distances are capped at the table diagonal, so submitting an uncertain ball is never worse than omitting it. Missing balls, extra balls, malformed rows, and rows at wrong timestamps all incur the maximum-distance penalty.\n- The score is max(0, 1 − RMS / 0.85 m) over all scored points. The 0.85 m threshold puts frozen data-free baselines at zero: a constant layout derived from the examples measures 0.91 m RMS and random in-bounds guessing 1.35 m.\n- A missing, symlinked, or unparseable predictions file zeroes the result; rows for the public example clips are ignored without penalty.", "verification_bullets": ["The score is max(0, 1 − RMS / 0.85 m) over all scored points. The 0.85 m threshold puts frozen data-free baselines at zero: a constant layout derived from the examples measures 0.91 m RMS and random in-bounds guessing 1.35 m.", "A missing, symlinked, or unparseable predictions file zeroes the result; rows for the public example clips are ignored without penalty."], "safeguards": ["The score is max(0, 1 − RMS / 0.85 m) over all scored points. The 0.85 m threshold puts frozen data-free baselines at zero: a constant layout derived from the examples measures 0.91 m RMS and random in-bounds guessing 1.35 m.", "A missing, symlinked, or unparseable predictions file zeroes the result; rows for the public example clips are ignored without penalty."], "environment": {"Base image": "python:3.11-slim-bookworm", "Tools available": "FFmpeg, Python 3", "Compute": "8 CPUs · 32 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "cp-algorithms Hungarian algorithm (CC BY-SA 4.0): The minimum-cost assignment routine in the hidden scorer is adapted from Andrey Lopatin's implementation published by cp-algorithms; the agent-visible diagnostic uses SciPy's implementation instead. Everything else is original to this task: the clips (procedurally generated from a first-party snooker-physics simulation), scorer, diagnostics, and coordinate contract.", "url": "https://cp-algorithms.com/graph/hungarian-algorithm.html"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.4688, "std": 0.055, "steps": "425", "in_tok": "158.0M", "out_tok": "403k", "avg_cost": "$88.76", "avg_time": "6.5h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.4262, "std": 0.081, "steps": "364", "in_tok": "141.7M", "out_tok": "236k", "avg_cost": "$151.76", "avg_time": "4.5h"}, {"rank": 3, "model": "Grok 4.6", "score": 0.2009, "std": 0.142, "steps": "591", "in_tok": "146.0M", "out_tok": "1.0M", "avg_cost": "$176.62", "avg_time": "9.6h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.0778, "std": 0.077, "steps": "539", "in_tok": "230.8M", "out_tok": "594k", "avg_cost": "$73.28", "avg_time": "17.7h"}, {"rank": 5, "model": "Qwen3.8-Max", "score": 0.0176, "std": 0.039, "steps": "669", "in_tok": "222.7M", "out_tok": "1.7M", "avg_cost": "$83.53", "avg_time": "19.3h"}, {"rank": 6, "model": "Inkling", "score": 0.0107, "std": 0.015, "steps": "87", "in_tok": "2.6M", "out_tok": "41k", "avg_cost": "$2.68", "avg_time": "1.2h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0, "std": 0.0, "steps": "581", "in_tok": "242.0M", "out_tok": "587k", "avg_cost": "$4.25", "avg_time": "12.0h"}, {"rank": 8, "model": "Gemini 3.7 Flash", "score": 0.0, "std": 0.0, "steps": "262", "in_tok": "54.3M", "out_tok": "348k", "avg_cost": "$7.16", "avg_time": "1.6h"}, {"rank": 9, "model": "Kimi K3", "score": 0.0, "std": 0.0, "steps": "774", "in_tok": "331.1M", "out_tok": "926k", "avg_cost": "$122.31", "avg_time": "19.8h"}, {"rank": 10, "model": "Muse Spark 1.2", "score": 0.0, "std": 0.0, "steps": "314", "in_tok": "78.4M", "out_tok": "444k", "avg_cost": "$17.08", "avg_time": "1.6h"}], "best_score": 0.4688, "mean_score": 0.1202, "n_models": 10}, {"slug": "spice-sim-rust", "number": 29, "title": "SPICE Circuit Simulator in Rust", "url": "https://www.frontierswe.com/tasks/spice-sim-rust", "categories": ["Implementation"], "summary": "Build a SPICE circuit simulator in Rust (BSIM3-SOI plus the classic Berkeley device set) that matches ngspice's numbers across seven analysis types.", "scoring": "Byte exact", "toolchains": ["Rust", "Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "2 CPUs · 4 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Rust, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "SPICE is the lingua franca of analog circuit simulation: a netlist describes devices and their connections, and the simulator builds and solves the nonlinear system they imply: operating points via Newton iteration, transient responses via adaptive time-stepping, small-signal AC, pole-zero, noise, and sensitivity analyses on top. Getting the numbers right means simultaneously getting the device physics right (down to the BSIM3-SOI MOSFET model, one of the more intricate compact models in the classic Berkeley lineage), the numerical methods right, and the netlist language right.\nThis task grades the simulator differentially against a live ngspice: correctness is defined as producing the same numbers the real tool produces on decks drawn from ngspice's own regression suite, not as matching any stored text.", "task": "Implement a SPICE simulator in the Cargo project at /app , with sources only as .rs files under src/ . cargo build --release compiles them into /app/target/release/spice-sim , offline. Run from a netlist's own directory as spice-sim --batch <netlist.cir> , it executes the analyses the netlist requests, prints results to stdout and diagnostics to stderr, and exits 0.\n- Support the standard analyses: .op , .dc , .tran , .ac , .pz , .noise , and .sens .\n- Cover the BSIM3-SOI MOSFET model and the classic Berkeley SPICE3 device set; modern compact models (BSIM4, HiSIM, HICUM, BSIMSOI) are out of scope.\n- Match ngspice numerically, not byte-for-byte; table layout does not matter.\n- Standard-library Rust only, with serde /serde_json available as vendored crates; no external tools.", "task_bullets": ["Support the standard analyses: .op , .dc , .tran , .ac , .pz , .noise , and .sens .", "Cover the BSIM3-SOI MOSFET model and the classic Berkeley SPICE3 device set; modern compact models (BSIM4, HiSIM, HICUM, BSIMSOI) are out of scope.", "Match ngspice numerically, not byte-for-byte; table layout does not matter.", "Standard-library Rust only, with serde /serde_json available as vendored crates; no external tools."], "agent_gets": "A ready-to-build Cargo project at /app and the scope-defining suite under suite/ : 114 netlists curated from the ngspice-42 regression tree, each paired with a .gold reference generated by the real ngspice at image build. ./build_and_test.sh (all decks) or ./build_and_test.sh <name> (one) builds and self-checks. The machine is offline; Rust 1.83.0 and the vendored crates are preinstalled, and ngspice itself is relocated off the agent's PATH and locked root-only.", "verification": "A clean-room verifier, separate from the agent's environment, reconstructs the project from a pristine scaffold plus only the agent's src/*.rs sources (and Cargo.toml ), rebuilds it offline as a non-root user, and grades it differentially against the live, root-only ngspice. Every graded deck is first numerically mutated with a hidden fixed seed (same topology, different component values), and its expected output regenerated by ngspice at verify time, so there are no stored answers to replay (four decks that cannot be safely mutated are graded on nominal goldens).\n- The comparator matches computed values, not text: waveforms under calibrated RMS and pointwise tolerances, scalars under relative tolerance.\n- Reward is the fraction of graded decks whose numbers match; build failure, a missing binary, or matching zero decks all score 0.\n- The candidate binary is copied out of /app , which is locked unreadable during grading, and runs as the nobody user, defense-in-depth against reading the baked goldens.\n- Per-deck caps of 60 s (ngspice) and 90 s (candidate) inside a 3000-second suite budget; the rebuild is capped at 600 s.", "verification_bullets": ["The comparator matches computed values, not text: waveforms under calibrated RMS and pointwise tolerances, scalars under relative tolerance.", "Reward is the fraction of graded decks whose numbers match; build failure, a missing binary, or matching zero decks all score 0.", "The candidate binary is copied out of /app , which is locked unreadable during grading, and runs as the nobody user, defense-in-depth against reading the baked goldens.", "Per-deck caps of 60 s (ngspice) and 90 s (candidate) inside a 3000-second suite budget; the rebuild is capped at 600 s."], "safeguards": ["Reward is the fraction of graded decks whose numbers match; build failure, a missing binary, or matching zero decks all score 0."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Rust, C/C++ toolchain, Python 3", "Compute": "2 CPUs · 4 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "ngspice regression test suite (BSD-3-Clause (project license; mixed open-source components)): The .cir netlists and support files from the tests/ tree of the ngspice-42 release, cloned at image build from the pinned tag and curated down to the graded device families. No upstream golden outputs are used; references are generated live by ngspice.", "url": "https://git.code.sf.net/p/ngspice/ngspice"}, {"title": "ngspice (BSD-3-Clause (project license; mixed open-source components)): The Ubuntu 24.04 package (ngspice=42+ds-3build1 ) is the differential oracle: relocated to a root-only path off the agent's PATH so it can be neither discovered nor executed, and run by the verifier to produce live golden output for every graded netlist.", "url": "https://ngspice.sourceforge.io"}, {"title": "Rust toolchain and vendored crates (MIT OR Apache-2.0 (plus permissive licenses on transitive deps)): Rust 1.83.0 via rustup, with serde and serde_json (and transitive dependencies) baked into an offline crate mirror so the agent's cargo build --release works without network access.", "url": "https://crates.io"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.7394, "std": 0.076, "steps": "423", "in_tok": "166.3M", "out_tok": "869k", "avg_cost": "$104.28", "avg_time": "5.0h"}, {"rank": 2, "model": "Grok 4.6", "score": 0.499, "std": 0.046, "steps": "2,486", "in_tok": "694.8M", "out_tok": "2.4M", "avg_cost": "$699.22", "avg_time": "16.3h"}, {"rank": 3, "model": "Gemini 3.7 Flash", "score": 0.4889, "std": 0.094, "steps": "2,471", "in_tok": "1014.2M", "out_tok": "1.8M", "avg_cost": "$100.88", "avg_time": "8.3h"}, {"rank": 4, "model": "GLM-5.3", "score": 0.4404, "std": 0.085, "steps": "1,479", "in_tok": "660.6M", "out_tok": "1.5M", "avg_cost": "$181.86", "avg_time": "16.6h"}, {"rank": 5, "model": "GPT-5.6", "score": 0.3697, "std": 0.072, "steps": "797", "in_tok": "316.5M", "out_tok": "409k", "avg_cost": "$325.29", "avg_time": "4.6h"}, {"rank": 6, "model": "Qwen3.8-Max", "score": 0.3192, "std": 0.04, "steps": "999", "in_tok": "434.6M", "out_tok": "1.8M", "avg_cost": "$121.27", "avg_time": "19.3h"}, {"rank": 7, "model": "Muse Spark 1.2", "score": 0.0727, "std": 0.035, "steps": "300", "in_tok": "97.8M", "out_tok": "465k", "avg_cost": "$17.96", "avg_time": "1.8h"}, {"rank": 8, "model": "DeepSeek V4 Flash Exp", "score": 0.0, "std": 0.0, "steps": "2,413", "in_tok": "1113.2M", "out_tok": "2.0M", "avg_cost": "$18.59", "avg_time": "15.7h"}, {"rank": 9, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "110", "in_tok": "7.3M", "out_tok": "25k", "avg_cost": "$7.01", "avg_time": "0.8h"}, {"rank": 10, "model": "Kimi K3", "score": 0.0, "std": 0.0, "steps": "1,317", "in_tok": "545.4M", "out_tok": "1.8M", "avg_cost": "$198.35", "avg_time": "19.8h"}], "best_score": 0.7394, "mean_score": 0.2929, "n_models": 10}, {"slug": "gba-stepper", "number": 30, "title": "Stepper Music Sequencer GBA", "url": "https://www.frontierswe.com/tasks/gba-stepper", "categories": ["Visual Reasoning", "Implementation"], "summary": "Clone a Game Boy Advance music tracker from black-box probes alone: pixel-exact screens and sample-exact PSG audio.", "scoring": "Pixel match", "toolchains": ["Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "8 CPUs · 8 GB RAM", "gpu": null, "base_image": "devkitpro/devkitarm:latest", "tools": "FFmpeg, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "The Game Boy Advance generates music through four programmable sound generator (PSG) channels driven directly by hardware registers (square waves, a programmable wavetable, and a noise channel), with every parameter change landing on an exact frame boundary. A step sequencer built on this hardware is a dense knot of state: pattern grids, per-channel parameter editors, sound banks, tempo logic, and a playback engine whose audible output depends on register-level synthesis details and frame-accurate timing.\nThis task turns a real, shipped GBA homebrew tracker into a clean-room reverse-engineering exercise. The reference ROM is never readable by the agent: it lives root-only inside the sandbox and can only be observed through a probe tool that runs keystroke scripts against it and returns what it drew and played. Everything the clone must reproduce has to be inferred from those observations.", "task": "The agent must reimplement the 4-channel PSG step sequencer from scratch as its own GBA ROM: the exact same screens, pixels, and controls, and above all the exact same sound and timing. The source lives in /app (starting from src/main.c ) and make -C /app must produce /app/tracker.gba .\n- Observe the reference as a black box with ref-probe <script.txt> <out_dir> , which captures screenshots (shot_*.png ), audio traces (listen_*.npy ), and frame-sequence recordings with an .mp4 preview for any keystroke script the agent writes.\n- Match the UI pixel for pixel across pattern grids, parameter editors, sound banks, and transport screens.\n- Match the PSG audio synthesis and frame-accurate sequencer timing: the audible output, not just the visuals.\n- Iterate with ./run-tests.sh , which builds the ROM and reports, per script, how close its frames and audio are to the reference.", "task_bullets": ["Observe the reference as a black box with ref-probe <script.txt> <out_dir> , which captures screenshots (shot_*.png ), audio traces (listen_*.npy ), and frame-sequence recordings with an .mp4 preview for any keystroke script the agent writes.", "Match the UI pixel for pixel across pattern grids, parameter editors, sound banks, and transport screens.", "Match the PSG audio synthesis and frame-accurate sequencer timing: the audible output, not just the visuals.", "Iterate with ./run-tests.sh , which builds the ROM and reports, per script, how close its frames and audio are to the reference."], "agent_gets": "A ready-to-build GBA project at /app : a Makefile , a starter src/main.c , sample keystroke scripts in /app/scripts/ , the ref-probe client, capture-and-compare tooling in /app/tools/ (the same capture code the verifier uses), and /app/README.md documenting the script format and workflow. The reference ROM itself is runnable but never readable; only its captured frames and audio are observable. The image preinstalls the devkitARM GBA toolchain, the mGBA emulator with Python bindings, and an audio-analysis stack (librosa , scipy , soundfile ). The machine is offline.", "verification": "A clean-room verifier in a separate container rebuilds the clone from captured source (make -C /app , run as the non-root agent user) and captures both the reference and the clone on a hidden suite of keystroke scripts: seeded, full-range variants of the public samples that sweep every scale, parameter, sound bank, tempo, and sustained 30-60 second songs. The reward is 0.25 × visual + 0.75 × audio , deliberately weighted toward the harder dimension.\n- Visual closeness is an exponential of per-pixel MSE on every captured frame; a bit-identical clone scores 1.0. Audio closeness combines log-mel spectral distance, RMS level ratio, and trace length ratio, with symmetric silence gating so a silent clone scores zero on audible events.\n- A self-check captures the reference twice and requires both dimensions to agree above 0.999; a failed self-check marks the trial invalid as a harness fault rather than scoring the agent zero.\n- A failed build or missing ROM zeroes the result, and tracker.gba is rejected if it is a symlink or resolves outside /app ; the clone is fed to the emulator purely as data, with no agent process alive during capture.", "verification_bullets": ["Visual closeness is an exponential of per-pixel MSE on every captured frame; a bit-identical clone scores 1.0. Audio closeness combines log-mel spectral distance, RMS level ratio, and trace length ratio, with symmetric silence gating so a silent clone scores zero on audible events.", "A self-check captures the reference twice and requires both dimensions to agree above 0.999; a failed self-check marks the trial invalid as a harness fault rather than scoring the agent zero.", "A failed build or missing ROM zeroes the result, and tracker.gba is rejected if it is a symlink or resolves outside /app ; the clone is fed to the emulator purely as data, with no agent process alive during capture."], "safeguards": ["Visual closeness is an exponential of per-pixel MSE on every captured frame; a bit-identical clone scores 1.0. Audio closeness combines log-mel spectral distance, RMS level ratio, and trace length ratio, with symmetric silence gating so a silent clone scores zero on audible events.", "A self-check captures the reference twice and requires both dimensions to agree above 0.999; a failed self-check marks the trial invalid as a harness fault rather than scoring the agent zero.", "A failed build or missing ROM zeroes the result, and tracker.gba is rejected if it is a symlink or resolves outside /app ; the clone is fed to the emulator purely as data, with no agent process alive during capture."], "environment": {"Base image": "devkitpro/devkitarm:latest", "Tools available": "FFmpeg, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 8 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "STEPPER (by Bad Diode) (MIT): The black-box reference ROM (v1.9, 96 KiB). It is baked root-only into the verifier side of the image and never enters the agent workspace; the agent observes it only through ref-probe captures. No STEPPER or Bad Diode branding appears in any agent-visible surface, and no bytes of the ROM reach the graded deliverable.", "url": "https://git.badd10de.dev/stepper"}, {"title": "mGBA (MPL-2.0): Version 0.10.5, built from source into the image (headless core plus Python bindings); used by both the agent for probing and the verifier for capture and grading. Fetched at image build, not redistributed in the task repository.", "url": "https://github.com/mgba-emu/mgba"}, {"title": "devkitPro / devkitARM (GPL/LGPL (libgba: LGPL with static-linking exception)): The GBA homebrew toolchain base image (arm-none-eabi GCC and binutils plus libgba) that both the agent and the verifier build with. Pulled at image build, not redistributed in the task repository.", "url": "https://devkitpro.org"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.9443, "std": 0.023, "steps": "1,853", "in_tok": "819.9M", "out_tok": "1.7M", "avg_cost": "$451.30", "avg_time": "15.8h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.6671, "std": 0.113, "steps": "1,430", "in_tok": "525.2M", "out_tok": "644k", "avg_cost": "$532.83", "avg_time": "9.3h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.3947, "std": 0.172, "steps": "1,923", "in_tok": "914.2M", "out_tok": "1.9M", "avg_cost": "$250.66", "avg_time": "14.3h"}, {"rank": 4, "model": "Grok 4.6", "score": 0.3262, "std": 0.142, "steps": "1,319", "in_tok": "342.9M", "out_tok": "2.2M", "avg_cost": "$346.56", "avg_time": "10.1h"}, {"rank": 5, "model": "Kimi K3", "score": 0.2844, "std": 0.173, "steps": "1,376", "in_tok": "579.8M", "out_tok": "1.8M", "avg_cost": "$213.37", "avg_time": "19.9h"}, {"rank": 6, "model": "Gemini 3.7 Flash", "score": 0.1791, "std": 0.043, "steps": "1,456", "in_tok": "558.2M", "out_tok": "1.2M", "avg_cost": "$52.61", "avg_time": "4.4h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.0751, "std": 0.046, "steps": "2,861", "in_tok": "1261.3M", "out_tok": "1.9M", "avg_cost": "$20.53", "avg_time": "12.8h"}, {"rank": 8, "model": "Qwen3.8-Max", "score": 0.0162, "std": 0.01, "steps": "1,027", "in_tok": "394.1M", "out_tok": "2.8M", "avg_cost": "$123.43", "avg_time": "19.9h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0035, "std": 0.008, "steps": "356", "in_tok": "90.6M", "out_tok": "672k", "avg_cost": "$30.14", "avg_time": "3.7h"}, {"rank": 10, "model": "Inkling", "score": 0.0, "std": 0.0, "steps": "157", "in_tok": "7.3M", "out_tok": "29k", "avg_cost": "$7.06", "avg_time": "0.3h"}], "best_score": 0.9443, "mean_score": 0.2891, "n_models": 10}, {"slug": "synthetic-music-diarization", "number": 31, "title": "Synthetic Music Diarization", "url": "https://www.frontierswe.com/tasks/synthetic-music-diarization", "categories": ["AI Research"], "summary": "Build an offline music diarizer that transcribes instrument notes with MIDI pitches and segments singer activity in short audio clips.", "scoring": "Hidden tests", "toolchains": ["Python"], "compute": {"label": "CPU", "kind": "cpu", "compute": "4 CPUs · 32 GB RAM", "gpu": null, "base_image": "python:3.13-slim-trixie", "tools": "PyTorch, FFmpeg, Python 3", "time_limit": "20h"}, "background": "Music diarization asks \"who or what is sounding, and when\": for instrumental audio that means transcribing every note — instrument label, MIDI pitch, onset, and offset — and for vocal audio it means segmenting which singer is active over time. Polyphonic transcription is a long-standing hard problem: notes overlap across instruments with similar timbres, onsets must be localized to within a fraction of a second, and singer identity must be inferred without any enrollment data.\nThis task combines both problems in one offline command-line tool evaluated across four source datasets (BabySlakh, URMP, MusicNet, and vocadito) whose manifests are deliberately anonymized — IDs and filenames are opaque, so the diarizer cannot shortcut by recognizing which dataset a clip came from. It must genuinely analyze the audio, on CPU only, within a hard batch time limit.", "task": "Build an offline diarizer for short instrumental-music and solo-vocal WAV clips. For clips in the instrument_note family it must emit timestamped instrument-note events with integer MIDI pitches; for singer_segment clips it must emit singer-identity activity segments. All runtime files live in /app/diarizer .\n- The entrypoint must support python /app/diarizer/diarize.py --input-dir /path/to/songs --output /path/to/predictions.jsonl , reading a songs.jsonl manifest and writing exactly one JSON object per input ID.\n- Instrument events carry a label from a fixed 27-instrument vocabulary plus a MIDI note; singer events use labels singer_a through singer_l .\n- Production batches may contain up to 233 clips, must finish within 900 seconds, and must produce at most 8 MiB of JSONL.\n- Missing, extra, or duplicate IDs invalidate the output, and labels must not be inferred from IDs or file paths — the manifests are intentionally opaque.", "task_bullets": ["The entrypoint must support python /app/diarizer/diarize.py --input-dir /path/to/songs --output /path/to/predictions.jsonl , reading a songs.jsonl manifest and writing exactly one JSON object per input ID.", "Instrument events carry a label from a fixed 27-instrument vocabulary plus a MIDI note; singer events use labels singer_a through singer_l .", "Production batches may contain up to 233 clips, must finish within 900 seconds, and must produce at most 8 MiB of JSONL.", "Missing, extra, or duplicate IDs invalidate the output, and labels must not be inferred from IDs or file paths — the manifests are intentionally opaque."], "agent_gets": "Labeled development data under /app/data in three splits — train (with gzipped labels), val , and public — comprising over 11 hours of WAV audio derived from BabySlakh, URMP, and MusicNet (instrument-note clips) and vocadito (solo-vocal singer-segment clips), all resampled excerpts with per-file hash-validated integrity. The interface and label schemas are documented in /app/README.md , and a starter stub lives at /app/diarizer/diarize.py . The preinstalled Python 3.13 stack is CPU-only: torch and torchaudio 2.8.0, librosa , soundfile , scipy , scikit-learn , and numpy . The machine is offline with no GPU; a sandbox-timer CLI reports the remaining budget.", "verification": "A root-owned clean-room verifier stages a hidden 233-clip replay set drawn from the same task families, deletes the visible /app/data , and runs the submitted diarizer as the unprivileged agent user under a 900-second timeout — with strace monitoring for any attempt to write the verifier's reward artifacts. Before staging, a manifest-privacy check confirms the replay exposes no source metadata.\n- An instrument note matches only with equal label and MIDI note, onset error at most 0.12 s, and interval IoU at least 0.35; singer segments match with equal label and IoU at least 0.35.\n- Each family is scored by event F1 blended with temporal-boundary quality; instrument-note transcription is weighted 75% and singer segmentation 25%, and a gate scales the instrument contribution down until its component reaches 0.20.\n- A contract multiplier penalizes invalid events and zeroes the score for structural errors (missing, extra, or duplicate IDs).\n- A crashed or timed-out diarizer, oversized output, or an attempted reward-file write zeroes the result; submission sources are also scanned for suspicious references to verifier paths.", "verification_bullets": ["An instrument note matches only with equal label and MIDI note, onset error at most 0.12 s, and interval IoU at least 0.35; singer segments match with equal label and IoU at least 0.35.", "Each family is scored by event F1 blended with temporal-boundary quality; instrument-note transcription is weighted 75% and singer segmentation 25%, and a gate scales the instrument contribution down until its component reaches 0.20.", "A contract multiplier penalizes invalid events and zeroes the score for structural errors (missing, extra, or duplicate IDs).", "A crashed or timed-out diarizer, oversized output, or an attempted reward-file write zeroes the result; submission sources are also scanned for suspicious references to verifier paths."], "safeguards": ["A contract multiplier penalizes invalid events and zeroes the score for structural errors (missing, extra, or duplicate IDs).", "A crashed or timed-out diarizer, oversized output, or an attempted reward-file write zeroes the result; submission sources are also scanned for suspicious references to verifier paths."], "environment": {"Base image": "python:3.13-slim-trixie", "Tools available": "PyTorch, FFmpeg, Python 3", "Compute": "4 CPUs · 32 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "BabySlakh / Slakh2100 (CC-BY-4.0): Synthesized multi-track audio by Ethan Manilow et al.; excerpt WAVs and derived instrument-note annotations are redistributed in the task data.", "url": "https://zenodo.org/records/4603870"}, {"title": "URMP (CC0-1.0): The University of Rochester Multi-Modal Music Performance Dataset (Bochen Li, Xinzhao Liu, Karthik Dinesh et al.), released through Dryad; the task uses excerpts prepared from the YourMT3 16 kHz archive, distributed under CC-BY-4.0.", "url": "https://doi.org/10.5061/dryad.ng3r749"}, {"title": "MusicNet (CC-BY-4.0): Classical recordings with note annotations by John Thickstun, Zaid Harchaoui, and Sham Kakade; excerpt WAVs and derived labels are included.", "url": "https://zenodo.org/records/5120004"}, {"title": "vocadito (CC-BY-4.0): Solo-vocal recordings by Rachel Bittner, Katherine Pasalo, Juan José Bosch, Gabriel Meseguer Brocal, and David Rubinstein; the source of the singer-segment clips. Only task-specific excerpts and derived annotations are included; preparation and verification code is original to this benchmark.", "url": "https://zenodo.org/records/5578807"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.514, "std": 0.026, "steps": "166", "in_tok": "19.0M", "out_tok": "122k", "avg_cost": "$12.45", "avg_time": "17.3h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.4946, "std": 0.022, "steps": "337", "in_tok": "72.2M", "out_tok": "153k", "avg_cost": "$66.37", "avg_time": "13.7h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.4554, "std": 0.027, "steps": "365", "in_tok": "80.4M", "out_tok": "272k", "avg_cost": "$28.41", "avg_time": "18.6h"}, {"rank": 4, "model": "Gemini 3.7 Flash", "score": 0.4425, "std": 0.031, "steps": "419", "in_tok": "128.9M", "out_tok": "253k", "avg_cost": "$23.34", "avg_time": "8.7h"}, {"rank": 5, "model": "Qwen3.8-Max", "score": 0.4265, "std": 0.027, "steps": "479", "in_tok": "97.8M", "out_tok": "527k", "avg_cost": "$28.25", "avg_time": "18.9h"}, {"rank": 6, "model": "Kimi K3", "score": 0.3999, "std": 0.027, "steps": "263", "in_tok": "40.9M", "out_tok": "207k", "avg_cost": "$16.40", "avg_time": "18.9h"}, {"rank": 7, "model": "DeepSeek V4 Flash Exp", "score": 0.3901, "std": 0.056, "steps": "494", "in_tok": "115.3M", "out_tok": "294k", "avg_cost": "$2.06", "avg_time": "18.5h"}, {"rank": 8, "model": "Muse Spark 1.2", "score": 0.3626, "std": 0.015, "steps": "339", "in_tok": "105.8M", "out_tok": "303k", "avg_cost": "$27.77", "avg_time": "4.9h"}, {"rank": 9, "model": "Grok 4.6", "score": 0.3418, "std": 0.193, "steps": "1,125", "in_tok": "274.3M", "out_tok": "1.3M", "avg_cost": "$270.37", "avg_time": "19.4h"}, {"rank": 10, "model": "Inkling", "score": 0.0541, "std": 0.089, "steps": "139", "in_tok": "8.2M", "out_tok": "42k", "avg_cost": "$7.97", "avg_time": "0.8h"}], "best_score": 0.514, "mean_score": 0.3881, "n_models": 10}, {"slug": "verilog-sim-swift", "number": 32, "title": "Verilog Simulator in Swift", "url": "https://www.frontierswe.com/tasks/verilog-sim-swift", "categories": ["Implementation"], "summary": "Implement a Verilog-2005 simulator in Swift, graded head-to-head against live Icarus Verilog on ~1,500 regression designs.", "scoring": "Hidden tests", "toolchains": ["Swift", "Python", "C/C++"], "compute": {"label": "CPU", "kind": "cpu", "compute": "2 CPUs · 4 GB RAM", "gpu": null, "base_image": "ubuntu:24.04", "tools": "Swift, Icarus Verilog, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Simulating Verilog means implementing a language and a scheduler at once: four-state logic on arbitrary bit widths, the standard's intricate self-determination rules for expression width and signedness, blocking versus non-blocking assignment semantics, event-driven scheduling with delays, and the system tasks ($display and friends) through which a design reports its own behavior. Corner cases in width, sign extension, and part-selects are exactly where independent simulators disagree, and where a correct one must not.\nThe task is cross-domain by design: hardware-description-language semantics, implemented in Swift, judged against Icarus Verilog (an established open-source simulator) running live on the same designs.", "task": "Implement a Verilog simulator in the Swift package at /app , writing sources only as .swift files under Sources/vsim/ (standard library plus Foundation only, with Package.swift left as provided), and building offline with swift build -c release into /app/.build/release/vsim .\n- Invoked as vsim <design.v> , it reads a self-contained, self-driving Verilog-2005 design, runs it to completion, prints its output to stdout and diagnostics to stderr, and exits 0.\n- Output is compared ignoring trailing whitespace and tool-diagnostic lines: what matters is what the design itself prints.\n- The example designs in ivtest/ (listed in ivtest/manifest.tsv ) define the scope, from full behavioral regression tests to generated operator/width/sign-corner expression testbenches.", "task_bullets": ["Invoked as vsim <design.v> , it reads a self-contained, self-driving Verilog-2005 design, runs it to completion, prints its output to stdout and diagnostics to stderr, and exits 0.", "Output is compared ignoring trailing whitespace and tool-diagnostic lines: what matters is what the design itself prints.", "The example designs in ivtest/ (listed in ivtest/manifest.tsv ) define the scope, from full behavioral regression tests to generated operator/width/sign-corner expression testbenches."], "agent_gets": "A ready-to-build Swift package at /app and the ivtest/ corpus: 1,528 designs listed in manifest.tsv , drawn from the Icarus Verilog regression suite (ivltests/ ) and from VlogHammer expression modules wrapped as self-driving testbenches (vlh/ ), each paired with its expected output under ivtest/goldens/ . build_and_test.sh builds the simulator and checks it against the corpus. The Swift toolchain is preinstalled and the machine is offline; Icarus Verilog itself is locked root-only, unreachable by the agent.", "verification": "A clean-room verifier, separate from the agent's environment, reconstructs the project from a pristine scaffold plus only the agent's .swift sources, rebuilds it offline as a non-root user (600-second cap), and grades it differentially against live Icarus Verilog: for every graded design, the root-only iverilog /vvp toolchain computes the expected output at verify time, so there are no stored answers to replay, and the candidate's output must match it under the whitespace/diagnostic-insensitive comparator.\n- Reward is the fraction of graded designs passed. Designs Icarus Verilog itself cannot run are skipped, as are non-differential goldens (bare pass/fail verdicts reconstructible from the source); those cannot earn credit without real simulation.\n- An unmodified starting scaffold, a project that does not build offline, a missing binary, or zero passed designs all score 0.\n- Per-design caps of 10 seconds for each side inside a 3300-second suite budget.", "verification_bullets": ["Reward is the fraction of graded designs passed. Designs Icarus Verilog itself cannot run are skipped, as are non-differential goldens (bare pass/fail verdicts reconstructible from the source); those cannot earn credit without real simulation.", "An unmodified starting scaffold, a project that does not build offline, a missing binary, or zero passed designs all score 0.", "Per-design caps of 10 seconds for each side inside a 3300-second suite budget."], "safeguards": ["An unmodified starting scaffold, a project that does not build offline, a missing binary, or zero passed designs all score 0."], "environment": {"Base image": "ubuntu:24.04", "Tools available": "Swift, Icarus Verilog, C/C++ toolchain, Python 3", "Compute": "2 CPUs · 4 GB RAM", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "Icarus Verilog regression designs (ivtest) (GPL-2.0): The Verilog design files under ivltests/ , cloned from a pinned upstream commit at image build into both the root-only graded corpus and the agent-visible copy, from which the held-out graded designs are removed. No upstream golden outputs are used; expected output is computed live from iverilog at verify time.", "url": "https://github.com/steveicarus/iverilog"}, {"title": "VlogHammer (ISC): The auto-generated combinational expression modules (operator/width/sign/shift/concat/part-select corner cases), regenerated deterministically at image build from a pinned commit and wrapped as self-contained Verilog-2005 $display testbenches; the synthesis-equivalence flow is not used.", "url": "https://github.com/YosysHQ/VlogHammer"}, {"title": "Icarus Verilog / OSS CAD Suite (ISC (packaging); GPL-2.0 (iverilog/vvp)): A pinned oss-cad-suite release provides iverilog and vvp , which serve both as the differential oracle computing live goldens and as the reference candidate on the oracle run; root/verifier-only, never reachable by the agent.", "url": "https://github.com/YosysHQ/oss-cad-suite-build"}, {"title": "Swift toolchain (Apache-2.0 with Runtime Library Exception): The official swift.org Linux release (6.1.3), fetched at image build and frozen into the image for offline builds.", "url": "https://swift.org"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.9219, "std": 0.01, "steps": "975", "in_tok": "408.9M", "out_tok": "856k", "avg_cost": "$225.25", "avg_time": "8.8h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.8503, "std": 0.024, "steps": "657", "in_tok": "272.9M", "out_tok": "321k", "avg_cost": "$279.87", "avg_time": "6.6h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.7884, "std": 0.029, "steps": "1,925", "in_tok": "871.2M", "out_tok": "1.2M", "avg_cost": "$239.03", "avg_time": "19.8h"}, {"rank": 4, "model": "Kimi K3", "score": 0.724, "std": 0.079, "steps": "1,509", "in_tok": "659.9M", "out_tok": "1.4M", "avg_cost": "$229.52", "avg_time": "19.9h"}, {"rank": 5, "model": "Grok 4.6", "score": 0.7195, "std": 0.124, "steps": "1,015", "in_tok": "261.0M", "out_tok": "1.9M", "avg_cost": "$272.94", "avg_time": "12.2h"}, {"rank": 6, "model": "Gemini 3.7 Flash", "score": 0.6863, "std": 0.21, "steps": "1,491", "in_tok": "609.9M", "out_tok": "1.4M", "avg_cost": "$69.60", "avg_time": "19.4h"}, {"rank": 7, "model": "Qwen3.8-Max", "score": 0.4154, "std": 0.204, "steps": "1,116", "in_tok": "482.3M", "out_tok": "1.5M", "avg_cost": "$130.78", "avg_time": "19.8h"}, {"rank": 8, "model": "DeepSeek V4 Flash Exp", "score": 0.3979, "std": 0.1, "steps": "3,046", "in_tok": "1410.5M", "out_tok": "1.8M", "avg_cost": "$22.57", "avg_time": "19.1h"}, {"rank": 9, "model": "Muse Spark 1.2", "score": 0.0188, "std": 0.019, "steps": "262", "in_tok": "69.8M", "out_tok": "391k", "avg_cost": "$13.15", "avg_time": "1.4h"}, {"rank": 10, "model": "Inkling", "score": 0.0003, "std": 0.001, "steps": "173", "in_tok": "10.1M", "out_tok": "31k", "avg_cost": "$9.68", "avg_time": "1.4h"}], "best_score": 0.9219, "mean_score": 0.5523, "n_models": 10}, {"slug": "torcs-ai-bot", "number": 33, "title": "Vision-only TORCS Racing Bot", "url": "https://www.frontierswe.com/tasks/torcs-ai-bot", "categories": ["Visual Reasoning", "AI Research"], "summary": "Race a simulated car around tracks it has never seen, from the forward camera image alone, no telemetry.", "scoring": "Hidden tests", "toolchains": ["CUDA", "Python", "C/C++"], "compute": {"label": "1× A10G", "kind": "gpu", "compute": "8 CPUs · 16 GB RAM", "gpu": "1× A10G", "base_image": "ubuntu:22.04", "tools": "CUDA, PyTorch, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Driving a race car from pixels alone is a classic hard problem in embodied AI: the policy must infer speed, track position, and upcoming curvature from a single forward camera (no speedometer, no track sensors, no HUD) and turn that into steering, throttle, and brake commands fast enough to hold a racing line. Doing it on tracks the policy has never seen rules out memorizing any one circuit; the solution has to generalize.\nThe task builds on a patched, headless build of the TORCS racing simulator with a lock-step control loop: the engine blocks for the bot's action each step and runs faster than real time, so the agent can roll out thousands of episodes to hand-tune a controller, collect data, or train a model; the approach is entirely open.", "task": "The agent must implement act(obs) in /app/bot.py : a policy that takes the camera observation and returns driving controls, completing laps as fast as it can, including on tracks it has not seen.\n- The only input is obs[\"frame\"] , a 480×640 RGB forward road camera image; speed, track position, and lap progress must be inferred from the pixels.\n- The bot returns steer in [-1, 1] plus accel and brake in [0, 1]; gear and clutch are automatic.\n- The camera publishes a new image every second control step (and the first frames are black), so motion estimation from pixels must account for the frame cadence.\n- An episode runs up to 21,000 control steps (~430 s of sim time); a crawling car reaches the step limit before finishing a lap. Each control step must return within ~30 seconds, including the first (model loading counts).\n- Module-level state and one-time setup (e.g. loading a trained model) are allowed; the policy may be hand-written, learned, or anything in between.", "task_bullets": ["The only input is obs[\"frame\"] , a 480×640 RGB forward road camera image; speed, track position, and lap progress must be inferred from the pixels.", "The bot returns steer in [-1, 1] plus accel and brake in [0, 1]; gear and clutch are automatic.", "The camera publishes a new image every second control step (and the first frames are black), so motion estimation from pixels must account for the frame cadence.", "An episode runs up to 21,000 control steps (~430 s of sim time); a crawling car reaches the step limit before finishing a lap. Each control step must return within ~30 seconds, including the first (model loading counts).", "Module-level state and one-time setup (e.g. loading a trained model) are allowed; the policy may be hand-written, learned, or anything in between."], "agent_gets": "A workspace at /app with the bot.py stub, a Gymnasium-style harness (game_harness ) that runs the sim headless with reset /step /restart , a runner (python3 /app/run_bot.py ), a minimal demo (harness_demo.py ), a sample camera frame, and /app/README.md documenting the observation/control interface and episode rules. Five practice tracks are available in the sandbox (g-track-1 , eroad , ruudskogen , aalborg , alpine-1 ); the evaluation tracks are not among them. The image preinstalls numpy , scipy , scikit-image , Pillow , OpenCV, and torch with torchvision , and the box has an A10G GPU for training. The machine is offline.", "verification": "A clean-room verifier in a separate container restores the harness from a root-only pristine copy (keeping the agent's own files), unlocks two held-out tracks that never appear in the agent workspace (corkscrew and street-1 ) and races the bot 3 times on each, de-rooted as the non-root agent user with pixels-only observations. Scoring reads only root-written telemetry from the verifier's own harness, never anything the agent wrote.\n- A completed lap maps to a reward by piecewise-linear interpolation over five per-track lap-time anchors: 1.0 for superhuman pace, 0.8 for a skilled human, 0.6 for a published imitation-learning bot, 0.4 for this build's reference sensor-based bot, and 0.2 for barely finishing within the step budget. Street-1's human and imitation anchors come from published TORCS results; corkscrew has no published record, so its rungs are set from an internal reference lap. The 0.4 rung is measured on this exact build.\n- A DNF scores at most 0.2 times the fraction of the track covered, always below the completion floor. The final reward is the mean over runs, then over the two tracks.\n- A bot that fails to answer within the 30-second per-step cap is declared dead; the engine and all agent processes are killed before scoring. Harness faults mark the trial invalid rather than scoring the agent zero.", "verification_bullets": ["A completed lap maps to a reward by piecewise-linear interpolation over five per-track lap-time anchors: 1.0 for superhuman pace, 0.8 for a skilled human, 0.6 for a published imitation-learning bot, 0.4 for this build's reference sensor-based bot, and 0.2 for barely finishing within the step budget. Street-1's human and imitation anchors come from published TORCS results; corkscrew has no published record, so its rungs are set from an internal reference lap. The 0.4 rung is measured on this exact build.", "A DNF scores at most 0.2 times the fraction of the track covered, always below the completion floor. The final reward is the mean over runs, then over the two tracks.", "A bot that fails to answer within the 30-second per-step cap is declared dead; the engine and all agent processes are killed before scoring. Harness faults mark the trial invalid rather than scoring the agent zero."], "safeguards": ["A bot that fails to answer within the 30-second per-step cap is declared dead; the engine and all agent processes are killed before scoring. Harness faults mark the trial invalid rather than scoring the agent zero."], "environment": {"Base image": "ubuntu:22.04", "Tools available": "CUDA, PyTorch, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 16 GB RAM", "GPU": "1× A10G", "Time limit": "20h"}, "changes_from_v1": "", "references": [{"title": "TORCS (The Open Racing Car Simulator) (GPL-2.0-or-later (content: GPL-2.0 / Free Art License)): A trimmed TORCS 1.3.7 engine tree is vendored and built into the image. Non-free NFS3-derived car artwork, GIMP texture sources, and background music were pruned, so the image ships only GPL-2 and Free-Art-License content; the full license text is retained in the vendored tree.", "url": "https://torcs.org"}, {"title": "SCR championship server (scr_server driver) (GPL-2.0): The Simulated Car Racing championship server patch for TORCS 1.3.7 (originally by Loiacono, Cardamone, and Lanzi), exposing the sensor/actuator interface the task's bot drives through.", "url": "https://github.com/barisdemirdelen/scr-torcs-1.3.7"}, {"title": "pyTORCS-docker (No license published (only GPL-derivative patches retained)): Origin of the trimmed torcs/ subtree and three engine patches (main-menu skip, countdown removal, a shared-memory frame channel) plus pre-configured race XML. Because the repository publishes no license, none of its original authored material is redistributed: its README and Dockerfiles were removed; what remains is derivative work of GPL-2.0 TORCS, redistributed under GPL-2.0-or-later.", "url": "https://github.com/gerkone/pyTORCS-docker"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.2829, "std": 0.075, "steps": "457", "in_tok": "144.5M", "out_tok": "316k", "avg_cost": "$78.81", "avg_time": "14.2h"}, {"rank": 2, "model": "Qwen3.8-Max", "score": 0.0348, "std": 0.043, "steps": "488", "in_tok": "82.9M", "out_tok": "779k", "avg_cost": "$34.01", "avg_time": "19.6h"}, {"rank": 3, "model": "GLM-5.3", "score": 0.0343, "std": 0.012, "steps": "443", "in_tok": "182.6M", "out_tok": "353k", "avg_cost": "$82.22", "avg_time": "19.0h"}, {"rank": 4, "model": "Grok 4.6", "score": 0.0341, "std": 0.05, "steps": "544", "in_tok": "125.0M", "out_tok": "942k", "avg_cost": "$151.43", "avg_time": "15.9h"}, {"rank": 5, "model": "DeepSeek V4 Flash Exp", "score": 0.0277, "std": 0.027, "steps": "579", "in_tok": "216.4M", "out_tok": "447k", "avg_cost": "$3.74", "avg_time": "17.4h"}, {"rank": 6, "model": "Kimi K3", "score": 0.027, "std": 0.024, "steps": "483", "in_tok": "146.1M", "out_tok": "383k", "avg_cost": "$54.88", "avg_time": "19.8h"}, {"rank": 7, "model": "GPT-5.6", "score": 0.0241, "std": 0.054, "steps": "299", "in_tok": "80.1M", "out_tok": "123k", "avg_cost": "$76.20", "avg_time": "10.8h"}, {"rank": 8, "model": "Inkling", "score": 0.0186, "std": 0.022, "steps": "248", "in_tok": "26.2M", "out_tok": "75k", "avg_cost": "$25.15", "avg_time": "1.8h"}, {"rank": 9, "model": "Gemini 3.7 Flash", "score": 0.0128, "std": 0.007, "steps": "246", "in_tok": "47.9M", "out_tok": "237k", "avg_cost": "$9.88", "avg_time": "4.0h"}, {"rank": 10, "model": "Muse Spark 1.2", "score": 0.0063, "std": 0.007, "steps": "324", "in_tok": "86.0M", "out_tok": "371k", "avg_cost": "$41.48", "avg_time": "11.1h"}], "best_score": 0.2829, "mean_score": 0.0503, "n_models": 10}, {"slug": "modular-stack-wan21", "number": 34, "title": "Wan 2.1 on MAX/Mojo", "url": "https://www.frontierswe.com/tasks/modular-stack-wan21", "categories": ["Implementation"], "summary": "Port Wan 2.1 text-to-video inference from PyTorch to Modular's MAX/Mojo stack, matching reference frames pixel by pixel.", "scoring": "Hidden tests", "toolchains": ["Mojo", "CUDA", "Python", "C/C++"], "compute": {"label": "1× H100", "kind": "gpu", "compute": "8 CPUs · 128 GB RAM", "gpu": "1× H100", "base_image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", "tools": "CUDA, MAX SDK, Mojo, CMake, C/C++ toolchain, Python 3", "time_limit": "20h"}, "background": "Wan 2.1 T2V-1.3B is a text-to-video diffusion model (a diffusion transformer paired with a 3D VAE and a text encoder) whose inference pipeline spans text encoding, iterative denoising, and video decoding on the GPU. Porting it off PyTorch means re-expressing every model component on a different ML stack: Modular's MAX graph and inference APIs, with custom Mojo kernels where the built-in operations fall short.\nThe port is measured against the original: the agent's pipeline must reproduce the PyTorch reference's output frames to high pixel fidelity, working entirely offline from the reference source, the pretrained weights, and the bundled MAX API documentation. PyTorch itself is not installed; the target stack is the only stack available.", "task": "Implement Wan 2.1 T2V-1.3B text-to-video generation on Modular's MAX/Mojo stack. The deliverable is the Python package at /app/wan21_max/ , whose entrypoint module wan_pipeline.py must export generate_video(prompt, height, width, num_frames, num_steps, seed) returning a list of PIL Images, one per frame.\n- The signature stays stable and everything derives from the arguments; sources are .py modules (plus .mojo kernels for custom ops) under /app/wan21_max/ .\n- Everything the pipeline needs at generation time must live in the package: /app/reference/ and the .py files under /app/weights/ are removed before scoring.\n- The port must be built on the MAX SDK; torch , transformers , and diffusers are hard library constraints: they are not installed and must not be used.\n- A short generation (5 frames, 4 steps) must complete within 10 minutes including any first-call compilation.", "task_bullets": ["The signature stays stable and everything derives from the arguments; sources are .py modules (plus .mojo kernels for custom ops) under /app/wan21_max/ .", "Everything the pipeline needs at generation time must live in the package: /app/reference/ and the .py files under /app/weights/ are removed before scoring.", "The port must be built on the MAX SDK; torch , transformers , and diffusers are hard library constraints: they are not installed and must not be used.", "A short generation (5 frames, 4 steps) must complete within 10 minutes including any first-call compilation."], "agent_gets": "The package scaffold at /app/wan21_max/ and an /app/README.md spelling out the exact API, the workload family, what counts as correct output, and the library constraints. For porting: the Wan 2.1 PyTorch reference source at /app/reference/ (read-only), the pretrained T2V-1.3B weights in diffusers format at /app/weights/ , and the Modular MAX API documentation at /app/max_docs/ . For self-checking: sample workloads with baked reference frames under /app/examples/ and /app/verify_correctness.py , which generates each sample workload, compares output against the reference frames, and enforces the per-generation time budget. The machine has one H100 GPU, is offline, ships the MAX SDK and Mojo toolchain with warmed compile caches, and exposes a sandbox-timer CLI for the remaining budget.", "verification": "A clean-room verifier runs in a fresh container on the captured workspace. It reconstructs the scored package from a pristine scaffold plus only the agent's wan21_max/ files into a root-owned, read-only directory, strips the reference source and stray Python files as disclosed, then runs a gate sequence: a source-level anti-cheat verdict (banned imports, verifier-internals references, shell or dynamic-import reach, and a required MAX SDK import), an import check, and the disclosed smoke gate. Only then does it generate frames for nine hidden workloads as a non-root user. Root then scores the saved frames against root-only reference frames; no candidate code runs during scoring.\n- Per workload, validity gates require the exact frame count and size, no symlinked frames (which would otherwise echo back the references), and no blank frames; credit is then a continuous ramp on mean per-frame PSNR: 0 at 10 dB, 1.0 at the disclosed 25 dB bar.\n- The reward is the geometric mean of per-workload credits, with a small 0.02 per-workload floor so a single hard workload cannot zero an otherwise strong port.\n- Banned-library imports, a missing MAX import, an unimportable entrypoint, or a failed smoke test (crash, timeout, or blank frames) zero the result; workloads that miss the generation stage's start deadline degrade to partial credit rather than erroring the run.", "verification_bullets": ["Per workload, validity gates require the exact frame count and size, no symlinked frames (which would otherwise echo back the references), and no blank frames; credit is then a continuous ramp on mean per-frame PSNR: 0 at 10 dB, 1.0 at the disclosed 25 dB bar.", "The reward is the geometric mean of per-workload credits, with a small 0.02 per-workload floor so a single hard workload cannot zero an otherwise strong port.", "Banned-library imports, a missing MAX import, an unimportable entrypoint, or a failed smoke test (crash, timeout, or blank frames) zero the result; workloads that miss the generation stage's start deadline degrade to partial credit rather than erroring the run."], "safeguards": ["The reward is the geometric mean of per-workload credits, with a small 0.02 per-workload floor so a single hard workload cannot zero an otherwise strong port.", "Banned-library imports, a missing MAX import, an unimportable entrypoint, or a failed smoke test (crash, timeout, or blank frames) zero the result; workloads that miss the generation stage's start deadline degrade to partial credit rather than erroring the run."], "environment": {"Base image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", "Tools available": "CUDA, MAX SDK, Mojo, CMake, C/C++ toolchain, Python 3", "Compute": "8 CPUs · 128 GB RAM", "GPU": "1× H100", "Time limit": "20h"}, "changes_from_v1": "This task is an improved revision of Wan 2.1 on MAX/Mojo from FrontierSWE V1.\n- Scoring changed from a geometric-mean speedup against a PyTorch baseline (gated on an all-or-nothing 25 dB PSNR check where one failed workload zeroed everything) to a pure-correctness reward: a geometric mean of continuous per-workload PSNR credits ramping from 10 dB to the disclosed 25 dB bar, with a small per-workload floor so partial ports earn partial credit.\n- The PyTorch ban became structural: V1 scanned .py files for banned imports; V2 uninstalls torch, transformers, and diffusers from the image entirely (verified in preflight), so the constraint cannot be routed around at runtime.\n- Verification moved to a clean-room separate container with privilege separation: the scored package is reconstructed from a pristine scaffold plus only the agent's wan21_max/ files into a root-owned read-only directory, candidate code runs de-rooted, reference frames stay root-only, and the scorer rejects symlinked frames.\n- The smoke gate grew from 120 seconds to 10 minutes for the short generation, explicitly covering first-call Mojo compilation, and the Mojo compile cache was made agent-writable after measuring that a root-owned cache silently cost 20-30 seconds per cold call; the generation stage now degrades to partial credit at its start deadline instead of failing hard.\n- The agent budget grew from 4 hours to 20 hours, and both the agent and verifier images are pinned by content hash, with the verifier additionally pinned to the same H100 GPU class that the PSNR scoring runs on.\n- The deliverable was restructured from a single /app/submission/candidate_pipeline.py into the /app/wan21_max/ package with the full contract moved into /app/README.md , and the /app/.timer file interface was replaced by the standard sandbox-timer CLI.", "references": [{"title": "Wan 2.1 (Apache-2.0): The PyTorch reference implementation, cloned at image build to /app/reference/ for the agent to read while porting the pipeline to MAX.", "url": "https://github.com/Wan-Video/Wan2.1"}, {"title": "Wan 2.1 T2V-1.3B weights (diffusers format) (Apache-2.0): The pretrained weights the candidate pipeline loads, fetched at image build and exposed at /app/weights . The same weights, run through the diffusers WanPipeline , generated the baked reference frames used for scoring and the visible examples.", "url": "https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B-Diffusers"}, {"title": "diffusers / transformers / PyTorch (Apache-2.0 (diffusers, transformers); BSD-3-Clause (PyTorch)): The stock PyTorch WanPipeline stack the port is measured against; it generated the baked reference frames out-of-band at pinned versions. The stack is not installed in the task image: the deliverable must not use it, and the ban is structural (absent, with no egress to fetch it).", "url": "https://github.com/huggingface/diffusers"}, {"title": "Modular MAX / Mojo SDK (MAX Platform / Community License): The MAX/Mojo toolchain the task targets, pip-installed from public PyPI at image build (not vendored or modified), plus its API documentation fetched from the public docs.modular.com endpoints into /app/max_docs/ .", "url": "https://pypi.org/project/modular/"}], "results": [{"rank": 1, "model": "Claude Fable 5.1", "score": 0.9246, "std": 0.007, "steps": "279", "in_tok": "96.7M", "out_tok": "337k", "avg_cost": "$56.47", "avg_time": "7.4h"}, {"rank": 2, "model": "GPT-5.6", "score": 0.9017, "std": 0.021, "steps": "281", "in_tok": "73.9M", "out_tok": "127k", "avg_cost": "$65.97", "avg_time": "4.3h"}, {"rank": 3, "model": "Kimi K3", "score": 0.7866, "std": 0.281, "steps": "343", "in_tok": "99.3M", "out_tok": "417k", "avg_cost": "$39.10", "avg_time": "9.2h"}, {"rank": 4, "model": "Grok 4.6", "score": 0.6431, "std": 0.359, "steps": "452", "in_tok": "108.5M", "out_tok": "1.1M", "avg_cost": "$119.46", "avg_time": "12.8h"}, {"rank": 5, "model": "GLM-5.3", "score": 0.5702, "std": 0.467, "steps": "450", "in_tok": "155.9M", "out_tok": "420k", "avg_cost": "$54.84", "avg_time": "17.5h"}, {"rank": 6, "model": "DeepSeek V4 Flash Exp", "score": 0.3511, "std": 0.089, "steps": "552", "in_tok": "213.1M", "out_tok": "423k", "avg_cost": "$3.66", "avg_time": "17.8h"}, {"rank": 7, "model": "Qwen3.8-Max", "score": 0.2491, "std": 0.207, "steps": "443", "in_tok": "109.6M", "out_tok": "1.4M", "avg_cost": "$40.44", "avg_time": "19.7h"}, {"rank": 8, "model": "Gemini 3.7 Flash", "score": 0.0331, "std": 0.002, "steps": "520", "in_tok": "185.3M", "out_tok": "609k", "avg_cost": "$31.93", "avg_time": "8.5h"}, {"rank": 9, "model": "Inkling", "score": 0.0228, "std": 0.021, "steps": "161", "in_tok": "14.9M", "out_tok": "29k", "avg_cost": "$14.28", "avg_time": "1.3h"}, {"rank": 10, "model": "Muse Spark 1.2", "score": 0.0115, "std": 0.016, "steps": "297", "in_tok": "94.1M", "out_tok": "425k", "avg_cost": "$21.15", "avg_time": "3.3h"}], "best_score": 0.9246, "mean_score": 0.4494, "n_models": 10}]}; | |