source/docs/programbench-five-task-manifest.json

{
  "schema_version": 1,
  "name": "beam-pi-programbench-five-task-pilot",
  "status": "provisional_selection_calibration_pending",
  "created_at_utc": "2026-10-09T11:37:30.043627+00:00",
  "benchmark": {
    "repository": "https://github.com/facebookresearch/ProgramBench",
    "commit": "27f02157c785f8da3647aa6dbbe6b9137f99f10e"
  },
  "test_dataset": {
    "repository": "programbench/ProgramBench-Tests",
    "revision": "de0ddfb637590c7ecb54fa0b5301f6dc7dfbcee5",
    "metadata_url": "https://huggingface.co/api/datasets/programbench/ProgramBench-Tests/revision/de0ddfb637590c7ecb54fa0b5301f6dc7dfbcee5",
    "private": false,
    "gated": false,
    "archive_contents_accessed": false
  },
  "selection": {
    "task_count": 5,
    "method": "Purposive CLI feasibility sample chosen from public project identity, task language and difficulty metadata, without hidden test contents, outcomes, or model performance. Five behavioral domains; 3 Rust projects and 2 C projects.",
    "statistically_representative": false,
    "anthropic_166_task_membership_verified": false,
    "estimated_runtime_seconds": null,
    "runtime_note": "See beam-pi-five-task-runtime-estimate.md; throughput probes are measured, while episode and grading runtimes remain forecasts."
  },
  "boundaries": {
    "experiments_launched": false,
    "containers_pulled_or_run": false,
    "hidden_test_contents_or_outcomes_inspected": false,
    "solver_visibility": "This manifest and evaluator metadata are controller/evaluator inputs. Do not expose upstream source or test manifests to solver containers.",
    "benchmark_model_calls_made": false,
    "separate_synthetic_throughput_probe_calls_made": 14,
    "execution_status_note": "Selection-stage observations only; current execution status is recorded in study run artifacts."
  },
  "tasks": [
    {
      "instance_id": "burntsushi__ripgrep.3b7fd44",
      "upstream_repository": "BurntSushi/ripgrep",
      "upstream_commit": "3b7fd442a6f3aa73f650e763d7cbb902c03d700e",
      "original_language": "Rust",
      "upstream_difficulty_label": "medium",
      "behavioral_domain": "regex search and filesystem behavior",
      "selection_reason": "Practical search utility with regex, directory traversal, and ignore behavior; supplies a broader CLI reconstruction task.",
      "task_metadata": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/burntsushi__ripgrep.3b7fd44/task.yaml",
        "sha256": "f32651f3ff776dbf28b97d94cd11442c68e616233a8a78a4093b039b437f630b"
      },
      "public_test_manifest": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/burntsushi__ripgrep.3b7fd44/tests.json",
        "sha256": "19d9b8440bc839042b7b1c47efc12f0eb899ef20bd3150ff75a76900159bfecd",
        "raw_branch_count": 13,
        "raw_test_entry_count": 2538,
        "count_definition": "Sum of lengths of branch tests lists, before ignored-branch or ignored-test filtering. Counts are public metadata, not observed outcomes or an evaluated denominator."
      },
      "task_cleanroom_image": {
        "repository": "programbench/burntsushi_1776_ripgrep.3b7fd44",
        "discovery_tag": "task_cleanroom_v6",
        "platform": "linux/amd64",
        "digest": "sha256:b4f1765b097dc4a49dbf8aa77809a94c0092a1cb7a4777f2324453c928665219",
        "immutable_reference": "programbench/burntsushi_1776_ripgrep.3b7fd44@sha256:b4f1765b097dc4a49dbf8aa77809a94c0092a1cb7a4777f2324453c928665219",
        "registry_metadata_url": "https://hub.docker.com/v2/repositories/programbench/burntsushi_1776_ripgrep.3b7fd44/tags/task_cleanroom_v6",
        "tag_last_updated": "2026-06-18T17:54:01.962621Z",
        "registry_reported_size_bytes": 1134048487,
        "availability_evidence": "Anonymous Docker Hub tag metadata only; not pulled or executed."
      },
      "reference_calibration": {
        "status": "pending",
        "golden_verified": false,
        "reference_pass_fraction": null,
        "frozen_evaluation_mask": null
      }
    },
    {
      "instance_id": "jqlang__jq.b33a763",
      "upstream_repository": "jqlang/jq",
      "upstream_commit": "b33a7634ba34ffa7ce7368cc0ebf5ca40b54c7e6",
      "original_language": "C",
      "upstream_difficulty_label": "medium",
      "behavioral_domain": "JSON transformation language",
      "selection_reason": "Exercises parsing and structured transformations; supplies a second broader CLI reconstruction task.",
      "task_metadata": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/jqlang__jq.b33a763/task.yaml",
        "sha256": "4fa67ff06341527e45df605d99cff00c3524756c349f80bf4650600011a588cd"
      },
      "public_test_manifest": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/jqlang__jq.b33a763/tests.json",
        "sha256": "bb5b8cc4f6d3814f12cb1e4ee71dc793270b0d5802a509ca73ee8f2c7bc16439",
        "raw_branch_count": 12,
        "raw_test_entry_count": 6796,
        "count_definition": "Sum of lengths of branch tests lists, before ignored-branch or ignored-test filtering. Counts are public metadata, not observed outcomes or an evaluated denominator."
      },
      "task_cleanroom_image": {
        "repository": "programbench/jqlang_1776_jq.b33a763",
        "discovery_tag": "task_cleanroom_v6",
        "platform": "linux/amd64",
        "digest": "sha256:fcfd26a12dc63122864220c7d6ee1993a9cc23fa30ec9607e587d145b2cbd3fa",
        "immutable_reference": "programbench/jqlang_1776_jq.b33a763@sha256:fcfd26a12dc63122864220c7d6ee1993a9cc23fa30ec9607e587d145b2cbd3fa",
        "registry_metadata_url": "https://hub.docker.com/v2/repositories/programbench/jqlang_1776_jq.b33a763/tags/task_cleanroom_v6",
        "tag_last_updated": "2026-06-18T17:56:19.570314Z",
        "registry_reported_size_bytes": 926327914,
        "availability_evidence": "Anonymous Docker Hub tag metadata only; not pulled or executed."
      },
      "reference_calibration": {
        "status": "pending",
        "golden_verified": false,
        "reference_pass_fraction": null,
        "frozen_evaluation_mask": null
      }
    },
    {
      "instance_id": "burntsushi__xsv.f430466",
      "upstream_repository": "BurntSushi/xsv",
      "upstream_commit": "f4304660231d7907798e831411871d2679d335c3",
      "original_language": "Rust",
      "upstream_difficulty_label": "medium",
      "behavioral_domain": "CSV processing",
      "selection_reason": "Adds tabular data processing with a command toolkit distinct from regex search and JSON expressions.",
      "task_metadata": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/burntsushi__xsv.f430466/task.yaml",
        "sha256": "3df5ea9ab5f1bd9904637fce1619e090db15b6745b58a12ac0ac1664aa4b9483"
      },
      "public_test_manifest": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/burntsushi__xsv.f430466/tests.json",
        "sha256": "42f1318cf96cb4c9d904674fb330f48c45625a8081c567778f7cdd5fed1dffbb",
        "raw_branch_count": 12,
        "raw_test_entry_count": 1323,
        "count_definition": "Sum of lengths of branch tests lists, before ignored-branch or ignored-test filtering. Counts are public metadata, not observed outcomes or an evaluated denominator."
      },
      "task_cleanroom_image": {
        "repository": "programbench/burntsushi_1776_xsv.f430466",
        "discovery_tag": "task_cleanroom_v6",
        "platform": "linux/amd64",
        "digest": "sha256:b6b343719edfc37364aac9f5c565d68e89f977b35719d541183ba20bb8e5ad0a",
        "immutable_reference": "programbench/burntsushi_1776_xsv.f430466@sha256:b6b343719edfc37364aac9f5c565d68e89f977b35719d541183ba20bb8e5ad0a",
        "registry_metadata_url": "https://hub.docker.com/v2/repositories/programbench/burntsushi_1776_xsv.f430466/tags/task_cleanroom_v6",
        "tag_last_updated": "2026-06-18T17:54:04.599601Z",
        "registry_reported_size_bytes": 1147924363,
        "availability_evidence": "Anonymous Docker Hub tag metadata only; not pulled or executed."
      },
      "reference_calibration": {
        "status": "pending",
        "golden_verified": false,
        "reference_pass_fraction": null,
        "frozen_evaluation_mask": null
      }
    },
    {
      "instance_id": "cmatsuoka__figlet.202a0a8",
      "upstream_repository": "cmatsuoka/figlet",
      "upstream_commit": "202a0a8110650a943f1125f536b3bb455cf72ee1",
      "original_language": "C",
      "upstream_difficulty_label": "easy",
      "behavioral_domain": "text rendering",
      "selection_reason": "Adds character-based banner rendering and an upstream easy-labeled task with a different observable output surface.",
      "task_metadata": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/cmatsuoka__figlet.202a0a8/task.yaml",
        "sha256": "5eee58ea736ecba056cceeb64c4bc070760fc7da4ab067ae34a98a5a0f70d48c"
      },
      "public_test_manifest": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/cmatsuoka__figlet.202a0a8/tests.json",
        "sha256": "a87ad8ff6ebd24561039f56938f93d234963329d9a33bedb975f159f96d5f9b0",
        "raw_branch_count": 12,
        "raw_test_entry_count": 1044,
        "count_definition": "Sum of lengths of branch tests lists, before ignored-branch or ignored-test filtering. Counts are public metadata, not observed outcomes or an evaluated denominator."
      },
      "task_cleanroom_image": {
        "repository": "programbench/cmatsuoka_1776_figlet.202a0a8",
        "discovery_tag": "task_cleanroom_v6",
        "platform": "linux/amd64",
        "digest": "sha256:cb7cb91db6bbea1e2753a3c05aa338c1154e7e81712b10a3d9aff0c676c6ea46",
        "immutable_reference": "programbench/cmatsuoka_1776_figlet.202a0a8@sha256:cb7cb91db6bbea1e2753a3c05aa338c1154e7e81712b10a3d9aff0c676c6ea46",
        "registry_metadata_url": "https://hub.docker.com/v2/repositories/programbench/cmatsuoka_1776_figlet.202a0a8/tags/task_cleanroom_v6",
        "tag_last_updated": "2026-06-18T17:54:20.066575Z",
        "registry_reported_size_bytes": 884811922,
        "availability_evidence": "Anonymous Docker Hub tag metadata only; not pulled or executed."
      },
      "reference_calibration": {
        "status": "pending",
        "golden_verified": false,
        "reference_pass_fraction": null,
        "frozen_evaluation_mask": null
      }
    },
    {
      "instance_id": "sharkdp__hexyl.2e26437",
      "upstream_repository": "sharkdp/hexyl",
      "upstream_commit": "2e2643782d6ced9b5ac75596169a79127d8e535a",
      "original_language": "Rust",
      "upstream_difficulty_label": "medium",
      "behavioral_domain": "binary display",
      "selection_reason": "Adds byte-oriented display and formatting, providing a focused CLI surface alongside the broader data processors.",
      "task_metadata": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/sharkdp__hexyl.2e26437/task.yaml",
        "sha256": "78a7a46acc7ac5e236d1a9aac9a50958291da6a9639431a3ed39d87e2058805b"
      },
      "public_test_manifest": {
        "url": "https://github.com/facebookresearch/ProgramBench/blob/27f02157c785f8da3647aa6dbbe6b9137f99f10e/src/programbench/data/tasks/sharkdp__hexyl.2e26437/tests.json",
        "sha256": "bcae0262b4096f9e8d1df2fd8212dc94baa21043de3a26531648b8f2882fe52c",
        "raw_branch_count": 11,
        "raw_test_entry_count": 974,
        "count_definition": "Sum of lengths of branch tests lists, before ignored-branch or ignored-test filtering. Counts are public metadata, not observed outcomes or an evaluated denominator."
      },
      "task_cleanroom_image": {
        "repository": "programbench/sharkdp_1776_hexyl.2e26437",
        "discovery_tag": "task_cleanroom_v6",
        "platform": "linux/amd64",
        "digest": "sha256:7122034cf2cd85eeb9e55def4a47131ea5472a5f2b650fed50b61b381d6cde39",
        "immutable_reference": "programbench/sharkdp_1776_hexyl.2e26437@sha256:7122034cf2cd85eeb9e55def4a47131ea5472a5f2b650fed50b61b381d6cde39",
        "registry_metadata_url": "https://hub.docker.com/v2/repositories/programbench/sharkdp_1776_hexyl.2e26437/tags/task_cleanroom_v6",
        "tag_last_updated": "2026-06-18T17:58:02.47938Z",
        "registry_reported_size_bytes": 974463372,
        "availability_evidence": "Anonymous Docker Hub tag metadata only; not pulled or executed."
      },
      "reference_calibration": {
        "status": "pending",
        "golden_verified": false,
        "reference_pass_fraction": null,
        "frozen_evaluation_mask": null
      }
    }
  ],
  "study": {
    "tasks": 5,
    "conditions": 3,
    "repetitions": 1,
    "episodes": 15,
    "context_token_cap_per_episode": 1000000,
    "wall_clock_cap_seconds_per_episode": 7200,
    "forecast_global_inflight_request_cap": 5,
    "api_token_cap_per_episode": 8000000,
    "benchmark_api_token_cap": 120000000,
    "validation_api_token_cap": 10000000,
    "global_api_token_cap": 130000000
  }
}