Buckets:

download
raw
28 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;τ²-bench Environment&quot;,&quot;local&quot;:&quot;τ²-bench-environment&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Web UI&quot;,&quot;local&quot;:&quot;web-ui&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Tools&quot;,&quot;local&quot;:&quot;tools&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;The simulated user&quot;,&quot;local&quot;:&quot;the-simulated-user&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Quick start&quot;,&quot;local&quot;:&quot;quick-start&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Running the server&quot;,&quot;local&quot;:&quot;running-the-server&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Notes&quot;,&quot;local&quot;:&quot;notes&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/openenv/main/en/_app/immutable/entry/start.B22qDl-k.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/chunks/Bc-J0DBZ.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/chunks/D2-C6oRw.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/entry/app.Du7tmpJJ.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/chunks/DZFNMJnC.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/chunks/Cs50_Xpk.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/chunks/BH1xZHjK.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/nodes/0.C4MPJlme.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/nodes/2.Bvl1Fj2X.js" rel="modulepreload">
<link href="/docs/openenv/main/en/_app/immutable/chunks/zPQIlmx-.js" rel="modulepreload">
<!--ndi2hg--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;τ²-bench Environment&quot;,&quot;local&quot;:&quot;τ²-bench-environment&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Web UI&quot;,&quot;local&quot;:&quot;web-ui&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Tools&quot;,&quot;local&quot;:&quot;tools&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;The simulated user&quot;,&quot;local&quot;:&quot;the-simulated-user&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Quick start&quot;,&quot;local&quot;:&quot;quick-start&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Running the server&quot;,&quot;local&quot;:&quot;running-the-server&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Notes&quot;,&quot;local&quot;:&quot;notes&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/openenv/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="τ²-bench-environment" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#τ²-bench-environment"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>τ²-bench Environment</span></h1><!--]--><!----> <p><a href="https://github.com/sierra-research/tau2-bench" rel="nofollow">τ²-bench</a> as an OpenEnv environment. Each episode is a customer-service conversation: an LLM-simulated user comes with a request, and the agent has to solve it with the domain’s tools while following the domain’s policy. The user replies to what the agent says, changes their mind, or pushes for things the policy does not allow. When the conversation ends, τ²-bench’s own evaluator scores it, mostly from the final state of the database.</p> <p>That makes it multi-turn (a real back-and-forth with a user), multi-step (several tool calls per turn), and verifiable. It works for evaluation and as an RL environment.</p> <table><thead><tr><th>Domain</th><th>Tasks (train / test)</th><th>Tools</th><th>Reward</th></tr></thead><tbody><tr><td><code>airline</code></td><td>30 / 20</td><td>book, cancel, change flights, baggage, certificates…</td><td>database state + information given to the user</td></tr><tr><td><code>retail</code></td><td>74 / 40</td><td>orders, returns, exchanges, addresses, payments…</td><td>database state + natural-language assertions (judged by an LLM)</td></tr><tr><td><code>telecom</code></td><td>74 / 40</td><td>account and line management; the user also has tools on their phone</td><td>assertions on the final state of the account and the phone</td></tr></tbody></table> <p><code>telecom-workflow</code>, <code>banking_knowledge</code> and <code>mock</code> are also available (<code>banking_knowledge</code> needs τ²-bench’s <code>knowledge</code> extra).</p> <!--[1--><h2 class="relative group"><a id="web-ui" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#web-ui"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Web UI</span></h2><!--]--><!----> <p>The Space opens on a τ²-bench tab. The domain, the split and the simulated customer’s model can be changed at the top, and there are four views:</p> <ul><li><strong>Tasks</strong>: an explorer of the split’s tasks, with search and filters (changes the database or read-only). A task page shows what the agent does not see (why the customer calls, what they know, how they behave) and what the conversation is scored on.</li> <li><strong>Run a model</strong>: pick an agent model on Hugging Face Inference Providers (the list shows each model’s providers and price) and watch the conversation, with each tool call as a collapsible row and τ²-bench’s reward breakdown at the end. A run can be stopped.</li> <li><strong>Play as the agent</strong>: talk to the simulated customer yourself and call the domain’s tools from a form, then see your reward.</li> <li><strong>Runs</strong>: the runs of your session. Open one to read it again or download it as JSON, or compare two to four side by side.</li></ul> <p>On a Space, visitors sign in with Hugging Face, and their conversations run on their own <a href="https://huggingface.co/docs/inference-providers" rel="nofollow">Inference Providers</a> credits (the Space asks for the <code>inference-api</code> scope). Running a conversation requires signing in, and browsing the tasks doesn’t.</p> <p>The runs live in the browser session and are not kept across restarts. The generic OpenEnv playground stays available as a second tab.</p> <!--[1--><h2 class="relative group"><a id="tools" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tools"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Tools</span></h2><!--]--><!----> <p>The agent acts through MCP tools:</p> <ul><li><strong>The domain’s tools</strong>, e.g. <code>get_user_details</code>, <code>search_direct_flight</code>, <code>book_reservation</code>. They read and write the task’s database.</li> <li><strong><code>respond_to_user(message)</code></strong>: says something to the user and returns their reply.</li> <li><strong><code>done()</code></strong>: ends the conversation from the agent’s side.</li></ul> <p>The episode ends when the user is done (or the agent calls <code>done</code>), and the last observation carries <code>done=True</code>, the reward, and τ²-bench’s breakdown in <code>metadata["reward_info"]</code>.</p> <!--[1--><h2 class="relative group"><a id="the-simulated-user" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#the-simulated-user"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>The simulated user</span></h2><!--]--><!----> <p>The user (and the judge for retail’s natural-language assertions) is an LLM. It runs on <a href="https://huggingface.co/docs/inference-providers" rel="nofollow">Hugging Face Inference Providers</a> by default, with <code>deepseek-ai/DeepSeek-V4.1-Flash</code>:</p> <table><thead><tr><th><code>TAU2_USER_PROVIDER</code></th><th>Credential</th><th>Default <code>TAU2_USER_MODEL</code></th></tr></thead><tbody><tr><td><code>hf</code> (default)</td><td><code>HF_TOKEN</code></td><td><code>deepseek-ai/DeepSeek-V4.1-Flash</code></td></tr><tr><td><code>openai</code></td><td><code>OPENAI_API_KEY</code></td><td><code>gpt-4.1</code></td></tr><tr><td><code>anthropic</code></td><td><code>ANTHROPIC_API_KEY</code></td><td><code>claude-sonnet-4-5</code></td></tr></tbody></table> <p>Any model of the provider works through <code>TAU2_USER_MODEL</code> (for <code>hf</code>, any <a href="https://huggingface.co/inference/models" rel="nofollow">Inference Providers chat model</a>). Non-reasoning models answer fastest, which matters when the user is called thousands of times during training.</p> <blockquote class="note"><p>τ²-bench’s published results use <code>gpt-4.1</code> as the user. Scores with a different user model are not comparable to the leaderboard.</p></blockquote> <!--[1--><h2 class="relative group"><a id="quick-start" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quick-start"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Quick start</span></h2><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> openenv.core.env_server.mcp_types <span class="hljs-keyword">import</span> CallToolAction
<span class="hljs-keyword">from</span> tau2_env <span class="hljs-keyword">import</span> Tau2Env
<span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span> Tau2Env(base_url=<span class="hljs-string">&quot;http://localhost:8000&quot;</span>) <span class="hljs-keyword">as</span> env:
result = <span class="hljs-keyword">await</span> env.reset(task_id=<span class="hljs-string">&quot;2&quot;</span>)
<span class="hljs-built_in">print</span>(result.observation.metadata[<span class="hljs-string">&quot;user_message&quot;</span>]) <span class="hljs-comment"># the user&#x27;s opening message</span>
policy = result.observation.metadata[<span class="hljs-string">&quot;policy&quot;</span>] <span class="hljs-comment"># what the agent must follow</span>
tools = <span class="hljs-keyword">await</span> env.list_tools()
reply = <span class="hljs-keyword">await</span> env.call_tool(
<span class="hljs-string">&quot;respond_to_user&quot;</span>, message=<span class="hljs-string">&quot;Happy to help. What&#x27;s your user id?&quot;</span>
)
user = <span class="hljs-keyword">await</span> env.call_tool(<span class="hljs-string">&quot;get_user_details&quot;</span>, user_id=<span class="hljs-string">&quot;noah_muller_9847&quot;</span>)
<span class="hljs-comment"># `call_tool` returns the tool&#x27;s output only. `step` also returns whether the</span>
<span class="hljs-comment"># episode is over and, once it is, the reward.</span>
result = <span class="hljs-keyword">await</span> env.step(CallToolAction(tool_name=<span class="hljs-string">&quot;done&quot;</span>, arguments={}))
<span class="hljs-built_in">print</span>(result.done, result.reward, result.observation.metadata[<span class="hljs-string">&quot;reward_info&quot;</span>])<!----></pre></div><!----> <p><code>reset()</code> takes <code>task_id</code> to run a given task, or picks one of the split at random (<code>seed</code> makes it reproducible). Give the agent the policy as its system prompt. With the default <code>hf</code> provider, <code>reset(hf_token=...)</code> makes the session’s user and judge run on your token, which is how to use a server that has none, like the public Space (<code>base_url="https://sergiopaniego-tau2-env.hf.space"</code>).</p> <!--[1--><h2 class="relative group"><a id="running-the-server" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#running-the-server"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Running the server</span></h2><!--]--><!----> <p>τ²-bench’s tasks, databases and policies live in its repository rather than in the package, so outside Docker fetch them once (the commit matches the <code>tau2</code> pin in <code>pyproject.toml</code>) and point <code>TAU2_DATA_DIR</code> at them:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->git <span class="hljs-built_in">clone</span> --filter=blob:none --no-checkout https://github.com/sierra-research/tau2-bench
git -C tau2-bench sparse-checkout <span class="hljs-built_in">set</span> --no-cone data/tau2/domains data/tau2/user_simulator
git -C tau2-bench checkout 5bfa7e37b36656b37dc6d022156be6563c1007f3
<span class="hljs-built_in">cd</span> envs/tau2_env
TAU2_DATA_DIR=../../tau2-bench/data HF_TOKEN=hf_... ENABLE_WEB_INTERFACE=<span class="hljs-literal">true</span> uv run server<!----></pre></div><!----> <p>The Docker image does this at build time.</p> <table><thead><tr><th>Variable</th><th>Default</th><th></th></tr></thead><tbody><tr><td><code>TAU2_DOMAIN</code></td><td><code>airline</code></td><td>τ²-bench domain</td></tr><tr><td><code>TAU2_SPLIT</code></td><td><code>test</code></td><td><code>train</code>, <code>test</code> or <code>base</code> (all tasks)</td></tr><tr><td><code>TAU2_USER_PROVIDER</code></td><td><code>hf</code></td><td><code>hf</code>, <code>openai</code> or <code>anthropic</code></td></tr><tr><td><code>TAU2_USER_MODEL</code></td><td>the provider’s default</td><td>model for the user and the judge</td></tr><tr><td><code>TAU2_DATA_DIR</code></td><td>set in the Docker image</td><td>τ²-bench’s <code>data</code> folder</td></tr><tr><td><code>MAX_CONCURRENT_ENVS</code></td><td><code>8</code></td><td>API sessions at once</td></tr></tbody></table> <p>With Docker:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->docker build -t tau2-env -f envs/tau2_env/server/Dockerfile envs/tau2_env
docker run -p 8000:8000 -e HF_TOKEN=hf_... tau2-env<!----></pre></div><!----> <p>On a Space, the UI’s runs use each visitor’s own token once they sign in, so the Space needs no secret. They always run on Inference Providers, whatever <code>TAU2_USER_PROVIDER</code> is. API clients pass their own with <code>reset(hf_token=...)</code>, so an <code>HF_TOKEN</code> secret is only a fallback for clients that don’t, and the UI never uses it. Without credentials the server and the task explorer still run, and <code>reset()</code> explains what is missing.</p> <p>The image starts from a Python 3.12 base rather than <code>openenv-base</code>, because τ²-bench requires Python 3.12+.</p> <!--[1--><h2 class="relative group"><a id="notes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#notes"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Notes</span></h2><!--]--><!----> <ul><li>τ²-bench is installed from GitHub, pinned to a commit. The <code>tau2</code> package on PyPI is an unrelated project.</li> <li>To evaluate Claude Code on τ²-bench, see <a href="https://huggingface.co/docs/openenv/tutorials/claude-code-harness" rel="nofollow">Evaluate Claude Code in an Environment</a>.</li> <li>Train on the <code>train</code> split and evaluate on <code>test</code>, so training does not leak into the benchmark.</li> <li>litellm has no prices for Inference Providers models, so τ²-bench reports the cost of those runs as $0.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/openenv/blob/main/docs/source/environments/tau2.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_xquke6 = {
base: "/docs/openenv/main/en",
assets: "/docs/openenv/main/en"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/openenv/main/en/_app/immutable/entry/start.B22qDl-k.js"),
import("/docs/openenv/main/en/_app/immutable/entry/app.Du7tmpJJ.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 2],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
28 kB
·
Xet hash:
f19cd47209cc55c2d84930e0de29e74382e01e03211e5bba3182aaccde00ce01

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.