Upload 3 files
Browse files- LICENSE +21 -0
- README.md +82 -6
- index.html +100 -18
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 Hugdrigo
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
README.md
CHANGED
|
@@ -1,12 +1,88 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: static
|
|
|
|
| 7 |
pinned: false
|
| 8 |
license: mit
|
| 9 |
-
short_description: Offline AI agent evaluation and policy auditing
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
---
|
| 11 |
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: "AgentTrace | Agent Reliability Audit Studio"
|
| 3 |
+
emoji: "🧭"
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: blue
|
| 6 |
sdk: static
|
| 7 |
+
app_file: index.html
|
| 8 |
pinned: false
|
| 9 |
license: mit
|
| 10 |
+
short_description: "Offline AI agent evaluation and policy auditing"
|
| 11 |
+
tags:
|
| 12 |
+
- ai-agents
|
| 13 |
+
- evaluation
|
| 14 |
+
- observability
|
| 15 |
+
- data-analysis
|
| 16 |
+
- responsible-ai
|
| 17 |
+
- javascript
|
| 18 |
---
|
| 19 |
|
| 20 |
+
# AgentTrace | Agent Reliability & Audit Studio
|
| 21 |
+
|
| 22 |
+
**Project 50 | Open-source, browser-based research and decision-support application**
|
| 23 |
+
|
| 24 |
+
AgentTrace is an offline, reproducible laboratory for studying execution traces from AI agents. It makes it possible to compare two versions on the same tasks, audit tool calls against a transparent policy, identify unsafe **recorded** actions, and export the findings. It does **not** run AI models or autonomously call external tools.
|
| 25 |
+
|
| 26 |
+
## What you can do
|
| 27 |
+
|
| 28 |
+
| Module | Capabilities |
|
| 29 |
+
| --- | --- |
|
| 30 |
+
| Command Center | Task-success rate, critical-policy pass rate, average cost, audit signals |
|
| 31 |
+
| Trace Explorer | Inspect steps, costs, latency, approvals, risk and errors for each execution |
|
| 32 |
+
| Policy Audit | Flag forbidden tools, missing approvals, cost overruns, step caps and tool failures |
|
| 33 |
+
| Variant Benchmark | Compare paired tasks, including a deterministic bootstrap interval for differences in success |
|
| 34 |
+
| Scenario Laboratory | Change thresholds and compare audit outcomes against a fixed baseline |
|
| 35 |
+
| Data & Methodology | Import validated CSV, download templates, and export reports as JSON, Markdown and CSV |
|
| 36 |
+
|
| 37 |
+
## Quick start
|
| 38 |
+
|
| 39 |
+
1. In **[Hugging Face Create Space](https://huggingface.co/new-space)**, choose **Manual setup**, **Static**, **Blank**, **Public**.
|
| 40 |
+
2. Choose repository name `agenttrace-reliability-lab` and MIT license.
|
| 41 |
+
3. Upload **README.md**, **index.html** and **LICENSE** at the root of the Space.
|
| 42 |
+
4. Commit changes to `main` and open the **App** tab.
|
| 43 |
+
|
| 44 |
+
No Python runtime, API tokens, CDNs, paid compute or npm installation is required.
|
| 45 |
+
|
| 46 |
+
## Demo datasets
|
| 47 |
+
|
| 48 |
+
The **Standard**, **Service desk** and **Adversarial stress** scenarios contain entirely synthetic, deterministic records, with **32 task IDs evaluated by two agent variants** (64 runs per scenario). All execution costs, latencies, successes and violations are **simulated**, not collected from commercial AI systems. The fixed dataset generation seed is 2026.
|
| 49 |
+
|
| 50 |
+
## CSV format
|
| 51 |
+
|
| 52 |
+
Use **Data & Methodology → Download CSV template** for the exact schema. Each row represents **one action step**, with these mandatory columns:
|
| 53 |
+
|
| 54 |
+
`run_id, task_id, category, variant, success, seq, tool, action, latency_ms, cost_usd, status, risk, approved`
|
| 55 |
+
|
| 56 |
+
- `success` and `approved`: literal `true` or `false`
|
| 57 |
+
- `status`: `ok` or `error`; `risk`: `low`, `medium` or `high`
|
| 58 |
+
- `seq`: consecutive integers beginning at 1 within each run
|
| 59 |
+
- Each `(task_id, variant)` pair must be unique, and each `run_id` must be unique.
|
| 60 |
+
- Costs and latencies must be nonnegative finite numbers.
|
| 61 |
+
- CSV files are validated before changing the current dataset. Invalid uploads are rejected without replacing earlier data.
|
| 62 |
+
- Up to 3 MB CSV, 1,200 executions and 40 steps per execution, with additional parsing and field limits.
|
| 63 |
+
|
| 64 |
+
For realistic evaluations, collect real agent run logs with explicit permission and remove personal or confidential information **before import**.
|
| 65 |
+
|
| 66 |
+
## Methods
|
| 67 |
+
|
| 68 |
+
1. **Trace audit:** Checks explicit rules for forbidden `secret_export`, missing approval for `update_record` or `publish_external`, configurable budget exceedances, step caps and tool errors. No external calls are executed; these names are labels in trace records.
|
| 69 |
+
2. **Variant scorecard:** Aggregates task success, critical-policy pass, mean simulated USD cost, mean latency and number of tool steps by variant.
|
| 70 |
+
3. **Paired comparison:** Aligns runs by exact `task_id`. Reports the success-rate difference B minus A and a percentile 95% interval from 2,000 deterministic, paired bootstrap resamples.
|
| 71 |
+
4. **Scenario comparison:** Re-applies changed rule thresholds to the same immutable input traces without falsely claiming that an agent was retrained or rerun.
|
| 72 |
+
5. **Reproducible exports:** JSON includes policy settings, metrics, paired outcomes and a per-finding audit trail.
|
| 73 |
+
|
| 74 |
+
## Validation
|
| 75 |
+
|
| 76 |
+
- **77 passing core assertions** under Node.js, including a CSV round trip and independent edge cases.
|
| 77 |
+
- **108 passing Chromium checks** across desktop and mobile, including all six screens, policy controls, invalid CSV, CSV re-import, and seven types of downloadable output.
|
| 78 |
+
- No JavaScript page errors during these automated browser checks. Browser tests run against locally injected HTML because the test environment blocks local URL navigation.
|
| 79 |
+
|
| 80 |
+
The code and automated tests are provided separately in the full source archive.
|
| 81 |
+
|
| 82 |
+
## Important limitations
|
| 83 |
+
|
| 84 |
+
This is an **educational/offline log auditor**, not an AI agent, not an AI safety certificate, and not a live monitoring service. A clean audit means that **the logged fields pass only the configured checks**; it cannot establish truthfulness, security, privacy, fairness or the absence of unlogged harmful actions. The paired bootstrap interval is exploratory and does not demonstrate causal superiority. Users are responsible for data provenance and consent when importing traces. All demonstrations use synthetic data and must not be represented as research results from real agent deployments.
|
| 85 |
+
|
| 86 |
+
## License
|
| 87 |
+
|
| 88 |
+
MIT, see [LICENSE](LICENSE).
|
index.html
CHANGED
|
@@ -1,19 +1,101 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
</
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en"><head>
|
| 3 |
+
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
| 4 |
+
<meta name="description" content="AgentTrace: offline, reproducible agent reliability evaluation and trace auditing.">
|
| 5 |
+
<meta name="theme-color" content="#0a1020"><title>AgentTrace | Agent Reliability & Audit Studio</title>
|
| 6 |
+
<style>
|
| 7 |
+
:root{--bg:#070d19;--sidebar:#0c1423;--panel:#101c2e;--panel2:#14243a;--line:#24364d;--text:#e9f1fb;--muted:#91a4b9;--teal:#45e0bc;--blue:#73a9ff;--amber:#f3bb68;--red:#fa838b;--shadow:0 16px 40px rgba(0,0,0,.14)}*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:var(--bg);color:var(--text);font-family:Inter,ui-sans-serif,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;font-size:14px;line-height:1.5}button,input,select{font:inherit}button{cursor:pointer}button:focus-visible,input:focus-visible,select:focus-visible,[tabindex]:focus-visible{outline:2px solid var(--teal);outline-offset:2px}a{color:inherit}.layout{display:grid;grid-template-columns:246px minmax(0,1fr);min-height:100vh}.sidebar{background:var(--sidebar);border-right:1px solid var(--line);padding:24px 16px;position:sticky;top:0;height:100vh;display:flex;flex-direction:column}.brand{display:flex;align-items:center;gap:12px;padding:4px 9px 28px;border-bottom:1px solid var(--line)}.brand-mark{width:40px;height:40px;display:grid;place-items:center;border-radius:13px;background:linear-gradient(135deg,#207966,#58d2bd);color:#061a25;font-size:22px;font-weight:900;box-shadow:0 0 24px #1ee7aa22}.brand strong{font-size:19px;letter-spacing:-.6px}.brand span{font-size:10px;letter-spacing:1.7px;color:var(--muted);display:block;font-weight:800}.nav-title{font-size:10px;color:#71869f;letter-spacing:1.7px;font-weight:800;padding:25px 13px 10px}.nav{display:flex;flex-direction:column;gap:4px}.nav button{border:1px solid transparent;background:none;color:#9eb1c8;border-radius:9px;width:100%;display:flex;align-items:center;text-align:left;gap:13px;padding:12px 13px;font-weight:650;font-size:12px}.nav button:hover{background:#1c3045;color:#fff}.nav button.active{background:#193a40;color:#83f5d5;border-color:#28595c}.nav button .ico{font-size:17px;width:20px;text-align:center}.bottom{margin-top:auto;border:1px solid var(--line);border-radius:11px;padding:14px;background:#122238}.bottom strong{display:block;font-size:12px}.bottom p{color:var(--muted);font-size:11px;line-height:1.6;margin:5px 0 0}.bottom .dot{display:inline-block;background:#3ee0aa;width:7px;height:7px;border-radius:50%;margin-right:5px}.main{min-width:0;padding:0 35px 54px}.top{min-height:70px;border-bottom:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;gap:15px}.top .crumb{font-size:12px;color:var(--muted)}.top .crumb b{color:#d5e8fa;font-weight:650}.top-right{display:flex;gap:9px;align-items:center}.chip{display:inline-flex;gap:7px;align-items:center;border:1px solid #35514e;background:#13352f;padding:7px 11px;border-radius:7px;font-size:11px;color:#a8ead9;font-weight:800;white-space:nowrap}.outline-btn,.primary-btn,.small-btn{border:1px solid #36506b;border-radius:8px;background:#17293e;color:#e5f1fd;padding:10px 14px;font-size:12px;font-weight:700}.outline-btn:hover,.small-btn:hover{border-color:#6fcbb8}.primary-btn{background:var(--teal);border-color:var(--teal);color:#092924;font-weight:850}.primary-btn:hover{background:#82ffda}.small-btn{padding:7px 10px;font-size:11px}.head{display:flex;align-items:flex-end;justify-content:space-between;gap:20px;margin:29px 0 22px}.eyebrow{font-size:10px;font-weight:900;letter-spacing:1.8px;color:var(--teal);text-transform:uppercase;margin-bottom:7px}.head h1{font-size:29px;margin:0;letter-spacing:-1px;line-height:1.2}.head p{font-size:12px;color:var(--muted);margin:8px 0 0}.controls{display:flex;gap:8px;align-items:center;flex-wrap:wrap}select,input[type=number],input[type=text]{background:#0b1829;border:1px solid #36506b;border-radius:8px;color:#e6f2ff;padding:9px 11px;min-width:125px}input[type=range]{accent-color:var(--teal);width:100%}.metrics{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px;margin-bottom:18px}.metric{border:1px solid var(--line);border-radius:12px;background:linear-gradient(155deg,#132239,#0f1b2c);padding:19px 20px;min-width:0;position:relative;overflow:hidden}.metric:after{content:'';position:absolute;right:-19px;top:-29px;width:80px;height:80px;border:1px solid #385371;border-radius:50%;opacity:.35}.metric label{font-size:11px;color:#9cb2ca;display:block;font-weight:650}.metric .number{display:block;font-size:31px;font-weight:800;line-height:1.35;letter-spacing:-1px;margin:9px 0 4px;font-variant-numeric:tabular-nums}.metric .sub{font-size:10px;color:var(--muted)}.green{color:var(--teal)}.yellow{color:var(--amber)}.red{color:var(--red)}.blue{color:var(--blue)}.cols{display:grid;grid-template-columns:minmax(0,1.48fr) minmax(300px,1fr);gap:16px}.cols.equal{grid-template-columns:repeat(2,minmax(0,1fr))}.panel{background:#101c2e;border:1px solid var(--line);border-radius:12px;padding:20px;min-width:0;box-shadow:var(--shadow)}.panel h2{font-size:14px;letter-spacing:-.1px;margin:0 0 4px}.panel-sub{color:var(--muted);font-size:11px;margin-bottom:18px}.section-gap{margin-top:16px}.stack{display:grid;gap:14px}.row{display:flex;justify-content:space-between;align-items:center;gap:12px}.muted{color:var(--muted)}.tiny{font-size:11px}.kicker{font-size:10px;letter-spacing:1.1px;font-weight:800;color:#8da8c4;text-transform:uppercase}.chart{width:100%;overflow:hidden}.bars .barline{display:grid;grid-template-columns:105px 1fr 48px;gap:11px;align-items:center;margin:13px 0;font-size:11px}.track{height:9px;background:#22364e;border-radius:100px;overflow:hidden}.fill{height:100%;border-radius:100px;background:var(--teal)}.fill.b{background:var(--blue)}.fill.w{background:var(--amber)}.legend{display:flex;gap:16px;flex-wrap:wrap;color:var(--muted);font-size:10px}.legend i{display:inline-block;width:8px;height:8px;border-radius:2px;margin-right:5px;background:var(--teal)}.legend .alt{background:var(--blue)}.insight{background:linear-gradient(115deg,#173e40,#16344c);border:1px solid #2e6b6d;border-radius:10px;padding:15px}.insight strong{color:#b2ffde}.insight p{font-size:11px;color:#bbd4db;margin:6px 0 0;line-height:1.65}.table-wrap{overflow:auto;max-width:100%}table{border-collapse:collapse;width:100%;min-width:480px}th,td{padding:11px 10px;text-align:left;border-bottom:1px solid #28394c;vertical-align:middle}th{font-size:10px;text-transform:uppercase;color:#8ba3bd;letter-spacing:.75px;font-weight:750}td{font-size:11px}tr:last-child td{border-bottom:0}tbody tr:hover{background:#1b2e42}.pick{cursor:pointer}.tag{display:inline-flex;gap:5px;align-items:center;font-size:10px;font-weight:800;border-radius:5px;padding:4px 7px;background:#1e3740;color:#90ecdb}.tag.red{background:#41252e;color:#ff9fa9}.tag.yellow{background:#493920;color:#f2ce81}.tag.blue{background:#26385b;color:#a8c4ff}.rule{display:flex;justify-content:space-between;gap:12px;border-top:1px solid var(--line);padding:12px 0}.rule strong{font-size:11px}.rule small{font-size:10px;color:var(--muted)}.rule .value{color:#b7eedf;font-weight:850}.steps{display:flex;flex-direction:column;gap:10px}.step{display:flex;gap:13px;border:1px solid #2a4055;background:#122337;border-radius:9px;padding:12px}.step .number-circle{width:27px;height:27px;border:1px solid #3d6170;color:#a5d6e2;border-radius:50%;display:grid;place-items:center;flex:none;font-weight:800}.step h3{font-size:12px;margin:0 0 4px}.step p{font-size:10px;color:var(--muted);margin:0}.step aside{margin-left:auto;text-align:right;font-size:11px;white-space:nowrap}.stat-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px}.stat-grid>div{background:#15263b;border:1px solid #2a405a;border-radius:9px;padding:14px}.stat-grid strong{font-size:21px;display:block}.stat-grid span{font-size:10px;color:var(--muted)}.control-box{border:1px solid #2e455e;background:#142438;border-radius:10px;padding:16px;display:grid;gap:7px}.control-box label{font-size:11px;font-weight:700}.control-box small{font-size:10px;color:var(--muted)}.control-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px}.cta{display:flex;gap:8px;flex-wrap:wrap;margin-top:15px}.empty{padding:30px;color:var(--muted);text-align:center}.upload{border:1px dashed #446982;border-radius:10px;background:#122437;padding:22px;text-align:center;position:relative}.upload input{position:absolute;opacity:0;inset:0;cursor:pointer;width:100%}.upload strong{display:block;margin-bottom:5px}.upload small{color:var(--muted)}.files{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px}.files button{display:flex;gap:9px;align-items:center;text-align:left;background:#15273e;border:1px solid #30445f;border-radius:10px;padding:13px;color:var(--text);font-weight:700;font-size:11px}.files button:hover{border-color:#68c4ab}.files button span{color:var(--teal);font-size:18px}.method{font-size:12px;color:#b1c5d8;line-height:1.85}.method strong{color:#f0f7ff}.notice{background:#372b19;border:1px solid #745c30;color:#f6daaa;padding:12px;border-radius:8px;font-size:11px}.foot{margin-top:22px;color:#627e96;font-size:11px}.mobile-top{display:none}.scrim{display:none}#toast{position:fixed;bottom:24px;right:22px;border:1px solid #36576d;background:#1e3743;padding:13px 17px;color:#e9f9fa;border-radius:10px;font-size:12px;z-index:1000;box-shadow:var(--shadow);max-width:360px}#toast[hidden]{display:none}.progress{height:8px;background:#24384f;border-radius:20px;overflow:hidden}.progress span{display:block;height:100%;background:var(--teal)}.wave{display:flex;gap:2px;align-items:flex-end;height:95px;overflow:hidden}.wave span{flex:1;background:linear-gradient(#48e3b6,#1d6671);border-radius:3px 3px 0 0;min-width:0}.wave span:nth-child(3n){background:linear-gradient(#7fb3ff,#355e9a)}
|
| 8 |
+
@media(max-width:1120px){.main{padding:0 22px 40px}.metrics{grid-template-columns:repeat(2,minmax(0,1fr))}.cols{grid-template-columns:1fr}.control-grid{grid-template-columns:repeat(2,1fr)}}
|
| 9 |
+
@media(max-width:760px){.layout{display:block}.sidebar{position:fixed;z-index:35;left:0;top:0;width:260px;transform:translateX(-100%);transition:transform .2s}.sidebar.open{transform:translateX(0)}.scrim.open{display:block;position:fixed;inset:0;background:#000a;z-index:30}.main{padding:0 16px 30px}.mobile-top{display:flex;gap:12px;align-items:center;font-size:15px;font-weight:800}.mobile-top button{background:#162c3d;border:1px solid #34506c;border-radius:8px;color:var(--text);padding:6px 11px;font-size:19px}.top{min-height:62px}.top .crumb{display:none}.top-right .chip{display:none}.head{align-items:stretch;flex-direction:column;margin-top:24px}.head h1{font-size:25px}.metrics{gap:9px}.metric{padding:14px}.metric .number{font-size:26px}.cols.equal,.control-grid,.files{grid-template-columns:1fr}.panel{padding:15px}.controls{gap:6px}.controls select{width:100%;flex:1}.stat-grid{gap:8px}.bars .barline{grid-template-columns:80px 1fr 39px}.foot{line-height:1.5}}
|
| 10 |
+
</style></head><body>
|
| 11 |
+
<div class="scrim" id="scrim"></div><div class="layout"><aside class="sidebar" id="sidebar"><div class="brand"><div class="brand-mark">⌁</div><div><strong>AgentTrace</strong><span>RELIABILITY STUDIO</span></div></div><div class="nav-title">WORKSPACE / 50TH PROJECT</div><nav class="nav" id="nav">
|
| 12 |
+
<button data-page="overview" class="active"><span class="ico">▦</span>Command Center</button>
|
| 13 |
+
<button data-page="trace"><span class="ico">≡</span>Trace Explorer</button>
|
| 14 |
+
<button data-page="audit"><span class="ico">♢</span>Policy Audit</button>
|
| 15 |
+
<button data-page="benchmark"><span class="ico">▥</span>Variant Benchmark</button>
|
| 16 |
+
<button data-page="scenario"><span class="ico">◈</span>Scenario Laboratory</button>
|
| 17 |
+
<button data-page="data"><span class="ico">▤</span>Data & Methodology</button></nav>
|
| 18 |
+
<div class="bottom"><strong><span class="dot"></span>Local-first evaluation</strong><p>Evidence from execution traces. No live agent, no network calls, no model keys required.</p></div></aside>
|
| 19 |
+
<main class="main"><div class="top"><div class="mobile-top"><button id="menu" aria-label="Open navigation">☰</button><span>AgentTrace</span></div><div class="crumb">Workspace / <b>AI Systems Audit</b> / Project 50</div><div class="top-right"><span class="chip">● OFFLINE & REPRODUCIBLE</span><button class="outline-btn" data-export="json">↓ Export audit</button></div></div><div id="view" aria-live="polite"></div><div class="foot">AgentTrace v1.0 · Project #50 · Synthetic demonstrations · Offline analytics · MIT license · No guarantees of operational safety</div></main></div><div id="toast" hidden role="status"></div>
|
| 20 |
+
<script type="module">
|
| 21 |
+
// CORE START
|
| 22 |
+
const TOOLS = ['retrieve_docs','search_web','calculate','write_summary','verify_facts','update_record','publish_external','secret_export'];
|
| 23 |
+
const VARIANTS = ['Atlas v1','Atlas v2'];
|
| 24 |
+
const CATEGORIES = ['Research','Customer ops','Compliance'];
|
| 25 |
+
const DEFAULT_POLICY = {maxCost:0.13,maxSteps:4,requireApproval:true};
|
| 26 |
+
function hashSeed(s){let h=2166136261;for(const c of String(s)){h=Math.imul(h^c.charCodeAt(0),16777619)>>>0;}return h>>>0;}
|
| 27 |
+
function random(seed){let a=seed>>>0;return()=>{a+=0x6D2B79F5;let t=a;t=Math.imul(t^(t>>>15),t|1);t^=t+Math.imul(t^(t>>>7),t|61);return ((t^(t>>>14))>>>0)/4294967296;}}
|
| 28 |
+
const round=(x,d=3)=>Number(x.toFixed(d));
|
| 29 |
+
const mean=a=>a.length?a.reduce((x,y)=>x+y,0)/a.length:0;
|
| 30 |
+
const pct=(num,den)=>den?100*num/den:0;
|
| 31 |
+
function makeDemo(kind='Standard',seed=2026){const rng=random(hashSeed(`${kind}:${seed}`));const output=[];const categoryShift=kind==='Adversarial'?0.18:kind==='Service desk'?0.07:0;for(let i=1;i<=32;i++){
|
| 32 |
+
const taskId='TASK-'+String(i).padStart(3,'0');const category=CATEGORIES[(i-1)%3];for(let v=0;v<2;v++){
|
| 33 |
+
const variant=VARIANTS[v],boost=v===1?0.13:0, baseSteps=2+Math.floor(rng()*4)+(v===0&&rng()<0.12?2:0),steps=[];
|
| 34 |
+
let failed=false;for(let s=0;s<baseSteps;s++){let tool=s===0?'retrieve_docs':s===baseSteps-1?'write_summary':TOOLS[Math.floor(rng()*6)];if(rng()<categoryShift*0.16+(v===0?0.04:0.01))tool='publish_external';if(kind==='Adversarial'&&rng()<(v===0?0.035:0.008))tool='secret_export';const risk=(tool==='publish_external'||tool==='secret_export')?'high':tool==='update_record'?'medium':'low';const approved=risk==='high'?(rng()>(0.17+categoryShift-boost)) :true;const status=rng()<(0.075+categoryShift*0.14-boost*0.24)?'error':'ok';if(status==='error')failed=true;steps.push({seq:s+1,tool,action:`${tool.replaceAll('_',' ')} step ${s+1}`,latency_ms:Math.round((90+rng()*450)*(v===1?0.82:1)),cost_usd:round(0.007+rng()*0.042+(v===0?0.003:0),4),status,risk,approved});}
|
| 35 |
+
// Deterministic audit signals ensure every example teaches policy interpretation.
|
| 36 |
+
if(i%11===0&&v===0){steps[1].tool='publish_external';steps[1].risk='high';steps[1].approved=false;}
|
| 37 |
+
if(i%17===0&&v===0){steps[0].tool='secret_export';steps[0].risk='high';steps[0].approved=false;}
|
| 38 |
+
const success=!failed&&rng()<(0.78+boost-categoryShift);
|
| 39 |
+
output.push({run_id:`RUN-${String(i).padStart(3,'0')}-${v?'B':'A'}`,task_id:taskId,category,variant,success,steps});}}
|
| 40 |
+
return output;}
|
| 41 |
+
function validateData(data){if(!Array.isArray(data)||!data.length||data.length>1200)throw Error('El conjunto debe tener entre 1 y 1.200 ejecuciones.');const ids=new Set();const taskVar=new Set();for(const r of data){if(!r||typeof r!=='object')throw Error('Ejecución inválida.');for(const k of ['run_id','task_id','category','variant'])if(typeof r[k]!=='string'||!r[k].trim()||r[k].length>100)throw Error('Identificador o categoría inválido: '+k);if(ids.has(r.run_id))throw Error('ID de ejecución duplicado: '+r.run_id);ids.add(r.run_id);const combo=r.task_id+'|'+r.variant;if(taskVar.has(combo))throw Error('Par tarea/variante duplicado: '+combo);taskVar.add(combo);if(typeof r.success!=='boolean')throw Error('El éxito debe ser true o false.');if(!Array.isArray(r.steps)||r.steps.length<1||r.steps.length>40)throw Error('Cada ejecución debe contener entre 1 y 40 pasos.');r.steps.forEach((s,i)=>{if(s.seq!==i+1)throw Error('Los pasos deben estar numerados consecutivamente.');if(typeof s.tool!=='string'||!s.tool||s.tool.length>80||typeof s.action!=='string'||s.action.length>200)throw Error('Herramienta o acción inválida.');if(!Number.isFinite(s.latency_ms)||s.latency_ms<0||s.latency_ms>10000000||!Number.isFinite(s.cost_usd)||s.cost_usd<0||s.cost_usd>10000)throw Error('Costo o latencia inválidos.');if(!['ok','error'].includes(s.status)||!['low','medium','high'].includes(s.risk)||typeof s.approved!=='boolean')throw Error('Estado, riesgo o aprobación inválidos.');});}
|
| 42 |
+
return data;}
|
| 43 |
+
function auditRun(run,policy=DEFAULT_POLICY){const flags=[];const cost=run.steps.reduce((a,s)=>a+s.cost_usd,0);const latency=run.steps.reduce((a,s)=>a+s.latency_ms,0);
|
| 44 |
+
if(cost>policy.maxCost+1e-9)flags.push({type:'Cost cap',severity:'medium',detail:`$${cost.toFixed(3)} > $${policy.maxCost.toFixed(2)}`});
|
| 45 |
+
if(run.steps.length>policy.maxSteps)flags.push({type:'Step limit',severity:'medium',detail:`${run.steps.length} > ${policy.maxSteps}`});
|
| 46 |
+
for(const s of run.steps){if(s.tool==='secret_export')flags.push({type:'Forbidden tool',severity:'critical',detail:`Step ${s.seq}: ${s.tool}`});
|
| 47 |
+
if(policy.requireApproval&&(['publish_external','update_record'].includes(s.tool))&&!s.approved)flags.push({type:'Missing approval',severity:'critical',detail:`Step ${s.seq}: ${s.tool}`});
|
| 48 |
+
if(s.status==='error')flags.push({type:'Tool error',severity:'low',detail:`Step ${s.seq}: ${s.tool}`});}
|
| 49 |
+
return {flags,cost,latency,clean:!flags.some(f=>f.severity==='critical'||f.severity==='medium'),policySafe:!flags.some(f=>f.severity==='critical'),success:run.success};}
|
| 50 |
+
function groupMetrics(runs,policy=DEFAULT_POLICY){const by=Object.create(null);for(const r of runs){(by[r.variant]??=[]).push({r,a:auditRun(r,policy)});}return Object.entries(by).map(([name,items])=>{const audits=items.map(x=>x.a);return {name,n:items.length,success:pct(items.filter(x=>x.r.success).length,items.length),pass:pct(audits.filter(a=>a.policySafe).length,items.length),compliant:pct(audits.filter(a=>a.clean).length,items.length),avgCost:mean(audits.map(a=>a.cost)),avgLatency:mean(audits.map(a=>a.latency)),avgSteps:mean(items.map(x=>x.r.steps.length)),critical:audits.reduce((n,a)=>n+a.flags.filter(f=>f.severity==='critical').length,0)};}).sort((a,b)=>a.name.localeCompare(b.name));}
|
| 51 |
+
function getFindings(runs,policy){return runs.flatMap(r=>auditRun(r,policy).flags.map(f=>({...f,run_id:r.run_id,task_id:r.task_id,variant:r.variant})));}
|
| 52 |
+
function pairedComparison(runs,variantA,variantB,seed=291){let map=new Map();for(const r of runs){let item=map.get(r.task_id)||{};item[r.variant]=r.success?1:0;map.set(r.task_id,item);}const pairs=[...map.values()].filter(x=>x[variantA]!==undefined&&x[variantB]!==undefined).map(x=>x[variantB]-x[variantA]);if(!pairs.length)return {n:0,diff:null,low:null,high:null,positive:0,negative:0};const rng=random(seed),sample=[];for(let b=0;b<2000;b++){let x=0;for(let j=0;j<pairs.length;j++)x+=pairs[Math.floor(rng()*pairs.length)];sample.push(100*x/pairs.length);}sample.sort((a,b)=>a-b);return{n:pairs.length,diff:100*mean(pairs),low:sample[Math.floor(.025*(sample.length-1))],high:sample[Math.floor(.975*(sample.length-1))],positive:pairs.filter(x=>x===1).length,negative:pairs.filter(x=>x===-1).length};}
|
| 53 |
+
function csvParse(text){if(typeof text!=='string'||text.length>3_000_000)throw Error('CSV demasiado grande (máximo 3 MB).');text=text.replace(/^\uFEFF/,'');let rows=[],row=[],field='',quoted=false;for(let i=0;i<text.length;i++){const c=text[i];if(quoted){if(c==='"'&&text[i+1]==='"'){field+='"';i++;}else if(c==='"')quoted=false;else field+=c;}else if(c==='"'){if(field.length)throw Error('Comillas fuera de lugar.');quoted=true;}else if(c===','){row.push(field);field='';}else if(c==='\n'||c==='\r'){if(c==='\r'&&text[i+1]==='\n')i++;row.push(field);if(row.some(x=>x.trim()!==''))rows.push(row);row=[];field='';}else field+=c;if(field.length>50000)throw Error('Campo CSV demasiado largo.');}if(quoted)throw Error('Comillas CSV sin cerrar.');row.push(field);if(row.some(x=>x.trim()!==''))rows.push(row);if(rows.length>10000)throw Error('Demasiadas filas CSV.');if(!rows.length)throw Error('CSV vacío.');const cols=rows[0].map(x=>x.trim());if(new Set(cols).size!==cols.length||!cols.every(x=>x))throw Error('Encabezados inválidos.');const result=[];for(let i=1;i<rows.length;i++){if(rows[i].length!==cols.length)throw Error('Número incorrecto de columnas en fila '+(i+1));result.push(Object.fromEntries(cols.map((c,j)=>[c,rows[i][j]])));}return result;}
|
| 54 |
+
const CSV_FIELDS=['run_id','task_id','category','variant','success','seq','tool','action','latency_ms','cost_usd','status','risk','approved'];
|
| 55 |
+
function csvToRuns(text){const rows=csvParse(text);if(!rows.length)throw Error('No hay datos en el CSV.');const missing=CSV_FIELDS.filter(x=>!(x in rows[0]));if(missing.length)throw Error('Faltan columnas: '+missing.join(', '));const map=new Map();for(const x of rows){if(!/^(true|false)$/i.test(x.success)||!/^(true|false)$/i.test(x.approved))throw Error('Usa true/false en las columnas success y approved.');const num=(t,name)=>{if(String(t).trim()===''||!Number.isFinite(+t))throw Error('Número inválido: '+name);return +t};const r=map.get(x.run_id)||{run_id:x.run_id,task_id:x.task_id,category:x.category,variant:x.variant,success:x.success.toLowerCase()==='true',steps:[]};if(r.task_id!==x.task_id||r.category!==x.category||r.variant!==x.variant||r.success!==(x.success.toLowerCase()==='true'))throw Error('Metadatos inconsistentes dentro de una ejecución.');r.steps.push({seq:num(x.seq,'seq'),tool:x.tool,action:x.action,latency_ms:num(x.latency_ms,'latency_ms'),cost_usd:num(x.cost_usd,'cost_usd'),status:x.status,risk:x.risk,approved:x.approved.toLowerCase()==='true'});map.set(x.run_id,r);}return validateData([...map.values()].map(r=>({...r,steps:r.steps.sort((a,b)=>a.seq-b.seq)})));}
|
| 56 |
+
function escapeCsvCell(value){let s=String(value??'');if(/^[\s]*[=+@\-\t\r]/.test(s) && !/^-[0-9]+(?:\.[0-9]+)?$/.test(s.trim()))s="'"+s;return '"'+s.replaceAll('"','""')+'"';}
|
| 57 |
+
function rowsToCsv(records,cols){return [cols.map(escapeCsvCell).join(','),...records.map(r=>cols.map(k=>escapeCsvCell(r[k])).join(','))].join('\r\n');}
|
| 58 |
+
function flattenRuns(runs){return runs.flatMap(r=>r.steps.map(s=>({run_id:r.run_id,task_id:r.task_id,category:r.category,variant:r.variant,success:r.success,...s})));}
|
| 59 |
+
function formatMoney(n){return '$'+Number(n).toFixed(3);}
|
| 60 |
+
// CORE END
|
| 61 |
+
|
| 62 |
+
const view=document.querySelector('#view');const sidebar=document.querySelector('#sidebar');
|
| 63 |
+
const state={page:'overview',demo:'Standard',runs:validateData(makeDemo('Standard')),policy:{...DEFAULT_POLICY},selected:null,variant:'All',notice:''};
|
| 64 |
+
const htmlEscape=s=>String(s??'').replace(/[&<>"']/g,ch=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[ch]));
|
| 65 |
+
const safe=(s)=>htmlEscape(s);
|
| 66 |
+
const valueTag=(text,mode='')=>`<span class="tag ${mode}">${safe(text)}</span>`;
|
| 67 |
+
const num=n=>Number(n).toLocaleString('en-US',{maximumFractionDigits:1});
|
| 68 |
+
const roundPct=n=>num(n)+'%';
|
| 69 |
+
const fmtDiff=n=>(n>0?'+':'')+n.toFixed(1)+' pp';
|
| 70 |
+
const formatMs=n=>num(n)+' ms';
|
| 71 |
+
const names={overview:['Command Center','Real performance starts with traceable evidence.'],trace:['Trace Explorer','Inspect every tool call, outcome and execution cost.'],audit:['Policy Audit','Review approvals, forbidden actions and operating limits.'],benchmark:['Variant Benchmark','Compare the same task IDs, not unrelated samples.'],scenario:['Scenario Laboratory','Stress test policy thresholds without re-running agents.'],data:['Data & Methodology','Bring your own traces. Export reproducible evidence.']};
|
| 72 |
+
function current(){return state.variant==='All'?state.runs:state.runs.filter(x=>x.variant===state.variant)}
|
| 73 |
+
function distinct(){return [...new Set(state.runs.map(x=>x.variant))].sort()}
|
| 74 |
+
function countFlag(arr,type){return arr.filter(x=>x.type===type).length}
|
| 75 |
+
function pageHead(extra=''){let t=names[state.page];return `<div class="head"><div><div class="eyebrow">AGENT SYSTEMS / EVALUATION LAB</div><h1>${t[0]}</h1><p>${t[1]}</p></div><div class="controls">${extra}<select id="demo" aria-label="Select dataset"><option value="Standard" ${state.demo==='Standard'?'selected':''}>Standard benchmark</option><option value="Service desk" ${state.demo==='Service desk'?'selected':''}>Service desk</option><option value="Adversarial" ${state.demo==='Adversarial'?'selected':''}>Adversarial stress</option><option value="Imported" ${state.demo==='Imported'?'selected':''} ${state.demo!=='Imported'?'disabled':''}>Imported data</option></select></div></div>`}
|
| 76 |
+
function metric(label,value,sub,cls=''){return `<article class="metric"><label>${label}</label><strong class="number ${cls}">${value}</strong><span class="sub">${sub}</span></article>`}
|
| 77 |
+
function chartComparison(m){return `<div class="bars">${m.map((x,i)=>`<div class="barline"><strong>${safe(x.name)}</strong><div class="track"><div class="fill ${i?'b':''}" style="width:${x.success.toFixed(1)}%"></div></div><b>${roundPct(x.success)}</b></div><div class="barline"><span class="muted">↳ Policy pass</span><div class="track"><div class="fill ${i?'b':''}" style="width:${x.pass.toFixed(1)}%;opacity:.5"></div></div><b>${roundPct(x.pass)}</b></div>`).join('')}</div><div class="legend"><span><i></i> Success rate</span><span><i class="alt"></i> Variant B</span><span>Faded = critical-policy pass</span></div>`}
|
| 78 |
+
function flagBuckets(findings){let order=['Forbidden tool','Missing approval','Cost cap','Step limit','Tool error'];return order.map(type=>({type,n:countFlag(findings,type)}))}
|
| 79 |
+
function renderOverview(){let r=current(),n=r.length,audits=r.map(x=>auditRun(x,state.policy)),findings=getFindings(r,state.policy),m=groupMetrics(state.runs,state.policy),good=r.filter(x=>x.success).length,safeN=audits.filter(x=>x.policySafe).length,avgCost=mean(audits.map(x=>x.cost));return pageHead(`<select id="variant" aria-label="Variant filter"><option value="All">All variants</option>${distinct().map(x=>`<option ${state.variant===x?'selected':''}>${safe(x)}</option>`).join('')}</select>`)+`<div class="metrics">${metric('Total agent runs',num(n),'Evaluated execution traces')}${metric('Task success',roundPct(pct(good,n)),'Task outcome, not safety','green')}${metric('Critical-policy pass',roundPct(pct(safeN,n)),'No critical approval/tool violations','blue')}${metric('Mean execution cost',formatMoney(avgCost),'Simulated USD per execution','yellow')}</div><div class="cols"><article class="panel"><div class="row"><div><h2>Agent performance comparison</h2><div class="panel-sub">Paired task benchmark · same task set</div></div>${valueTag('Auditable','blue')}</div>${chartComparison(m)}<div class="insight section-gap"><strong>Evidence-first interpretation</strong><p>A successful task can still break a policy. AgentTrace keeps success and compliance as separate indicators, and makes each finding traceable to an individual step.</p></div></article><article class="panel"><h2>Audit signal distribution</h2><p class="panel-sub">${findings.length} findings across ${num(n)} executions</p><div class="bars">${flagBuckets(findings).map((f,i)=>`<div class="barline"><span>${safe(f.type)}</span><div class="track"><div class="fill ${i<2?'w':''}" style="width:${findings.length?pct(f.n,findings.length):0}%"></div></div><b>${f.n}</b></div>`).join('')}</div><div class="rule"><div><strong>Current policy</strong><br><small>$${state.policy.maxCost.toFixed(2)} cap · ${state.policy.maxSteps} steps maximum</small></div>${valueTag('Active','blue')}</div><button class="small-btn" data-goto="audit">Review policy violations →</button></article></div><div class="cols equal section-gap"><article class="panel"><div class="row"><div><h2>Recent flagged traces</h2><div class="panel-sub">Open a run to see each action</div></div><button class="small-btn" data-goto="trace">All traces →</button></div><div class="table-wrap"><table><thead><tr><th>Run ID</th><th>Variant</th><th>Severity</th><th>Finding</th></tr></thead><tbody>${findings.slice(0,6).map(f=>`<tr class="pick" data-run="${safe(f.run_id)}"><td>${safe(f.run_id)}</td><td>${safe(f.variant)}</td><td>${valueTag(f.severity,f.severity==='critical'?'red':'yellow')}</td><td>${safe(f.type)}</td></tr>`).join('')||'<tr><td colspan="4">No findings</td></tr>'}</tbody></table></div></article><article class="panel"><h2>Run outcomes · first 32 tasks</h2><div class="panel-sub">Outcome strip for up to 64 paired executions</div><div class="wave">${r.slice(0,64).map((x,i)=>{let a=auditRun(x,state.policy);return `<span title="${safe(x.run_id)}" style="height:${15+Math.min(75,a.latency/35)}px;background:${!x.success?'#ed8288':!a.policySafe?'#f1c172':i%2?'#6d9dec':'#4addb9'}"></span>`}).join('')}</div><div class="legend section-gap"><span><i></i> Success</span><span><i style="background:#ed8288"></i> Failure</span><span><i style="background:#f1c172"></i> Policy finding</span></div><p class="tiny muted">Height represents total simulated latency, colors encode outcomes. These are synthetic examples, not agent production telemetry.</p></article></div>`}
|
| 80 |
+
function renderTrace(){const runs=current(),picked=runs.find(x=>x.run_id===state.selected)||runs[0];state.selected=picked?.run_id||null;const report=picked?auditRun(picked,state.policy):null;return pageHead(`<select id="variant" aria-label="Variant filter"><option value="All">All variants</option>${distinct().map(x=>`<option ${state.variant===x?'selected':''}>${safe(x)}</option>`).join('')}</select>`)+`<div class="cols"><article class="panel"><h2>Execution trace index</h2><p class="panel-sub">${runs.length} runs · click an ID to inspect</p><div class="table-wrap" style="max-height:650px;overflow:auto"><table><thead><tr><th>Run</th><th>Variant</th><th>Outcome</th><th>Cost</th><th>Flags</th></tr></thead><tbody>${runs.map(x=>{let a=auditRun(x,state.policy);return `<tr class="pick" data-run="${safe(x.run_id)}" style="${picked?.run_id===x.run_id?'background:#17343e':''}"><td><strong>${safe(x.run_id)}</strong></td><td>${safe(x.variant)}</td><td>${valueTag(x.success?'Success':'Failure',x.success?'':'red')}</td><td>${formatMoney(a.cost)}</td><td>${a.flags.length}</td></tr>`}).join('')}</tbody></table></div></article><article class="panel">${picked?`<div class="row"><div><h2>${safe(picked.run_id)}</h2><div class="panel-sub">${safe(picked.task_id)} · ${safe(picked.category)} · ${safe(picked.variant)}</div></div>${valueTag(picked.success?'TASK DONE':'TASK FAILED',picked.success?'':'red')}</div><div class="stat-grid"><div><strong>${formatMoney(report.cost)}</strong><span>Total tool cost</span></div><div><strong>${formatMs(report.latency)}</strong><span>Total latency</span></div><div><strong>${picked.steps.length}</strong><span>Tool steps</span></div><div><strong class="${report.policySafe?'green':'red'}">${report.flags.length}</strong><span>Audit findings</span></div></div><h2 class="section-gap">Step-by-step trace</h2><div class="steps section-gap">${picked.steps.map(s=>`<div class="step"><div class="number-circle">${s.seq}</div><div><h3>${safe(s.tool)}</h3><p>${safe(s.action)}</p><div class="tiny muted">${safe(s.risk)} risk · ${s.approved?'approved':'not approved'}</div></div><aside>${valueTag(s.status,s.status==='ok'?'':'red')}<p>${formatMoney(s.cost_usd)} · ${formatMs(s.latency_ms)}</p></aside></div>`).join('')}</div><h2 class="section-gap">Audit findings</h2>${report.flags.map(f=>`<div class="rule"><div><strong>${safe(f.type)}</strong><br><small>${safe(f.detail)}</small></div>${valueTag(f.severity,f.severity==='critical'?'red':'yellow')}</div>`).join('')||`<p class="tiny muted">No active findings for this run.</p>`}`:'<div class="empty">No trace selected.</div>'}</article></div>`}
|
| 81 |
+
function renderAudit(){const runs=current(),f=getFindings(runs,state.policy),critical=f.filter(x=>x.severity==='critical');return pageHead()+`<div class="metrics">${metric('Critical findings',critical.length,'Forbidden actions or missing approvals','red')}${metric('Cost cap breaches',countFlag(f,'Cost cap'),'Current threshold applied','yellow')}${metric('Step limit breaches',countFlag(f,'Step limit'),'Current threshold applied')}${metric('Tool execution errors',countFlag(f,'Tool error'),'Trace-reported tool failures','blue')}</div><div class="cols"><article class="panel"><div class="row"><div><h2>Active audit policy</h2><p class="panel-sub">Controls change the current audit. Raw traces remain unchanged.</p></div>${valueTag('Rules engine')}</div>${policyControls()}<div class="notice section-gap">A clean flag list does not prove an agent is safe. This tool checks a limited, explicit ruleset on imported or synthetic action logs.</div></article><article class="panel"><h2>Most common violations</h2><div class="panel-sub">${f.length} active findings</div><div class="bars">${flagBuckets(f).map(x=>`<div class="barline"><span>${safe(x.type)}</span><div class="track"><div class="fill ${x.type.includes('approval')||x.type.includes('Forbidden')?'w':''}" style="width:${f.length?pct(x.n,f.length):0}%"></div></div><strong>${x.n}</strong></div>`).join('')}</div><div class="rule"><div><strong>Approval gate</strong><br><small>update_record and publish_external</small></div>${valueTag(state.policy.requireApproval?'Enforced':'Disabled',state.policy.requireApproval?'':'yellow')}</div><div class="rule"><div><strong>Always forbidden</strong><br><small>secret_export</small></div>${valueTag('Enforced')}</div></article></div><article class="panel section-gap"><h2>Detailed finding register</h2><p class="panel-sub">Click a row to open the complete trace. ${f.length} findings detected.</p><div class="table-wrap" style="max-height:410px;overflow:auto"><table><thead><tr><th>Run</th><th>Variant</th><th>Category</th><th>Finding</th><th>Evidence</th></tr></thead><tbody>${f.slice(0,160).map(x=>`<tr class="pick" data-run="${safe(x.run_id)}"><td>${safe(x.run_id)}</td><td>${safe(x.variant)}</td><td>${valueTag(x.severity,x.severity==='critical'?'red':x.severity==='medium'?'yellow':'blue')}</td><td>${safe(x.type)}</td><td>${safe(x.detail)}</td></tr>`).join('')||'<tr><td colspan="5">No active findings</td></tr>'}</tbody></table></div></article>`}
|
| 82 |
+
function policyControls(){return `<div class="control-grid"><div class="control-box"><label for="cost">Maximum cost / run</label><strong class="green" id="costLabel">$${state.policy.maxCost.toFixed(2)}</strong><input id="cost" type="range" min="0.05" max="0.50" step="0.01" value="${state.policy.maxCost}"><small>Actual simulated sum of step costs</small></div><div class="control-box"><label for="steps">Maximum steps</label><strong class="green" id="stepsLabel">${state.policy.maxSteps} steps</strong><input id="steps" type="range" min="2" max="12" step="1" value="${state.policy.maxSteps}"><small>Long traces trigger a finding</small></div><div class="control-box"><label for="approval">Approval requirement</label><div class="row"><strong class="green" id="approvalLabel">${state.policy.requireApproval?'ON':'OFF'}</strong><input type="checkbox" id="approval" ${state.policy.requireApproval?'checked':''} aria-label="Enforce approvals"></div><small>Applies to state changes and publishing</small></div></div>`}
|
| 83 |
+
function renderBenchmark(){const metrics=groupMetrics(state.runs,state.policy),[A,B]=distinct(),p=pairedComparison(state.runs,A,B);const successRange=p.n?`${fmtDiff(p.low)} to ${fmtDiff(p.high)}`:'not available';return pageHead()+`<div class="metrics">${metric('Paired task count',p.n,'Tasks shared by both variants')}${metric('Success difference',p.n?fmtDiff(p.diff):'N/A',`${safe(B||'B')} minus ${safe(A||'A')}`,p.diff>=0?'green':'red')}${metric('Bootstrap 95% interval',p.n?successRange:'N/A','Paired, 2,000 resamples')}${metric('Discordant task pairs',p.positive+p.negative,`${p.positive} favor B · ${p.negative} favor A`,'yellow')}</div><div class="cols"><article class="panel"><h2>Variant scorecard</h2><p class="panel-sub">Comparable synthetic task set, with paired outcomes</p><div class="table-wrap"><table><thead><tr><th>Metric</th>${metrics.map(x=>`<th>${safe(x.name)}</th>`).join('')}</tr></thead><tbody>${[['Task success',x=>roundPct(x.success)],['Critical-policy pass',x=>roundPct(x.pass)],['Full policy clean',x=>roundPct(x.compliant)],['Average cost',x=>formatMoney(x.avgCost)],['Average latency',x=>formatMs(x.avgLatency)],['Average steps',x=>num(x.avgSteps)],['Critical findings',x=>x.critical]].map(row=>`<tr><td>${row[0]}</td>${metrics.map(x=>`<td><strong>${row[1](x)}</strong></td>`).join('')}</tr>`).join('')}</tbody></table></div><div class="insight section-gap"><strong>How to read the interval</strong><p>${p.n?(p.low>0?'The paired bootstrap interval is entirely positive in this demonstration.':p.high<0?'The paired bootstrap interval is entirely negative in this demonstration.':'The bootstrap interval crosses zero. The available paired data do not establish a clear advantage.'): 'Two overlapping variants are required.'} These are exploratory intervals, not evidence from a live randomized experiment.</p></div></article><article class="panel"><h2>Success vs. policy compliance</h2><p class="panel-sub">The two goals are not interchangeable</p>${chartComparison(metrics)}<div class="rule"><div><strong>Paired analysis</strong><br><small>Match exactly on task_id</small></div>${valueTag(`${p.n} tasks`,'blue')}</div><div class="rule"><div><strong>Inference method</strong><br><small>Non-parametric paired bootstrap</small></div>${valueTag('95% CI','blue')}</div><div class="notice section-gap">Performance figures are computed from execution logs. They do not validate the truthfulness of model responses or establish causal superiority.</div></article></div>`}
|
| 84 |
+
function renderScenario(){const base={maxCost:.13,maxSteps:4,requireApproval:true},before=state.runs.map(x=>auditRun(x,base)),after=state.runs.map(x=>auditRun(x,state.policy));const safePct=a=>pct(a.filter(x=>x.policySafe).length,a.length);const cleanPct=a=>pct(a.filter(x=>x.clean).length,a.length);return pageHead()+`<div class="cols"><article class="panel"><h2>Policy sandbox</h2><p class="panel-sub">Compare your thresholds to an unchanged baseline of $0.13 / 4 steps / approval gate enabled.</p>${policyControls()}<div class="cta"><button class="small-btn" data-resetpolicy="1">↺ Reset baseline</button><button class="primary-btn" data-export="scenario">↓ Export comparison</button></div><div class="notice section-gap">This is a static re-audit, not a retraining run. Changing thresholds does not change success outcomes or agent behavior.</div></article><article class="panel"><h2>Scenario impact</h2><p class="panel-sub">Changes in flagged runs under two rule settings</p><div class="stat-grid"><div><strong>${roundPct(cleanPct(before))}</strong><span>Baseline clean executions</span></div><div><strong class="green">${roundPct(cleanPct(after))}</strong><span>Scenario clean executions</span></div><div><strong>${roundPct(safePct(before))}</strong><span>Baseline critical-policy pass</span></div><div><strong>${roundPct(safePct(after))}</strong><span>Scenario critical-policy pass</span></div></div><div class="section-gap"><h2>Findings comparison</h2><div class="bars">${['Cost cap','Step limit','Missing approval'].map(t=>{let beforeN=before.flatMap(x=>x.flags).filter(x=>x.type===t).length,afterN=after.flatMap(x=>x.flags).filter(x=>x.type===t).length;return `<div class="barline"><span>${t}</span><div class="track"><div class="fill" style="width:${pct(afterN,Math.max(beforeN,afterN,1))}%"></div></div><strong>${afterN}</strong></div>`}).join('')}</div></div></article></div>`}
|
| 85 |
+
function renderData(){const r=state.runs;const recordCount=r.reduce((a,x)=>a+x.steps.length,0);return pageHead()+`<div class="metrics">${metric('Loaded runs',r.length,'Unique run_id values')}${metric('Tool actions',recordCount,'Complete step-level observations')}${metric('Task IDs',new Set(r.map(x=>x.task_id)).size,'Distinct task keys')}${metric('Variants',distinct().length,'Systems available for comparison')}</div><div class="cols"><article class="panel"><h2>Bring your own execution traces</h2><p class="panel-sub">Upload CSV with a row per tool action; data stay inside your browser tab.</p><div class="upload"><input id="csvinput" type="file" accept=".csv,text/csv" aria-label="Upload CSV file"><strong>↑ Drop or choose a CSV trace file</strong><small>Max 3 MB · 10,000 lines · validated before replacement</small></div><div class="cta"><button class="small-btn" data-export="template">↓ Download CSV template</button><button class="small-btn" data-export="raw">↓ Export current traces</button></div><div class="rule section-gap"><div><strong>Dataset mode</strong><br><small>Current source of the metrics</small></div>${valueTag(state.demo==='Imported'?'Imported local CSV':'Synthetic demo','blue')}</div><div class="notice">Sensitive information should be anonymized before import. Uploaded traces remain in this browser session and are not automatically persisted after a reload.</div></article><article class="panel"><h2>Reproducible deliverables</h2><p class="panel-sub">Download a snapshot of the current data, settings and findings.</p><div class="files"><button data-export="json"><span>▧</span>Audit report · JSON</button><button data-export="audit"><span>≣</span>Policy findings · CSV</button><button data-export="raw"><span>≣</span>Trace log · CSV</button><button data-export="markdown"><span>▤</span>Research memo · Markdown</button><button data-export="benchmark"><span>▦</span>Variant scorecard · CSV</button><button data-export="scenario"><span>◈</span>Scenario compare · JSON</button></div><div class="rule section-gap"><div><strong>External services</strong><br><small>No API tokens or network connections</small></div>${valueTag('None','blue')}</div></article></div><article class="panel section-gap"><h2>Methods & limitations</h2><div class="panel-sub">Auditable, deterministic outputs with explicit assumptions</div><div class="cols equal"><div class="method"><p><strong>Trace model.</strong> Each run has a task ID, variant, binary task outcome, and ordered tool steps. Each step stores tool, latency, cost, status, risk and approval flag. Example datasets are generated from a fixed seed.</p><p><strong>Policy definitions.</strong> The critical audit flags are use of a forbidden tool and missing approval for privileged actions. Cost, step budgets and tool errors are tracked separately. A pass means only that the observed fields satisfy the configured checks.</p></div><div class="method"><p><strong>Paired benchmark.</strong> Both variants must share task_id. The difference in binary success is summarized with 2,000 paired bootstrap resamples using a fixed seed. No confidence adjustment for repeated exploration is applied.</p><p><strong>Important limits.</strong> This is an offline trace auditor, not an autonomous agent or a guarantee of safety, privacy, fairness or factual accuracy. Costs and latencies in examples are synthetic. The import format is intentionally constrained and is not a complete OpenTelemetry format.</p></div></div></article>`}
|
| 86 |
+
function render(){view.innerHTML=({overview:renderOverview,trace:renderTrace,audit:renderAudit,benchmark:renderBenchmark,scenario:renderScenario,data:renderData}[state.page])();document.querySelectorAll('#nav button').forEach(b=>b.classList.toggle('active',b.dataset.page===state.page));}
|
| 87 |
+
function toast(t){let e=document.querySelector('#toast');e.textContent=t;e.hidden=false;clearTimeout(toast.timer);toast.timer=setTimeout(()=>e.hidden=true,5000)}
|
| 88 |
+
function closeNav(){sidebar.classList.remove('open');document.querySelector('#scrim').classList.remove('open')}
|
| 89 |
+
function navigate(page){if(!(page in names))return;state.page=page;closeNav();render();window.scrollTo({top:0,behavior:'instant'})}
|
| 90 |
+
function report(){const runs=state.runs;const metrics=groupMetrics(runs,state.policy);const vs=distinct();return {application:'AgentTrace',version:'1.0',project:50,mode:state.demo,synthetic:state.demo!=='Imported',policy:{...state.policy},run_count:runs.length,step_count:runs.reduce((n,r)=>n+r.steps.length,0),variant_metrics:metrics,paired_comparison:vs.length>=2?pairedComparison(runs,vs[0],vs[1]):null,findings:getFindings(runs,state.policy),limitations:['Checks only logged actions and configured rules','Demo traces are synthetic','Bootstrap results are exploratory','Pass is not proof of agent safety']};}
|
| 91 |
+
function saveDownload(name,content,type='text/plain;charset=utf-8'){const blob=new Blob([content],{type}),url=URL.createObjectURL(blob),a=document.createElement('a');a.href=url;a.download=name;document.body.appendChild(a);a.click();a.remove();setTimeout(()=>URL.revokeObjectURL(url),800);}
|
| 92 |
+
function doExport(mode){let r=report();if(mode==='template'){const sample=flattenRuns(makeDemo('Standard')).slice(0,5);saveDownload('agenttrace_template.csv',rowsToCsv(sample,CSV_FIELDS),'text/csv;charset=utf-8');return;}if(mode==='json'){saveDownload('agenttrace_audit.json',JSON.stringify(r,null,2),'application/json');return;}if(mode==='raw'){saveDownload('agenttrace_actions.csv',rowsToCsv(flattenRuns(state.runs),CSV_FIELDS),'text/csv;charset=utf-8');return;}if(mode==='audit'){saveDownload('agenttrace_findings.csv',rowsToCsv(r.findings,['run_id','task_id','variant','type','severity','detail']),'text/csv;charset=utf-8');return;}if(mode==='benchmark'){saveDownload('agenttrace_benchmark.csv',rowsToCsv(r.variant_metrics,['name','n','success','pass','compliant','avgCost','avgLatency','avgSteps','critical']),'text/csv;charset=utf-8');return;}if(mode==='scenario'){let base={maxCost:.13,maxSteps:4,requireApproval:true},count=p=>state.runs.reduce((a,x)=>a+auditRun(x,p).flags.length,0);saveDownload('agenttrace_scenario.json',JSON.stringify({baseline:{policy:base,findings:count(base)},scenario:{policy:state.policy,findings:count(state.policy)}},null,2),'application/json');return;}if(mode==='markdown'){const lines=['# AgentTrace Audit Report','','- Source: '+(r.synthetic?'Synthetic demonstration':'Locally imported CSV'),'- Executions: '+r.run_count,'- Tool steps: '+r.step_count,'','## Results',...r.variant_metrics.map(x=>`- ${x.name}: ${x.success.toFixed(1)}% task success, ${x.pass.toFixed(1)}% critical-policy pass, $${x.avgCost.toFixed(3)} mean cost`),'','## Policy findings','- Total: '+r.findings.length,'','## Methodological cautions','- Rule compliance is not safety certification.','- All demonstrations are synthetic.','- Paired bootstrap intervals are exploratory.'];saveDownload('agenttrace_research_memo.md',lines.join('\n'),'text/markdown;charset=utf-8');return;}throw Error('Unsupported export');}
|
| 93 |
+
document.querySelector('#nav').addEventListener('click',e=>{let b=e.target.closest('button[data-page]');if(b)navigate(b.dataset.page)});
|
| 94 |
+
document.querySelector('#menu').addEventListener('click',()=>{sidebar.classList.toggle('open');document.querySelector('#scrim').classList.toggle('open')});document.querySelector('#scrim').addEventListener('click',closeNav);
|
| 95 |
+
document.addEventListener('click',e=>{let b=e.target.closest('button');if(!b){let tr=e.target.closest('tr[data-run]');if(tr){state.selected=tr.dataset.run;state.variant='All';navigate('trace')}return;}try{if(b.dataset.goto)navigate(b.dataset.goto);else if(b.dataset.export)doExport(b.dataset.export);else if(b.dataset.resetpolicy){state.policy={...DEFAULT_POLICY};render();}else if(b.dataset.run){state.selected=b.dataset.run;state.variant='All';navigate('trace');}}catch(err){toast('Error: '+err.message)}let tr=e.target.closest('tr[data-run]');if(tr){state.selected=tr.dataset.run;state.variant='All';navigate('trace')}});
|
| 96 |
+
document.addEventListener('change',async e=>{const t=e.target;try{if(t.id==='demo'&&t.value!=='Imported'){state.demo=t.value;state.runs=validateData(makeDemo(t.value));state.variant='All';state.selected=null;render();toast('Dataset changed: '+state.demo)}else if(t.id==='variant'){state.variant=t.value;state.selected=null;render()}else if(t.id==='cost'){state.policy.maxCost=+t.value;render()}else if(t.id==='steps'){state.policy.maxSteps=+t.value;render()}else if(t.id==='approval'){state.policy.requireApproval=t.checked;render()}else if(t.id==='csvinput'&&t.files.length){const f=t.files[0];if(f.size>3_000_000)throw Error('CSV file is too large (3 MB max).');const input=await f.text();const parsed=csvToRuns(input);state.runs=parsed;state.demo='Imported';state.variant='All';state.selected=null;render();toast('Successfully imported '+parsed.length+' validated runs.')}}catch(err){toast('Import rejected: '+err.message);if(t.id==='csvinput')t.value=''}});
|
| 97 |
+
// Changes to threshold labels are cosmetic until the range change is committed.
|
| 98 |
+
document.addEventListener('input',e=>{if(e.target.id==='cost'){let x=document.querySelector('#costLabel');if(x)x.textContent='$'+Number(e.target.value).toFixed(2)}else if(e.target.id==='steps'){let x=document.querySelector('#stepsLabel');if(x)x.textContent=e.target.value+' steps'}});
|
| 99 |
+
render();
|
| 100 |
+
|
| 101 |
+
</script></body></html>
|