Add files using upload-large-folder tool
afa0cbf verified about 2 months ago
40.4 kB
use anyhow::Result;
use app_test_support::ChatGptAuthFixture;
use app_test_support::DEFAULT_CLIENT_NAME;
use app_test_support::MockResponsesConfig;
use app_test_support::TestAppServer;
use app_test_support::create_final_assistant_message_sse_response;
use app_test_support::create_mock_responses_server_sequence;
use app_test_support::to_response;
use app_test_support::write_chatgpt_auth;
use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url;
use codex_app_server_protocol::ApprovalsReviewer;
use codex_app_server_protocol::AskForApproval;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SandboxPolicy;
use codex_app_server_protocol::ServerRequest;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadReadParams;
use codex_app_server_protocol::ThreadReadResponse;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::ThreadStartResponse;
use codex_app_server_protocol::TurnSettingsUpdateParams;
use codex_app_server_protocol::TurnSettingsUpdateResponse;
use codex_app_server_protocol::TurnSettingsUpdateStatus;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
use codex_app_server_protocol::UserInput;
use codex_config::types::AuthCredentialsStoreMode;
use codex_config::types::OtelExporterKind;
use codex_config::types::OtelHttpProtocol;
use codex_core::config::ConfigBuilder;
use codex_core_plugins::loader::curated_plugin_cache_version;
use codex_core_plugins::store::PluginStore;
use codex_features::Feature;
use codex_models_manager::bundled_models_response;
use codex_plugin::PluginId;
use codex_protocol::openai_models::ModelsResponse;
use codex_protocol::openai_models::ReasoningEffort;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
use core_test_support::skip_if_remote;
use core_test_support::skip_if_wine_exec;
use pretty_assertions::assert_eq;
use serde_json::Value;
use serde_json::json;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::time::Duration;
use tempfile::TempDir;
use tokio::time::timeout;
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
use wiremock::matchers::method;
use wiremock::matchers::path;
use wiremock::matchers::path_regex;

const SERVICE_VERSION: &str = "0.0.0-test";

#[tokio::test]
async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
    skip_if_no_network!(Ok(()));
    const PRIVATE: &str = "guardian-private-content";
    const READ_TIMEOUT: Duration = Duration::from_secs(60);
    let server = responses::start_mock_server().await;
    let codex_home = TempDir::new()?;
    mount_analytics_capture(&server, codex_home.path()).await?;
    MockResponsesConfig::new(&server.uri())
        .with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri()))
        .with_provider_config("supports_websockets = false")
        .enable_feature(Feature::GuardianApproval)
        .disable_feature(Feature::Apps)
        .write(codex_home.path())?;
    let parent_tool = |id: &str| {
        responses::sse_response(responses::sse(vec![
            responses::ev_response_created(id),
            responses::ev_function_call(
                id,
                "exec_command",
                &json!({
                    "cmd": format!("echo {id} > {id}.txt"),
                    "sandbox_permissions": "require_escalated",
                    "justification": "Exercise Guardian analytics",
                })
                .to_string(),
            ),
            responses::ev_completed(id),
        ]))
    };
    let requests = responses::mount_response_sequence(
        &server,
        vec![
            parent_tool("parent-first"),
            responses::sse_response(responses::sse(vec![
                responses::ev_response_created("guardian-first"),
                responses::ev_web_search_call_added_partial("guardian-search", "in_progress"),
                responses::ev_web_search_call_done("guardian-search", "completed", PRIVATE),
                responses::ev_reasoning_item("guardian-reasoning", &[PRIVATE], &[PRIVATE]),
                responses::ev_assistant_message(
                    "guardian-answer",
                    &json!({"outcome": "deny", "rationale": PRIVATE}).to_string(),
                ),
                responses::ev_completed("guardian-first"),
            ])),
            parent_tool("parent-second"),
            responses::sse_response(responses::sse_failed(
                "guardian-failed",
                "invalid_prompt",
                PRIVATE,
            )),
            parent_tool("parent-third"),
            responses::sse_response(responses::sse(vec![responses::ev_response_created(
                "guardian-pending",
            )]))
            .set_delay(READ_TIMEOUT),
        ],
    )
    .await;
    let mut app_server = TestAppServer::builder()
        .with_codex_home(codex_home.path())
        .without_managed_config()
        // Basic Guardian command approvals require host-native paths.
        .without_auto_env()
        .build_initialized_with_timeout(READ_TIMEOUT)
        .await?;
    let ThreadStartResponse { thread, .. } = app_server
        .request(|request_id| ClientRequest::ThreadStart {
            request_id,
            params: ThreadStartParams {
                approval_policy: Some(AskForApproval::OnRequest),
                approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
                ..Default::default()
            },
        })
        .await?;
    let request_id = app_server
        .send_turn_start_request(TurnStartParams {
            thread_id: thread.id.clone(),
            input: vec![UserInput::Text {
                text: "Review three commands".to_string(),
                text_elements: Vec::new(),
            }],
            effort: Some(ReasoningEffort::High),
            ..Default::default()
        })
        .await?;
    let TurnStartResponse { turn } =
        timeout(READ_TIMEOUT, app_server.read_response(request_id)).await??;
    let reviews = timeout(READ_TIMEOUT, async {
        loop {
            let reviews = requests
                .requests()
                .into_iter()
                .map(|request| request.body_json()["client_metadata"].clone())
                .filter(|metadata| metadata["x-openai-subagent"] == "guardian")
                .collect::<Vec<_>>();
            if reviews.len() == 3 {
                break reviews;
            }
            tokio::time::sleep(Duration::from_millis(25)).await;
        }
    })
    .await?;
    app_server
        .interrupt_turn_and_wait_for_aborted(thread.id.clone(), turn.id, READ_TIMEOUT)
        .await?;
    wait_for_matching_analytics_event(&server, READ_TIMEOUT, |event| {
        event["event_type"] == "codex_turn_event"
            && event["event_params"]["turn_id"] == reviews[2]["turn_id"]
    })
    .await?;
    timeout(READ_TIMEOUT, app_server.shutdown_gracefully()).await??;
    let events = captured_analytics_events(&server).await;
    assert!(!serde_json::to_string(&events)?.contains(PRIVATE));
    let parent_command = events
        .iter()
        .find(|event| {
            event["event_type"] == "codex_command_execution_event"
                && event["event_params"]["thread_id"] == thread.id
                && event["event_params"]["item_id"] == "parent-first"
        })
        .expect("denied parent command analytics");
    let params = &parent_command["event_params"];
    assert_eq!(
        json!([params["model_slug"], params["reasoning_effort"]]),
        json!(["mock-model", "high"])
    );
    let children = events
        .iter()
        .filter(|event| event["event_params"]["subagent_source"] == "guardian")
        .collect::<Vec<_>>();
    for event in &children {
        let params = &event["event_params"];
        assert_eq!(
            json!([
                params["thread_id"],
                params["session_id"],
                params["parent_thread_id"],
                params["thread_source"]
            ]),
            json!([
                reviews[0]["thread_id"],
                thread.session_id,
                thread.id,
                "guardian_review"
            ])
        );
    }
    let turns = children
        .iter()
        .filter(|event| event["event_type"] == "codex_turn_event")
        .map(|event| {
            let params = &event["event_params"];
            let started_at = params["started_at"].as_u64().expect("turn start time");
            let completed_at = params["completed_at"]
                .as_u64()
                .expect("turn completion time");
            assert!(started_at > 0 && completed_at >= started_at);
            assert!(params["duration_ms"].is_u64());
            assert_eq!(params["guardian_v2_enabled"], false);
            json!([
                params["turn_id"],
                params["status"],
                params["turn_error"],
                params["total_tool_call_count"],
                params["web_search_count"]
            ])
        })
        .collect::<Vec<_>>();
    assert_eq!(
        turns,
        vec![
            json!([reviews[0]["turn_id"], "completed", null, 1, 1]),
            json!([reviews[1]["turn_id"], "failed", "other", 0, 0]),
            json!([reviews[2]["turn_id"], "interrupted", null, 0, 0]),
        ]
    );
    let tools = children
        .iter()
        .filter(|event| event["event_type"] == "codex_web_search_event")
        .collect::<Vec<_>>();
    let [tool] = tools.as_slice() else {
        anyhow::bail!(
            "expected one Guardian web search event, got {}",
            tools.len()
        );
    };
    let params = &tool["event_params"];
    let started_at = params["started_at_ms"].as_u64().expect("tool start time");
    let completed_at = params["completed_at_ms"]
        .as_u64()
        .expect("tool completion time");
    assert!(started_at > 0 && completed_at >= started_at);
    assert_eq!(params["duration_ms"], completed_at - started_at);
    assert_eq!(
        json!([
            params["turn_id"],
            params["item_id"],
            params["terminal_status"]
        ]),
        json!([reviews[0]["turn_id"], "guardian-search", "completed"])
    );
    Ok(())
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn multi_agent_v2_tools_emit_collaborator_analytics() -> Result<()> {
    const PARENT_PROMPT: &str = "exercise collaborator analytics";
    const READ_TIMEOUT: Duration = Duration::from_secs(30);
    let calls = [
        (
            "spawn_agent",
            json!({"task_name": "worker", "message": "PRIVATE_CHILD", "fork_turns": "none"}),
        ),
        (
            "send_message",
            json!({"target": "worker", "message": "PRIVATE_MESSAGE"}),
        ),
        (
            "followup_task",
            json!({"target": "worker", "message": "PRIVATE_FOLLOWUP"}),
        ),
        ("interrupt_agent", json!({"target": "worker"})),
        ("list_agents", json!({})),
        (
            "send_message",
            json!({"target": "missing", "message": "PRIVATE_MESSAGE"}),
        ),
        (
            "followup_task",
            json!({"target": "/root", "message": "PRIVATE_FOLLOWUP"}),
        ),
        ("interrupt_agent", json!({"target": "/root"})),
        ("list_agents", json!({"unexpected": true})),
        ("send_message", json!({})),
    ];
    let server = responses::start_mock_server().await;
    for (index, (tool, args)) in calls.iter().enumerate() {
        let response_id = format!("parent-response-{index}");
        responses::mount_sse_once_match(
            &server,
            |request: &wiremock::Request| {
                String::from_utf8_lossy(&request.body).contains(PARENT_PROMPT)
            },
            responses::sse(vec![
                responses::ev_response_created(&response_id),
                responses::ev_function_call_with_namespace(
                    &format!("call-{index}"),
                    "collaboration",
                    tool,
                    &args.to_string(),
                ),
                responses::ev_completed(&response_id),
            ]),
        )
        .await;
    }
    // Child turns and the final parent request only need an ordinary completion.
    Mock::given(method("POST"))
        .and(path_regex(".*/responses$"))
        .respond_with(
            ResponseTemplate::new(200)
                .insert_header("content-type", "text/event-stream")
                .set_body_string(responses::sse(vec![responses::ev_completed(
                    "response-done",
                )])),
        )
        .with_priority(10)
        .mount(&server)
        .await;

    let codex_home = TempDir::new()?;
    MockResponsesConfig::new(&server.uri())
        .enable_feature(Feature::MultiAgentV2)
        .with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri()))
        .write(codex_home.path())?;
    mount_analytics_capture(&server, codex_home.path()).await?;
    let mut app_server = TestAppServer::builder()
        .with_codex_home(codex_home.path())
        .without_managed_config()
        .build_initialized()
        .await?;
    let thread = app_server
        .start_thread(ThreadStartParams::default())
        .await?
        .thread;
    let completed = app_server
        .start_turn_and_wait_for_completion(TurnStartParams {
            thread_id: thread.id.clone(),
            input: vec![UserInput::Text {
                text: PARENT_PROMPT.to_string(),
                text_elements: Vec::new(),
            }],
            ..Default::default()
        })
        .await?;
    let read: ThreadReadResponse = app_server
        .request(|request_id| ClientRequest::ThreadRead {
            request_id,
            params: ThreadReadParams {
                thread_id: thread.id.clone(),
                include_turns: true,
            },
        })
        .await?;
    let items = &read
        .thread
        .turns
        .iter()
        .find(|turn| turn.id == completed.turn.id)
        .expect("completed parent turn should be readable")
        .items;
    let child_thread_id = items
        .iter()
        .find_map(|item| match item {
            ThreadItem::SubAgentActivity {
                id,
                agent_thread_id,
                ..
            } if id == "call-0" => Some(agent_thread_id.clone()),
            _ => None,
        })
        .expect("spawn should retain its public activity item");
    assert!(
        !items
            .iter()
            .any(|item| matches!(item, ThreadItem::CollabAgentToolCall { .. }))
    );
    let activity_ids = items
        .iter()
        .filter_map(|item| match item {
            ThreadItem::SubAgentActivity { id, .. } if id.starts_with("call-") => Some(id.as_str()),
            _ => None,
        })
        .collect::<Vec<_>>();
    assert_eq!(activity_ids, vec!["call-0", "call-1", "call-2", "call-3"]);

    for (index, (tool, _)) in calls.iter().enumerate() {
        let event = wait_for_matching_analytics_event(&server, READ_TIMEOUT, |event| {
            event["event_type"] == "codex_collab_agent_tool_call_event"
                && event["event_params"]["item_id"] == format!("call-{index}")
        })
        .await?;
        let params = &event["event_params"];
        let receivers = match index {
            0..=3 => vec![child_thread_id.clone()],
            6 | 7 => vec![thread.id.clone()],
            _ => Vec::new(),
        };
        assert_eq!(
            json!({
                "tool": params["tool_name"],
                "sender": params["sender_thread_id"],
                "receivers": params["receiver_thread_ids"],
                "receiverCount": params["receiver_thread_count"],
                "turn": params["turn_id"],
                "status": params["terminal_status"],
                "origin": params["originating_response_id"],
                "failure": params["failure_kind"],
                "hasDuration": params["execution_duration_ms"].is_u64(),
            }),
            json!({
                "tool": tool,
                "sender": thread.id,
                "receivers": receivers,
                "receiverCount": receivers.len(),
                "turn": completed.turn.id,
                "status": if index < 5 { "completed" } else { "failed" },
                "origin": format!("parent-response-{index}"),
                "failure": if index < 5 { None } else { Some("tool_error") },
                "hasDuration": true,
            })
        );
        assert_eq!(params["duration_ms"], params["execution_duration_ms"]);
        assert!(!event.to_string().contains("PRIVATE_"));
    }
    let turn_event = wait_for_matching_analytics_event(&server, READ_TIMEOUT, |event| {
        event["event_type"] == "codex_turn_event"
            && event["event_params"]["turn_id"] == completed.turn.id
    })
    .await?;
    assert_eq!(
        json!({
            "total": turn_event["event_params"]["total_tool_call_count"],
            "subagent": turn_event["event_params"]["subagent_tool_call_count"],
            "dynamic": turn_event["event_params"]["dynamic_tool_call_count"],
        }),
        json!({"total": calls.len(), "subagent": calls.len(), "dynamic": 0})
    );

    Ok(())
}

fn set_metrics_exporter(config: &mut codex_core::config::Config) {
    config.otel.metrics_exporter = OtelExporterKind::OtlpHttp {
        endpoint: "http://localhost:4318".to_string(),
        headers: HashMap::new(),
        protocol: OtelHttpProtocol::Json,
        tls: None,
    };
}

#[tokio::test]
async fn app_server_default_analytics_disabled_without_flag() -> Result<()> {
    let codex_home = TempDir::new()?;
    let mut config = ConfigBuilder::default()
        .codex_home(codex_home.path().to_path_buf())
        .build()
        .await?;
    set_metrics_exporter(&mut config);
    config.analytics_enabled = None;

    let provider = codex_core::otel_init::build_provider(
        &config,
        SERVICE_VERSION,
        Some("codex-app-server"),
        /*default_analytics_enabled*/ false,
    )
    .map_err(|err| anyhow::anyhow!(err.to_string()))?;

    // With analytics unset in the config and the default flag is false, metrics are disabled.
    // A provider may still exist for non-metrics telemetry, so check metrics specifically.
    let has_metrics = provider.as_ref().and_then(|otel| otel.metrics()).is_some();
    assert_eq!(has_metrics, false);
    Ok(())
}

#[tokio::test]
async fn app_server_default_analytics_enabled_with_flag() -> Result<()> {
    let codex_home = TempDir::new()?;
    let mut config = ConfigBuilder::default()
        .codex_home(codex_home.path().to_path_buf())
        .build()
        .await?;
    set_metrics_exporter(&mut config);
    config.analytics_enabled = None;

    let provider = codex_core::otel_init::build_provider(
        &config,
        SERVICE_VERSION,
        Some("codex-app-server"),
        /*default_analytics_enabled*/ true,
    )
    .map_err(|err| anyhow::anyhow!(err.to_string()))?;

    // With analytics unset in the config and the default flag is true, metrics are enabled.
    let has_metrics = provider.as_ref().and_then(|otel| otel.metrics()).is_some();
    assert_eq!(has_metrics, true);
    Ok(())
}

pub(crate) async fn mount_analytics_capture(server: &MockServer, codex_home: &Path) -> Result<()> {
    Mock::given(method("POST"))
        .and(path("/codex/analytics-events/events"))
        .respond_with(ResponseTemplate::new(200))
        .mount(server)
        .await;

    write_chatgpt_auth(
        codex_home,
        ChatGptAuthFixture::new("chatgpt-token")
            .account_id("account-123")
            .chatgpt_user_id("user-123")
            .chatgpt_account_id("account-123"),
        AuthCredentialsStoreMode::File,
    )?;

    Ok(())
}

pub(crate) async fn captured_analytics_events(server: &MockServer) -> Vec<Value> {
    server
        .received_requests()
        .await
        .unwrap_or_default()
        .into_iter()
        .filter(|request| request.url.path() == "/codex/analytics-events/events")
        .flat_map(|request| {
            let payload: Value = serde_json::from_slice(&request.body).expect("analytics payload");
            payload["events"]
                .as_array()
                .expect("analytics events")
                .clone()
        })
        .collect()
}

pub(crate) async fn wait_for_analytics_payload(
    server: &MockServer,
    read_timeout: Duration,
) -> Result<Value> {
    let body = timeout(read_timeout, async {
        loop {
            let Some(requests) = server.received_requests().await else {
                tokio::time::sleep(Duration::from_millis(25)).await;
                continue;
            };
            if let Some(request) = requests.iter().find(|request| {
                request.method == "POST" && request.url.path() == "/codex/analytics-events/events"
            }) {
                break request.body.clone();
            }
            tokio::time::sleep(Duration::from_millis(25)).await;
        }
    })
    .await?;
    serde_json::from_slice(&body).map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))
}

pub(crate) async fn wait_for_analytics_event(
    server: &MockServer,
    read_timeout: Duration,
    event_type: &str,
) -> Result<Value> {
    wait_for_matching_analytics_event(server, read_timeout, |event| {
        event["event_type"] == event_type
    })
    .await
}

pub(crate) async fn wait_for_goal_event(
    server: &MockServer,
    read_timeout: Duration,
    event_kind: &str,
    goal_status: &str,
) -> Result<Value> {
    wait_for_matching_analytics_event(server, read_timeout, |event| {
        event["event_type"] == "codex_goal_event"
            && event["event_params"]["event_kind"] == event_kind
            && event["event_params"]["goal_status"] == goal_status
    })
    .await
}

pub(crate) async fn wait_for_matching_analytics_event(
    server: &MockServer,
    read_timeout: Duration,
    matches: impl Fn(&Value) -> bool,
) -> Result<Value> {
    timeout(read_timeout, async {
        loop {
            let Some(requests) = server.received_requests().await else {
                tokio::time::sleep(Duration::from_millis(25)).await;
                continue;
            };
            for request in &requests {
                if request.method != "POST"
                    || request.url.path() != "/codex/analytics-events/events"
                {
                    continue;
                }
                let payload: Value = serde_json::from_slice(&request.body)
                    .map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))?;
                let Some(events) = payload["events"].as_array() else {
                    continue;
                };
                if let Some(event) = events.iter().find(|event| matches(event)) {
                    return Ok::<Value, anyhow::Error>(event.clone());
                }
            }
            tokio::time::sleep(Duration::from_millis(25)).await;
        }
    })
    .await?
}

pub(crate) fn thread_initialized_event(payload: &Value) -> Result<&Value> {
    let events = payload["events"]
        .as_array()
        .ok_or_else(|| anyhow::anyhow!("analytics payload missing events array"))?;
    events
        .iter()
        .find(|event| event["event_type"] == "codex_thread_initialized")
        .ok_or_else(|| anyhow::anyhow!("codex_thread_initialized event should be present"))
}

pub(crate) fn assert_basic_thread_initialized_event(
    event: &Value,
    thread_id: &str,
    session_id: &str,
    expected_product_client_id: &str,
    expected_model: &str,
    initialization_mode: &str,
    expected_thread_source: &str,
) {
    assert_eq!(event["event_params"]["thread_id"], thread_id);
    assert_eq!(event["event_params"]["session_id"], session_id);
    assert_eq!(
        event["event_params"]["app_server_client"]["product_client_id"],
        expected_product_client_id
    );
    assert_eq!(
        event["event_params"]["app_server_client"]["client_name"],
        DEFAULT_CLIENT_NAME
    );
    assert_eq!(
        event["event_params"]["app_server_client"]["rpc_transport"],
        "stdio"
    );
    assert_eq!(event["event_params"]["model"], expected_model);
    assert_eq!(event["event_params"]["ephemeral"], false);
    assert_eq!(
        event["event_params"]["thread_source"],
        expected_thread_source
    );
    assert_eq!(
        event["event_params"]["subagent_source"],
        serde_json::Value::Null
    );
    assert_eq!(
        event["event_params"]["parent_thread_id"],
        serde_json::Value::Null
    );
    assert_eq!(
        event["event_params"]["initialization_mode"],
        initialization_mode
    );
    assert!(event["event_params"]["created_at"].as_u64().is_some());
}

const METRICS_PLUGIN_ID: &str = "sample@openai-curated";
const TEST_CURATED_PLUGIN_SHA: &str = "0123456789abcdef0123456789abcdef01234567";

fn write_curated_metrics_plugin(codex_home: &Path) -> Result<PathBuf> {
    let plugin_id = PluginId::parse(METRICS_PLUGIN_ID)?;
    let plugin_root = PluginStore::new(codex_home.to_path_buf()).plugin_root(
        &plugin_id,
        &curated_plugin_cache_version(TEST_CURATED_PLUGIN_SHA),
    );
    let script_path = plugin_root.join("scripts/run.sh");
    std::fs::create_dir_all(plugin_root.join(".codex-plugin"))?;
    std::fs::create_dir_all(script_path.parent().expect("script path has parent"))?;
    std::fs::write(
        plugin_root.join(".codex-plugin/plugin.json"),
        r#"{"name":"sample","version":"0.1.0"}"#,
    )?;
    std::fs::write(
        plugin_root.join("analytics.yaml"),
        r#"version: 1
operations:
  scan:
    path: ./scripts/run.sh
    measurements:
      findings:
        dimensions:
          severity: [high, low]
      files_scanned: {}
"#,
    )?;
    std::fs::write(
        &script_path,
        r#"test -n "$CODEX_PLUGIN_METRICS_OUTPUT"
sleep "${1:-0.3}"
while [ -n "${2:-}" ] && [ ! -f "$2" ]; do sleep 0.01; done
printf '%s' '{"version":1,"measurements":[{"name":"findings","value":3,"dimensions":{"severity":"high"}},{"name":"files_scanned","value":17}]}' > "$CODEX_PLUGIN_METRICS_OUTPUT"
"#,
    )?;

    let curated_repo = codex_home.join(".tmp/plugins");
    std::fs::create_dir_all(curated_repo.join(".agents/plugins"))?;
    std::fs::write(
        curated_repo.join(".agents/plugins/marketplace.json"),
        r#"{
  "name": "openai-curated",
  "plugins": [
    {"name": "sample", "source": {"source": "local", "path": "./plugins/sample"}}
  ]
}"#,
    )?;
    std::fs::write(
        codex_home.join(".tmp/plugins.sha"),
        format!("{TEST_CURATED_PLUGIN_SHA}\n"),
    )?;
    Ok(script_path.into_path_buf())
}

async fn assert_plugin_measurement_analytics(remote: bool, background: bool) -> Result<()> {
    skip_if_no_network!(Ok(()));
    skip_if_remote!(
        Ok(()),
        "trusted plugin metrics fixture uses a local Codex home cache"
    );
    skip_if_wine_exec!(Ok(()), "plugin metrics fixture is Unix-only");

    let codex_home = TempDir::new()?;
    let script_path = write_curated_metrics_plugin(codex_home.path())?.canonicalize()?;
    let release_path = codex_home.path().join("release-command");
    let mut command = vec![
        "/bin/sh".to_string(),
        script_path.to_string_lossy().into_owned(),
    ];
    if background {
        command.push("1.0".to_string());
        command.push(release_path.to_string_lossy().into_owned());
    }
    let call_id = "curated-plugin-metrics";
    let arguments = serde_json::to_string(&json!({
        "cmd": shlex::try_join(command.iter().map(String::as_str))?,
        "yield_time_ms": if background { 10 } else { 1_000 },
    }))?;
    let command_response = responses::sse(vec![
        responses::ev_response_created("resp-1"),
        responses::ev_function_call(call_id, "exec_command", &arguments),
        responses::ev_completed("resp-1"),
    ]);
    let final_response = create_final_assistant_message_sse_response("done")?;
    let pause_response = |id: &str| {
        responses::sse(vec![
            responses::ev_response_created(id),
            responses::ev_function_call(
                id,
                "request_user_input",
                &json!({
                    "questions": [{
                        "id": "continue", "header": "Continue", "question": "Continue?",
                        "options": [
                            {"label": "Yes", "description": "Continue the turn."},
                            {"label": "No", "description": "Stop the turn."}
                        ]
                    }]
                })
                .to_string(),
            ),
            responses::ev_completed(id),
        ])
    };
    let mut response_sequence = vec![pause_response("before-command"), command_response];
    if background {
        response_sequence.push(pause_response("after-launch"));
    }
    response_sequence.push(final_response);
    let server = create_mock_responses_server_sequence(response_sequence).await;

    let analytics_server = responses::start_mock_server().await;
    write_mock_responses_config_toml_with_chatgpt_base_url(
        codex_home.path(),
        &server.uri(),
        &analytics_server.uri(),
    )?;
    let config_path = codex_home.path().join("config.toml");
    let config = std::fs::read_to_string(&config_path)?;
    let model = bundled_models_response()?
        .models
        .into_iter()
        .find(|model| model.slug == "gpt-5.4")
        .expect("bundled gpt-5.4 model");
    let models = [
        ("initial-model", ReasoningEffort::Low),
        ("invoking-model", ReasoningEffort::High),
    ]
    .into_iter()
    .map(|(slug, effort)| {
        let mut model = model.clone();
        model.slug = slug.to_string();
        model.default_reasoning_level = Some(effort);
        model
    })
    .collect();
    let catalog_path = codex_home.path().join("measurement-models.json");
    std::fs::write(
        &catalog_path,
        serde_json::to_vec(&ModelsResponse { models })?,
    )?;
    let catalog_config = format!(
        "model_catalog_json = {}",
        serde_json::to_string(&catalog_path)?
    );
    std::fs::write(
        config_path,
        format!(
            r#"{catalog_config}
{config}
[features]
step_model_switching = true
default_mode_request_user_input = true
plugins = true
remote_plugin = false
unified_exec = true
shell_zsh_fork = false
unified_exec_zsh_fork = false
[plugins."{METRICS_PLUGIN_ID}"]
enabled = true
"#,
        ),
    )?;
    mount_analytics_capture(&analytics_server, codex_home.path()).await?;

    let mut builder = TestAppServer::builder()
        .with_codex_home(codex_home.path())
        .without_managed_config();
    if remote {
        builder = builder.with_exec_server_delay(Duration::ZERO);
    }
    let mut mcp = builder.build().await?;
    if remote {
        assert_eq!(
            mcp.auto_env_params()?.environment_id,
            codex_exec_server::REMOTE_ENVIRONMENT_ID
        );
    }
    timeout(Duration::from_secs(10), mcp.initialize()).await??;
    let thread_request = mcp
        .send_thread_start_request_with_auto_env(ThreadStartParams {
            model: Some("initial-model".to_string()),
            service_name: Some("codex_work_desktop".to_string()),
            ..Default::default()
        })
        .await?;
    let thread_response: JSONRPCResponse = timeout(
        Duration::from_secs(10),
        mcp.read_stream_until_response_message(RequestId::Integer(thread_request)),
    )
    .await??;
    let ThreadStartResponse { thread, .. } = to_response(thread_response)?;
    let thread_id = thread.id.clone();

    let turn_request = mcp
        .send_turn_start_request(TurnStartParams {
            thread_id: thread.id,
            input: vec![UserInput::Text {
                text: "run the curated plugin metrics script".to_string(),
                text_elements: Vec::new(),
            }],
            sandbox_policy: Some(SandboxPolicy::ReadOnly {
                network_access: false,
            }),
            ..Default::default()
        })
        .await?;
    let turn_response = timeout(
        Duration::from_secs(10),
        mcp.read_stream_until_response_message(RequestId::Integer(turn_request)),
    )
    .await??;
    let turn_response: TurnStartResponse = to_response(turn_response)?;
    for model in if background {
        vec!["invoking-model", "initial-model"]
    } else {
        vec!["invoking-model"]
    } {
        let request = timeout(
            Duration::from_secs(10),
            mcp.read_stream_until_request_message(),
        )
        .await??;
        let ServerRequest::ToolRequestUserInput { request_id, .. } = request else {
            anyhow::bail!("expected request_user_input, received {request:?}");
        };
        let response: TurnSettingsUpdateResponse = mcp
            .request(|request_id| ClientRequest::TurnSettingsUpdate {
                request_id,
                params: TurnSettingsUpdateParams {
                    thread_id: thread_id.clone(),
                    turn_id: turn_response.turn.id.clone(),
                    model: Some(model.to_string()),
                    ..Default::default()
                },
            })
            .await?;
        assert_eq!(response.status, TurnSettingsUpdateStatus::Applied);
        mcp.send_response(
            request_id,
            json!({"answers": {"continue": {"answers": ["Yes"]}}}),
        )
        .await?;
    }
    let completed_turn = timeout(
        Duration::from_secs(10),
        mcp.read_stream_until_notification_message("turn/completed"),
    )
    .await??;
    let turn_id = completed_turn
        .params
        .as_ref()
        .and_then(|params| params["turn"]["id"].as_str())
        .expect("completed turn id");

    if background {
        std::fs::write(release_path, "release")?;
    }

    for measurement_name in ["findings", "files_scanned"] {
        wait_for_matching_analytics_event(&analytics_server, Duration::from_secs(10), |event| {
            event["event_type"] == "codex_plugin_measurement_event"
                && event["event_params"]["item_id"] == call_id
                && event["event_params"]["measurement_name"] == measurement_name
        })
        .await?;
    }
    let command_event =
        wait_for_matching_analytics_event(&analytics_server, Duration::from_secs(10), |event| {
            event["event_type"] == "codex_command_execution_event"
                && event["event_params"]["item_id"] == call_id
        })
        .await?;
    assert_eq!(
        json!({
            "model_slug": command_event["event_params"]["model_slug"],
            "reasoning_effort": command_event["event_params"]["reasoning_effort"],
            "plugin_id": command_event["event_params"]["plugin_id"],
            "script_path": command_event["event_params"]["script_path"],
            "item_id": command_event["event_params"]["item_id"],
            "exit_code": command_event["event_params"]["exit_code"],
        }),
        json!({
            "model_slug": "invoking-model",
            "reasoning_effort": "high",
            "plugin_id": METRICS_PLUGIN_ID,
            "script_path": "scripts/run.sh",
            "item_id": call_id,
            "exit_code": 0,
        })
    );
    let mut measurement_events = Vec::new();
    for request in analytics_server
        .received_requests()
        .await
        .unwrap_or_default()
    {
        if request.method != "POST" || request.url.path() != "/codex/analytics-events/events" {
            continue;
        }
        let payload: Value = serde_json::from_slice(&request.body)?;
        let Some(events) = payload["events"].as_array() else {
            continue;
        };
        measurement_events.extend(
            events
                .iter()
                .filter(|event| {
                    event["event_type"] == "codex_plugin_measurement_event"
                        && event["event_params"]["item_id"] == call_id
                })
                .cloned(),
        );
    }
    measurement_events.sort_by(|left, right| {
        left["event_params"]["measurement_name"]
            .as_str()
            .cmp(&right["event_params"]["measurement_name"].as_str())
    });
    assert_eq!(measurement_events.len(), 2);
    let execution_id = measurement_events[0]["event_params"]["execution_id"]
        .as_str()
        .expect("measurement execution id");
    assert!(!execution_id.is_empty());
    assert_eq!(
        measurement_events[1]["event_params"]["execution_id"].as_str(),
        Some(execution_id)
    );
    assert_eq!(
        measurement_events
            .iter()
            .map(|event| json!({
                "plugin_id": event["event_params"]["plugin_id"],
                "operation": event["event_params"]["operation"],
                "measurement_name": event["event_params"]["measurement_name"],
                "number_value": event["event_params"]["number_value"],
                "dimensions": event["event_params"]["dimensions"],
                "item_id": event["event_params"]["item_id"],
            }))
            .collect::<Vec<_>>(),
        vec![
            json!({
                "plugin_id": METRICS_PLUGIN_ID,
                "operation": "scan",
                "measurement_name": "files_scanned",
                "number_value": 17.0,
                "dimensions": null,
                "item_id": call_id,
            }),
            json!({
                "plugin_id": METRICS_PLUGIN_ID,
                "operation": "scan",
                "measurement_name": "findings",
                "number_value": 3.0,
                "dimensions": {"severity": "high"},
                "item_id": call_id,
            }),
        ]
    );
    for event in &measurement_events {
        let event_params = event["event_params"]
            .as_object()
            .expect("measurement event params");
        let mut field_names = event_params.keys().map(String::as_str).collect::<Vec<_>>();
        field_names.sort_unstable();
        assert_eq!(
            field_names,
            vec![
                "dimensions",
                "execution_id",
                "item_id",
                "measurement_name",
                "model_slug",
                "number_value",
                "operation",
                "originator",
                "plugin_id",
                "reasoning_effort",
                "thread_id",
                "turn_id",
            ]
        );
        assert_eq!(event_params["thread_id"], thread_id);
        assert_eq!(event_params["turn_id"], turn_id);
        assert_eq!(event_params["originator"], "codex_work_desktop");
        assert_eq!(event_params["model_slug"], "invoking-model");
        assert_eq!(event_params["reasoning_effort"], "high");
    }

    Ok(())
}

#[cfg_attr(windows, ignore = "plugin metrics fixture is Unix-only")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_plugin_script_emits_measurement_analytics() -> Result<()> {
    assert_plugin_measurement_analytics(/*remote*/ false, /*background*/ false).await
}

#[cfg_attr(windows, ignore = "plugin metrics fixture is Unix-only")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remote_unified_plugin_script_emits_measurement_analytics() -> Result<()> {
    assert_plugin_measurement_analytics(/*remote*/ true, /*background*/ false).await
}

#[cfg_attr(windows, ignore = "plugin metrics fixture is Unix-only")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_background_plugin_script_emits_measurements_after_turn_completion() -> Result<()> {
    assert_plugin_measurement_analytics(/*remote*/ false, /*background*/ true).await
}

#[cfg_attr(windows, ignore = "plugin metrics fixture is Unix-only")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remote_unified_background_plugin_script_emits_measurements_after_turn_completion()
-> Result<()> {
    assert_plugin_measurement_analytics(/*remote*/ true, /*background*/ true).await
}