From 2dde0bf62d7f8f57385669e5897ca813ec0bedc6 Mon Sep 17 00:00:00 2001 From: Alex Daley Date: Tue, 7 Jul 2026 18:06:56 -0400 Subject: [PATCH 1/2] Emit plugin install suggestion outcomes from core --- .../tools/handlers/request_plugin_install.rs | 91 ++++++--- .../tests/suite/request_plugin_install.rs | 188 +++++++++++++++--- 2 files changed, 230 insertions(+), 49 deletions(-) diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install.rs b/codex-rs/core/src/tools/handlers/request_plugin_install.rs index e44ed41fa4d8..e3b9ec07afc5 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install.rs @@ -4,6 +4,10 @@ use std::sync::Arc; use codex_analytics::PluginInstallRequestSource; use codex_analytics::PluginInstallRequested; use codex_analytics::PluginInstallRequestedPlugin; +use codex_analytics::PluginInstallSuggestionOutcome; +use codex_analytics::PluginInstallSuggestionOutcomeTool; +use codex_analytics::PluginInstallSuggestionResponseAction; +use codex_analytics::PluginInstallSuggestionToolType; use codex_analytics::build_track_events_context; use codex_config::types::ToolSuggestDisabledTool; use codex_core_plugins::remote::REMOTE_GLOBAL_MARKETPLACE_NAME; @@ -180,23 +184,24 @@ impl RequestPluginInstallHandler { let tool_type = tool.tool_type(); let suggestion_id = format!("request_plugin_install_{call_id}"); + let source = match self.presentation { + ToolSuggestPresentation::ListTool => PluginInstallRequestSource::LegacyDiscovery, + ToolSuggestPresentation::RecommendationContext => { + PluginInstallRequestSource::EndpointRecommendation + } + }; + let tracking = build_track_events_context( + turn.model_info.slug.clone(), + session.thread_id.to_string(), + turn.sub_id.clone(), + turn.originator.clone(), + ); if let DiscoverableTool::Plugin(plugin) = &tool { - let source = match self.presentation { - ToolSuggestPresentation::ListTool => PluginInstallRequestSource::LegacyDiscovery, - ToolSuggestPresentation::RecommendationContext => { - PluginInstallRequestSource::EndpointRecommendation - } - }; session .services .analytics_events_client .track_plugin_install_requested( - build_track_events_context( - turn.model_info.slug.clone(), - session.thread_id.to_string(), - turn.sub_id.clone(), - turn.originator.clone(), - ), + tracking.clone(), PluginInstallRequested { suggestion_id: suggestion_id.clone(), plugins: vec![PluginInstallRequestedPlugin { @@ -210,7 +215,7 @@ impl RequestPluginInstallHandler { ); } - let request_id = RequestId::String(suggestion_id.into()); + let request_id = RequestId::String(suggestion_id.clone().into()); let request = build_request_plugin_install_elicitation_request(suggest_reason, &tool); let elicitation = session .request_mcp_server_elicitation( @@ -243,24 +248,64 @@ impl RequestPluginInstallHandler { } if elicitation.sent { - let tool_type = match tool_type { - DiscoverableToolType::Connector => "connector", - DiscoverableToolType::Plugin => "plugin", + let (outcome_tool_type, tool_type_name) = match tool_type { + DiscoverableToolType::Connector => { + (PluginInstallSuggestionToolType::Connector, "connector") + } + DiscoverableToolType::Plugin => (PluginInstallSuggestionToolType::Plugin, "plugin"), }; - let response_action = match response.as_ref().map(|response| &response.action) { - Some(ElicitationAction::Accept) => "accept", - Some(ElicitationAction::Decline) => "decline", - Some(ElicitationAction::Cancel) => "cancel", - None => "unavailable", + let (response_action, response_action_name) = + match response.as_ref().map(|response| &response.action) { + Some(ElicitationAction::Accept) => { + (PluginInstallSuggestionResponseAction::Accept, "accept") + } + Some(ElicitationAction::Decline) => { + (PluginInstallSuggestionResponseAction::Decline, "decline") + } + Some(ElicitationAction::Cancel) => { + (PluginInstallSuggestionResponseAction::Cancel, "cancel") + } + None => ( + PluginInstallSuggestionResponseAction::Unavailable, + "unavailable", + ), + }; + let (remote_plugin_id, connector_ids) = match &tool { + DiscoverableTool::Connector(connector) => (None, vec![connector.id.clone()]), + DiscoverableTool::Plugin(plugin) => ( + plugin.remote_plugin_id.clone(), + plugin.app_connector_ids.clone(), + ), }; turn.session_telemetry.record_plugin_install_suggestion( - tool_type, + tool_type_name, tool.id(), tool.name(), - response_action, + response_action_name, user_confirmed, completed, ); + session + .services + .analytics_events_client + .track_plugin_install_suggestion_outcome( + tracking, + PluginInstallSuggestionOutcome { + suggestion_id, + source, + tools: vec![PluginInstallSuggestionOutcomeTool { + tool_type: outcome_tool_type, + tool_id: tool.id().to_string(), + tool_name: tool.name().to_string(), + remote_plugin_id, + connector_ids, + selected: user_confirmed, + }], + response_action, + user_confirmed, + completed, + }, + ); } let content = serde_json::to_string(&RequestPluginInstallResult { diff --git a/codex-rs/core/tests/suite/request_plugin_install.rs b/codex-rs/core/tests/suite/request_plugin_install.rs index da10cd1149a2..b8d8736048e5 100644 --- a/codex-rs/core/tests/suite/request_plugin_install.rs +++ b/codex-rs/core/tests/suite/request_plugin_install.rs @@ -199,6 +199,32 @@ async fn resolve_install_elicitation( Ok(()) } +async fn wait_for_analytics_event(server: &wiremock::MockServer, event_type: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let requests = server.received_requests().await.unwrap_or_default(); + if let Some(event) = requests + .into_iter() + .filter(|request| request.url.path() == "/codex/analytics-events/events") + .find_map(|request| { + let payload: Value = serde_json::from_slice(&request.body).ok()?; + payload["events"].as_array().and_then(|events| { + events + .iter() + .find(|event| event["event_type"] == event_type) + .cloned() + }) + }) + { + return event; + } + if Instant::now() >= deadline { + panic!("timed out waiting for {event_type} analytics"); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + async fn mount_remote_calendar_recommendation(server: &wiremock::MockServer) { Mock::given(method("GET")) .and(path("/ps/plugins/suggested")) @@ -328,6 +354,80 @@ async fn explicit_false_preserves_legacy_workflow() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn legacy_connector_install_emits_attributed_suggestion_outcome() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let apps_server = AppsTestServer::mount(&server).await?; + mount_recommendations( + &server, + ResponseTemplate::new(200).set_body_json(json!({"enabled": false, "plugins": []})), + ) + .await; + let call_id = "install-gmail"; + let _mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + call_id, + REQUEST_PLUGIN_INSTALL_TOOL_NAME, + &serde_json::to_string(&json!({ + "tool_type": "connector", + "action_type": "install", + "tool_id": DISCOVERABLE_GMAIL_ID, + "suggest_reason": "Use Gmail for this request" + }))?, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + let test = build_test(&server, &apps_server).await?; + + let elicitation = start_install_turn(&test, "use Gmail").await?; + resolve_install_elicitation(&test, elicitation, ElicitationAction::Decline).await?; + + let outcome_event = + wait_for_analytics_event(&server, "codex_plugin_install_suggestion_outcome").await; + let thread_id = outcome_event["event_params"]["thread_id"].clone(); + let turn_id = outcome_event["event_params"]["turn_id"].clone(); + assert_eq!( + outcome_event, + json!({ + "event_type": "codex_plugin_install_suggestion_outcome", + "event_params": { + "suggestion_id": "request_plugin_install_install-gmail", + "source": "legacy_discovery", + "tools": [{ + "tool_type": "connector", + "tool_id": DISCOVERABLE_GMAIL_ID, + "tool_name": DISCOVERABLE_GMAIL_ID, + "remote_plugin_id": null, + "connector_ids": [DISCOVERABLE_GMAIL_ID], + "selected": false, + }], + "response_action": "decline", + "user_confirmed": false, + "completed": false, + "thread_id": thread_id, + "turn_id": turn_id, + "model_slug": "gpt-5.4", + "product_client_id": codex_login::default_client::originator().value, + } + }) + ); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn endpoint_mode_injects_candidates_hides_list_and_rejects_invented_ids() -> Result<()> { skip_if_no_network!(Ok(())); @@ -476,29 +576,7 @@ async fn run_remote_plugin_install_metadata_case() -> Result<()> { assert_eq!(meta["remote_plugin_id"], REMOTE_PLUGIN_ID); assert_eq!(meta["app_connector_ids"], json!([APP_CONNECTOR_ID])); - let deadline = Instant::now() + Duration::from_secs(10); - let analytics_event = loop { - let requests = server.received_requests().await.unwrap_or_default(); - if let Some(event) = requests - .into_iter() - .filter(|request| request.url.path() == "/codex/analytics-events/events") - .find_map(|request| { - let payload: Value = serde_json::from_slice(&request.body).ok()?; - payload["events"].as_array().and_then(|events| { - events - .iter() - .find(|event| event["event_type"] == "codex_plugin_install_requested") - .cloned() - }) - }) - { - break event; - } - if Instant::now() >= deadline { - panic!("timed out waiting for plugin install request analytics"); - } - tokio::time::sleep(Duration::from_millis(50)).await; - }; + let analytics_event = wait_for_analytics_event(&server, "codex_plugin_install_requested").await; let thread_id = analytics_event["event_params"]["thread_id"].clone(); let turn_id = analytics_event["event_params"]["turn_id"].clone(); assert_eq!( @@ -514,8 +592,8 @@ async fn run_remote_plugin_install_metadata_case() -> Result<()> { "connector_ids": [APP_CONNECTOR_ID], }], "source": "endpoint_recommendation", - "thread_id": thread_id, - "turn_id": turn_id, + "thread_id": thread_id.clone(), + "turn_id": turn_id.clone(), "model_slug": "gpt-5.4", "product_client_id": codex_login::default_client::originator().value, } @@ -524,6 +602,34 @@ async fn run_remote_plugin_install_metadata_case() -> Result<()> { resolve_install_elicitation(&test, elicitation, ElicitationAction::Decline).await?; + let outcome_event = + wait_for_analytics_event(&server, "codex_plugin_install_suggestion_outcome").await; + assert_eq!( + outcome_event, + json!({ + "event_type": "codex_plugin_install_suggestion_outcome", + "event_params": { + "suggestion_id": "request_plugin_install_install-github", + "source": "endpoint_recommendation", + "tools": [{ + "tool_type": "plugin", + "tool_id": "github@openai-curated-remote", + "tool_name": "GitHub", + "remote_plugin_id": REMOTE_PLUGIN_ID, + "connector_ids": [APP_CONNECTOR_ID], + "selected": false, + }], + "response_action": "decline", + "user_confirmed": false, + "completed": false, + "thread_id": thread_id, + "turn_id": turn_id, + "model_slug": "gpt-5.4", + "product_client_id": codex_login::default_client::originator().value, + } + }) + ); + let requests = mock.requests(); assert_eq!(requests.len(), 2); for request in requests { @@ -591,6 +697,37 @@ async fn run_remote_plugin_install_refresh_case(refreshed_tools: RefreshedAppsTo drop(initial_remote_installed_plugins); resolve_install_elicitation(&test, elicitation, ElicitationAction::Accept).await?; + let completed = matches!(refreshed_tools, RefreshedAppsTools::Available); + let outcome_event = + wait_for_analytics_event(&server, "codex_plugin_install_suggestion_outcome").await; + let thread_id = outcome_event["event_params"]["thread_id"].clone(); + let turn_id = outcome_event["event_params"]["turn_id"].clone(); + assert_eq!( + outcome_event, + json!({ + "event_type": "codex_plugin_install_suggestion_outcome", + "event_params": { + "suggestion_id": "request_plugin_install_install-calendar", + "source": "endpoint_recommendation", + "tools": [{ + "tool_type": "plugin", + "tool_id": REMOTE_CALENDAR_PLUGIN_CONFIG_ID, + "tool_name": "Calendar", + "remote_plugin_id": REMOTE_CALENDAR_PLUGIN_ID, + "connector_ids": [CALENDAR_CONNECTOR_ID], + "selected": true, + }], + "response_action": "accept", + "user_confirmed": true, + "completed": completed, + "thread_id": thread_id, + "turn_id": turn_id, + "model_slug": "gpt-5.4", + "product_client_id": codex_login::default_client::originator().value, + } + }) + ); + let requests = mock.requests(); assert_eq!(requests.len(), 2); assert!( @@ -599,7 +736,6 @@ async fn run_remote_plugin_install_refresh_case(refreshed_tools: RefreshedAppsTo .is_none(), "calendar tool should be absent before the remote install" ); - let completed = matches!(refreshed_tools, RefreshedAppsTools::Available); assert_eq!( serde_json::from_str::( &requests[1] From baa3df615802b39deb4dbbbb6722f17da41ab3ad Mon Sep 17 00:00:00 2001 From: Alex Daley Date: Tue, 7 Jul 2026 18:40:40 -0400 Subject: [PATCH 2/2] Verify remote plugin suggestion completion --- .../tools/handlers/request_plugin_install.rs | 46 +++++++------ .../tests/suite/request_plugin_install.rs | 64 +++++++++++++++---- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install.rs b/codex-rs/core/src/tools/handlers/request_plugin_install.rs index e3b9ec07afc5..f145ec0c9b22 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install.rs @@ -14,6 +14,7 @@ use codex_core_plugins::remote::REMOTE_GLOBAL_MARKETPLACE_NAME; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_rmcp_client::ElicitationAction; use codex_rmcp_client::ElicitationResponse; +use codex_tools::DiscoverablePluginInfo; use codex_tools::DiscoverableTool; use codex_tools::DiscoverableToolAction; use codex_tools::DiscoverableToolType; @@ -300,10 +301,10 @@ impl RequestPluginInstallHandler { remote_plugin_id, connector_ids, selected: user_confirmed, + completed, }], response_action, user_confirmed, - completed, }, ); } @@ -412,12 +413,9 @@ async fn verify_request_plugin_install_completed( }), DiscoverableTool::Plugin(plugin) => { if is_remote_plugin_install_suggestion(&plugin.id) { - let (_, accessible_connectors) = tokio::join!( + let (completed, _) = tokio::join!( refresh_remote_installed_plugins_cache_after_install( - session, - turn, - auth, - plugin.id.as_str(), + session, turn, auth, plugin, ), refresh_missing_requested_connectors( turn, @@ -427,12 +425,7 @@ async fn verify_request_plugin_install_completed( plugin.id.as_str(), ) ); - return accessible_connectors.is_some_and(|accessible_connectors| { - all_requested_connectors_picked_up( - &plugin.app_connector_ids, - &accessible_connectors, - ) - }); + return completed; } session.reload_user_config_layer().await; @@ -459,11 +452,18 @@ async fn refresh_remote_installed_plugins_cache_after_install( session: &crate::session::session::Session, turn: &crate::session::turn_context::TurnContext, auth: Option<&codex_login::CodexAuth>, - tool_id: &str, -) { + plugin: &DiscoverablePluginInfo, +) -> bool { + let Some(remote_plugin_id) = plugin.remote_plugin_id.as_deref() else { + warn!( + "remote plugin install suggestion did not include a remote plugin id for {}", + plugin.id + ); + return false; + }; let plugins_manager = &session.services.plugins_manager; let plugins_config = turn.config.plugins_config_input(); - if let Err(err) = plugins_manager + match plugins_manager .build_and_cache_remote_installed_plugin_marketplaces( &plugins_config, auth, @@ -472,9 +472,19 @@ async fn refresh_remote_installed_plugins_cache_after_install( ) .await { - warn!( - "failed to refresh remote installed plugins cache after plugin install request for {tool_id}: {err:#}" - ); + Ok(marketplaces) => marketplaces + .into_iter() + .flat_map(|marketplace| marketplace.plugins) + .any(|installed_plugin| { + installed_plugin.remote_plugin_id == remote_plugin_id && installed_plugin.installed + }), + Err(err) => { + warn!( + "failed to refresh remote installed plugins cache after plugin install request for {}: {err:#}", + plugin.id + ); + false + } } } diff --git a/codex-rs/core/tests/suite/request_plugin_install.rs b/codex-rs/core/tests/suite/request_plugin_install.rs index b8d8736048e5..0e9d781294c9 100644 --- a/codex-rs/core/tests/suite/request_plugin_install.rs +++ b/codex-rs/core/tests/suite/request_plugin_install.rs @@ -414,6 +414,7 @@ async fn legacy_connector_install_emits_attributed_suggestion_outcome() -> Resul "remote_plugin_id": null, "connector_ids": [DISCOVERABLE_GMAIL_ID], "selected": false, + "completed": false, }], "response_action": "decline", "user_confirmed": false, @@ -618,6 +619,7 @@ async fn run_remote_plugin_install_metadata_case() -> Result<()> { "remote_plugin_id": REMOTE_PLUGIN_ID, "connector_ids": [APP_CONNECTOR_ID], "selected": false, + "completed": false, }], "response_action": "decline", "user_confirmed": false, @@ -646,15 +648,37 @@ enum RefreshedAppsTools { Missing, } +#[derive(Clone, Copy)] +enum RefreshedPluginInventory { + Installed, + Missing, +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn remote_plugin_install_refreshes_plugin_and_apps_tool_caches() -> Result<()> { +async fn remote_plugin_completion_uses_refreshed_plugin_inventory() -> Result<()> { skip_if_no_network!(Ok(())); - run_remote_plugin_install_refresh_case(RefreshedAppsTools::Available).await?; - run_remote_plugin_install_refresh_case(RefreshedAppsTools::Missing).await + run_remote_plugin_install_refresh_case( + RefreshedPluginInventory::Installed, + RefreshedAppsTools::Available, + ) + .await?; + run_remote_plugin_install_refresh_case( + RefreshedPluginInventory::Installed, + RefreshedAppsTools::Missing, + ) + .await?; + run_remote_plugin_install_refresh_case( + RefreshedPluginInventory::Missing, + RefreshedAppsTools::Available, + ) + .await } -async fn run_remote_plugin_install_refresh_case(refreshed_tools: RefreshedAppsTools) -> Result<()> { +async fn run_remote_plugin_install_refresh_case( + refreshed_plugin_inventory: RefreshedPluginInventory, + refreshed_tools: RefreshedAppsTools, +) -> Result<()> { let server = start_mock_server().await; let apps_server = match refreshed_tools { RefreshedAppsTools::Available => { @@ -693,11 +717,20 @@ async fn run_remote_plugin_install_refresh_case(refreshed_tools: RefreshedAppsTo let test = build_test(&server, &apps_server).await?; let elicitation = start_install_turn(&test, "use Calendar").await?; - mount_remote_calendar_installed_plugins(&server).await; - drop(initial_remote_installed_plugins); + if matches!( + refreshed_plugin_inventory, + RefreshedPluginInventory::Installed + ) { + mount_remote_calendar_installed_plugins(&server).await; + drop(initial_remote_installed_plugins); + } resolve_install_elicitation(&test, elicitation, ElicitationAction::Accept).await?; - let completed = matches!(refreshed_tools, RefreshedAppsTools::Available); + let completed = matches!( + refreshed_plugin_inventory, + RefreshedPluginInventory::Installed + ); + let apps_tools_available = matches!(refreshed_tools, RefreshedAppsTools::Available); let outcome_event = wait_for_analytics_event(&server, "codex_plugin_install_suggestion_outcome").await; let thread_id = outcome_event["event_params"]["thread_id"].clone(); @@ -716,6 +749,7 @@ async fn run_remote_plugin_install_refresh_case(refreshed_tools: RefreshedAppsTo "remote_plugin_id": REMOTE_CALENDAR_PLUGIN_ID, "connector_ids": [CALENDAR_CONNECTOR_ID], "selected": true, + "completed": completed, }], "response_action": "accept", "user_confirmed": true, @@ -756,15 +790,17 @@ async fn run_remote_plugin_install_refresh_case(refreshed_tools: RefreshedAppsTo requests[1] .tool_by_name(CALENDAR_NAMESPACE, CALENDAR_CREATE_EVENT_TOOL) .is_some(), - completed, + apps_tools_available, "the resumed router should reflect the refreshed Apps tools" ); - assert!( - !tool_names(&requests[1].body_json()) - .iter() - .any(|name| name == REQUEST_PLUGIN_INSTALL_TOOL_NAME), - "the refreshed installed-plugin cache should filter the cached recommendation" - ); + if completed { + assert!( + !tool_names(&requests[1].body_json()) + .iter() + .any(|name| name == REQUEST_PLUGIN_INSTALL_TOOL_NAME), + "the refreshed installed-plugin cache should filter the cached recommendation" + ); + } Ok(()) }