From 5d73581629c1ceb1f488a80682662c1ff02823bb Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 10:57:31 +0800 Subject: [PATCH 01/13] refactor: add XHS-aligned content platform interface --- core/src/sites/content.rs | 350 ++++++++++++++++++++++++++++++++++++ core/src/sites/dy/mod.rs | 4 +- core/src/sites/dy/tools.rs | 143 ++++++++++++++- core/src/sites/mod.rs | 5 + core/src/sites/xhs/mod.rs | 2 +- core/src/sites/xhs/tools.rs | 126 ++++++++++++- 6 files changed, 624 insertions(+), 6 deletions(-) create mode 100644 core/src/sites/content.rs diff --git a/core/src/sites/content.rs b/core/src/sites/content.rs new file mode 100644 index 00000000..a4a1ecca --- /dev/null +++ b/core/src/sites/content.rs @@ -0,0 +1,350 @@ +//! XHS-shaped content-platform contract shared by research sites. +//! +//! Xiaohongshu is the reference implementation: platform adapters expose the +//! same four content operations and keep site-specific page/runtime details +//! behind this trait. Callers select an implementation by `site_id` instead of +//! branching on Douyin, TikTok, or XHS tool names. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde::Serialize; +use serde_json::{json, Value}; + +use crate::agent::{Backend as LlmProvider, Tool, ToolContext, ToolResult}; +use crate::cdp::PageSession; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentOperation { + GetNotes, + Search, + AuthorScan, + PageState, +} + +impl ContentOperation { + pub const fn tool_name(self) -> &'static str { + match self { + Self::GetNotes => "get_notes", + Self::Search => "search", + Self::AuthorScan => "author_scan", + Self::PageState => "page_state", + } + } +} + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct ContentCapabilities { + pub full_search: bool, + pub author_full_scan: bool, + pub search_filters: bool, + pub comments: bool, + pub comment_replies: bool, + pub media_download: bool, + pub ocr: bool, + pub audio_transcription: bool, + pub cross_run_history: bool, + pub artifacts: bool, +} + +/// The stable content interface. Its operation names and request vocabulary +/// follow the established XHS tools: content items are `notes`, result limits +/// are `num_notes`, and full scans can request comments/media/OCR/ASR. +#[async_trait] +pub trait ContentPlatform: Send + Sync { + fn site_id(&self) -> &'static str; + fn display_name(&self) -> &'static str; + fn capabilities(&self) -> ContentCapabilities; + fn input_schema(&self, operation: ContentOperation) -> Value; + + fn effective_input(&self, _operation: ContentOperation, input: &Value) -> Value { + input.clone() + } + + async fn get_notes(&self, input: Value, ctx: &ToolContext) -> anyhow::Result; + async fn search(&self, input: Value, ctx: &ToolContext) -> anyhow::Result; + async fn author_scan(&self, input: Value, ctx: &ToolContext) -> anyhow::Result; + async fn page_state(&self, input: Value, ctx: &ToolContext) -> anyhow::Result; +} + +struct ContentPlatformTool { + platform: Arc, + operation: ContentOperation, + description: String, +} + +impl ContentPlatformTool { + fn new(platform: Arc, operation: ContentOperation) -> Self { + let site = platform.display_name(); + let description = match operation { + ContentOperation::GetNotes => format!( + "Read one or more {site} notes by the locators returned from search or author_scan." + ), + ContentOperation::Search => format!( + "Research {site} by keyword. The default is an XHS-style full scan; preview=true returns note cards only." + ), + ContentOperation::AuthorScan => format!( + "Research one {site} author and their notes. The default is a full scan; preview=true returns note cards only." + ), + ContentOperation::PageState => { + format!("Read the current {site} page, login, and modal state.") + } + }; + Self { + platform, + operation, + description, + } + } +} + +#[async_trait] +impl Tool for ContentPlatformTool { + fn name(&self) -> &str { + self.operation.tool_name() + } + + fn description(&self) -> &str { + &self.description + } + + fn input_schema(&self) -> Value { + self.platform.input_schema(self.operation) + } + + fn defer_until_site(&self) -> &str { + self.platform.site_id() + } + + fn effective_input(&self, input: &Value) -> Value { + self.platform.effective_input(self.operation, input) + } + + async fn call(&self, input: Value, ctx: &ToolContext) -> anyhow::Result { + match self.operation { + ContentOperation::GetNotes => self.platform.get_notes(input, ctx).await, + ContentOperation::Search => self.platform.search(input, ctx).await, + ContentOperation::AuthorScan => self.platform.author_scan(input, ctx).await, + ContentOperation::PageState => self.platform.page_state(input, ctx).await, + } + } +} + +/// Build the common four-tool surface for any selected platform adapter. +pub fn content_platform_tools(platform: Arc) -> Vec> { + [ + ContentOperation::GetNotes, + ContentOperation::Search, + ContentOperation::AuthorScan, + ContentOperation::PageState, + ] + .into_iter() + .map(|operation| { + Arc::new(ContentPlatformTool::new(platform.clone(), operation)) as Arc + }) + .collect() +} + +/// Select the site implementation while keeping entrypoints independent from +/// platform modules. TikTok registers the same adapter in its integration PR. +pub fn select_content_platform( + site_id: &str, + page: Arc, + llm_provider: Option>, +) -> anyhow::Result> { + match site_id { + "xhs" => Ok(crate::sites::xhs::xhs_content_platform(page, llm_provider)), + "dy" => Ok(crate::sites::dy::dy_content_platform(page, llm_provider)), + _ => anyhow::bail!("site does not implement the content-platform contract: {site_id}"), + } +} + +pub fn empty_filters_schema() -> Value { + json!({ + "type": "object", + "description": "Platform search filters. This implementation currently accepts no filter keys.", + "properties": {}, + "additionalProperties": false + }) +} + +pub fn tokenized_note_locator_schema() -> Value { + json!({ + "type": "object", + "properties": { + "note_id": { "type": "string" }, + "xsec_token": { "type": "string" } + }, + "required": ["note_id", "xsec_token"], + "additionalProperties": false + }) +} + +pub fn video_note_locator_schema() -> Value { + json!({ + "type": "object", + "properties": { + "note_id": { "type": "string", "description": "Platform video/content id." }, + "url": { "type": "string", "description": "Canonical content URL when available." } + }, + "required": ["note_id"], + "additionalProperties": false + }) +} + +pub fn get_notes_input_schema( + locator_schema: Value, + default_comments: i64, + asr_enabled: bool, +) -> Value { + let mut schema = json!({ + "type": "object", + "properties": { + "notes": { + "type": "array", + "description": "Note locators previously returned by search or author_scan.", + "minItems": 1, + "maxItems": 20, + "items": locator_schema + }, + "num_comments": { + "type": "integer", + "description": "Comments to load per note; replies count toward the total. 0 skips comments.", + "default": default_comments, + "minimum": 0 + }, + "download_media": { + "type": "boolean", + "description": "Download note images/videos into the run dir and include local paths.", + "default": false + }, + "ocr": { + "type": "boolean", + "description": "Run local OCR on note images or a video note's cover.", + "default": false + }, + "transcribe_audio": { + "type": "boolean", + "description": "For video notes, download the video and transcribe audio while signed in with socai agent selected.", + "default": false + } + }, + "required": ["notes"], + "additionalProperties": false + }); + if !asr_enabled { + strip_hosted_transcription_schema(&mut schema); + } + schema +} + +pub fn search_input_schema( + filters_schema: Value, + default_notes: i64, + default_comments: i64, + asr_enabled: bool, +) -> Value { + let mut schema = json!({ + "type": "object", + "properties": { + "query": { "type": "string" }, + "filters": filters_schema, + "num_notes": { + "type": "integer", + "description": "Number of notes to collect and read. In preview mode, the number of cards to collect.", + "default": default_notes, + "minimum": 1 + }, + "num_comments": { + "type": "integer", + "description": "Comments to load per note. Ignored in preview mode.", + "default": default_comments, + "minimum": 0 + }, + "download_media": { + "type": "boolean", + "description": "Download note images/videos into the run dir and include local paths. Ignored in preview mode.", + "default": false + }, + "ocr": { + "type": "boolean", + "description": "Run local OCR on note images or video covers.", + "default": false + }, + "transcribe_audio": { + "type": "boolean", + "description": "For opened video notes, download the video and transcribe audio while signed in with socai agent selected. Ignored in preview mode.", + "default": false + }, + "preview": { + "type": "boolean", + "description": "Fast cards-only mode without opening notes or reading bodies/comments.", + "default": false + } + }, + "required": ["query"] + }); + if !asr_enabled { + strip_hosted_transcription_schema(&mut schema); + } + schema +} + +pub fn author_scan_input_schema(default_comments: i64, asr_enabled: bool) -> Value { + let mut schema = json!({ + "type": "object", + "properties": { + "author_id": { "type": "string", "description": "Platform author id or profile URL." }, + "num_notes": { "type": "integer", "minimum": 1 }, + "num_comments": { + "type": "integer", + "description": "Comments to load per note. Ignored in preview mode.", + "default": default_comments, + "minimum": 0 + }, + "preview": { + "type": "boolean", + "description": "Fast cards-only mode without opening notes.", + "default": false + }, + "download_media": { + "type": "boolean", + "description": "Download note images/videos and include local paths. Ignored in preview mode.", + "default": false + }, + "ocr": { + "type": "boolean", + "description": "Run local OCR on note images or video covers.", + "default": false + }, + "transcribe_audio": { + "type": "boolean", + "description": "For opened video notes, download the video and transcribe audio while signed in with socai agent selected. Ignored in preview mode.", + "default": false + } + }, + "required": ["author_id"] + }); + if !asr_enabled { + strip_hosted_transcription_schema(&mut schema); + } + schema +} + +pub fn page_state_input_schema() -> Value { + json!({"type": "object", "properties": {}, "additionalProperties": false}) +} + +pub fn strip_hosted_transcription_schema(schema: &mut Value) { + if let Some(properties) = schema.get_mut("properties").and_then(Value::as_object_mut) { + properties.remove("transcribe_audio"); + } +} + +pub fn strip_hosted_transcription_input(input: &mut Value) -> bool { + input + .as_object_mut() + .and_then(|object| object.remove("transcribe_audio")) + .and_then(|value| value.as_bool()) + .unwrap_or(false) +} diff --git a/core/src/sites/dy/mod.rs b/core/src/sites/dy/mod.rs index a11a139e..89ad2a1a 100644 --- a/core/src/sites/dy/mod.rs +++ b/core/src/sites/dy/mod.rs @@ -5,6 +5,6 @@ pub mod tools; pub use self::entities::DouyinVideoCard; pub use self::page::{DouyinPageRuntime, DOUYIN_HOME_URL}; pub use self::tools::{ - dy_agent_instructions, dy_agent_tools, dy_tools, dy_tools_with_llm_provider, DY_KNOWLEDGE, - DY_SITE, + dy_agent_instructions, dy_agent_tools, dy_content_platform, dy_default_agent_tools, dy_tools, + dy_tools_with_llm_provider, DY_KNOWLEDGE, DY_SITE, }; diff --git a/core/src/sites/dy/tools.rs b/core/src/sites/dy/tools.rs index bb2ad045..859f7e96 100644 --- a/core/src/sites/dy/tools.rs +++ b/core/src/sites/dy/tools.rs @@ -6,6 +6,11 @@ use serde_json::{json, Value}; use crate::agent::tool::ToolProgressSender; use crate::agent::{Backend as LlmProvider, Tool, ToolContext, ToolResult}; use crate::cdp::PageSession; +use crate::sites::content::{ + author_scan_input_schema, content_platform_tools, empty_filters_schema, get_notes_input_schema, + page_state_input_schema, search_input_schema, video_note_locator_schema, ContentCapabilities, + ContentOperation, ContentPlatform, +}; use crate::sites::dy::DouyinPageRuntime; use crate::sites::registry::{ required_string, ArgKind, BoxFuture, CommandArg, SiteCommand, SiteSpec, SlowWhen, @@ -38,6 +43,140 @@ pub async fn dy_agent_tools( Ok(dy_tools_with_llm_provider(page, Some(llm_provider))) } +struct DouyinContentPlatform { + tools: Vec>, +} + +impl DouyinContentPlatform { + fn tool(&self, operation: ContentOperation) -> anyhow::Result<&Arc> { + self.tools + .iter() + .find(|tool| tool.name() == operation.tool_name()) + .ok_or_else(|| { + anyhow::anyhow!( + "Douyin content-platform tool is missing: {}", + operation.tool_name() + ) + }) + } + + fn unsupported(operation: ContentOperation) -> ToolResult { + json_result(&json!({ + "ok": false, + "site": "dy", + "reason": "unsupported_content_operation", + "operation": operation.tool_name(), + })) + } +} + +#[async_trait] +impl ContentPlatform for DouyinContentPlatform { + fn site_id(&self) -> &'static str { + "dy" + } + + fn display_name(&self) -> &'static str { + "Douyin" + } + + fn capabilities(&self) -> ContentCapabilities { + ContentCapabilities { + full_search: false, + author_full_scan: false, + search_filters: false, + comments: false, + comment_replies: false, + media_download: false, + ocr: false, + audio_transcription: false, + cross_run_history: false, + artifacts: false, + } + } + + fn input_schema(&self, operation: ContentOperation) -> Value { + match operation { + ContentOperation::GetNotes => { + get_notes_input_schema(video_note_locator_schema(), 8, false) + } + ContentOperation::Search => search_input_schema(empty_filters_schema(), 10, 5, false), + ContentOperation::AuthorScan => author_scan_input_schema(5, false), + ContentOperation::PageState => page_state_input_schema(), + } + } + + fn effective_input(&self, operation: ContentOperation, input: &Value) -> Value { + let mut effective = input.clone(); + if operation == ContentOperation::Search && effective.get("num_notes").is_none() { + effective["num_notes"] = json!(10); + } + effective + } + + async fn get_notes(&self, _input: Value, _ctx: &ToolContext) -> anyhow::Result { + Ok(Self::unsupported(ContentOperation::GetNotes)) + } + + async fn search(&self, mut input: Value, ctx: &ToolContext) -> anyhow::Result { + if !input + .get("preview") + .and_then(Value::as_bool) + .unwrap_or(false) + { + return Ok(Self::unsupported(ContentOperation::Search)); + } + if let Some(num_notes) = input + .as_object_mut() + .and_then(|obj| obj.remove("num_notes")) + { + input["num"] = num_notes; + } + if let Some(object) = input.as_object_mut() { + for key in [ + "filters", + "num_comments", + "download_media", + "ocr", + "transcribe_audio", + "preview", + ] { + object.remove(key); + } + } + self.tool(ContentOperation::Search)?.call(input, ctx).await + } + + async fn author_scan(&self, _input: Value, _ctx: &ToolContext) -> anyhow::Result { + Ok(Self::unsupported(ContentOperation::AuthorScan)) + } + + async fn page_state(&self, input: Value, ctx: &ToolContext) -> anyhow::Result { + self.tool(ContentOperation::PageState)? + .call(input, ctx) + .await + } +} + +pub fn dy_content_platform( + page: Arc, + _llm_provider: Option>, +) -> Arc { + Arc::new(DouyinContentPlatform { + tools: dy_tools(page), + }) +} + +pub async fn dy_default_agent_tools( + page: Arc, + llm_provider: Arc, +) -> anyhow::Result>> { + Ok(content_platform_tools(dy_content_platform( + page, + Some(llm_provider), + ))) +} + pub fn dy_agent_instructions(extra: &str) -> String { let base = DY_KNOWLEDGE.trim().to_string(); let extra = extra.trim(); @@ -55,9 +194,9 @@ pub static DY_SITE: SiteSpec = SiteSpec { // timeout for the site's occasional 4-5 minute blank-page throttling. home_url: "", agent_tools: |page, llm| Box::pin(dy_agent_tools(page, llm)), - default_agent_tools: None, + default_agent_tools: Some(|page, llm| Box::pin(dy_default_agent_tools(page, llm))), agent_instructions: dy_agent_instructions, - default_agent_instructions: None, + default_agent_instructions: Some(dy_agent_instructions), commands: &[ SiteCommand { name: "search", diff --git a/core/src/sites/mod.rs b/core/src/sites/mod.rs index f8a80db9..6bb7dcbc 100644 --- a/core/src/sites/mod.rs +++ b/core/src/sites/mod.rs @@ -1,8 +1,13 @@ +pub mod content; pub mod dy; pub mod registry; pub mod runner; pub mod xhs; +pub use content::{ + content_platform_tools, select_content_platform, ContentCapabilities, ContentOperation, + ContentPlatform, +}; pub use registry::{ all_sites, find_site, required_string, AgentInstructionsFn, AgentToolsFn, ArgKind, BoxFuture, CommandArg, CommandRunFn, SiteCommand, SiteSpec, SlowWhen, diff --git a/core/src/sites/xhs/mod.rs b/core/src/sites/xhs/mod.rs index b81fb5fb..8d604b58 100644 --- a/core/src/sites/xhs/mod.rs +++ b/core/src/sites/xhs/mod.rs @@ -10,7 +10,7 @@ pub use self::history::{HistoryEntry, HistorySnapshot, XhsHistoryStore}; pub use self::page::{ReadNoteOptions, XhsPageRuntime, XHS_HOME_URL}; pub use self::tools::{ author_scan_command, close_open_note, ensure_search_ready, search_command, - xhs_agent_instructions, xhs_agent_tools, xhs_default_agent_tools, + xhs_agent_instructions, xhs_agent_tools, xhs_content_platform, xhs_default_agent_tools, xhs_macro_tools_with_llm_provider, xhs_tools, xhs_tools_with_llm_provider, XHS_KNOWLEDGE, XHS_SITE, }; diff --git a/core/src/sites/xhs/tools.rs b/core/src/sites/xhs/tools.rs index 3cd9ef39..598c58f0 100644 --- a/core/src/sites/xhs/tools.rs +++ b/core/src/sites/xhs/tools.rs @@ -25,6 +25,9 @@ use crate::media::{ use async_trait::async_trait; use serde_json::{json, Map, Value}; +use crate::sites::content::{ + content_platform_tools, ContentCapabilities, ContentOperation, ContentPlatform, +}; use crate::sites::registry::{ required_string, ArgKind, BoxFuture, CommandArg, SiteCommand, SiteSpec, SlowWhen, }; @@ -173,6 +176,123 @@ pub fn xhs_macro_tools_with_llm_provider( ] } +struct XhsContentPlatform { + tools: Vec>, +} + +impl XhsContentPlatform { + fn tool(&self, operation: ContentOperation) -> anyhow::Result<&Arc> { + self.tools + .iter() + .find(|tool| tool.name() == operation.tool_name()) + .ok_or_else(|| { + anyhow::anyhow!( + "XHS content-platform tool is missing: {}", + operation.tool_name() + ) + }) + } +} + +#[async_trait] +impl ContentPlatform for XhsContentPlatform { + fn site_id(&self) -> &'static str { + "xhs" + } + + fn display_name(&self) -> &'static str { + "Xiaohongshu" + } + + fn capabilities(&self) -> ContentCapabilities { + ContentCapabilities { + full_search: true, + author_full_scan: true, + search_filters: true, + comments: true, + comment_replies: true, + media_download: true, + ocr: true, + audio_transcription: crate::cloud::hosted_llm_selected(), + cross_run_history: true, + artifacts: true, + } + } + + fn input_schema(&self, operation: ContentOperation) -> Value { + self.tool(operation) + .map(|tool| tool.input_schema()) + .unwrap_or_else(|_| json!({"type": "object", "properties": {}})) + } + + fn effective_input(&self, operation: ContentOperation, input: &Value) -> Value { + self.tool(operation) + .map(|tool| tool.effective_input(input)) + .unwrap_or_else(|_| input.clone()) + } + + async fn get_notes(&self, input: Value, ctx: &ToolContext) -> anyhow::Result { + self.tool(ContentOperation::GetNotes)? + .call(input, ctx) + .await + } + + async fn search(&self, input: Value, ctx: &ToolContext) -> anyhow::Result { + self.tool(ContentOperation::Search)?.call(input, ctx).await + } + + async fn author_scan(&self, input: Value, ctx: &ToolContext) -> anyhow::Result { + self.tool(ContentOperation::AuthorScan)? + .call(input, ctx) + .await + } + + async fn page_state(&self, input: Value, ctx: &ToolContext) -> anyhow::Result { + self.tool(ContentOperation::PageState)? + .call(input, ctx) + .await + } +} + +/// XHS is the reference adapter and therefore owns product defaults for the +/// shared content surface: media download and OCR are enabled for app/TUI +/// calls, while ASR remains gated by the hosted-agent selection. +pub fn xhs_content_platform( + page: Arc, + llm_provider: Option>, +) -> Arc { + let history = Arc::new(XhsHistoryStore::open_default()); + let asr_enabled = crate::cloud::hosted_llm_selected(); + Arc::new(XhsContentPlatform { + tools: vec![ + Arc::new(GetNotesTool { + page: page.clone(), + llm_provider: llm_provider.clone(), + history: history.clone(), + always_download_media: true, + always_ocr: true, + asr_enabled, + }) as Arc, + Arc::new(SearchTool { + page: page.clone(), + llm_provider, + history: history.clone(), + always_download_media: true, + always_ocr: true, + asr_enabled, + }) as Arc, + Arc::new(AuthorScanTool { + page: page.clone(), + history, + always_download_media: true, + always_ocr: true, + asr_enabled, + }) as Arc, + Arc::new(PageStateTool { page }) as Arc, + ], + }) +} + pub async fn xhs_agent_tools( page: Arc, llm_provider: Arc, @@ -188,7 +308,11 @@ pub async fn xhs_default_agent_tools( // Macro tools are self-contained and perform their own navigation/typed // failure handling, so the default agent factory should not require the // current tab to already be on XHS. - Ok(xhs_macro_tools_with_llm_provider(page, Some(llm_provider))) + let platform = xhs_content_platform(page.clone(), Some(llm_provider)); + let mut tools = content_platform_tools(platform); + tools.push(Arc::new(WaitForLoginTool { page })); + tools.push(Arc::new(WaitForRateLimitTool)); + Ok(tools) } pub fn xhs_agent_instructions(extra: &str) -> String { From 310d807316e78ce9162148184f07985dd1648978 Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 11:03:23 +0800 Subject: [PATCH 02/13] refactor: select content adapters through site registry --- core/src/sites/content.rs | 12 +++++------- core/src/sites/dy/tools.rs | 1 + core/src/sites/mod.rs | 2 +- core/src/sites/registry.rs | 5 +++++ core/src/sites/xhs/page.rs | 3 ++- core/src/sites/xhs/tools.rs | 1 + 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core/src/sites/content.rs b/core/src/sites/content.rs index a4a1ecca..462d07e3 100644 --- a/core/src/sites/content.rs +++ b/core/src/sites/content.rs @@ -145,18 +145,16 @@ pub fn content_platform_tools(platform: Arc) -> Vec, llm_provider: Option>, ) -> anyhow::Result> { - match site_id { - "xhs" => Ok(crate::sites::xhs::xhs_content_platform(page, llm_provider)), - "dy" => Ok(crate::sites::dy::dy_content_platform(page, llm_provider)), - _ => anyhow::bail!("site does not implement the content-platform contract: {site_id}"), - } + let site = crate::sites::find_site(site_id) + .ok_or_else(|| anyhow::anyhow!("unknown content platform: {site_id}"))?; + Ok((site.content_platform)(page, llm_provider)) } pub fn empty_filters_schema() -> Value { diff --git a/core/src/sites/dy/tools.rs b/core/src/sites/dy/tools.rs index 859f7e96..89165f7e 100644 --- a/core/src/sites/dy/tools.rs +++ b/core/src/sites/dy/tools.rs @@ -193,6 +193,7 @@ pub static DY_SITE: SiteSpec = SiteSpec { // Let Douyin tools own first navigation so they can use a much longer // timeout for the site's occasional 4-5 minute blank-page throttling. home_url: "", + content_platform: dy_content_platform, agent_tools: |page, llm| Box::pin(dy_agent_tools(page, llm)), default_agent_tools: Some(|page, llm| Box::pin(dy_default_agent_tools(page, llm))), agent_instructions: dy_agent_instructions, diff --git a/core/src/sites/mod.rs b/core/src/sites/mod.rs index 6bb7dcbc..e30f999c 100644 --- a/core/src/sites/mod.rs +++ b/core/src/sites/mod.rs @@ -10,6 +10,6 @@ pub use content::{ }; pub use registry::{ all_sites, find_site, required_string, AgentInstructionsFn, AgentToolsFn, ArgKind, BoxFuture, - CommandArg, CommandRunFn, SiteCommand, SiteSpec, SlowWhen, + CommandArg, CommandRunFn, ContentPlatformFn, SiteCommand, SiteSpec, SlowWhen, }; pub use runner::{run_tool_command, PageHook, ToolCommand}; diff --git a/core/src/sites/registry.rs b/core/src/sites/registry.rs index a6d6c281..d213e53d 100644 --- a/core/src/sites/registry.rs +++ b/core/src/sites/registry.rs @@ -14,12 +14,15 @@ use serde_json::Value; use crate::agent::tool::ToolProgressSender; use crate::agent::{Backend as LlmProvider, Tool}; use crate::cdp::PageSession; +use crate::sites::content::ContentPlatform; pub type BoxFuture = Pin> + Send>>; /// Async factory: build the site's agent tools against a shared page. pub type AgentToolsFn = fn(Arc, Arc) -> BoxFuture>>; pub type AgentInstructionsFn = fn(&str) -> String; +pub type ContentPlatformFn = + fn(Arc, Option>) -> Arc; /// One-shot CLI/daemon command: `(page, JSON args, debug_snapshot, progress)` → JSON. pub type CommandRunFn = @@ -31,6 +34,8 @@ pub struct SiteSpec { pub id: &'static str, pub about: &'static str, pub home_url: &'static str, + /// XHS-shaped content adapter selected by this site's id. + pub content_platform: ContentPlatformFn, pub agent_tools: AgentToolsFn, /// Optional default tool surface for normal app/TUI agents. Sites can keep /// a broader command/debug surface in `agent_tools` while exposing a diff --git a/core/src/sites/xhs/page.rs b/core/src/sites/xhs/page.rs index 70964184..068d4e85 100644 --- a/core/src/sites/xhs/page.rs +++ b/core/src/sites/xhs/page.rs @@ -1591,7 +1591,8 @@ impl<'a> XhsPageRuntime<'a> { if options.include_media { let t_enrich = Instant::now(); - self.enrich_note_media(&mut note, options.max_images).await?; + self.enrich_note_media(&mut note, options.max_images) + .await?; perf.insert( "enrich_ms".into(), json!(t_enrich.elapsed().as_millis() as u64), diff --git a/core/src/sites/xhs/tools.rs b/core/src/sites/xhs/tools.rs index 598c58f0..c0764c80 100644 --- a/core/src/sites/xhs/tools.rs +++ b/core/src/sites/xhs/tools.rs @@ -331,6 +331,7 @@ pub static XHS_SITE: SiteSpec = SiteSpec { id: "xhs", about: "Xiaohongshu (xiaohongshu.com)", home_url: XHS_HOME_URL, + content_platform: xhs_content_platform, agent_tools: |page, llm| Box::pin(xhs_agent_tools(page, llm)), default_agent_tools: Some(|page, llm| Box::pin(xhs_default_agent_tools(page, llm))), agent_instructions: xhs_agent_instructions, From 6a44982b4efd54e2337ef5e56deeda47af953a97 Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 11:09:12 +0800 Subject: [PATCH 03/13] refactor: share XHS content enrichment defaults --- core/src/sites/content.rs | 27 +++++++++++++++++++++++++++ core/src/sites/mod.rs | 4 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/core/src/sites/content.rs b/core/src/sites/content.rs index 462d07e3..d708cbe5 100644 --- a/core/src/sites/content.rs +++ b/core/src/sites/content.rs @@ -333,6 +333,33 @@ pub fn page_state_input_schema() -> Value { json!({"type": "object", "properties": {}, "additionalProperties": false}) } +/// Apply the established XHS app/TUI defaults to every platform adapter. +/// Preview scans OCR covers in memory; full scans also retain downloaded media. +pub fn xhs_product_effective_input(operation: ContentOperation, input: &Value) -> Value { + let mut effective = input.clone(); + if operation == ContentOperation::Search && effective.get("num_notes").is_none() { + effective["num_notes"] = json!(10); + } + if operation == ContentOperation::PageState { + return effective; + } + + effective["ocr"] = json!(true); + let preview = effective + .get("preview") + .and_then(Value::as_bool) + .unwrap_or(false); + if preview { + if let Some(object) = effective.as_object_mut() { + object.remove("download_media"); + object.remove("transcribe_audio"); + } + } else { + effective["download_media"] = json!(true); + } + effective +} + pub fn strip_hosted_transcription_schema(schema: &mut Value) { if let Some(properties) = schema.get_mut("properties").and_then(Value::as_object_mut) { properties.remove("transcribe_audio"); diff --git a/core/src/sites/mod.rs b/core/src/sites/mod.rs index e30f999c..9807c397 100644 --- a/core/src/sites/mod.rs +++ b/core/src/sites/mod.rs @@ -5,8 +5,8 @@ pub mod runner; pub mod xhs; pub use content::{ - content_platform_tools, select_content_platform, ContentCapabilities, ContentOperation, - ContentPlatform, + content_platform_tools, select_content_platform, xhs_product_effective_input, + ContentCapabilities, ContentOperation, ContentPlatform, }; pub use registry::{ all_sites, find_site, required_string, AgentInstructionsFn, AgentToolsFn, ArgKind, BoxFuture, From 907661eb7d208c165cb17f60853a14c60fe4aadb Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 11:12:20 +0800 Subject: [PATCH 04/13] chore: keep content refactor diff focused --- core/src/sites/xhs/page.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/sites/xhs/page.rs b/core/src/sites/xhs/page.rs index 068d4e85..70964184 100644 --- a/core/src/sites/xhs/page.rs +++ b/core/src/sites/xhs/page.rs @@ -1591,8 +1591,7 @@ impl<'a> XhsPageRuntime<'a> { if options.include_media { let t_enrich = Instant::now(); - self.enrich_note_media(&mut note, options.max_images) - .await?; + self.enrich_note_media(&mut note, options.max_images).await?; perf.insert( "enrich_ms".into(), json!(t_enrich.elapsed().as_millis() as u64), From 8408251083a68543597c500ba8449e3953a1be9c Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 11:14:53 +0800 Subject: [PATCH 05/13] fix: expose only implemented content operations --- core/src/sites/content.rs | 4 ++++ core/src/sites/dy/mod.rs | 2 +- core/src/sites/dy/tools.rs | 30 ++++++++++-------------------- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/core/src/sites/content.rs b/core/src/sites/content.rs index d708cbe5..d85038ac 100644 --- a/core/src/sites/content.rs +++ b/core/src/sites/content.rs @@ -55,6 +55,9 @@ pub trait ContentPlatform: Send + Sync { fn site_id(&self) -> &'static str; fn display_name(&self) -> &'static str; fn capabilities(&self) -> ContentCapabilities; + fn supports_operation(&self, _operation: ContentOperation) -> bool { + true + } fn input_schema(&self, operation: ContentOperation) -> Value; fn effective_input(&self, _operation: ContentOperation, input: &Value) -> Value { @@ -139,6 +142,7 @@ pub fn content_platform_tools(platform: Arc) -> Vec }) diff --git a/core/src/sites/dy/mod.rs b/core/src/sites/dy/mod.rs index 89ad2a1a..8f254b67 100644 --- a/core/src/sites/dy/mod.rs +++ b/core/src/sites/dy/mod.rs @@ -5,6 +5,6 @@ pub mod tools; pub use self::entities::DouyinVideoCard; pub use self::page::{DouyinPageRuntime, DOUYIN_HOME_URL}; pub use self::tools::{ - dy_agent_instructions, dy_agent_tools, dy_content_platform, dy_default_agent_tools, dy_tools, + dy_agent_instructions, dy_agent_tools, dy_content_platform, dy_tools, dy_tools_with_llm_provider, DY_KNOWLEDGE, DY_SITE, }; diff --git a/core/src/sites/dy/tools.rs b/core/src/sites/dy/tools.rs index 89165f7e..a6940ef8 100644 --- a/core/src/sites/dy/tools.rs +++ b/core/src/sites/dy/tools.rs @@ -7,7 +7,7 @@ use crate::agent::tool::ToolProgressSender; use crate::agent::{Backend as LlmProvider, Tool, ToolContext, ToolResult}; use crate::cdp::PageSession; use crate::sites::content::{ - author_scan_input_schema, content_platform_tools, empty_filters_schema, get_notes_input_schema, + author_scan_input_schema, empty_filters_schema, get_notes_input_schema, page_state_input_schema, search_input_schema, video_note_locator_schema, ContentCapabilities, ContentOperation, ContentPlatform, }; @@ -95,6 +95,13 @@ impl ContentPlatform for DouyinContentPlatform { } } + fn supports_operation(&self, operation: ContentOperation) -> bool { + matches!( + operation, + ContentOperation::Search | ContentOperation::PageState + ) + } + fn input_schema(&self, operation: ContentOperation) -> Value { match operation { ContentOperation::GetNotes => { @@ -119,13 +126,6 @@ impl ContentPlatform for DouyinContentPlatform { } async fn search(&self, mut input: Value, ctx: &ToolContext) -> anyhow::Result { - if !input - .get("preview") - .and_then(Value::as_bool) - .unwrap_or(false) - { - return Ok(Self::unsupported(ContentOperation::Search)); - } if let Some(num_notes) = input .as_object_mut() .and_then(|obj| obj.remove("num_notes")) @@ -167,16 +167,6 @@ pub fn dy_content_platform( }) } -pub async fn dy_default_agent_tools( - page: Arc, - llm_provider: Arc, -) -> anyhow::Result>> { - Ok(content_platform_tools(dy_content_platform( - page, - Some(llm_provider), - ))) -} - pub fn dy_agent_instructions(extra: &str) -> String { let base = DY_KNOWLEDGE.trim().to_string(); let extra = extra.trim(); @@ -195,9 +185,9 @@ pub static DY_SITE: SiteSpec = SiteSpec { home_url: "", content_platform: dy_content_platform, agent_tools: |page, llm| Box::pin(dy_agent_tools(page, llm)), - default_agent_tools: Some(|page, llm| Box::pin(dy_default_agent_tools(page, llm))), + default_agent_tools: None, agent_instructions: dy_agent_instructions, - default_agent_instructions: Some(dy_agent_instructions), + default_agent_instructions: None, commands: &[ SiteCommand { name: "search", From f25d70c73dbda0c3b52d990f8fc30fb422594313 Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 11:17:16 +0800 Subject: [PATCH 06/13] fix: require reusable video note locators --- core/src/sites/content.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/sites/content.rs b/core/src/sites/content.rs index d85038ac..ab9b7a41 100644 --- a/core/src/sites/content.rs +++ b/core/src/sites/content.rs @@ -187,9 +187,9 @@ pub fn video_note_locator_schema() -> Value { "type": "object", "properties": { "note_id": { "type": "string", "description": "Platform video/content id." }, - "url": { "type": "string", "description": "Canonical content URL when available." } + "url": { "type": "string", "description": "Canonical content URL returned by search or author_scan." } }, - "required": ["note_id"], + "required": ["note_id", "url"], "additionalProperties": false }) } From 9b8e1ea4890d662e0e515a6b502cb0cfd22f1803 Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 11:28:38 +0800 Subject: [PATCH 07/13] docs: align author scan sampling with XHS --- core/src/sites/content.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/sites/content.rs b/core/src/sites/content.rs index ab9b7a41..77284006 100644 --- a/core/src/sites/content.rs +++ b/core/src/sites/content.rs @@ -297,7 +297,11 @@ pub fn author_scan_input_schema(default_comments: i64, asr_enabled: bool) -> Val "type": "object", "properties": { "author_id": { "type": "string", "description": "Platform author id or profile URL." }, - "num_notes": { "type": "integer", "minimum": 1 }, + "num_notes": { + "type": "integer", + "description": "Collect at least this many note cards by scrolling. Omit for the first visible profile screen.", + "minimum": 1 + }, "num_comments": { "type": "integer", "description": "Comments to load per note. Ignored in preview mode.", From 9daf942b067aa56c229e4abb3c09f717210e8e8a Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 11:36:31 +0800 Subject: [PATCH 08/13] fix: describe adapter capabilities accurately --- core/src/sites/content.rs | 6 +++++- core/src/sites/dy/tools.rs | 20 ++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/core/src/sites/content.rs b/core/src/sites/content.rs index 77284006..db325d38 100644 --- a/core/src/sites/content.rs +++ b/core/src/sites/content.rs @@ -79,13 +79,17 @@ struct ContentPlatformTool { impl ContentPlatformTool { fn new(platform: Arc, operation: ContentOperation) -> Self { let site = platform.display_name(); + let capabilities = platform.capabilities(); let description = match operation { ContentOperation::GetNotes => format!( "Read one or more {site} notes by the locators returned from search or author_scan." ), - ContentOperation::Search => format!( + ContentOperation::Search if capabilities.full_search => format!( "Research {site} by keyword. The default is an XHS-style full scan; preview=true returns note cards only." ), + ContentOperation::Search => { + format!("Search {site} by keyword and return visible note cards.") + } ContentOperation::AuthorScan => format!( "Research one {site} author and their notes. The default is a full scan; preview=true returns note cards only." ), diff --git a/core/src/sites/dy/tools.rs b/core/src/sites/dy/tools.rs index a6940ef8..fddad031 100644 --- a/core/src/sites/dy/tools.rs +++ b/core/src/sites/dy/tools.rs @@ -7,9 +7,8 @@ use crate::agent::tool::ToolProgressSender; use crate::agent::{Backend as LlmProvider, Tool, ToolContext, ToolResult}; use crate::cdp::PageSession; use crate::sites::content::{ - author_scan_input_schema, empty_filters_schema, get_notes_input_schema, - page_state_input_schema, search_input_schema, video_note_locator_schema, ContentCapabilities, - ContentOperation, ContentPlatform, + author_scan_input_schema, get_notes_input_schema, page_state_input_schema, + video_note_locator_schema, ContentCapabilities, ContentOperation, ContentPlatform, }; use crate::sites::dy::DouyinPageRuntime; use crate::sites::registry::{ @@ -107,7 +106,20 @@ impl ContentPlatform for DouyinContentPlatform { ContentOperation::GetNotes => { get_notes_input_schema(video_note_locator_schema(), 8, false) } - ContentOperation::Search => search_input_schema(empty_filters_schema(), 10, 5, false), + ContentOperation::Search => json!({ + "type": "object", + "properties": { + "query": { "type": "string" }, + "num_notes": { + "type": "integer", + "description": "Number of visible video cards to collect by scrolling.", + "default": 10, + "minimum": 1 + } + }, + "required": ["query"], + "additionalProperties": false + }), ContentOperation::AuthorScan => author_scan_input_schema(5, false), ContentOperation::PageState => page_state_input_schema(), } From 6f69fa6982c6109bab18682689bc1ef579e8c240 Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 19:40:29 +0800 Subject: [PATCH 09/13] feat: replace hosted ASR with local Qwen3 worker --- .github/workflows/release.yml | 44 ++- Cargo.lock | 357 +++++++++++++++++++- Cargo.toml | 1 + README.md | 14 +- app/package.json | 1 + app/scripts/prepare-asr-helper.mjs | 46 +++ app/src-tauri/tauri.conf.json | 7 +- asr/Cargo.toml | 19 ++ asr/src/main.rs | 307 +++++++++++++++++ cli/src/main.rs | 108 +++++- core/Cargo.toml | 6 +- core/src/cloud/asr.rs | 194 ----------- core/src/cloud/mod.rs | 2 - core/src/media/asr.rs | 507 +++++++++++++++++++++++++++++ core/src/media/audio.rs | 213 +----------- core/src/media/common.rs | 6 +- core/src/media/mod.rs | 19 +- core/src/media/processor.rs | 13 +- core/src/media/video.rs | 7 +- core/src/sites/content.rs | 16 +- core/src/sites/xhs/knowledge.md | 10 +- core/src/sites/xhs/page.rs | 4 +- core/src/sites/xhs/tools.rs | 122 ++++--- scripts/install-cli.ps1 | 5 + scripts/install-cli.sh | 5 + 25 files changed, 1514 insertions(+), 519 deletions(-) create mode 100644 app/scripts/prepare-asr-helper.mjs create mode 100644 asr/Cargo.toml create mode 100644 asr/src/main.rs delete mode 100644 core/src/cloud/asr.rs create mode 100644 core/src/media/asr.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4da4259..feab5f98 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -184,6 +184,8 @@ jobs: cargo build -p socai-cli --release --target aarch64-apple-darwin cargo build -p socai-cli --release --target x86_64-apple-darwin + cargo build -p socai-asr --release --target aarch64-apple-darwin + cargo build -p socai-asr --release --target x86_64-apple-darwin cli_dir="target/socai-cli-macos-universal" mkdir -p "${cli_dir}" @@ -191,15 +193,22 @@ jobs: target/aarch64-apple-darwin/release/socai \ target/x86_64-apple-darwin/release/socai \ -output "${cli_dir}/socai" + lipo -create \ + target/aarch64-apple-darwin/release/socai-asr \ + target/x86_64-apple-darwin/release/socai-asr \ + -output "${cli_dir}/socai-asr" chmod 0755 "${cli_dir}/socai" - - actual_archs="$(lipo -archs "${cli_dir}/socai")" - echo "${cli_dir}/socai architectures: ${actual_archs}" - for expected_arch in arm64 x86_64; do - if [[ " ${actual_archs} " != *" ${expected_arch} "* ]]; then - echo "expected ${cli_dir}/socai to include ${expected_arch}, got: ${actual_archs}" >&2 - exit 1 - fi + chmod 0755 "${cli_dir}/socai-asr" + + for binary in socai socai-asr; do + actual_archs="$(lipo -archs "${cli_dir}/${binary}")" + echo "${cli_dir}/${binary} architectures: ${actual_archs}" + for expected_arch in arm64 x86_64; do + if [[ " ${actual_archs} " != *" ${expected_arch} "* ]]; then + echo "expected ${cli_dir}/${binary} to include ${expected_arch}, got: ${actual_archs}" >&2 + exit 1 + fi + done done version_output="$("${cli_dir}/socai" --version)" @@ -219,18 +228,25 @@ jobs: BUILD_SHA="${SOCAI_BUILD_SHA:?SOCAI_BUILD_SHA not set by apply release version step}" cli_binary="target/socai-cli-macos-universal/socai" + asr_binary="target/socai-cli-macos-universal/socai-asr" mkdir -p dist-artifacts if [ ! -x "${cli_binary}" ]; then echo "no CLI binary found at ${cli_binary}" >&2 exit 1 fi + if [ ! -x "${asr_binary}" ]; then + echo "no local ASR helper found at ${asr_binary}" >&2 + exit 1 + fi package_dir="${RUNNER_TEMP}/socai-cli-package" rm -rf "${package_dir}" mkdir -p "${package_dir}" cp "${cli_binary}" "${package_dir}/socai" + cp "${asr_binary}" "${package_dir}/socai-asr" chmod 0755 "${package_dir}/socai" + chmod 0755 "${package_dir}/socai-asr" created_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" cat > "${package_dir}/manifest.json" < socai-cli-macos-universal.tar.gz.sha256) cp scripts/install-cli.sh dist-artifacts/install.sh (cd dist-artifacts && shasum -a 256 -c socai-cli-macos-universal.tar.gz.sha256) archive_listing="${RUNNER_TEMP}/socai-cli-archive-files.txt" tar -tzf dist-artifacts/socai-cli-macos-universal.tar.gz > "${archive_listing}" - for required_file in socai manifest.json; do + for required_file in socai socai-asr manifest.json; do if ! grep -Fxq "${required_file}" "${archive_listing}"; then echo "CLI archive missing ${required_file}" >&2 exit 1 @@ -700,7 +716,7 @@ jobs: SOCAI_PRO_BASE_URL: ${{ secrets.SOCAI_PRO_BASE_URL }} run: | $ErrorActionPreference = 'Stop' - cargo build -p socai-cli --release + cargo build -p socai-cli -p socai-asr --release $versionOutput = & .\target\release\socai.exe --version $versionOutput if ($versionOutput -ne "socai $env:VERSION") { @@ -762,6 +778,7 @@ jobs: Remove-Item -Recurse -Force $dist, $packageDir -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $dist, $packageDir | Out-Null Copy-Item .\target\release\socai.exe (Join-Path $packageDir 'socai.exe') + Copy-Item .\target\release\socai-asr.exe (Join-Path $packageDir 'socai-asr.exe') $createdAt = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') [ordered]@{ base_sha = $env:BASE_SHA @@ -772,7 +789,7 @@ jobs: } | ConvertTo-Json | Set-Content -Encoding utf8NoBOM (Join-Path $packageDir 'manifest.json') $archive = Join-Path $dist 'socai-cli-windows-x86_64.zip' - Compress-Archive -Path (Join-Path $packageDir 'socai.exe'), (Join-Path $packageDir 'manifest.json') -DestinationPath $archive + Compress-Archive -Path (Join-Path $packageDir 'socai.exe'), (Join-Path $packageDir 'socai-asr.exe'), (Join-Path $packageDir 'manifest.json') -DestinationPath $archive $hash = (Get-FileHash -Algorithm SHA256 $archive).Hash.ToLowerInvariant() $checksumPath = "$archive.sha256" [System.IO.File]::WriteAllText($checksumPath, "$hash socai-cli-windows-x86_64.zip`n", [System.Text.Encoding]::ASCII) @@ -789,6 +806,9 @@ jobs: if ($versionOutput -ne "socai $env:VERSION") { throw "expected archive CLI version 'socai $env:VERSION', got: $versionOutput" } + if (-not (Test-Path (Join-Path $verifyDir 'socai-asr.exe'))) { + throw 'archive missing socai-asr.exe' + } $manifest = Get-Content -Raw (Join-Path $verifyDir 'manifest.json') | ConvertFrom-Json if ($manifest.version -ne $env:VERSION) { throw 'manifest version mismatch' } if ($manifest.target -ne 'windows-x86_64') { throw 'manifest target mismatch' } diff --git a/Cargo.lock b/Cargo.lock index 32d2a5f6..0a24ecf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.8.12" @@ -466,6 +477,44 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cairo-rs" version = "0.18.5" @@ -665,6 +714,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -781,6 +840,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "cookie" version = "0.18.1" @@ -859,6 +924,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1089,6 +1169,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + [[package]] name = "der" version = "0.8.0" @@ -1180,6 +1266,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1457,6 +1544,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fast-float2" version = "0.2.4" @@ -1855,9 +1948,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -2127,6 +2222,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hmac-sha256" version = "1.1.14" @@ -2527,6 +2631,15 @@ dependencies = [ "cfb", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "inquire" version = "0.7.5" @@ -2817,6 +2930,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + [[package]] name = "libc" version = "0.2.186" @@ -2903,12 +3022,33 @@ dependencies = [ "imgref", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "lzma-rust2" version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "macro_rules_attribute" version = "0.2.2" @@ -3779,7 +3919,7 @@ dependencies = [ "ort-sys", "smallvec", "tracing", - "ureq", + "ureq 3.3.0", ] [[package]] @@ -3790,7 +3930,7 @@ checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" dependencies = [ "hmac-sha256", "lzma-rust2", - "ureq", + "ureq 3.3.0", ] [[package]] @@ -3886,6 +4026,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -4649,6 +4799,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -5098,6 +5249,29 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "sherpa-onnx" +version = "1.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4ef11f6b23c916a8e36c3cf65201f796d0e6aa30182a18d4f798d0b7f62547" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566aad07d0924a7ed897cc035cc67ad71cdfddb959717dbf445031d5d4b58d4d" +dependencies = [ + "bzip2 0.4.4", + "tar", + "ureq 2.12.1", + "zip 2.4.2", +] + [[package]] name = "shlex" version = "1.3.0" @@ -5207,6 +5381,17 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socai-asr" +version = "0.5.5" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "sherpa-onnx", + "symphonia", +] + [[package]] name = "socai-cli" version = "0.5.5" @@ -5234,6 +5419,7 @@ dependencies = [ "async-trait", "async-tungstenite", "base64 0.22.1", + "bzip2 0.6.1", "chrono", "dirs 5.0.1", "futures", @@ -5243,7 +5429,8 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "symphonia", + "sha2", + "tar", "tempfile", "thiserror 2.0.18", "tokio", @@ -5441,11 +5628,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" dependencies = [ "lazy_static", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-pcm", "symphonia-core", "symphonia-format-isomp4", + "symphonia-format-riff", "symphonia-metadata", ] +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + [[package]] name = "symphonia-core" version = "0.5.5" @@ -5472,6 +5696,18 @@ dependencies = [ "symphonia-utils-xiph", ] +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "symphonia-metadata" version = "0.5.5" @@ -6569,6 +6805,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "ureq" version = "3.3.0" @@ -6958,6 +7210,24 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webview2-com" version = "0.38.2" @@ -7740,6 +8010,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "y4m" version = "0.8.0" @@ -7815,6 +8094,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" @@ -7849,6 +8142,36 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.5.2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.4", + "hmac", + "indexmap 2.14.0", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.18", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + [[package]] name = "zip" version = "4.6.1" @@ -7899,6 +8222,34 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index ee6c1c06..0bd50213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "app/src-tauri", + "asr", "core", "cli", ] diff --git a/README.md b/README.md index 9343a183..2dc31fa4 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ If a prebuilt binary is unavailable for your platform, or you need a source buil ```bash git clone https://github.com/socai-io/socai.git cd socai +cargo install --path asr --force cargo install --path cli --force ``` @@ -170,7 +171,7 @@ socai xhs get-notes \ | `--preview` | Read only search-result or author-page post cards. | | `--download-media` | Download images and videos from opened posts and record local paths. | | `--ocr` | Run local OCR on post images or a video post's cover. | -| `--transcribe-audio` | Download opened videos and transcribe speech; requires signing in and selecting socai agent. | +| `--transcribe-audio` | Download opened videos and transcribe speech locally with Qwen3-ASR 0.6B Int8. | | `--filter ` | Apply a Xiaohongshu search-page filter; repeat to combine filters. | | `--pretty` | Pretty-print the final JSON result. | | `--debug-snapshot` | Save page DOM, accessibility trees, and screenshots for development diagnostics. | @@ -194,6 +195,17 @@ socai xhs search "Shanghai weekend activities" \ --filter sort=最新 ``` +### Local video transcription + +Install the local Qwen3-ASR model once before using `--transcribe-audio`: + +```bash +socai asr install +socai asr status --json +``` + +The compressed download is about 0.88 GB and the installed model uses about 0.95 GB. Audio stays on the device, and transcription does not require a socai account, credits, or an API key. + ## Browser and login modes socai supports four Chrome profile modes: diff --git a/app/package.json b/app/package.json index 60af3fc4..39418bc4 100644 --- a/app/package.json +++ b/app/package.json @@ -11,6 +11,7 @@ "dev:desktop:kill": "pkill -f 'target/debug/socai_ap[p]' 2>/dev/null; lsof -ti :1421 | xargs kill 2>/dev/null; for i in $(seq 1 20); do lsof -ti :1421 >/dev/null 2>&1 || break; sleep 0.1; done; true", "build": "tsc && vite build", "prepare:lark-cli": "node scripts/prepare-lark-cli.mjs", + "prepare:asr-helper": "node scripts/prepare-asr-helper.mjs", "build:mac": "tauri build --ci --target universal-apple-darwin --bundles app,dmg --config '{\"bundle\":{\"macOS\":{\"signingIdentity\":\"-\"}}}'", "build:mac:universal": "tauri build --ci --target universal-apple-darwin --bundles app,dmg --config '{\"bundle\":{\"macOS\":{\"signingIdentity\":\"-\"}}}'", "build:mac:apple-silicon": "tauri build --ci --target aarch64-apple-darwin --bundles app,dmg --config '{\"bundle\":{\"macOS\":{\"signingIdentity\":\"-\"}}}'", diff --git a/app/scripts/prepare-asr-helper.mjs b/app/scripts/prepare-asr-helper.mjs new file mode 100644 index 00000000..242d2482 --- /dev/null +++ b/app/scripts/prepare-asr-helper.mjs @@ -0,0 +1,46 @@ +import { chmodSync, copyFileSync, existsSync, mkdirSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +const APP_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const REPO_DIR = path.resolve(APP_DIR, ".."); +const BIN_DIR = path.join(APP_DIR, "src-tauri", "binaries"); +const explicitTarget = process.env.TAURI_ENV_TARGET_TRIPLE; +const release = process.env.TAURI_ENV_DEBUG === "false" || explicitTarget === "universal-apple-darwin"; +const profile = release ? "release" : "debug"; + +mkdirSync(BIN_DIR, { recursive: true }); + +function rustHost() { + const output = execFileSync("rustc", ["-vV"], { cwd: REPO_DIR, encoding: "utf8" }); + const line = output.split("\n").find((item) => item.startsWith("host: ")); + if (!line) throw new Error("rustc -vV did not report a host target"); + return line.slice("host: ".length).trim(); +} + +function build(target) { + const args = ["build", "-p", "socai-asr", "--target", target]; + if (release) args.push("--release"); + execFileSync("cargo", args, { cwd: REPO_DIR, stdio: "inherit" }); + const extension = target.includes("windows") ? ".exe" : ""; + const source = path.join(REPO_DIR, "target", target, profile, `socai-asr${extension}`); + if (!existsSync(source)) throw new Error(`ASR helper build missing: ${source}`); + const destination = path.join(BIN_DIR, `socai-asr-${target}${extension}`); + copyFileSync(source, destination); + if (!extension) chmodSync(destination, 0o755); + console.log(`[socai-asr] ready ${path.relative(APP_DIR, destination)}`); + return destination; +} + +const target = explicitTarget || rustHost(); +if (target === "universal-apple-darwin") { + const arm = build("aarch64-apple-darwin"); + const intel = build("x86_64-apple-darwin"); + const universal = path.join(BIN_DIR, "socai-asr-universal-apple-darwin"); + execFileSync("lipo", ["-create", arm, intel, "-output", universal], { stdio: "inherit" }); + chmodSync(universal, 0o755); + console.log(`[socai-asr] ready ${path.relative(APP_DIR, universal)}`); +} else { + build(target); +} diff --git a/app/src-tauri/tauri.conf.json b/app/src-tauri/tauri.conf.json index 0c3e5390..be6af9c5 100644 --- a/app/src-tauri/tauri.conf.json +++ b/app/src-tauri/tauri.conf.json @@ -4,9 +4,9 @@ "version": "0.5.5", "identifier": "com.socai.app", "build": { - "beforeDevCommand": "pnpm run prepare:lark-cli && pnpm run dev", + "beforeDevCommand": "pnpm run prepare:lark-cli && pnpm run prepare:asr-helper && pnpm run dev", "devUrl": "http://localhost:1421", - "beforeBuildCommand": "pnpm run prepare:lark-cli && pnpm run build", + "beforeBuildCommand": "pnpm run prepare:lark-cli && pnpm run prepare:asr-helper && pnpm run build", "frontendDist": "../dist" }, "app": { @@ -38,7 +38,8 @@ "active": true, "targets": "all", "externalBin": [ - "binaries/lark-cli" + "binaries/lark-cli", + "binaries/socai-asr" ], "resources": [ "third-party/lark-cli-LICENSE" diff --git a/asr/Cargo.toml b/asr/Cargo.toml new file mode 100644 index 00000000..bcb9fb2a --- /dev/null +++ b/asr/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "socai-asr" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "socai-asr" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +sherpa-onnx = "=1.13.7" +symphonia = { version = "0.5", default-features = false, features = ["aac", "isomp4", "mp3", "pcm", "wav"] } + +[lints] +workspace = true diff --git a/asr/src/main.rs b/asr/src/main.rs new file mode 100644 index 00000000..f1a9ba98 --- /dev/null +++ b/asr/src/main.rs @@ -0,0 +1,307 @@ +use std::fs::File; +use std::io::{BufRead, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context, Result}; +use serde::{Deserialize, Serialize}; +use sherpa_onnx::{ + LinearResampler, OfflineQwen3ASRModelConfig, OfflineRecognizer, OfflineRecognizerConfig, + VadModelConfig, VoiceActivityDetector, +}; +use symphonia::core::audio::SampleBuffer; +use symphonia::core::codecs::{Decoder, DecoderOptions}; +use symphonia::core::errors::Error as SymphoniaError; +use symphonia::core::formats::FormatOptions; +use symphonia::core::io::MediaSourceStream; +use symphonia::core::meta::MetadataOptions; +use symphonia::core::probe::Hint; + +const PROTOCOL_VERSION: u32 = 1; + +#[derive(Deserialize)] +struct Request { + protocol: u32, + id: u64, + path: String, + max_seconds: u64, +} + +#[derive(Serialize)] +struct Response { + protocol: u32, + id: u64, + ok: bool, + transcript: Option, + error: Option, +} + +struct ModelPaths { + conv_frontend: PathBuf, + encoder: PathBuf, + decoder: PathBuf, + tokenizer: PathBuf, + vad: PathBuf, +} + +impl ModelPaths { + fn new(root: &Path) -> Self { + Self { + conv_frontend: root.join("conv_frontend.onnx"), + encoder: root.join("encoder.int8.onnx"), + decoder: root.join("decoder.int8.onnx"), + tokenizer: root.join("tokenizer"), + vad: root.join("silero_vad.onnx"), + } + } +} + +fn main() -> Result<()> { + let model_dir = parse_model_dir()?; + let paths = ModelPaths::new(&model_dir); + let recognizer = create_recognizer(&paths)?; + let stdin = std::io::stdin(); + let mut stdout = std::io::BufWriter::new(std::io::stdout().lock()); + + for line in stdin.lock().lines() { + let line = line?; + if line.trim().is_empty() { + continue; + } + let response = match serde_json::from_str::(&line) { + Ok(request) if request.protocol == PROTOCOL_VERSION => { + match transcribe_file( + Path::new(&request.path), + request.max_seconds, + &paths, + &recognizer, + ) { + Ok(transcript) => Response { + protocol: PROTOCOL_VERSION, + id: request.id, + ok: true, + transcript: Some(transcript), + error: None, + }, + Err(error) => Response { + protocol: PROTOCOL_VERSION, + id: request.id, + ok: false, + transcript: None, + error: Some(format!("{error:#}")), + }, + } + } + Ok(request) => Response { + protocol: PROTOCOL_VERSION, + id: request.id, + ok: false, + transcript: None, + error: Some(format!( + "unsupported ASR protocol {}; expected {PROTOCOL_VERSION}", + request.protocol + )), + }, + Err(error) => Response { + protocol: PROTOCOL_VERSION, + id: 0, + ok: false, + transcript: None, + error: Some(format!("invalid ASR request: {error}")), + }, + }; + serde_json::to_writer(&mut stdout, &response)?; + stdout.write_all(b"\n")?; + stdout.flush()?; + } + Ok(()) +} + +fn parse_model_dir() -> Result { + let mut args = std::env::args_os().skip(1); + let mut serve = false; + let mut model_dir = None; + while let Some(arg) = args.next() { + if arg == "--serve" { + serve = true; + } else if arg == "--model-dir" { + model_dir = args.next().map(PathBuf::from); + } else { + anyhow::bail!("unknown argument: {}", PathBuf::from(arg).display()); + } + } + if !serve { + anyhow::bail!("socai-asr is an internal worker and requires --serve"); + } + model_dir.context("--model-dir is required") +} + +fn transcribe_file( + path: &Path, + max_seconds: u64, + paths: &ModelPaths, + recognizer: &OfflineRecognizer, +) -> Result { + let (samples, sample_rate) = decode_audio_file(path, max_seconds)?; + let samples = if sample_rate == 16_000 { + samples + } else { + let resampler = LinearResampler::create(sample_rate, 16_000) + .ok_or_else(|| anyhow!("failed to create {sample_rate} Hz to 16000 Hz resampler"))?; + resampler.resample(&samples, true) + }; + if samples.is_empty() { + anyhow::bail!("{} contains no decoded audio samples", path.display()); + } + + let mut vad_config = VadModelConfig::default(); + vad_config.silero_vad.model = Some(paths.vad.display().to_string()); + vad_config.silero_vad.threshold = 0.5; + vad_config.silero_vad.min_silence_duration = 0.2; + vad_config.silero_vad.min_speech_duration = 0.2; + vad_config.silero_vad.max_speech_duration = 20.0; + vad_config.silero_vad.window_size = 512; + vad_config.sample_rate = 16_000; + vad_config.num_threads = 1; + vad_config.provider = Some("cpu".into()); + let vad = VoiceActivityDetector::create(&vad_config, 30.0) + .ok_or_else(|| anyhow!("failed to initialize local Silero VAD"))?; + + let mut transcripts = Vec::new(); + for chunk in samples.chunks(512) { + vad.accept_waveform(chunk); + decode_ready_segments(&vad, recognizer, &mut transcripts); + } + vad.flush(); + decode_ready_segments(&vad, recognizer, &mut transcripts); + + if transcripts.is_empty() && samples.len() <= 30 * 16_000 { + decode_samples(recognizer, &samples, &mut transcripts); + } + let transcript = transcripts.join("\n").trim().to_string(); + if transcript.is_empty() { + anyhow::bail!("local Qwen3-ASR found no speech in {}", path.display()); + } + Ok(transcript) +} + +fn create_recognizer(paths: &ModelPaths) -> Result { + let mut config = OfflineRecognizerConfig::default(); + config.model_config.qwen3_asr = OfflineQwen3ASRModelConfig { + conv_frontend: Some(paths.conv_frontend.display().to_string()), + encoder: Some(paths.encoder.display().to_string()), + decoder: Some(paths.decoder.display().to_string()), + tokenizer: Some(paths.tokenizer.display().to_string()), + max_new_tokens: 512, + ..Default::default() + }; + config.model_config.tokens = Some(String::new()); + config.model_config.provider = Some("cpu".into()); + config.model_config.num_threads = std::thread::available_parallelism() + .map(|count| count.get().clamp(2, 4) as i32) + .unwrap_or(2); + OfflineRecognizer::create(&config).ok_or_else(|| anyhow!("failed to load Qwen3-ASR model")) +} + +fn decode_ready_segments( + vad: &VoiceActivityDetector, + recognizer: &OfflineRecognizer, + transcripts: &mut Vec, +) { + while let Some(segment) = vad.front() { + let samples = segment.samples().to_vec(); + drop(segment); + vad.pop(); + decode_samples(recognizer, &samples, transcripts); + } +} + +fn decode_samples(recognizer: &OfflineRecognizer, samples: &[f32], transcripts: &mut Vec) { + if samples.is_empty() { + return; + } + let stream = recognizer.create_stream(); + stream.accept_waveform(16_000, samples); + recognizer.decode(&stream); + if let Some(result) = stream.get_result() { + let text = result.text.trim(); + if !text.is_empty() { + transcripts.push(text.to_string()); + } + } +} + +fn decode_audio_file(path: &Path, max_seconds: u64) -> Result<(Vec, i32)> { + let file = File::open(path).with_context(|| format!("failed to open {}", path.display()))?; + let stream = MediaSourceStream::new(Box::new(file), Default::default()); + let mut hint = Hint::new(); + if let Some(ext) = path.extension().and_then(|ext| ext.to_str()) { + hint.with_extension(ext); + } + let probed = symphonia::default::get_probe() + .format( + &hint, + stream, + &FormatOptions::default(), + &MetadataOptions::default(), + ) + .with_context(|| format!("unsupported media container in {}", path.display()))?; + let mut format = probed.format; + let (track_id, mut decoder) = audio_decoder(&mut *format) + .ok_or_else(|| anyhow!("no supported audio track in {}", path.display()))?; + let mut mono = Vec::new(); + let mut sample_rate = None; + + loop { + let packet = match format.next_packet() { + Ok(packet) => packet, + Err(SymphoniaError::IoError(err)) + if err.kind() == std::io::ErrorKind::UnexpectedEof => + { + break; + } + Err(SymphoniaError::ResetRequired) => { + anyhow::bail!("audio stream changed format in {}", path.display()) + } + Err(err) => return Err(err.into()), + }; + if packet.track_id() != track_id { + continue; + } + let decoded = match decoder.decode(&packet) { + Ok(decoded) => decoded, + Err(SymphoniaError::DecodeError(_)) => continue, + Err(err) => return Err(err.into()), + }; + let spec = *decoded.spec(); + let rate = spec.rate as i32; + if sample_rate.is_some_and(|current| current != rate) { + anyhow::bail!("audio sample rate changed in {}", path.display()); + } + sample_rate = Some(rate); + let mut buffer = SampleBuffer::::new(decoded.capacity() as u64, spec); + buffer.copy_interleaved_ref(decoded); + let channels = spec.channels.count().max(1); + for frame in buffer.samples().chunks(channels) { + mono.push(frame.iter().copied().sum::() / frame.len() as f32); + if mono.len() >= max_seconds.saturating_mul(rate as u64) as usize { + return Ok((mono, rate)); + } + } + } + let rate = + sample_rate.ok_or_else(|| anyhow!("{} contains no decoded audio", path.display()))?; + Ok((mono, rate)) +} + +fn audio_decoder( + format: &mut dyn symphonia::core::formats::FormatReader, +) -> Option<(u32, Box)> { + for track in format.tracks() { + if let Ok(decoder) = + symphonia::default::get_codecs().make(&track.codec_params, &DecoderOptions::default()) + { + return Some((track.id, decoder)); + } + } + None +} diff --git a/cli/src/main.rs b/cli/src/main.rs index be74439b..5ffd6c0d 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -5,6 +5,7 @@ mod version; use anyhow::Result; use clap::{Arg, ArgAction, ArgMatches}; +use indicatif::{ProgressBar, ProgressStyle}; use serde_json::{Map, Value}; use socai_core::cloud as socai_pro; use socai_core::config as socai_config; @@ -92,6 +93,44 @@ fn build_cli() -> clap::Command { ), ), ) + .subcommand( + clap::Command::new("asr") + .about("Manage fully local Qwen3-ASR video transcription.") + .subcommand( + clap::Command::new("status") + .about("Show local ASR model readiness.") + .arg( + Arg::new("json") + .long("json") + .action(ArgAction::SetTrue) + .help("Print machine-readable JSON."), + ), + ) + .subcommand( + clap::Command::new("install") + .about("Download and verify Qwen3-ASR once for offline inference."), + ) + .subcommand( + clap::Command::new("transcribe") + .about("Transcribe a local audio or video file without an ASR API.") + .arg(Arg::new("path").required(true).value_name("FILE")) + .arg( + Arg::new("max-seconds") + .long("max-seconds") + .value_parser(clap::value_parser!(u64)) + .default_value("1200") + .help("Maximum leading audio duration to transcribe."), + ) + .arg( + Arg::new("json") + .long("json") + .action(ArgAction::SetTrue) + .help("Print machine-readable JSON."), + ), + ) + .subcommand_required(true) + .arg_required_else_help(true), + ) .subcommand(clap::Command::new("__daemon").hide(true)); for site in all_sites() { let mut site_cmd = clap::Command::new(site.id) @@ -237,7 +276,7 @@ async fn run_site_command( fn should_warn_for_update(subcommand: &str) -> bool { !matches!( subcommand, - "__daemon" | "update" | "version" | "config" | "pro" + "__daemon" | "update" | "version" | "config" | "pro" | "asr" ) } @@ -271,6 +310,7 @@ async fn main() -> Result<()> { "update" => version::run_update_command().await?, "config" => run_config_command(sub_matches)?, "pro" => run_pro_command(sub_matches).await?, + "asr" => run_asr_command(sub_matches).await?, "stop" => { // Graceful shutdown reaches whoever owns the IPC endpoint; the // sweep then kills any orphan daemon from any binary or SOCAI_HOME, @@ -303,6 +343,72 @@ async fn main() -> Result<()> { Ok(()) } +async fn run_asr_command(matches: &ArgMatches) -> Result<()> { + match matches.subcommand() { + Some(("install", _)) => { + let progress = ProgressBar::new(878_702_423); + progress.set_style( + ProgressStyle::with_template( + "{spinner:.white} {msg} [{bar:36.white/dim}] {bytes}/{total_bytes} {bytes_per_sec} {eta}", + )? + .progress_chars("=>-"), + ); + let mut stage = String::new(); + let progress_view = progress.clone(); + let status = socai_core::media::install_asr_model(move |event| { + if event.stage != stage { + stage = event.stage.clone(); + progress_view.set_message(match stage.as_str() { + "download" => "Downloading Qwen3-ASR", + "extract" => "Extracting Qwen3-ASR", + "vad" => "Downloading local VAD", + "complete" => "Local ASR ready", + _ => "Preparing local ASR", + }); + } + if let Some(total) = event.total_bytes { + progress_view.set_length(total); + progress_view.set_position(event.downloaded_bytes.min(total)); + } + }) + .await; + progress.finish_and_clear(); + let status = status?; + println!("{}", serde_json::to_string_pretty(&status)?); + } + Some(("transcribe", sub)) => { + let path = sub.get_one::("path").expect("path is required"); + let max_seconds = *sub + .get_one::("max-seconds") + .expect("max-seconds has a default"); + let spinner = ProgressBar::new_spinner(); + spinner.set_message("Transcribing locally with Qwen3-ASR"); + spinner.enable_steady_tick(std::time::Duration::from_millis(100)); + let result = socai_core::media::transcribe_local_file(path, max_seconds).await; + spinner.finish_and_clear(); + let transcript = result?; + if sub.get_flag("json") { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "model": "Qwen3-ASR-0.6B-Int8", + "runtime": "local/sherpa-onnx", + "path": path, + "transcript": transcript, + }))? + ); + } else { + println!("{transcript}"); + } + } + _ => { + let status = socai_core::media::asr_model_status()?; + println!("{}", serde_json::to_string_pretty(&status)?); + } + } + Ok(()) +} + async fn run_pro_command(matches: &ArgMatches) -> Result<()> { match matches.subcommand() { Some(("activate", sub)) => { diff --git a/core/Cargo.toml b/core/Cargo.toml index 2e98df2a..fd69cfb3 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -22,9 +22,9 @@ uuid = { version = "1", features = ["v4"] } chrono = { version = "0.4", features = ["serde"] } # Image compositing/resizing for batching note images into 2x2 vision grids. image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp"] } -# Pure-Rust mp4 demux for transcription: the AAC track is repacked as ADTS and -# uploaded as-is (cloud ASR accepts aac), so no decoder and no ffmpeg needed. -symphonia = { version = "0.5", default-features = false, features = ["isomp4"] } +sha2 = "0.10" +bzip2 = "0.6" +tar = "0.4" # Local OCR engine (PP-OCRv6 tiny via ONNX Runtime). Default features pull a # prebuilt ONNX Runtime at build time (download-binaries) + SIMD CPU kernels. # CPU-only by design: benchmarks showed CoreML/DirectML run PP-OCR slower, so we diff --git a/core/src/cloud/asr.rs b/core/src/cloud/asr.rs deleted file mode 100644 index 36e7fed2..00000000 --- a/core/src/cloud/asr.rs +++ /dev/null @@ -1,194 +0,0 @@ -use std::path::Path; -use std::time::{Duration, Instant}; - -use anyhow::{Context, Result}; -use serde::Deserialize; -use serde_json::json; -use tokio::io::AsyncReadExt; - -use super::auth::{bearer, configured_base_url, http_client, load_credentials}; - -const TASK_POLL_INTERVAL: Duration = Duration::from_secs(1); -const AUDIO_UPLOAD_TIMEOUT: Duration = Duration::from_secs(600); -const MAX_AUDIO_UPLOAD_BYTES: u64 = 128 * 1024 * 1024; -/// Consecutive poll failures tolerated before giving up on a submitted task. -const MAX_POLL_FAILURES: u32 = 3; - -#[derive(Debug, Deserialize)] -struct UploadUrlResponse { - task_id: String, - upload_url: String, - headers: std::collections::HashMap, -} - -#[derive(Debug, Deserialize)] -struct TaskResponse { - status: String, - transcript: Option, - error: Option, - provider_latency_ms: i64, -} - -#[derive(Debug, Clone)] -pub struct CloudAsrResult { - pub transcript: String, - pub provider_latency_ms: i64, - pub total_latency_ms: u128, -} - -pub async fn transcribe_audio_file( - path: &Path, - duration_s: i64, - timeout: Duration, - client_task_id: Option<&str>, -) -> Result { - let base_url = configured_base_url() - .ok_or_else(|| anyhow::anyhow!("socai server URL is not configured"))?; - let creds = load_credentials().ok_or_else(|| { - anyhow::anyhow!("sign in and select socai agent before requesting video transcription") - })?; - if creds.user_id.trim().is_empty() || !creds.hosted_llm_selected { - anyhow::bail!("sign in and select socai agent before requesting video transcription"); - } - // The extension matters: the server passes the filename through to - // DashScope, which detects the audio format from it. - let filename = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("audio.aac"); - let size_bytes = tokio::fs::metadata(path) - .await - .with_context(|| format!("failed to read metadata for {}", path.display()))? - .len(); - if size_bytes > MAX_AUDIO_UPLOAD_BYTES { - anyhow::bail!( - "audio upload is too large: {} bytes exceeds the 128 MiB limit", - size_bytes - ); - } - let client = http_client()?; - let started = Instant::now(); - let upload: UploadUrlResponse = bearer( - client - .post(format!("{base_url}/v1/asr/upload-url")) - .json(&json!({ - "filename": filename, - "content_type": "audio/aac", - "size_bytes": size_bytes, - "duration_s": duration_s.max(0), - "client_task_id": client_task_id.unwrap_or(""), - })), - &creds.device_token, - ) - .send() - .await? - .error_for_status()? - .json() - .await?; - - let file = tokio::fs::File::open(path) - .await - .with_context(|| format!("failed to open {} for upload", path.display()))?; - let stream = futures::stream::try_unfold(file, |mut file| async move { - let mut chunk = vec![0u8; 64 * 1024]; - let read = file.read(&mut chunk).await?; - if read == 0 { - return Ok::<_, std::io::Error>(None); - } - chunk.truncate(read); - Ok(Some((chunk, file))) - }); - let mut put = client - .put(&upload.upload_url) - .timeout(AUDIO_UPLOAD_TIMEOUT) - .header(reqwest::header::CONTENT_LENGTH, size_bytes) - .body(reqwest::Body::wrap_stream(stream)); - for (key, value) in &upload.headers { - put = put.header(key, value); - } - put.send().await?.error_for_status()?; - - bearer( - client.post(format!("{base_url}/v1/asr/tasks/{}/submit", upload.task_id)), - &creds.device_token, - ) - .send() - .await? - .error_for_status()?; - - let deadline = Instant::now() + timeout; - let mut poll_failures: u32 = 0; - loop { - // The upload is already done at this point; tolerate a few transient - // poll errors (5xx, network blips) instead of abandoning the task. - match poll_task(&client, &base_url, &upload.task_id, &creds.device_token).await { - Ok(task) => { - poll_failures = 0; - match task.status.as_str() { - "succeeded" => { - return Ok(CloudAsrResult { - transcript: task.transcript.unwrap_or_default(), - provider_latency_ms: task.provider_latency_ms, - total_latency_ms: started.elapsed().as_millis(), - }); - } - "failed" => { - let error = task.error.unwrap_or_else(|| "unknown error".into()); - // Fun-ASR reports an audio track it decoded fine but - // heard no speech in (music/SFX-only videos) as a - // failure. Surface the meaning, not the provider blob. - if error.contains("ASR_RESPONSE_HAVE_NO_WORDS") { - anyhow::bail!( - "no speech detected in the video's audio track (it likely \ - contains only music or sound effects), so there is no \ - spoken content to transcribe" - ); - } - anyhow::bail!("video transcription failed: {}", short_error(&error)); - } - _ => {} - } - } - Err(err) => { - poll_failures += 1; - if poll_failures >= MAX_POLL_FAILURES { - return Err(err.context("video transcription status polling failed")); - } - } - } - if Instant::now() >= deadline { - anyhow::bail!("video transcription timed out after {}s", timeout.as_secs()); - } - tokio::time::sleep(TASK_POLL_INTERVAL).await; - } -} - -async fn poll_task( - client: &reqwest::Client, - base_url: &str, - task_id: &str, - token: &str, -) -> Result { - Ok(bearer( - client.get(format!("{base_url}/v1/asr/tasks/{task_id}")), - token, - ) - .send() - .await? - .error_for_status()? - .json() - .await?) -} - -/// Trim a provider error to something an agent can read. Raw DashScope task -/// dumps run to kilobytes of nested JSON with signed URLs; the leading part -/// carries the task id + failure code, which is all a transcript_error needs. -fn short_error(error: &str) -> String { - const MAX: usize = 300; - let trimmed = error.trim(); - if trimmed.chars().count() <= MAX { - return trimmed.to_string(); - } - let head: String = trimmed.chars().take(MAX).collect(); - format!("{head}… (truncated)") -} diff --git a/core/src/cloud/mod.rs b/core/src/cloud/mod.rs index 681d07a3..ba46efd6 100644 --- a/core/src/cloud/mod.rs +++ b/core/src/cloud/mod.rs @@ -1,11 +1,9 @@ //! Managed socai services shared by the CLI and desktop app. -mod asr; mod auth; mod billing; mod browser; -pub use asr::{transcribe_audio_file, CloudAsrResult}; pub use auth::{ activate, activate_with_base_url, auth_session, hosted_llm_selected, llm_gateway_config, logout, pro_activated, redeem_invite, send_sms_code, set_hosted_llm_selected, status, diff --git a/core/src/media/asr.rs b/core/src/media/asr.rs new file mode 100644 index 00000000..9792b27a --- /dev/null +++ b/core/src/media/asr.rs @@ -0,0 +1,507 @@ +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::OnceLock; + +use anyhow::{anyhow, Context, Result}; +use bzip2::read::BzDecoder; +use futures::StreamExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tar::Archive; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines}; +use tokio::process::{Child, ChildStdin, ChildStdout, Command}; + +const MODEL_ID: &str = "sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25"; +const MODEL_ARCHIVE: &str = "sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2"; +const MODEL_URL: &str = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2"; +const MODEL_SHA256: &str = "393f8a14e2f5fb96746aaab342997a40641001fbd5bf9592a080a8329178ee96"; +const MODEL_ARCHIVE_BYTES: u64 = 878_702_423; +const VAD_FILE: &str = "silero_vad.onnx"; +const VAD_URL: &str = + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx"; +const VAD_SHA256: &str = "9e2449e1087496d8d4caba907f23e0bd3f78d91fa552479bb9c23ac09cbb1fd6"; +const PROTOCOL_VERSION: u32 = 1; + +#[derive(Debug, Clone, Serialize)] +pub struct AsrModelStatus { + pub model: &'static str, + pub runtime: &'static str, + pub installed: bool, + pub helper_available: bool, + pub available: bool, + pub model_dir: String, + pub helper_path: Option, + pub missing_files: Vec, + pub archive_bytes: u64, + pub license: &'static str, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AsrInstallProgress { + pub stage: String, + pub downloaded_bytes: u64, + pub total_bytes: Option, +} + +#[derive(Clone)] +struct ModelPaths { + root: PathBuf, + conv_frontend: PathBuf, + encoder: PathBuf, + decoder: PathBuf, + tokenizer: PathBuf, + vad: PathBuf, +} + +impl ModelPaths { + fn from_root(root: PathBuf) -> Self { + Self { + conv_frontend: root.join("conv_frontend.onnx"), + encoder: root.join("encoder.int8.onnx"), + decoder: root.join("decoder.int8.onnx"), + tokenizer: root.join("tokenizer"), + vad: root.join(VAD_FILE), + root, + } + } + + fn missing_files(&self) -> Vec { + let mut missing = Vec::new(); + for path in [&self.conv_frontend, &self.encoder, &self.decoder, &self.vad] { + if !path.is_file() || std::fs::metadata(path).is_ok_and(|meta| meta.len() == 0) { + missing.push(relative_display(&self.root, path)); + } + } + let tokenizer_ready = self.tokenizer.is_dir() + && std::fs::read_dir(&self.tokenizer) + .ok() + .and_then(|mut entries| entries.next()) + .is_some(); + if !tokenizer_ready { + missing.push("tokenizer/".into()); + } + missing + } +} + +#[derive(Serialize)] +struct WorkerRequest<'a> { + protocol: u32, + id: u64, + path: &'a str, + max_seconds: u64, +} + +#[derive(Deserialize)] +struct WorkerResponse { + protocol: u32, + id: u64, + ok: bool, + transcript: Option, + error: Option, +} + +struct AsrWorker { + child: Child, + stdin: ChildStdin, + stdout: Lines>, + next_id: u64, +} + +static WORKER: OnceLock>> = OnceLock::new(); + +pub fn asr_model_status() -> Result { + let paths = model_paths()?; + let missing_files = paths.missing_files(); + let helper_path = find_asr_helper(); + let installed = missing_files.is_empty(); + let helper_available = helper_path.is_some(); + Ok(AsrModelStatus { + model: "Qwen3-ASR-0.6B-Int8", + runtime: "local/sherpa-onnx", + installed, + helper_available, + available: installed && helper_available, + model_dir: paths.root.display().to_string(), + helper_path: helper_path.map(|path| path.display().to_string()), + missing_files, + archive_bytes: MODEL_ARCHIVE_BYTES, + license: "Apache-2.0", + }) +} + +pub fn local_asr_available() -> bool { + asr_model_status().is_ok_and(|status| status.available) +} + +pub async fn install_asr_model(mut progress: F) -> Result +where + F: FnMut(AsrInstallProgress) + Send, +{ + let status = asr_model_status()?; + if status.installed { + progress_event( + &mut progress, + "complete", + MODEL_ARCHIVE_BYTES, + Some(MODEL_ARCHIVE_BYTES), + ); + return Ok(status); + } + + let paths = model_paths()?; + if paths.root.exists() { + anyhow::bail!( + "local ASR model directory is incomplete: {}; missing: {}. Move it aside and run `socai asr install` again", + paths.root.display(), + status.missing_files.join(", ") + ); + } + let parent = paths + .root + .parent() + .ok_or_else(|| { + anyhow!( + "ASR model directory has no parent: {}", + paths.root.display() + ) + })? + .to_path_buf(); + tokio::fs::create_dir_all(&parent).await?; + let downloads = parent.join("downloads"); + tokio::fs::create_dir_all(&downloads).await?; + let archive = downloads.join(MODEL_ARCHIVE); + + let archive_ready = archive.is_file() + && verify_sha256(archive.clone(), MODEL_SHA256) + .await + .unwrap_or(false); + if archive_ready { + progress_event( + &mut progress, + "download", + MODEL_ARCHIVE_BYTES, + Some(MODEL_ARCHIVE_BYTES), + ); + } else { + if archive.exists() { + tokio::fs::remove_file(&archive).await.with_context(|| { + format!("failed to remove invalid ASR archive {}", archive.display()) + })?; + } + download_verified( + MODEL_URL, + &archive, + MODEL_SHA256, + Some(MODEL_ARCHIVE_BYTES), + "download", + &mut progress, + ) + .await?; + } + + progress_event(&mut progress, "extract", 0, None); + let staging = parent.join(format!(".{MODEL_ID}.install-{}", uuid::Uuid::new_v4())); + let archive_for_unpack = archive.clone(); + let staging_for_unpack = staging.clone(); + tokio::task::spawn_blocking(move || -> Result<()> { + std::fs::create_dir_all(&staging_for_unpack)?; + let file = File::open(&archive_for_unpack)?; + let decoder = BzDecoder::new(file); + Archive::new(decoder) + .unpack(&staging_for_unpack) + .context("failed to unpack Qwen3-ASR model archive")?; + Ok(()) + }) + .await + .context("ASR model extraction task panicked")??; + + let staged_root = staging.join(MODEL_ID); + let staged_paths = ModelPaths::from_root(staged_root.clone()); + let missing_without_vad: Vec<_> = staged_paths + .missing_files() + .into_iter() + .filter(|item| item != VAD_FILE) + .collect(); + if !missing_without_vad.is_empty() { + let _ = tokio::fs::remove_dir_all(&staging).await; + anyhow::bail!( + "downloaded Qwen3-ASR archive is incomplete; missing: {}", + missing_without_vad.join(", ") + ); + } + + let install_result = async { + download_verified( + VAD_URL, + &staged_paths.vad, + VAD_SHA256, + None, + "vad", + &mut progress, + ) + .await?; + if paths.root.exists() { + anyhow::bail!( + "ASR model directory appeared during install: {}", + paths.root.display() + ); + } + tokio::fs::rename(&staged_root, &paths.root) + .await + .with_context(|| format!("failed to install ASR model at {}", paths.root.display()))?; + Ok::<(), anyhow::Error>(()) + } + .await; + let _ = tokio::fs::remove_dir_all(&staging).await; + install_result?; + let _ = tokio::fs::remove_file(&archive).await; + + let status = asr_model_status()?; + if !status.installed { + anyhow::bail!( + "ASR model install completed with missing files: {}", + status.missing_files.join(", ") + ); + } + progress_event( + &mut progress, + "complete", + MODEL_ARCHIVE_BYTES, + Some(MODEL_ARCHIVE_BYTES), + ); + Ok(status) +} + +pub async fn transcribe_local_file(path: impl AsRef, max_seconds: u64) -> Result { + let status = asr_model_status()?; + if !status.installed { + anyhow::bail!( + "local Qwen3-ASR model is not installed; run `socai asr install` first (missing: {})", + status.missing_files.join(", ") + ); + } + let helper = find_asr_helper() + .context("local ASR helper is unavailable; reinstall socai or set SOCAI_ASR_HELPER")?; + let path = path.as_ref(); + let path = path + .canonicalize() + .with_context(|| format!("failed to resolve media path {}", path.display()))?; + let path_text = path + .to_str() + .ok_or_else(|| anyhow!("media path is not valid UTF-8: {}", path.display()))?; + + let worker_slot = WORKER.get_or_init(|| tokio::sync::Mutex::new(None)); + let mut slot = worker_slot.lock().await; + let stopped = match slot.as_mut() { + Some(worker) => worker.child.try_wait()?.is_some(), + None => false, + }; + if stopped { + *slot = None; + } + if slot.is_none() { + *slot = Some(start_worker(&helper, &status.model_dir).await?); + } + slot.as_mut() + .expect("worker initialized") + .transcribe(path_text, max_seconds) + .await +} + +impl AsrWorker { + async fn transcribe(&mut self, path: &str, max_seconds: u64) -> Result { + self.next_id = self.next_id.saturating_add(1); + let id = self.next_id; + let mut request = serde_json::to_vec(&WorkerRequest { + protocol: PROTOCOL_VERSION, + id, + path, + max_seconds, + })?; + request.push(b'\n'); + self.stdin.write_all(&request).await?; + self.stdin.flush().await?; + let line = self + .stdout + .next_line() + .await? + .context("local ASR helper exited without a response")?; + let response: WorkerResponse = serde_json::from_str(&line) + .with_context(|| format!("invalid local ASR helper response: {line}"))?; + if response.protocol != PROTOCOL_VERSION || response.id != id { + anyhow::bail!( + "local ASR protocol mismatch: expected v{PROTOCOL_VERSION} request {id}, got v{} request {}", + response.protocol, + response.id + ); + } + if response.ok { + response + .transcript + .filter(|text| !text.trim().is_empty()) + .context("local ASR helper returned an empty transcript") + } else { + anyhow::bail!( + "{}", + response.error.unwrap_or_else(|| "local ASR failed".into()) + ) + } + } +} + +async fn start_worker(helper: &Path, model_dir: &str) -> Result { + let mut child = Command::new(helper) + .arg("--serve") + .arg("--model-dir") + .arg(model_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("failed to start local ASR helper {}", helper.display()))?; + let stdin = child + .stdin + .take() + .context("ASR helper stdin is unavailable")?; + let stdout = child + .stdout + .take() + .context("ASR helper stdout is unavailable")?; + Ok(AsrWorker { + child, + stdin, + stdout: BufReader::new(stdout).lines(), + next_id: 0, + }) +} + +fn find_asr_helper() -> Option { + let filename = if cfg!(windows) { + "socai-asr.exe" + } else { + "socai-asr" + }; + let mut candidates = Vec::new(); + if let Some(path) = std::env::var_os("SOCAI_ASR_HELPER") { + candidates.push(PathBuf::from(path)); + } + if let Ok(executable) = std::env::current_exe() { + if let Some(dir) = executable.parent() { + candidates.push(dir.join(filename)); + if dir.file_name().is_some_and(|name| name == "deps") { + if let Some(profile_dir) = dir.parent() { + candidates.push(profile_dir.join(filename)); + } + } + } + } + if let Some(workspace) = Path::new(env!("CARGO_MANIFEST_DIR")).parent() { + candidates.push(workspace.join("target").join("debug").join(filename)); + candidates.push(workspace.join("target").join("release").join(filename)); + } + candidates.into_iter().find(|path| path.is_file()) +} + +fn model_paths() -> Result { + let root = if let Some(path) = std::env::var_os("SOCAI_ASR_MODEL_DIR") { + PathBuf::from(path) + } else if let Some(home) = std::env::var_os("SOCAI_HOME") { + PathBuf::from(home).join("models").join(MODEL_ID) + } else { + dirs::home_dir() + .context("could not resolve home directory for local ASR model")? + .join(".socai") + .join("models") + .join(MODEL_ID) + }; + Ok(ModelPaths::from_root(root)) +} + +async fn download_verified( + url: &str, + target: &Path, + expected_sha256: &str, + expected_size: Option, + stage: &str, + progress: &mut F, +) -> Result<()> +where + F: FnMut(AsrInstallProgress) + Send, +{ + let part = target.with_extension(format!( + "{}.part", + target + .extension() + .and_then(|value| value.to_str()) + .unwrap_or("download") + )); + if part.exists() { + tokio::fs::remove_file(&part).await?; + } + let response = reqwest::Client::new() + .get(url) + .send() + .await? + .error_for_status()?; + let total = response.content_length().or(expected_size); + let mut stream = response.bytes_stream(); + let mut file = tokio::fs::File::create(&part).await?; + let mut hasher = Sha256::new(); + let mut downloaded = 0u64; + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + file.write_all(&chunk).await?; + hasher.update(&chunk); + downloaded += chunk.len() as u64; + progress_event(progress, stage, downloaded, total); + } + file.flush().await?; + drop(file); + let digest = format!("{:x}", hasher.finalize()); + if digest != expected_sha256 { + let _ = tokio::fs::remove_file(&part).await; + anyhow::bail!("checksum mismatch for {url}: expected {expected_sha256}, got {digest}"); + } + tokio::fs::rename(&part, target).await?; + Ok(()) +} + +async fn verify_sha256(path: PathBuf, expected: &'static str) -> Result { + tokio::task::spawn_blocking(move || -> Result { + let mut file = File::open(&path)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 1024 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize()) == expected) + }) + .await + .context("ASR checksum task panicked")? +} + +fn progress_event(progress: &mut F, stage: &str, downloaded_bytes: u64, total_bytes: Option) +where + F: FnMut(AsrInstallProgress), +{ + progress(AsrInstallProgress { + stage: stage.to_string(), + downloaded_bytes, + total_bytes, + }); +} + +fn relative_display(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .display() + .to_string() +} diff --git a/core/src/media/audio.rs b/core/src/media/audio.rs index eb510a84..e9a5cd8d 100644 --- a/core/src/media/audio.rs +++ b/core/src/media/audio.rs @@ -1,21 +1,15 @@ -use std::io::Write; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::time::{Duration, Instant}; use anyhow::{Context, Result}; -use symphonia::core::codecs::CODEC_TYPE_AAC; -use symphonia::core::errors::Error as SymphoniaError; -use symphonia::core::formats::FormatOptions; -use symphonia::core::io::MediaSourceStream; -use symphonia::core::meta::MetadataOptions; -use symphonia::core::probe::Hint; -use crate::media::common::{ensure_dir, url_suffix, MediaUnavailable}; +use crate::media::asr::transcribe_local_file; +use crate::media::common::{url_suffix, MediaUnavailable}; use crate::media::processor::MediaProcessor; impl MediaProcessor { - /// Transcribe a video/audio source through socai's managed cloud ASR — the - /// only transcription path (local whisper was removed as uncontrollable). + /// Transcribe a video/audio source with the locally installed + /// Qwen3-ASR 0.6B Int8 model. Inference never calls an ASR service. pub async fn transcribe_audio(&self, source: &str, referer: &str) -> Result { let t0 = Instant::now(); let result = self.transcribe_audio_inner(source, referer).await; @@ -24,34 +18,23 @@ impl MediaProcessor { } async fn transcribe_audio_inner(&self, source: &str, referer: &str) -> Result { - if !self.config.use_cloud_asr { + if !self.config.use_local_asr { anyhow::bail!(MediaUnavailable( - "video transcription requires a signed-in account with socai agent selected".into() + "local video transcription is disabled for this media request".into() )); } let source_path = self.local_audio_source(source, referer).await?; - // Real clip duration (the clip is already capped at - // max_audio_seconds); the server uses it for usage accounting. - let (aac, duration) = self.extract_audio_aac(&source_path).await?; - let duration_s = duration.ceil() as i64; - let result = crate::cloud::transcribe_audio_file( - &aac, - duration_s, + tokio::time::timeout( Duration::from_secs(self.config.asr_timeout_s.max(60)), - self.billing_task_id.as_deref(), + transcribe_local_file(&source_path, self.config.max_audio_seconds), ) - .await?; - self.timing.record( - "cloud_asr_total", - Duration::from_millis(result.total_latency_ms as u64), - ); - if result.provider_latency_ms > 0 { - self.timing.record( - "cloud_asr_provider", - Duration::from_millis(result.provider_latency_ms as u64), - ); - } - Ok(result.transcript.trim().to_string()) + .await + .with_context(|| { + format!( + "local Qwen3-ASR timed out after {}s", + self.config.asr_timeout_s.max(60) + ) + })? } async fn local_audio_source(&self, source: &str, referer: &str) -> Result { @@ -66,168 +49,4 @@ impl MediaProcessor { Ok(PathBuf::from(value)) } } - - /// Extract the source's AAC track into an ADTS `.aac` file without - /// re-encoding, capped at `max_audio_seconds`. Cloud ASR (Fun-ASR) accepts - /// aac as-is, so no local decode is needed. Returns the file path and its - /// duration in seconds. - async fn extract_audio_aac(&self, source_path: &Path) -> Result<(PathBuf, f64)> { - let out = self.audio_output_path(source_path)?; - let source = source_path.to_path_buf(); - let target = out.clone(); - let max_seconds = self.config.max_audio_seconds; - let duration = - tokio::task::spawn_blocking(move || demux_aac_to_adts(&source, &target, max_seconds)) - .await - .context("audio demux task panicked")??; - Ok((out, duration)) - } - - fn audio_output_path(&self, source_path: &Path) -> Result { - let dir = if source_path.starts_with(&self.config.base_dir) { - source_path - .parent() - .unwrap_or_else(|| self.config.base_dir.as_path()) - .to_path_buf() - } else { - self.config.base_dir.join("audio") - }; - let dir = ensure_dir(&dir)?; - Ok(dir.join("audio.aac")) - } -} - -/// Copy the first AAC track of `source` into `target` as raw ADTS frames -/// (one 7-byte header per packet, no re-encoding), keeping at most -/// `max_seconds` of audio. Returns the written duration in seconds. -fn demux_aac_to_adts(source: &Path, target: &Path, max_seconds: u64) -> Result { - let file = std::fs::File::open(source) - .with_context(|| format!("failed to open {}", source.display()))?; - let stream = MediaSourceStream::new(Box::new(file), Default::default()); - let mut hint = Hint::new(); - if let Some(ext) = source.extension().and_then(|ext| ext.to_str()) { - hint.with_extension(ext); - } - let probed = symphonia::default::get_probe() - .format( - &hint, - stream, - &FormatOptions::default(), - &MetadataOptions::default(), - ) - .with_context(|| format!("unsupported media container in {}", source.display()))?; - let mut format = probed.format; - let track = format - .tracks() - .iter() - .find(|track| track.codec_params.codec == CODEC_TYPE_AAC) - .ok_or_else(|| MediaUnavailable(format!("no AAC audio track in {}", source.display())))?; - let track_id = track.id; - let sample_rate = track - .codec_params - .sample_rate - .ok_or_else(|| MediaUnavailable("AAC track has no sample rate".into()))?; - let asc = track - .codec_params - .extra_data - .as_deref() - .ok_or_else(|| MediaUnavailable("AAC track has no decoder config".into()))?; - let header = AdtsHeader::from_asc(asc)?; - let mut writer = std::io::BufWriter::new( - std::fs::File::create(target) - .with_context(|| format!("failed to create {}", target.display()))?, - ); - // The emitted ADTS header declares one 1024-sample AAC frame per packet. - // Bound the output using that exact duration contract, which is also what - // socai-server validates after upload. Using the source packet timestamp - // allowed the last frame to cross the configured limit while the client - // still reported the capped value. - let max_samples = max_seconds.saturating_mul(u64::from(sample_rate)); - let mut written_samples = 0u64; - loop { - let packet = match format.next_packet() { - Ok(packet) => packet, - Err(SymphoniaError::IoError(err)) - if err.kind() == std::io::ErrorKind::UnexpectedEof => - { - break - } - Err(err) => return Err(err.into()), - }; - if packet.track_id() != track_id { - continue; - } - const SAMPLES_PER_AAC_FRAME: u64 = 1024; - if written_samples.saturating_add(SAMPLES_PER_AAC_FRAME) > max_samples { - break; - } - writer.write_all(&header.for_frame(packet.data.len())?)?; - writer.write_all(&packet.data)?; - written_samples += SAMPLES_PER_AAC_FRAME; - } - writer.flush()?; - if written_samples == 0 { - anyhow::bail!(MediaUnavailable(format!( - "AAC track in {} contains no audio packets", - source.display() - ))); - } - Ok(written_samples as f64 / f64::from(sample_rate)) -} - -/// Fixed part of an ADTS header for an AAC-LC stream; only the per-frame -/// length bits vary between frames. -struct AdtsHeader { - sample_rate_index: u8, - channel_config: u8, -} - -impl AdtsHeader { - /// Read the sample rate index and channel configuration from the track's - /// AudioSpecificConfig (the mp4 `esds` decoder config). - fn from_asc(asc: &[u8]) -> Result { - if asc.len() < 2 { - anyhow::bail!(MediaUnavailable("AAC decoder config is too short".into())); - } - let object_type = asc[0] >> 3; - if object_type == 31 { - // Escape-coded object type shifts every following field. - anyhow::bail!(MediaUnavailable(format!( - "unsupported AAC object type {object_type}" - ))); - } - let sample_rate_index = ((asc[0] & 0x7) << 1) | (asc[1] >> 7); - if sample_rate_index >= 13 { - anyhow::bail!(MediaUnavailable( - "AAC stream uses an explicit sample rate, unsupported in ADTS".into(), - )); - } - let channel_config = (asc[1] >> 3) & 0xF; - if !(1..=7).contains(&channel_config) { - anyhow::bail!(MediaUnavailable(format!( - "unsupported AAC channel configuration {channel_config}" - ))); - } - Ok(Self { - sample_rate_index, - channel_config, - }) - } - - fn for_frame(&self, payload_len: usize) -> Result<[u8; 7]> { - let frame_len = payload_len + 7; - if frame_len > 0x1FFF { - anyhow::bail!("AAC frame too large for ADTS: {payload_len} bytes"); - } - let frame_len = frame_len as u16; - Ok([ - 0xFF, // syncword - 0xF1, // syncword end, MPEG-4, layer 0, no CRC - (1 << 6) | (self.sample_rate_index << 2) | (self.channel_config >> 2), // AAC-LC - ((self.channel_config & 0x3) << 6) | ((frame_len >> 11) as u8 & 0x3), - (frame_len >> 3) as u8, - (((frame_len & 0x7) as u8) << 5) | 0x1F, // buffer fullness (VBR) - 0xFC, // buffer fullness end, 1 frame - ]) - } } diff --git a/core/src/media/common.rs b/core/src/media/common.rs index 9f0d7d1d..afd535ce 100644 --- a/core/src/media/common.rs +++ b/core/src/media/common.rs @@ -12,12 +12,12 @@ AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123 Safari/537.36"; pub struct MediaConfig { pub base_dir: PathBuf, pub request_timeout_s: u64, - /// Wall-clock cap for one cloud ASR round trip (upload + poll). + /// Wall-clock cap for one local ASR decode. pub asr_timeout_s: u64, pub max_audio_seconds: u64, pub use_ocr: bool, pub use_vision: bool, - pub use_cloud_asr: bool, + pub use_local_asr: bool, pub vision_concurrency: usize, } @@ -30,7 +30,7 @@ impl MediaConfig { max_audio_seconds: 20 * 60, use_ocr: true, use_vision: true, - use_cloud_asr: false, + use_local_asr: false, vision_concurrency: 3, } } diff --git a/core/src/media/mod.rs b/core/src/media/mod.rs index c56ba54a..a6ff799b 100644 --- a/core/src/media/mod.rs +++ b/core/src/media/mod.rs @@ -1,11 +1,12 @@ //! Optional local/media processing used by site runtimes. //! //! Nothing here shells out to external media tools: video covers come from -//! their own CDN URL, audio transcription is cloud-only (socai takes the -//! demuxed aac as-is), and OCR runs in-process. Site runtimes can opt into -//! this crate for heavier media enrichment while keeping plain DOM extraction -//! fast and portable. +//! their own CDN URL, OCR runs in-process, and Qwen3-ASR audio transcription +//! runs in a bundled local worker. Site runtimes can opt into this crate for +//! heavier media enrichment while keeping plain DOM extraction fast and +//! portable. +mod asr; mod audio; mod background; mod common; @@ -16,15 +17,19 @@ mod processor; mod timing; mod video; -pub use self::background::{ - begin_background_media_generation, cancel_background_media_for_run, - current_background_media_generation, subscribe_background_media_events, BackgroundMediaEvent, +pub use self::asr::{ + asr_model_status, install_asr_model, local_asr_available, transcribe_local_file, + AsrInstallProgress, AsrModelStatus, }; pub(crate) use self::background::{ background_media_run_is_cancelled, background_video_download_semaphore, emit_background_media_event, reserve_background_video_download, subscribe_background_media_cancellation, wait_for_background_media_cancellation, }; +pub use self::background::{ + begin_background_media_generation, cancel_background_media_for_run, + current_background_media_generation, subscribe_background_media_events, BackgroundMediaEvent, +}; pub use self::common::{MediaConfig, MediaUnavailable}; pub use self::ocr::diagnostics as ocr_diagnostics; pub(crate) use self::ocr::ocr_images_bytes; diff --git a/core/src/media/processor.rs b/core/src/media/processor.rs index ddb66f97..a390f913 100644 --- a/core/src/media/processor.rs +++ b/core/src/media/processor.rs @@ -19,7 +19,6 @@ pub struct MediaProcessor { pub(crate) llm_provider: Option>, pub(crate) client: reqwest::Client, pub(crate) timing: Arc, - pub(crate) billing_task_id: Option, } impl MediaProcessor { @@ -34,7 +33,6 @@ impl MediaProcessor { llm_provider, client, timing: Arc::new(TimingRecord::default()), - billing_task_id: None, }) } @@ -52,15 +50,8 @@ impl MediaProcessor { self.timing.clone() } - pub fn set_cloud_asr(&mut self, enabled: bool) { - self.config.use_cloud_asr = enabled; - } - - pub fn set_billing_task_id(&mut self, task_id: Option<&str>) { - self.billing_task_id = task_id - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string); + pub fn set_local_asr(&mut self, enabled: bool) { + self.config.use_local_asr = enabled; } pub fn timing_summary(&self) -> Value { diff --git a/core/src/media/video.rs b/core/src/media/video.rs index 66bc04c0..7305681b 100644 --- a/core/src/media/video.rs +++ b/core/src/media/video.rs @@ -295,10 +295,9 @@ impl MediaProcessor { .await; } - // Inline transcription only when cloud ASR is enabled for this run; - // attempting it without an eligible hosted-agent session would stamp - // every enriched video with the same availability error. - if !source.is_empty() && self.config.use_cloud_asr { + // Inline transcription only when local ASR is enabled for this run; + // tool schemas hide this option until the model is installed. + if !source.is_empty() && self.config.use_local_asr { if let Some(map) = result.as_object_mut() { map.remove("transcript_error"); } diff --git a/core/src/sites/content.rs b/core/src/sites/content.rs index db325d38..96d872e5 100644 --- a/core/src/sites/content.rs +++ b/core/src/sites/content.rs @@ -231,7 +231,7 @@ pub fn get_notes_input_schema( }, "transcribe_audio": { "type": "boolean", - "description": "For video notes, download the video and transcribe audio while signed in with socai agent selected.", + "description": "For video notes, download the video and transcribe audio with the locally installed Qwen3-ASR model.", "default": false } }, @@ -239,7 +239,7 @@ pub fn get_notes_input_schema( "additionalProperties": false }); if !asr_enabled { - strip_hosted_transcription_schema(&mut schema); + strip_unavailable_transcription_schema(&mut schema); } schema } @@ -279,7 +279,7 @@ pub fn search_input_schema( }, "transcribe_audio": { "type": "boolean", - "description": "For opened video notes, download the video and transcribe audio while signed in with socai agent selected. Ignored in preview mode.", + "description": "For opened video notes, download the video and transcribe audio with the locally installed Qwen3-ASR model. Ignored in preview mode.", "default": false }, "preview": { @@ -291,7 +291,7 @@ pub fn search_input_schema( "required": ["query"] }); if !asr_enabled { - strip_hosted_transcription_schema(&mut schema); + strip_unavailable_transcription_schema(&mut schema); } schema } @@ -329,14 +329,14 @@ pub fn author_scan_input_schema(default_comments: i64, asr_enabled: bool) -> Val }, "transcribe_audio": { "type": "boolean", - "description": "For opened video notes, download the video and transcribe audio while signed in with socai agent selected. Ignored in preview mode.", + "description": "For opened video notes, download the video and transcribe audio with the locally installed Qwen3-ASR model. Ignored in preview mode.", "default": false } }, "required": ["author_id"] }); if !asr_enabled { - strip_hosted_transcription_schema(&mut schema); + strip_unavailable_transcription_schema(&mut schema); } schema } @@ -372,13 +372,13 @@ pub fn xhs_product_effective_input(operation: ContentOperation, input: &Value) - effective } -pub fn strip_hosted_transcription_schema(schema: &mut Value) { +pub fn strip_unavailable_transcription_schema(schema: &mut Value) { if let Some(properties) = schema.get_mut("properties").and_then(Value::as_object_mut) { properties.remove("transcribe_audio"); } } -pub fn strip_hosted_transcription_input(input: &mut Value) -> bool { +pub fn strip_unavailable_transcription_input(input: &mut Value) -> bool { input .as_object_mut() .and_then(|object| object.remove("transcribe_audio")) diff --git a/core/src/sites/xhs/knowledge.md b/core/src/sites/xhs/knowledge.md index be6bf2ef..b75a780a 100644 --- a/core/src/sites/xhs/knowledge.md +++ b/core/src/sites/xhs/knowledge.md @@ -137,11 +137,11 @@ Shared options (same meaning for both): pipelined behind the browse loop so it's near-free). Each note gets `ocr_text` as a per-image array (cover first); implies `download_media`; in `preview` mode it OCRs each card's cover only. -- `transcribe_audio=true` — transcribe a video note's audio in the cloud, - attaching `video.transcript`. This is available only when the user is signed - in and has selected socai agent in the model picker. If it is unavailable, - fails, or the user asks how to enable it, tell them to sign in and select - socai agent. A +- `transcribe_audio=true` — transcribe a video note's audio with the local + Qwen3-ASR 0.6B Int8 model and attach `video.transcript`. The option appears + after `socai asr install` completes. Inference runs on the device without an + ASR API, account, credits, or API key. If the option is unavailable, tell the + user to install the model and retry. A `transcript_error` saying no speech was detected means the video genuinely has no narration — report that as the answer. diff --git a/core/src/sites/xhs/page.rs b/core/src/sites/xhs/page.rs index 70964184..016a6894 100644 --- a/core/src/sites/xhs/page.rs +++ b/core/src/sites/xhs/page.rs @@ -102,8 +102,8 @@ pub struct ReadNoteOptions { /// Implies `download_media` (the caller forces it on), since OCR reads the /// saved files. pub ocr: bool, - /// Transcribe downloaded video notes through the configured cloud ASR - /// server. Implies `download_media` and `download_video_file` at the caller. + /// Transcribe downloaded video notes with the local Qwen3-ASR model. + /// Implies `download_media` and `download_video_file` at the caller. pub transcribe_audio: bool, /// Optional additional hydration settle after body content first appears. /// The page script always applies a small baseline settle because XHS can diff --git a/core/src/sites/xhs/tools.rs b/core/src/sites/xhs/tools.rs index c0764c80..f7069357 100644 --- a/core/src/sites/xhs/tools.rs +++ b/core/src/sites/xhs/tools.rs @@ -83,7 +83,7 @@ pub fn xhs_tools_with_llm_provider( llm_provider: Option>, ) -> Vec> { let history = Arc::new(XhsHistoryStore::open_default()); - let asr_enabled = crate::cloud::hosted_llm_selected(); + let asr_enabled = crate::media::local_asr_available(); vec![ Arc::new(GetNotesTool { page: page.clone(), @@ -143,7 +143,7 @@ pub fn xhs_macro_tools_with_llm_provider( llm_provider: Option>, ) -> Vec> { let history = Arc::new(XhsHistoryStore::open_default()); - let asr_enabled = crate::cloud::hosted_llm_selected(); + let asr_enabled = crate::media::local_asr_available(); // The app/TUI agent interface always downloads note media so the offline // files are on hand for deeper analysis, and always OCRs every image; the // CLI keeps its --download-media / --ocr opt-ins via the full tool set above. @@ -213,7 +213,7 @@ impl ContentPlatform for XhsContentPlatform { comment_replies: true, media_download: true, ocr: true, - audio_transcription: crate::cloud::hosted_llm_selected(), + audio_transcription: crate::media::local_asr_available(), cross_run_history: true, artifacts: true, } @@ -256,13 +256,13 @@ impl ContentPlatform for XhsContentPlatform { /// XHS is the reference adapter and therefore owns product defaults for the /// shared content surface: media download and OCR are enabled for app/TUI -/// calls, while ASR remains gated by the hosted-agent selection. +/// calls, while ASR is exposed after its local model is installed. pub fn xhs_content_platform( page: Arc, llm_provider: Option>, ) -> Arc { let history = Arc::new(XhsHistoryStore::open_default()); - let asr_enabled = crate::cloud::hosted_llm_selected(); + let asr_enabled = crate::media::local_asr_available(); Arc::new(XhsContentPlatform { tools: vec![ Arc::new(GetNotesTool { @@ -378,7 +378,7 @@ pub static XHS_SITE: SiteSpec = SiteSpec { key: "transcribe_audio", long: Some("transcribe-audio"), value_name: "TRANSCRIBE_AUDIO", - help: "For video notes, transcribe audio while signed in with socai agent selected.", + help: "For video notes, transcribe audio with the locally installed Qwen3-ASR model.", required: false, kind: ArgKind::Flag, }, @@ -455,7 +455,7 @@ pub static XHS_SITE: SiteSpec = SiteSpec { long: Some("transcribe-audio"), value_name: "TRANSCRIBE_AUDIO", help: "For opened video notes, download the video file and transcribe audio \ - while signed in with socai agent selected. Ignored with --preview.", + with the locally installed Qwen3-ASR model. Ignored with --preview.", required: false, kind: ArgKind::Flag, }, @@ -531,7 +531,7 @@ pub static XHS_SITE: SiteSpec = SiteSpec { long: Some("transcribe-audio"), value_name: "TRANSCRIBE_AUDIO", help: "For opened video notes, download the video file and transcribe audio \ - while signed in with socai agent selected. Ignored with --preview.", + with the locally installed Qwen3-ASR model. Ignored with --preview.", required: false, kind: ArgKind::Flag, }, @@ -992,31 +992,30 @@ fn read_note_options(input: &Value) -> ReadNoteOptions { } } -/// Tool args available only to signed-in users while socai agent is selected. -const HOSTED_AGENT_ARG_KEYS: &[&str] = &["transcribe_audio"]; +/// Tool args available only after the local model has been installed. +const LOCAL_ASR_ARG_KEYS: &[&str] = &["transcribe_audio"]; -/// Attached when an unavailable call asked for a hosted-agent-only argument, -/// so the agent can relay the remedy instead of failing the whole call. -const HOSTED_AGENT_SKIP_NOTE: &str = "video transcription requires a signed-in account \ - with socai agent selected; the call ran without transcription."; +/// Attached when a stale call requested ASR before local model installation. +const LOCAL_ASR_SKIP_NOTE: &str = "video transcription requires the local Qwen3-ASR model; \ + run `socai asr install`, then retry. The call ran without transcription."; -/// Hide hosted-agent-only properties when the current session cannot use them. -fn strip_hosted_agent_schema(schema: &mut Value) { +/// Hide local-ASR properties until the current installation can use them. +fn strip_unavailable_asr_schema(schema: &mut Value) { if let Some(props) = schema.get_mut("properties").and_then(Value::as_object_mut) { - for key in HOSTED_AGENT_ARG_KEYS { + for key in LOCAL_ASR_ARG_KEYS { props.remove(*key); } } } -/// Drop hosted-agent-only args when unavailable (a stale conversation can -/// still carry one). Returns whether the argument was actually requested. -fn strip_hosted_agent_input(input: &mut Value) -> bool { +/// Drop unavailable local-ASR args from stale conversations. Returns whether +/// transcription was actually requested. +fn strip_unavailable_asr_input(input: &mut Value) -> bool { let Some(obj) = input.as_object_mut() else { return false; }; let mut requested = false; - for key in HOSTED_AGENT_ARG_KEYS { + for key in LOCAL_ASR_ARG_KEYS { requested |= obj .remove(*key) .and_then(|value| value.as_bool()) @@ -1025,14 +1024,14 @@ fn strip_hosted_agent_input(input: &mut Value) -> bool { requested } -fn attach_hosted_agent_skip_note(payload: &mut Value, skipped: bool) { +fn attach_local_asr_skip_note(payload: &mut Value, skipped: bool) { if !skipped { return; } if let Some(obj) = payload.as_object_mut() { obj.insert( "transcribe_audio_skipped".into(), - json!(HOSTED_AGENT_SKIP_NOTE), + json!(LOCAL_ASR_SKIP_NOTE), ); } } @@ -1068,8 +1067,7 @@ fn media_for( ) -> anyhow::Result> { if include_media || transcribe_audio { let mut media = MediaProcessor::for_run_dir(ctx.output_dir(), llm_provider)?; - media.set_cloud_asr(transcribe_audio); - media.set_billing_task_id(ctx.billing_task_id.as_deref()); + media.set_local_asr(transcribe_audio); Ok(Some(media)) } else { Ok(None) @@ -1322,7 +1320,7 @@ async fn scan_card_note( // itself would incorrectly skip an upgrade requested later in the same // agent run (for example, a plain read followed by transcribe_audio=true). // In particular, that could reuse a pre-upgrade entity carrying the old - // ffmpeg transcription error instead of retrying through cloud ASR. + // ffmpeg transcription error instead of retrying through local ASR. let processed_in_run = !card.note_id.is_empty() && ctx.has_processed_note(&card.note_id, level, requested_media); if !card.note_id.is_empty() @@ -1366,7 +1364,7 @@ async fn scan_card_note( include_media, download_media, download_video_file: download_video_file_inline, - // Scans never transcribe inline: the caller runs cloud ASR in a + // Scans never transcribe inline: the caller runs local ASR in a // background task (spawn_note_transcribe) so it overlaps the // next note's read + download. The dedup check above still uses // the real `transcribe_audio` flag, so a cache hit returns the @@ -1690,14 +1688,12 @@ async fn join_note_ocr( timings } -/// Max cloud ASR tasks in flight at once. Transcription is network-bound -/// (upload + provider poll), so this bounds concurrent load on socai-server -/// while still letting note N's transcription overlap the read + download of -/// note N+1. -const ASR_PIPELINE_CONCURRENCY: usize = 2; +/// Max local ASR tasks in flight. Qwen3-ASR shares one in-memory recognizer; +/// serial decoding avoids competing CPU work and duplicate model pressure. +const ASR_PIPELINE_CONCURRENCY: usize = 1; /// Spawn a background task that transcribes a freshly-read video note's -/// already-downloaded video file through cloud ASR. `None` when there's +/// already-downloaded video file through local ASR. `None` when there's /// nothing to transcribe (not a fresh successful read, no media processor, no /// downloaded video, or the cached entity already carries a transcript). fn spawn_note_transcribe( @@ -1909,7 +1905,7 @@ pub fn note_data_record( } } record.insert("stats".into(), Value::Object(stats)); - // Video audio transcript (cloud ASR), so the app's note viewer can show + // Video audio transcript from local ASR, so the app's note viewer can show // the spoken content alongside the media. if let Some(transcript) = entity .get("video") @@ -2482,7 +2478,7 @@ const LEAN_NOTE_FIELDS: &[&str] = &[ // Per-note OCR summary (joined from each image's ocr_text). Only present // when the scan ran with `ocr`; the per-image texts stay in the artifact. "ocr_text", - // Video audio transcript from cloud ASR. The full video object stays in the + // Video audio transcript from local ASR. The full video object stays in the // artifact; this keeps the usable text in the compact result. "audio_transcript", ]; @@ -3290,7 +3286,7 @@ impl Tool for GetNotesTool { }, "transcribe_audio": { "type": "boolean", - "description": "For video notes, download the video and transcribe audio while signed in with socai agent selected.", + "description": "For video notes, download the video and transcribe audio with the locally installed Qwen3-ASR model.", "default": false } }, @@ -3298,7 +3294,7 @@ impl Tool for GetNotesTool { "additionalProperties": false }); if !self.asr_enabled { - strip_hosted_agent_schema(&mut schema); + strip_unavailable_asr_schema(&mut schema); } schema } @@ -3308,7 +3304,7 @@ impl Tool for GetNotesTool { } async fn call(&self, mut input: Value, ctx: &ToolContext) -> anyhow::Result { - let asr_skipped = !self.asr_enabled && strip_hosted_agent_input(&mut input); + let asr_skipped = !self.asr_enabled && strip_unavailable_asr_input(&mut input); let targets = direct_note_refs(&input)?; let gate = XhsPageRuntime::new(&self.page); let login = match gate.login_gate(true).await { @@ -3519,7 +3515,7 @@ impl Tool for GetNotesTool { .map(|rel| ctx.run_dir.join(rel).to_string_lossy().into_owned()); lean_scan_payload(&mut payload); attach_artifact_pointer(&mut payload, artifact_path, ARTIFACT_EXTRA_NOTE_PROPERTIES); - attach_hosted_agent_skip_note(&mut payload, asr_skipped); + attach_local_asr_skip_note(&mut payload, asr_skipped); Ok(json_result(&payload)) } } @@ -3529,8 +3525,8 @@ pub struct ReadNoteTool { page: Arc, llm_provider: Option>, history: Arc, - /// Signed in with socai agent selected; when false, managed-ASR arguments - /// are hidden from the schema and skipped at runtime. + /// Whether the local Qwen3-ASR model is installed; when false, ASR + /// arguments are hidden from the schema and skipped at runtime. asr_enabled: bool, } @@ -3561,13 +3557,13 @@ impl Tool for ReadNoteTool { } }); if !self.asr_enabled { - strip_hosted_agent_schema(&mut schema); + strip_unavailable_asr_schema(&mut schema); } schema } async fn call(&self, mut input: Value, ctx: &ToolContext) -> anyhow::Result { - let asr_skipped = !self.asr_enabled && strip_hosted_agent_input(&mut input); + let asr_skipped = !self.asr_enabled && strip_unavailable_asr_input(&mut input); let note_id = get_str(&input, "note_id").map(str::to_string); let index = input .get("index") @@ -3646,7 +3642,7 @@ impl Tool for ReadNoteTool { if let Some(perf) = value.as_object_mut().and_then(|map| map.remove("perf")) { write_run_perf_file(ctx, "read.json", &json!({ "read": perf })); } - attach_hosted_agent_skip_note(&mut value, asr_skipped); + attach_local_asr_skip_note(&mut value, asr_skipped); Ok(json_result(&value)) } } @@ -3684,13 +3680,13 @@ impl Tool for ExtractNoteTool { } }); if !self.asr_enabled { - strip_hosted_agent_schema(&mut schema); + strip_unavailable_asr_schema(&mut schema); } schema } async fn call(&self, mut input: Value, ctx: &ToolContext) -> anyhow::Result { - let asr_skipped = !self.asr_enabled && strip_hosted_agent_input(&mut input); + let asr_skipped = !self.asr_enabled && strip_unavailable_asr_input(&mut input); let wait_seconds = get_f64(&input, "wait_seconds", 8.0); let options = read_note_options(&input); let xhs = XhsPageRuntime::new_with_media( @@ -3711,7 +3707,7 @@ impl Tool for ExtractNoteTool { attach_top_comments(&xhs, &mut value).await; self.history .record(&value, &options.level, options.include_media); - attach_hosted_agent_skip_note(&mut value, asr_skipped); + attach_local_asr_skip_note(&mut value, asr_skipped); Ok(json_result(&value)) } } @@ -4248,7 +4244,7 @@ impl Tool for SearchTool { }, "transcribe_audio": { "type": "boolean", - "description": "For opened video notes, download the video file and transcribe audio while signed in with socai agent selected. Ignored in preview mode.", + "description": "For opened video notes, download the video file and transcribe audio with the locally installed Qwen3-ASR model. Ignored in preview mode.", "default": false }, "preview": { @@ -4260,7 +4256,7 @@ impl Tool for SearchTool { "required": ["query"] }); if !self.asr_enabled { - strip_hosted_agent_schema(&mut schema); + strip_unavailable_asr_schema(&mut schema); } schema } @@ -4275,7 +4271,7 @@ impl Tool for SearchTool { } async fn call(&self, mut input: Value, ctx: &ToolContext) -> anyhow::Result { - let asr_skipped = !self.asr_enabled && strip_hosted_agent_input(&mut input); + let asr_skipped = !self.asr_enabled && strip_unavailable_asr_input(&mut input); let query = get_str(&input, "query") .ok_or_else(|| anyhow::anyhow!("missing query"))? .to_string(); @@ -4536,7 +4532,7 @@ impl Tool for SearchTool { let mut seen: std::collections::HashSet = std::collections::HashSet::new(); let mut cursor = 0usize; let mut stalls = 0usize; - // OCR and cloud ASR run in the background so they overlap the next + // OCR and local ASR run in the background so they overlap the next // note's read + download; tasks are joined after the browse loop. let ocr_sem = Arc::new(tokio::sync::Semaphore::new(OCR_PIPELINE_CONCURRENCY)); let mut pending_ocr: Vec<(usize, tokio::task::JoinHandle)> = Vec::new(); @@ -4995,7 +4991,7 @@ impl Tool for SearchTool { // agent/CLI output stays small, then point at the artifact for the rest. lean_scan_payload(&mut payload); attach_artifact_pointer(&mut payload, artifact_path, ARTIFACT_EXTRA_NOTE_PROPERTIES); - attach_hosted_agent_skip_note(&mut payload, asr_skipped); + attach_local_asr_skip_note(&mut payload, asr_skipped); Ok(json_result(&payload)) } } @@ -5083,14 +5079,14 @@ impl Tool for AuthorScanTool { }, "transcribe_audio": { "type": "boolean", - "description": "For opened video notes, download the video file and transcribe audio while signed in with socai agent selected. Ignored in preview mode.", + "description": "For opened video notes, download the video file and transcribe audio with the locally installed Qwen3-ASR model. Ignored in preview mode.", "default": false } }, "required": ["author_id"] }); if !self.asr_enabled { - strip_hosted_agent_schema(&mut schema); + strip_unavailable_asr_schema(&mut schema); } schema } @@ -5100,7 +5096,7 @@ impl Tool for AuthorScanTool { } async fn call(&self, mut input: Value, ctx: &ToolContext) -> anyhow::Result { - let asr_skipped = !self.asr_enabled && strip_hosted_agent_input(&mut input); + let asr_skipped = !self.asr_enabled && strip_unavailable_asr_input(&mut input); let author_id = get_str(&input, "author_id") .map(str::trim) .filter(|id| !id.is_empty()) @@ -5212,7 +5208,7 @@ impl Tool for AuthorScanTool { let mut notes: Vec = Vec::new(); let mut stop_reason = String::new(); if !preview { - // OCR and cloud ASR run in the background so they overlap the next + // OCR and local ASR run in the background so they overlap the next // note's read + download; tasks are joined after the loop. let ocr_sem = Arc::new(tokio::sync::Semaphore::new(OCR_PIPELINE_CONCURRENCY)); let mut pending_ocr: Vec<(usize, tokio::task::JoinHandle)> = Vec::new(); @@ -5432,7 +5428,7 @@ impl Tool for AuthorScanTool { // agent/CLI output stays small, then point at the artifact for the rest. lean_scan_payload(&mut payload); attach_artifact_pointer(&mut payload, artifact_path, ARTIFACT_EXTRA_NOTE_PROPERTIES); - attach_hosted_agent_skip_note(&mut payload, asr_skipped); + attach_local_asr_skip_note(&mut payload, asr_skipped); Ok(json_result(&payload)) } } @@ -5529,7 +5525,7 @@ mod tests { } #[test] - fn strip_hosted_agent_schema_hides_asr_args() { + fn strip_unavailable_asr_schema_hides_asr_args() { let mut schema = json!({ "type": "object", "properties": { @@ -5537,22 +5533,22 @@ mod tests { "transcribe_audio": { "type": "boolean" } } }); - strip_hosted_agent_schema(&mut schema); + strip_unavailable_asr_schema(&mut schema); assert!(schema["properties"].get("transcribe_audio").is_none()); assert!(schema["properties"].get("query").is_some()); } #[test] - fn strip_hosted_agent_input_drops_args_and_reports_requests() { + fn strip_unavailable_asr_input_drops_args_and_reports_requests() { let mut input = json!({ "query": "咖啡", "transcribe_audio": true }); - assert!(strip_hosted_agent_input(&mut input)); + assert!(strip_unavailable_asr_input(&mut input)); assert!(input.get("transcribe_audio").is_none()); assert_eq!(input["query"], json!("咖啡")); // Not requested (absent or false) → no skip note owed. let mut plain = json!({ "query": "咖啡" }); - assert!(!strip_hosted_agent_input(&mut plain)); + assert!(!strip_unavailable_asr_input(&mut plain)); let mut off = json!({ "query": "咖啡", "transcribe_audio": false }); - assert!(!strip_hosted_agent_input(&mut off)); + assert!(!strip_unavailable_asr_input(&mut off)); } } diff --git a/scripts/install-cli.ps1 b/scripts/install-cli.ps1 index 95253488..7f9382c7 100644 --- a/scripts/install-cli.ps1 +++ b/scripts/install-cli.ps1 @@ -48,9 +48,13 @@ try { Expand-Archive -Force -Path $ArchivePath -DestinationPath $UnpackDir $SourceExe = Join-Path $UnpackDir 'socai.exe' + $SourceAsrExe = Join-Path $UnpackDir 'socai-asr.exe' if (-not (Test-Path -LiteralPath $SourceExe)) { throw 'release archive did not contain socai.exe' } + if (-not (Test-Path -LiteralPath $SourceAsrExe)) { + throw 'release archive did not contain socai-asr.exe' + } $DestExe = Join-Path $InstallDir 'socai.exe' if (Test-Path -LiteralPath $DestExe) { @@ -71,6 +75,7 @@ try { } } Copy-Item -Force -LiteralPath $SourceExe -Destination $DestExe + Copy-Item -Force -LiteralPath $SourceAsrExe -Destination (Join-Path $InstallDir 'socai-asr.exe') Write-Host "installed socai to $DestExe" & $DestExe --version diff --git a/scripts/install-cli.sh b/scripts/install-cli.sh index 43f2bb1a..fa14f5a8 100755 --- a/scripts/install-cli.sh +++ b/scripts/install-cli.sh @@ -73,8 +73,13 @@ if [ ! -f "$unpack_dir/socai" ]; then echo "release archive did not contain ./socai" >&2 exit 1 fi +if [ ! -f "$unpack_dir/socai-asr" ]; then + echo "release archive did not contain ./socai-asr" >&2 + exit 1 +fi install -m 0755 "$unpack_dir/socai" "$install_dir/socai" +install -m 0755 "$unpack_dir/socai-asr" "$install_dir/socai-asr" printf 'installed socai to %s\n' "$install_dir/socai" "$install_dir/socai" --version From f2b1ef78fcc6cad4770eb4de8d1131b3a7c58043 Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 20:11:50 +0800 Subject: [PATCH 10/13] fix: reset local ASR worker after timeout --- app/src/main.ts | 2 +- core/src/media/asr.rs | 48 +++++++++++++++++++++++++++++++++++++---- core/src/media/audio.rs | 15 +++++-------- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/app/src/main.ts b/app/src/main.ts index 011173f9..18a2a10e 100644 --- a/app/src/main.ts +++ b/app/src/main.ts @@ -137,7 +137,7 @@ export interface NoteData { comments?: NoteComment[]; // top comments captured with the read media?: NoteMedia[]; // media[0] === cover media_dir?: string; // run-relative folder, when src paths are relative - transcript?: string; // video audio transcript (cloud ASR) + transcript?: string; // video audio transcript (local ASR) saved?: boolean; // Tolerate extra fields the archive may carry. [key: string]: unknown; diff --git a/core/src/media/asr.rs b/core/src/media/asr.rs index 9792b27a..73b1c933 100644 --- a/core/src/media/asr.rs +++ b/core/src/media/asr.rs @@ -3,6 +3,7 @@ use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::OnceLock; +use std::time::Duration; use anyhow::{anyhow, Context, Result}; use bzip2::read::BzDecoder; @@ -276,6 +277,22 @@ where } pub async fn transcribe_local_file(path: impl AsRef, max_seconds: u64) -> Result { + transcribe_local_file_inner(path.as_ref(), max_seconds, None).await +} + +pub(crate) async fn transcribe_local_file_with_timeout( + path: impl AsRef, + max_seconds: u64, + timeout: Duration, +) -> Result { + transcribe_local_file_inner(path.as_ref(), max_seconds, Some(timeout)).await +} + +async fn transcribe_local_file_inner( + path: &Path, + max_seconds: u64, + timeout: Option, +) -> Result { let status = asr_model_status()?; if !status.installed { anyhow::bail!( @@ -285,7 +302,6 @@ pub async fn transcribe_local_file(path: impl AsRef, max_seconds: u64) -> } let helper = find_asr_helper() .context("local ASR helper is unavailable; reinstall socai or set SOCAI_ASR_HELPER")?; - let path = path.as_ref(); let path = path .canonicalize() .with_context(|| format!("failed to resolve media path {}", path.display()))?; @@ -305,10 +321,34 @@ pub async fn transcribe_local_file(path: impl AsRef, max_seconds: u64) -> if slot.is_none() { *slot = Some(start_worker(&helper, &status.model_dir).await?); } - slot.as_mut() - .expect("worker initialized") - .transcribe(path_text, max_seconds) + let result = match timeout { + Some(timeout) => match tokio::time::timeout( + timeout, + slot.as_mut() + .expect("worker initialized") + .transcribe(path_text, max_seconds), + ) .await + { + Ok(result) => result, + Err(_) => Err(anyhow!( + "local Qwen3-ASR timed out after {}s", + timeout.as_secs() + )), + }, + None => { + slot.as_mut() + .expect("worker initialized") + .transcribe(path_text, max_seconds) + .await + } + }; + if result.is_err() { + // A transport or protocol error can leave a late response in stdout. + // Drop the worker so the next request starts with a clean protocol stream. + *slot = None; + } + result } impl AsrWorker { diff --git a/core/src/media/audio.rs b/core/src/media/audio.rs index e9a5cd8d..8aedc476 100644 --- a/core/src/media/audio.rs +++ b/core/src/media/audio.rs @@ -1,9 +1,9 @@ use std::path::PathBuf; use std::time::{Duration, Instant}; -use anyhow::{Context, Result}; +use anyhow::Result; -use crate::media::asr::transcribe_local_file; +use crate::media::asr::transcribe_local_file_with_timeout; use crate::media::common::{url_suffix, MediaUnavailable}; use crate::media::processor::MediaProcessor; @@ -24,17 +24,12 @@ impl MediaProcessor { )); } let source_path = self.local_audio_source(source, referer).await?; - tokio::time::timeout( + transcribe_local_file_with_timeout( + &source_path, + self.config.max_audio_seconds, Duration::from_secs(self.config.asr_timeout_s.max(60)), - transcribe_local_file(&source_path, self.config.max_audio_seconds), ) .await - .with_context(|| { - format!( - "local Qwen3-ASR timed out after {}s", - self.config.asr_timeout_s.max(60) - ) - })? } async fn local_audio_source(&self, source: &str, referer: &str) -> Result { From 4ca72e2ac7e6c07ce00b45331a5ff09301cf0a5b Mon Sep 17 00:00:00 2001 From: Asklv Date: Wed, 2 Sep 2026 20:27:58 +0800 Subject: [PATCH 11/13] fix: harden local ASR installation --- .cargo/config.toml | 2 + .github/workflows/release.yml | 2 + README.md | 1 + app/scripts/prepare-asr-helper.mjs | 10 +++ app/src-tauri/src/commands.rs | 20 +++++ app/src-tauri/src/lib.rs | 2 + app/src/lib/i18n.ts | 23 ++++++ app/src/panels/settings.ts | 91 +++++++++++++++++++++ app/src/styles.css | 1 + asr/src/main.rs | 72 ++++++++++++----- core/src/media/asr.rs | 41 ++++++++-- scripts/prepare-sherpa-onnx-libs.mjs | 117 +++++++++++++++++++++++++++ 12 files changed, 359 insertions(+), 23 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 scripts/prepare-sherpa-onnx-libs.mjs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..6d22c209 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[env] +SHERPA_ONNX_ARCHIVE_DIR = { value = "../target/sherpa-onnx-archives", relative = true } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index feab5f98..bc64be2b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -182,6 +182,7 @@ jobs: run: | set -euo pipefail + node scripts/prepare-sherpa-onnx-libs.mjs aarch64-apple-darwin x86_64-apple-darwin cargo build -p socai-cli --release --target aarch64-apple-darwin cargo build -p socai-cli --release --target x86_64-apple-darwin cargo build -p socai-asr --release --target aarch64-apple-darwin @@ -716,6 +717,7 @@ jobs: SOCAI_PRO_BASE_URL: ${{ secrets.SOCAI_PRO_BASE_URL }} run: | $ErrorActionPreference = 'Stop' + node scripts/prepare-sherpa-onnx-libs.mjs x86_64-pc-windows-msvc cargo build -p socai-cli -p socai-asr --release $versionOutput = & .\target\release\socai.exe --version $versionOutput diff --git a/README.md b/README.md index 2dc31fa4..a978b92c 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ If a prebuilt binary is unavailable for your platform, or you need a source buil ```bash git clone https://github.com/socai-io/socai.git cd socai +node scripts/prepare-sherpa-onnx-libs.mjs cargo install --path asr --force cargo install --path cli --force ``` diff --git a/app/scripts/prepare-asr-helper.mjs b/app/scripts/prepare-asr-helper.mjs index 242d2482..03606690 100644 --- a/app/scripts/prepare-asr-helper.mjs +++ b/app/scripts/prepare-asr-helper.mjs @@ -35,6 +35,11 @@ function build(target) { const target = explicitTarget || rustHost(); if (target === "universal-apple-darwin") { + execFileSync( + process.execPath, + [path.join(REPO_DIR, "scripts", "prepare-sherpa-onnx-libs.mjs"), "aarch64-apple-darwin", "x86_64-apple-darwin"], + { cwd: REPO_DIR, stdio: "inherit" }, + ); const arm = build("aarch64-apple-darwin"); const intel = build("x86_64-apple-darwin"); const universal = path.join(BIN_DIR, "socai-asr-universal-apple-darwin"); @@ -42,5 +47,10 @@ if (target === "universal-apple-darwin") { chmodSync(universal, 0o755); console.log(`[socai-asr] ready ${path.relative(APP_DIR, universal)}`); } else { + execFileSync( + process.execPath, + [path.join(REPO_DIR, "scripts", "prepare-sherpa-onnx-libs.mjs"), target], + { cwd: REPO_DIR, stdio: "inherit" }, + ); build(target); } diff --git a/app/src-tauri/src/commands.rs b/app/src-tauri/src/commands.rs index 48965c68..25db069c 100644 --- a/app/src-tauri/src/commands.rs +++ b/app/src-tauri/src/commands.rs @@ -2929,6 +2929,8 @@ pub struct DesktopConfig { output_dir: String, /// Resolved default run-artifact root (shown as the input placeholder). output_dir_default: String, + /// Local video-transcription model and bundled helper readiness. + asr: socai_core::media::AsrModelStatus, } #[tauri::command] @@ -2945,9 +2947,27 @@ pub fn config_get() -> Result { chrome_profile_dir_default: default_managed_profile_dir(), output_dir: config.runs.dir.unwrap_or_default(), output_dir_default: default_runs_root_display(), + asr: socai_core::media::asr_model_status().map_err(|err| format!("{err:#}"))?, }) } +#[tauri::command] +pub fn asr_model_status() -> Result { + socai_core::media::asr_model_status().map_err(|err| format!("{err:#}")) +} + +#[tauri::command] +pub async fn asr_model_install( + app: AppHandle, +) -> Result { + let progress_app = app.clone(); + socai_core::media::install_asr_model(move |progress| { + let _ = progress_app.emit("asr:model-progress", progress); + }) + .await + .map_err(|err| format!("{err:#}")) +} + #[tauri::command] pub async fn pro_activate( telemetry: State<'_, DesktopTelemetry>, diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index d5d86442..6e28c292 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -352,6 +352,8 @@ pub fn run() { commands::config_get, commands::config_set, commands::config_unset, + commands::asr_model_status, + commands::asr_model_install, commands::pro_activate, commands::auth_session, commands::auth_sms_send, diff --git a/app/src/lib/i18n.ts b/app/src/lib/i18n.ts index c50053f9..6ecbc0ed 100644 --- a/app/src/lib/i18n.ts +++ b/app/src/lib/i18n.ts @@ -281,6 +281,29 @@ const messages = { en: "where run reports, traces, and screenshots are saved.", zh: "运行报告、轨迹和截图的保存位置。", }, + "settings.asr": { en: "local video transcription", zh: "本地视频转写" }, + "settings.asrReady": { en: "ready", zh: "已就绪" }, + "settings.asrInstall": { en: "install model", zh: "安装模型" }, + "settings.asrInstalling": { en: "installing…", zh: "安装中…" }, + "settings.asrPreparing": { en: "preparing download…", zh: "正在准备下载…" }, + "settings.asrDownloading": { + en: "downloading model… {percent}%", + zh: "正在下载模型… {percent}%", + }, + "settings.asrExtracting": { en: "extracting model…", zh: "正在解压模型…" }, + "settings.asrFinalizing": { en: "installing voice detection…", zh: "正在安装语音检测模型…" }, + "settings.asrHint": { + en: "Qwen3-ASR runs on this device. The download is about 0.88 GB and audio is never sent to an ASR service.", + zh: "Qwen3-ASR 在本机运行，下载约 0.88 GB，音频不会发送到在线转写服务。", + }, + "settings.asrHelperMissing": { + en: "reinstall app", + zh: "请重新安装应用", + }, + "settings.asrInstallFailed": { + en: "model installation failed. check the network and try again.", + zh: "模型安装失败，请检查网络后重试。", + }, "settings.inviteCode": { en: "invite code", zh: "邀请码" }, "settings.enter": { en: "enter", zh: "输入" }, "settings.loginForInvite": { en: "sign in first", zh: "请先登录" }, diff --git a/app/src/panels/settings.ts b/app/src/panels/settings.ts index 40f1edbc..e8a81bbc 100644 --- a/app/src/panels/settings.ts +++ b/app/src/panels/settings.ts @@ -12,6 +12,7 @@ //! display preference kept in localStorage via `setTimezone`. import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; import type { ShellState } from "../main"; import { esc } from "../lib/html"; import { @@ -31,6 +32,21 @@ interface DesktopConfig { chrome_profile_dir_default: string; output_dir: string; output_dir_default: string; + asr: AsrModelStatus; +} + +interface AsrModelStatus { + model: string; + installed: boolean; + helper_available: boolean; + available: boolean; + archive_bytes: number; +} + +interface AsrInstallProgress { + stage: "download" | "extract" | "vad" | "complete" | string; + downloaded_bytes: number; + total_bytes: number | null; } interface SettingsDraft { @@ -70,6 +86,10 @@ export namespace settingsMenu { let draft: SettingsDraft | null = null; let status: SaveStatus = ""; let inviteMessage = ""; + let asrInstalling = false; + let asrInstallError = ""; + let asrProgress: AsrInstallProgress | null = null; + let asrProgressListenerStarted = false; let statusTimer: number | null = null; let appVersion = ""; @@ -182,6 +202,7 @@ export namespace settingsMenu { return `