diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..e76d092c --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[env] +SHERPA_ONNX_ARCHIVE_DIR = { value = "target/sherpa-onnx-archives", relative = true } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4da4259..bc64be2b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -182,8 +182,11 @@ jobs: run: | set -euo pipefail + node scripts/prepare-sherpa-onnx-libs.mjs aarch64-apple-darwin x86_64-apple-darwin cargo build -p socai-cli --release --target aarch64-apple-darwin cargo build -p socai-cli --release --target x86_64-apple-darwin + cargo build -p socai-asr --release --target aarch64-apple-darwin + cargo build -p socai-asr --release --target x86_64-apple-darwin cli_dir="target/socai-cli-macos-universal" mkdir -p "${cli_dir}" @@ -191,15 +194,22 @@ jobs: target/aarch64-apple-darwin/release/socai \ target/x86_64-apple-darwin/release/socai \ -output "${cli_dir}/socai" + lipo -create \ + target/aarch64-apple-darwin/release/socai-asr \ + target/x86_64-apple-darwin/release/socai-asr \ + -output "${cli_dir}/socai-asr" chmod 0755 "${cli_dir}/socai" - - actual_archs="$(lipo -archs "${cli_dir}/socai")" - echo "${cli_dir}/socai architectures: ${actual_archs}" - for expected_arch in arm64 x86_64; do - if [[ " ${actual_archs} " != *" ${expected_arch} "* ]]; then - echo "expected ${cli_dir}/socai to include ${expected_arch}, got: ${actual_archs}" >&2 - exit 1 - fi + chmod 0755 "${cli_dir}/socai-asr" + + for binary in socai socai-asr; do + actual_archs="$(lipo -archs "${cli_dir}/${binary}")" + echo "${cli_dir}/${binary} architectures: ${actual_archs}" + for expected_arch in arm64 x86_64; do + if [[ " ${actual_archs} " != *" ${expected_arch} "* ]]; then + echo "expected ${cli_dir}/${binary} to include ${expected_arch}, got: ${actual_archs}" >&2 + exit 1 + fi + done done version_output="$("${cli_dir}/socai" --version)" @@ -219,18 +229,25 @@ jobs: BUILD_SHA="${SOCAI_BUILD_SHA:?SOCAI_BUILD_SHA not set by apply release version step}" cli_binary="target/socai-cli-macos-universal/socai" + asr_binary="target/socai-cli-macos-universal/socai-asr" mkdir -p dist-artifacts if [ ! -x "${cli_binary}" ]; then echo "no CLI binary found at ${cli_binary}" >&2 exit 1 fi + if [ ! -x "${asr_binary}" ]; then + echo "no local ASR helper found at ${asr_binary}" >&2 + exit 1 + fi package_dir="${RUNNER_TEMP}/socai-cli-package" rm -rf "${package_dir}" mkdir -p "${package_dir}" cp "${cli_binary}" "${package_dir}/socai" + cp "${asr_binary}" "${package_dir}/socai-asr" chmod 0755 "${package_dir}/socai" + chmod 0755 "${package_dir}/socai-asr" created_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" cat > "${package_dir}/manifest.json" < socai-cli-macos-universal.tar.gz.sha256) cp scripts/install-cli.sh dist-artifacts/install.sh (cd dist-artifacts && shasum -a 256 -c socai-cli-macos-universal.tar.gz.sha256) archive_listing="${RUNNER_TEMP}/socai-cli-archive-files.txt" tar -tzf dist-artifacts/socai-cli-macos-universal.tar.gz > "${archive_listing}" - for required_file in socai manifest.json; do + for required_file in socai socai-asr manifest.json; do if ! grep -Fxq "${required_file}" "${archive_listing}"; then echo "CLI archive missing ${required_file}" >&2 exit 1 @@ -700,7 +717,8 @@ jobs: SOCAI_PRO_BASE_URL: ${{ secrets.SOCAI_PRO_BASE_URL }} run: | $ErrorActionPreference = 'Stop' - cargo build -p socai-cli --release + node scripts/prepare-sherpa-onnx-libs.mjs x86_64-pc-windows-msvc + cargo build -p socai-cli -p socai-asr --release $versionOutput = & .\target\release\socai.exe --version $versionOutput if ($versionOutput -ne "socai $env:VERSION") { @@ -762,6 +780,7 @@ jobs: Remove-Item -Recurse -Force $dist, $packageDir -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $dist, $packageDir | Out-Null Copy-Item .\target\release\socai.exe (Join-Path $packageDir 'socai.exe') + Copy-Item .\target\release\socai-asr.exe (Join-Path $packageDir 'socai-asr.exe') $createdAt = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') [ordered]@{ base_sha = $env:BASE_SHA @@ -772,7 +791,7 @@ jobs: } | ConvertTo-Json | Set-Content -Encoding utf8NoBOM (Join-Path $packageDir 'manifest.json') $archive = Join-Path $dist 'socai-cli-windows-x86_64.zip' - Compress-Archive -Path (Join-Path $packageDir 'socai.exe'), (Join-Path $packageDir 'manifest.json') -DestinationPath $archive + Compress-Archive -Path (Join-Path $packageDir 'socai.exe'), (Join-Path $packageDir 'socai-asr.exe'), (Join-Path $packageDir 'manifest.json') -DestinationPath $archive $hash = (Get-FileHash -Algorithm SHA256 $archive).Hash.ToLowerInvariant() $checksumPath = "$archive.sha256" [System.IO.File]::WriteAllText($checksumPath, "$hash socai-cli-windows-x86_64.zip`n", [System.Text.Encoding]::ASCII) @@ -789,6 +808,9 @@ jobs: if ($versionOutput -ne "socai $env:VERSION") { throw "expected archive CLI version 'socai $env:VERSION', got: $versionOutput" } + if (-not (Test-Path (Join-Path $verifyDir 'socai-asr.exe'))) { + throw 'archive missing socai-asr.exe' + } $manifest = Get-Content -Raw (Join-Path $verifyDir 'manifest.json') | ConvertFrom-Json if ($manifest.version -ne $env:VERSION) { throw 'manifest version mismatch' } if ($manifest.target -ne 'windows-x86_64') { throw 'manifest target mismatch' } diff --git a/Cargo.lock b/Cargo.lock index 32d2a5f6..0a24ecf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.8.12" @@ -466,6 +477,44 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cairo-rs" version = "0.18.5" @@ -665,6 +714,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -781,6 +840,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "cookie" version = "0.18.1" @@ -859,6 +924,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1089,6 +1169,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + [[package]] name = "der" version = "0.8.0" @@ -1180,6 +1266,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1457,6 +1544,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fast-float2" version = "0.2.4" @@ -1855,9 +1948,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -2127,6 +2222,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hmac-sha256" version = "1.1.14" @@ -2527,6 +2631,15 @@ dependencies = [ "cfb", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "inquire" version = "0.7.5" @@ -2817,6 +2930,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + [[package]] name = "libc" version = "0.2.186" @@ -2903,12 +3022,33 @@ dependencies = [ "imgref", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "lzma-rust2" version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "macro_rules_attribute" version = "0.2.2" @@ -3779,7 +3919,7 @@ dependencies = [ "ort-sys", "smallvec", "tracing", - "ureq", + "ureq 3.3.0", ] [[package]] @@ -3790,7 +3930,7 @@ checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" dependencies = [ "hmac-sha256", "lzma-rust2", - "ureq", + "ureq 3.3.0", ] [[package]] @@ -3886,6 +4026,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -4649,6 +4799,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -5098,6 +5249,29 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "sherpa-onnx" +version = "1.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4ef11f6b23c916a8e36c3cf65201f796d0e6aa30182a18d4f798d0b7f62547" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566aad07d0924a7ed897cc035cc67ad71cdfddb959717dbf445031d5d4b58d4d" +dependencies = [ + "bzip2 0.4.4", + "tar", + "ureq 2.12.1", + "zip 2.4.2", +] + [[package]] name = "shlex" version = "1.3.0" @@ -5207,6 +5381,17 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socai-asr" +version = "0.5.5" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "sherpa-onnx", + "symphonia", +] + [[package]] name = "socai-cli" version = "0.5.5" @@ -5234,6 +5419,7 @@ dependencies = [ "async-trait", "async-tungstenite", "base64 0.22.1", + "bzip2 0.6.1", "chrono", "dirs 5.0.1", "futures", @@ -5243,7 +5429,8 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "symphonia", + "sha2", + "tar", "tempfile", "thiserror 2.0.18", "tokio", @@ -5441,11 +5628,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" dependencies = [ "lazy_static", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-pcm", "symphonia-core", "symphonia-format-isomp4", + "symphonia-format-riff", "symphonia-metadata", ] +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + [[package]] name = "symphonia-core" version = "0.5.5" @@ -5472,6 +5696,18 @@ dependencies = [ "symphonia-utils-xiph", ] +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "symphonia-metadata" version = "0.5.5" @@ -6569,6 +6805,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "ureq" version = "3.3.0" @@ -6958,6 +7210,24 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webview2-com" version = "0.38.2" @@ -7740,6 +8010,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "y4m" version = "0.8.0" @@ -7815,6 +8094,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" @@ -7849,6 +8142,36 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.5.2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.4", + "hmac", + "indexmap 2.14.0", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.18", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + [[package]] name = "zip" version = "4.6.1" @@ -7899,6 +8222,34 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index ee6c1c06..0bd50213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "app/src-tauri", + "asr", "core", "cli", ] diff --git a/README.md b/README.md index 9343a183..a978b92c 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ If a prebuilt binary is unavailable for your platform, or you need a source buil ```bash git clone https://github.com/socai-io/socai.git cd socai +node scripts/prepare-sherpa-onnx-libs.mjs +cargo install --path asr --force cargo install --path cli --force ``` @@ -170,7 +172,7 @@ socai xhs get-notes \ | `--preview` | Read only search-result or author-page post cards. | | `--download-media` | Download images and videos from opened posts and record local paths. | | `--ocr` | Run local OCR on post images or a video post's cover. | -| `--transcribe-audio` | Download opened videos and transcribe speech; requires signing in and selecting socai agent. | +| `--transcribe-audio` | Download opened videos and transcribe speech locally with Qwen3-ASR 0.6B Int8. | | `--filter ` | Apply a Xiaohongshu search-page filter; repeat to combine filters. | | `--pretty` | Pretty-print the final JSON result. | | `--debug-snapshot` | Save page DOM, accessibility trees, and screenshots for development diagnostics. | @@ -194,6 +196,17 @@ socai xhs search "Shanghai weekend activities" \ --filter sort=最新 ``` +### Local video transcription + +Install the local Qwen3-ASR model once before using `--transcribe-audio`: + +```bash +socai asr install +socai asr status --json +``` + +The compressed download is about 0.88 GB and the installed model uses about 0.95 GB. Audio stays on the device, and transcription does not require a socai account, credits, or an API key. + ## Browser and login modes socai supports four Chrome profile modes: diff --git a/app/package.json b/app/package.json index 60af3fc4..39418bc4 100644 --- a/app/package.json +++ b/app/package.json @@ -11,6 +11,7 @@ "dev:desktop:kill": "pkill -f 'target/debug/socai_ap[p]' 2>/dev/null; lsof -ti :1421 | xargs kill 2>/dev/null; for i in $(seq 1 20); do lsof -ti :1421 >/dev/null 2>&1 || break; sleep 0.1; done; true", "build": "tsc && vite build", "prepare:lark-cli": "node scripts/prepare-lark-cli.mjs", + "prepare:asr-helper": "node scripts/prepare-asr-helper.mjs", "build:mac": "tauri build --ci --target universal-apple-darwin --bundles app,dmg --config '{\"bundle\":{\"macOS\":{\"signingIdentity\":\"-\"}}}'", "build:mac:universal": "tauri build --ci --target universal-apple-darwin --bundles app,dmg --config '{\"bundle\":{\"macOS\":{\"signingIdentity\":\"-\"}}}'", "build:mac:apple-silicon": "tauri build --ci --target aarch64-apple-darwin --bundles app,dmg --config '{\"bundle\":{\"macOS\":{\"signingIdentity\":\"-\"}}}'", diff --git a/app/scripts/prepare-asr-helper.mjs b/app/scripts/prepare-asr-helper.mjs new file mode 100644 index 00000000..03606690 --- /dev/null +++ b/app/scripts/prepare-asr-helper.mjs @@ -0,0 +1,56 @@ +import { chmodSync, copyFileSync, existsSync, mkdirSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +const APP_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const REPO_DIR = path.resolve(APP_DIR, ".."); +const BIN_DIR = path.join(APP_DIR, "src-tauri", "binaries"); +const explicitTarget = process.env.TAURI_ENV_TARGET_TRIPLE; +const release = process.env.TAURI_ENV_DEBUG === "false" || explicitTarget === "universal-apple-darwin"; +const profile = release ? "release" : "debug"; + +mkdirSync(BIN_DIR, { recursive: true }); + +function rustHost() { + const output = execFileSync("rustc", ["-vV"], { cwd: REPO_DIR, encoding: "utf8" }); + const line = output.split("\n").find((item) => item.startsWith("host: ")); + if (!line) throw new Error("rustc -vV did not report a host target"); + return line.slice("host: ".length).trim(); +} + +function build(target) { + const args = ["build", "-p", "socai-asr", "--target", target]; + if (release) args.push("--release"); + execFileSync("cargo", args, { cwd: REPO_DIR, stdio: "inherit" }); + const extension = target.includes("windows") ? ".exe" : ""; + const source = path.join(REPO_DIR, "target", target, profile, `socai-asr${extension}`); + if (!existsSync(source)) throw new Error(`ASR helper build missing: ${source}`); + const destination = path.join(BIN_DIR, `socai-asr-${target}${extension}`); + copyFileSync(source, destination); + if (!extension) chmodSync(destination, 0o755); + console.log(`[socai-asr] ready ${path.relative(APP_DIR, destination)}`); + return destination; +} + +const target = explicitTarget || rustHost(); +if (target === "universal-apple-darwin") { + execFileSync( + process.execPath, + [path.join(REPO_DIR, "scripts", "prepare-sherpa-onnx-libs.mjs"), "aarch64-apple-darwin", "x86_64-apple-darwin"], + { cwd: REPO_DIR, stdio: "inherit" }, + ); + const arm = build("aarch64-apple-darwin"); + const intel = build("x86_64-apple-darwin"); + const universal = path.join(BIN_DIR, "socai-asr-universal-apple-darwin"); + execFileSync("lipo", ["-create", arm, intel, "-output", universal], { stdio: "inherit" }); + chmodSync(universal, 0o755); + console.log(`[socai-asr] ready ${path.relative(APP_DIR, universal)}`); +} else { + execFileSync( + process.execPath, + [path.join(REPO_DIR, "scripts", "prepare-sherpa-onnx-libs.mjs"), target], + { cwd: REPO_DIR, stdio: "inherit" }, + ); + build(target); +} diff --git a/app/src-tauri/src/commands.rs b/app/src-tauri/src/commands.rs index 48965c68..25db069c 100644 --- a/app/src-tauri/src/commands.rs +++ b/app/src-tauri/src/commands.rs @@ -2929,6 +2929,8 @@ pub struct DesktopConfig { output_dir: String, /// Resolved default run-artifact root (shown as the input placeholder). output_dir_default: String, + /// Local video-transcription model and bundled helper readiness. + asr: socai_core::media::AsrModelStatus, } #[tauri::command] @@ -2945,9 +2947,27 @@ pub fn config_get() -> Result { chrome_profile_dir_default: default_managed_profile_dir(), output_dir: config.runs.dir.unwrap_or_default(), output_dir_default: default_runs_root_display(), + asr: socai_core::media::asr_model_status().map_err(|err| format!("{err:#}"))?, }) } +#[tauri::command] +pub fn asr_model_status() -> Result { + socai_core::media::asr_model_status().map_err(|err| format!("{err:#}")) +} + +#[tauri::command] +pub async fn asr_model_install( + app: AppHandle, +) -> Result { + let progress_app = app.clone(); + socai_core::media::install_asr_model(move |progress| { + let _ = progress_app.emit("asr:model-progress", progress); + }) + .await + .map_err(|err| format!("{err:#}")) +} + #[tauri::command] pub async fn pro_activate( telemetry: State<'_, DesktopTelemetry>, diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index d5d86442..6e28c292 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -352,6 +352,8 @@ pub fn run() { commands::config_get, commands::config_set, commands::config_unset, + commands::asr_model_status, + commands::asr_model_install, commands::pro_activate, commands::auth_session, commands::auth_sms_send, diff --git a/app/src-tauri/tauri.conf.json b/app/src-tauri/tauri.conf.json index 0c3e5390..be6af9c5 100644 --- a/app/src-tauri/tauri.conf.json +++ b/app/src-tauri/tauri.conf.json @@ -4,9 +4,9 @@ "version": "0.5.5", "identifier": "com.socai.app", "build": { - "beforeDevCommand": "pnpm run prepare:lark-cli && pnpm run dev", + "beforeDevCommand": "pnpm run prepare:lark-cli && pnpm run prepare:asr-helper && pnpm run dev", "devUrl": "http://localhost:1421", - "beforeBuildCommand": "pnpm run prepare:lark-cli && pnpm run build", + "beforeBuildCommand": "pnpm run prepare:lark-cli && pnpm run prepare:asr-helper && pnpm run build", "frontendDist": "../dist" }, "app": { @@ -38,7 +38,8 @@ "active": true, "targets": "all", "externalBin": [ - "binaries/lark-cli" + "binaries/lark-cli", + "binaries/socai-asr" ], "resources": [ "third-party/lark-cli-LICENSE" diff --git a/app/src/lib/i18n.ts b/app/src/lib/i18n.ts index c50053f9..6ecbc0ed 100644 --- a/app/src/lib/i18n.ts +++ b/app/src/lib/i18n.ts @@ -281,6 +281,29 @@ const messages = { en: "where run reports, traces, and screenshots are saved.", zh: "运行报告、轨迹和截图的保存位置。", }, + "settings.asr": { en: "local video transcription", zh: "本地视频转写" }, + "settings.asrReady": { en: "ready", zh: "已就绪" }, + "settings.asrInstall": { en: "install model", zh: "安装模型" }, + "settings.asrInstalling": { en: "installing…", zh: "安装中…" }, + "settings.asrPreparing": { en: "preparing download…", zh: "正在准备下载…" }, + "settings.asrDownloading": { + en: "downloading model… {percent}%", + zh: "正在下载模型… {percent}%", + }, + "settings.asrExtracting": { en: "extracting model…", zh: "正在解压模型…" }, + "settings.asrFinalizing": { en: "installing voice detection…", zh: "正在安装语音检测模型…" }, + "settings.asrHint": { + en: "Qwen3-ASR runs on this device. The download is about 0.88 GB and audio is never sent to an ASR service.", + zh: "Qwen3-ASR 在本机运行,下载约 0.88 GB,音频不会发送到在线转写服务。", + }, + "settings.asrHelperMissing": { + en: "reinstall app", + zh: "请重新安装应用", + }, + "settings.asrInstallFailed": { + en: "model installation failed. check the network and try again.", + zh: "模型安装失败,请检查网络后重试。", + }, "settings.inviteCode": { en: "invite code", zh: "邀请码" }, "settings.enter": { en: "enter", zh: "输入" }, "settings.loginForInvite": { en: "sign in first", zh: "请先登录" }, diff --git a/app/src/main.ts b/app/src/main.ts index 011173f9..18a2a10e 100644 --- a/app/src/main.ts +++ b/app/src/main.ts @@ -137,7 +137,7 @@ export interface NoteData { comments?: NoteComment[]; // top comments captured with the read media?: NoteMedia[]; // media[0] === cover media_dir?: string; // run-relative folder, when src paths are relative - transcript?: string; // video audio transcript (cloud ASR) + transcript?: string; // video audio transcript (local ASR) saved?: boolean; // Tolerate extra fields the archive may carry. [key: string]: unknown; diff --git a/app/src/panels/settings.ts b/app/src/panels/settings.ts index 40f1edbc..e8a81bbc 100644 --- a/app/src/panels/settings.ts +++ b/app/src/panels/settings.ts @@ -12,6 +12,7 @@ //! display preference kept in localStorage via `setTimezone`. import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; import type { ShellState } from "../main"; import { esc } from "../lib/html"; import { @@ -31,6 +32,21 @@ interface DesktopConfig { chrome_profile_dir_default: string; output_dir: string; output_dir_default: string; + asr: AsrModelStatus; +} + +interface AsrModelStatus { + model: string; + installed: boolean; + helper_available: boolean; + available: boolean; + archive_bytes: number; +} + +interface AsrInstallProgress { + stage: "download" | "extract" | "vad" | "complete" | string; + downloaded_bytes: number; + total_bytes: number | null; } interface SettingsDraft { @@ -70,6 +86,10 @@ export namespace settingsMenu { let draft: SettingsDraft | null = null; let status: SaveStatus = ""; let inviteMessage = ""; + let asrInstalling = false; + let asrInstallError = ""; + let asrProgress: AsrInstallProgress | null = null; + let asrProgressListenerStarted = false; let statusTimer: number | null = null; let appVersion = ""; @@ -182,6 +202,7 @@ export namespace settingsMenu { return `