From ec3b078ef16d80b565f59f1ab92737285d2496bf Mon Sep 17 00:00:00 2001 From: UnoxyRich Date: Sat, 13 Jun 2026 10:49:47 +0800 Subject: [PATCH 01/12] Add check for KUBE_CONFIG_DATA before deployment --- .github/workflows/deploy-runtime.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-runtime.yml b/.github/workflows/deploy-runtime.yml index 11df07f2..bc88dd2c 100644 --- a/.github/workflows/deploy-runtime.yml +++ b/.github/workflows/deploy-runtime.yml @@ -8,7 +8,6 @@ on: jobs: deploy: - if: ${{ secrets.KUBE_CONFIG_DATA != '' }} runs-on: ubuntu-latest permissions: contents: read @@ -17,11 +16,25 @@ jobs: - uses: azure/setup-kubectl@v4 with: version: latest + - name: Check KUBE_CONFIG_DATA + id: check_secret + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG_DATA }} + run: | + if [ -z "$KUBE_CONFIG" ]; then + echo "KUBE_CONFIG_DATA secret is not set. Skipping deployment." + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi - name: Configure kubeconfig + if: steps.check_secret.outputs.skip != 'true' run: | mkdir -p "$HOME/.kube" echo "${{ secrets.KUBE_CONFIG_DATA }}" | base64 --decode > "$HOME/.kube/config" - name: Deploy production overlay + if: steps.check_secret.outputs.skip != 'true' run: kubectl apply -k infra/k8s/overlays/production - name: Wait for rollout + if: steps.check_secret.outputs.skip != 'true' run: kubectl rollout status deployment/syntax-senpai-runtime -n syntax-senpai --timeout=180s From c0220ee0c794dc1a2a5a6aaf6f7a5224ab53d1b7 Mon Sep 17 00:00:00 2001 From: UnoxyRich Date: Fri, 12 Jun 2026 22:35:44 +0800 Subject: [PATCH 02/12] feat: add NVIDIA NIM provider and improve token/cost metering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces the NVIDIA NIM provider (OpenAI-compatible, integrate.api.nvidia.com) with 6 model presets and wires it into the provider factory, env config, and the renderer's Settings UI. Also fixes the cost/context meter: `estimateCost` now returns null for unpriced models so the UI shows "—" instead of $0.00, and the streaming path captures the terminal `done` usage chunk so prompt-token counts (and thus the context-fill meter) are accurate during tool calls. Co-Authored-By: Claude Sonnet 4.6 --- apps/desktop/src/main/ipc/provider.ts | 9 +++ apps/desktop/src/renderer/src/App.vue | 31 +++++++- apps/desktop/src/renderer/src/stores/chat.ts | 34 +++++++-- .../SyntaxSenpai.xcodeproj/project.pbxproj | 4 +- packages/ai-core/src/providers/index.ts | 8 ++- packages/ai-core/src/providers/nvidia.ts | 70 +++++++++++++++++++ packages/ai-core/src/runtime.ts | 2 + 7 files changed, 148 insertions(+), 10 deletions(-) create mode 100644 packages/ai-core/src/providers/nvidia.ts diff --git a/apps/desktop/src/main/ipc/provider.ts b/apps/desktop/src/main/ipc/provider.ts index ae8003a7..9f0515f5 100644 --- a/apps/desktop/src/main/ipc/provider.ts +++ b/apps/desktop/src/main/ipc/provider.ts @@ -23,6 +23,7 @@ const OPENAI_COMPATIBLE_BASE_URLS: Record = { perplexity: 'https://api.perplexity.ai', huggingface: 'https://router.huggingface.co/v1', 'github-models': 'https://models.inference.ai.azure.com', + nvidia: 'https://integrate.api.nvidia.com/v1', lmstudio: 'http://127.0.0.1:1234/v1', } @@ -115,6 +116,14 @@ const FALLBACK_MODELS: Record { id: 'openai/gpt-4o-mini', displayName: 'GPT-4o Mini' }, { id: 'meta/Llama-3.3-70B-Instruct', displayName: 'Llama 3.3 70B' }, ], + nvidia: [ + { id: 'meta/llama-3.3-70b-instruct', displayName: 'Llama 3.3 70B Instruct' }, + { id: 'nvidia/llama-3.3-nemotron-super-49b-v1.5', displayName: 'Nemotron Super 49B v1.5' }, + { id: 'qwen/qwen3-coder-480b-a35b-instruct', displayName: 'Qwen3 Coder 480B' }, + { id: 'openai/gpt-oss-20b', displayName: 'GPT-OSS 20B' }, + { id: 'moonshotai/kimi-k2.6', displayName: 'Kimi K2.6' }, + { id: 'meta/llama-3.1-8b-instruct', displayName: 'Llama 3.1 8B (Fast)' }, + ], lmstudio: [{ id: 'local-model', displayName: 'Detected Local Model' }], ollama: [ { id: 'llama-3.1-8b', displayName: 'Llama 3.1 8B' }, diff --git a/apps/desktop/src/renderer/src/App.vue b/apps/desktop/src/renderer/src/App.vue index c3fb5746..8e653074 100644 --- a/apps/desktop/src/renderer/src/App.vue +++ b/apps/desktop/src/renderer/src/App.vue @@ -1143,6 +1143,7 @@ const providerOrder = [ 'xai', 'huggingface', 'github-models', + 'nvidia', ] const providerMetadata = [ { @@ -1271,6 +1272,18 @@ const providerMetadata = [ { id: 'meta/Llama-3.3-70B-Instruct', displayName: 'Llama 3.3 70B' }, ], }, + { + id: 'nvidia', + displayName: 'NVIDIA NIM', + models: [ + { id: 'meta/llama-3.3-70b-instruct', displayName: 'Llama 3.3 70B Instruct' }, + { id: 'nvidia/llama-3.3-nemotron-super-49b-v1.5', displayName: 'Nemotron Super 49B v1.5' }, + { id: 'qwen/qwen3-coder-480b-a35b-instruct', displayName: 'Qwen3 Coder 480B' }, + { id: 'openai/gpt-oss-20b', displayName: 'GPT-OSS 20B' }, + { id: 'moonshotai/kimi-k2.6', displayName: 'Kimi K2.6' }, + { id: 'meta/llama-3.1-8b-instruct', displayName: 'Llama 3.1 8B (Fast)' }, + ], + }, ] const providers = providerOrder .map((id) => providerMetadata.find((provider) => provider.id === id)) @@ -1304,6 +1317,12 @@ const MODEL_CONTEXT_WINDOWS: Record = { 'command-r-plus': 128000, 'command-r': 128000, 'command-a-03-2025': 256000, + 'meta/llama-3.3-70b-instruct': 128000, + 'nvidia/llama-3.3-nemotron-super-49b-v1.5': 128000, + 'qwen/qwen3-coder-480b-a35b-instruct': 256000, + 'openai/gpt-oss-20b': 128000, + 'moonshotai/kimi-k2.6': 256000, + 'meta/llama-3.1-8b-instruct': 128000, } const colorPresets: Array<{ @@ -2039,6 +2058,12 @@ const contextWindowSize = computed(() => ) const estimatedTokensUsed = computed(() => { + // Prefer the real prompt-token count from the last round-trip — it reflects + // the actual context sent (system prompt + tool defs + history). Fall back to + // a chars/4 estimate before the first response, or for providers that report + // no usage (e.g. ollama, lmstudio). + const real = store.usageTotals.lastPromptTokens + if (real > 0) return real const totalChars = store.messages.reduce((sum: number, m: any) => { const content = typeof m.content === 'string' ? m.content : JSON.stringify(m.content || '') return sum + content.length @@ -5939,7 +5964,7 @@ async function handleImportData() {
- ≈ {{ formatLocalizedCost(store.usageTotals.costUsd, locale) }} + ≈ {{ store.usageTotals.costUnpriced && store.usageTotals.costUsd === 0 + ? '—' + : formatLocalizedCost(store.usageTotals.costUsd, locale) }}
{ match: /deepseek/i, input: 0.00014, output: 0.00028 }, { match: /llama-3\.1-70b/i, input: 0.00059, output: 0.00079 }, { match: /mixtral-8x7b/i, input: 0.00024, output: 0.00024 }, + // NVIDIA NIM hosted open models (estimates; tune as NIM pricing firms up). + { match: /nemotron/i, input: 0.0009, output: 0.0009 }, + { match: /qwen3-coder/i, input: 0.002, output: 0.002 }, + { match: /gpt-oss/i, input: 0.0002, output: 0.0006 }, + { match: /kimi-k2/i, input: 0.0006, output: 0.0025 }, + { match: /llama-3\.3-70b/i, input: 0.0009, output: 0.0009 }, + { match: /llama-3\.1-8b/i, input: 0.0002, output: 0.0002 }, ] // 根据模型名称粗略估算本轮对话成本，主要给界面展示用，不追求精确计费。 -function estimateCost(model: string, promptTokens: number, completionTokens: number): number { +// Returns null when no pricing row matches so callers can show "unpriced" +// (—) instead of a misleading $0.00. +function estimateCost(model: string, promptTokens: number, completionTokens: number): number | null { const row = MODEL_COST_PER_1K.find((r) => r.match.test(model || '')) - if (!row) return 0 + if (!row) return null return (promptTokens / 1000) * row.input + (completionTokens / 1000) * row.output } @@ -248,6 +257,7 @@ const DEFAULT_MODEL_BY_PROVIDER: Record = { huggingface: 'meta-llama/Llama-3.3-70B-Instruct', 'github-models': 'openai/gpt-4o-mini', ollama: 'llama3.1:8b', + nvidia: 'meta/llama-3.3-70b-instruct', } const AFFECTION_STORAGE_KEY = 'syntax-senpai-affection' @@ -1068,12 +1078,18 @@ export const useChatStore = defineStore('chat', () => { totalTokens: 0, costUsd: 0, turns: 0, + // Real prompt-token count of the most recent round-trip — drives the + // context meter (exact context fill incl. system prompt + tool defs). + lastPromptTokens: 0, + // True once a token-bearing turn used a model with no pricing row, so the + // UI shows "—" rather than a misleading $0.00. + costUnpriced: false, }) let proactiveChatTimer: ReturnType | null = null let onlineProactiveTimer: ReturnType | null = null function resetUsageTotals() { - usageTotals.value = { promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0, turns: 0 } + usageTotals.value = { promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0, turns: 0, lastPromptTokens: 0, costUnpriced: false } } function recordUsage(model: string, usage?: { promptTokens?: number; completionTokens?: number; totalTokens?: number }) { @@ -1081,12 +1097,15 @@ export const useChatStore = defineStore('chat', () => { const p = usage.promptTokens || 0 const c = usage.completionTokens || 0 const t = usage.totalTokens || p + c + const cost = estimateCost(model, p, c) usageTotals.value = { promptTokens: usageTotals.value.promptTokens + p, completionTokens: usageTotals.value.completionTokens + c, totalTokens: usageTotals.value.totalTokens + t, - costUsd: usageTotals.value.costUsd + estimateCost(model, p, c), + costUsd: usageTotals.value.costUsd + (cost ?? 0), turns: usageTotals.value.turns + 1, + lastPromptTokens: p || usageTotals.value.lastPromptTokens, + costUnpriced: usageTotals.value.costUnpriced || (cost === null && t > 0), } } @@ -1373,6 +1392,7 @@ export const useChatStore = defineStore('chat', () => { let reasoningContent = '' const toolCalls: Array<{ id: string; name: string; arguments: Record }> = [] let synthId = 0 + let usage: { promptTokens?: number; completionTokens?: number; totalTokens?: number } | undefined for await (const chunk of provider.stream(finalReq) as AsyncIterable) { if (req.signal?.aborted) break @@ -1391,6 +1411,10 @@ export const useChatStore = defineStore('chat', () => { ? (tc.arguments as Record) : {}, }) + } else if (chunk.type === 'done' && chunk.usage) { + // Providers emit the terminal usage chunk last (openai.ts / anthropic.ts). + // Capturing it here is what makes the cost/token meter work in tools mode. + usage = chunk.usage } else if (chunk.type === 'error') { throw new Error(chunk.error || 'stream error') } @@ -1401,7 +1425,7 @@ export const useChatStore = defineStore('chat', () => { content, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, reasoningContent: reasoningContent || undefined, - usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, + usage: usage ?? { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, finishReason: toolCalls.length > 0 ? 'tool_calls' : 'stop', } } diff --git a/apps/mobile/ios/SyntaxSenpai.xcodeproj/project.pbxproj b/apps/mobile/ios/SyntaxSenpai.xcodeproj/project.pbxproj index f48b65c2..b3738921 100644 --- a/apps/mobile/ios/SyntaxSenpai.xcodeproj/project.pbxproj +++ b/apps/mobile/ios/SyntaxSenpai.xcodeproj/project.pbxproj @@ -372,7 +372,7 @@ ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_DEBUG"; PRODUCT_BUNDLE_IDENTIFIER = com.syntaxsenpai.app; - PRODUCT_NAME = SyntaxSenpai; + PRODUCT_NAME = "SyntaxSenpai"; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_OBJC_BRIDGING_HEADER = "SyntaxSenpai/SyntaxSenpai-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; @@ -407,7 +407,7 @@ ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_RELEASE"; PRODUCT_BUNDLE_IDENTIFIER = com.syntaxsenpai.app; - PRODUCT_NAME = SyntaxSenpai; + PRODUCT_NAME = "SyntaxSenpai"; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_OBJC_BRIDGING_HEADER = "SyntaxSenpai/SyntaxSenpai-Bridging-Header.h"; SWIFT_VERSION = 5.0; diff --git a/packages/ai-core/src/providers/index.ts b/packages/ai-core/src/providers/index.ts index d7331500..e1e327f5 100644 --- a/packages/ai-core/src/providers/index.ts +++ b/packages/ai-core/src/providers/index.ts @@ -27,6 +27,7 @@ export { GitHubModelsProvider, createGitHubModelsProvider } from "./github-model export { XAIGrokProvider, createXAIGrokProvider } from "./xai-grok"; export { AzureOpenAIProvider, createAzureOpenAIProvider } from "./azure-openai"; export { FireworksProvider, createFireworksProvider } from "./fireworks-ai"; +export { NvidiaProvider, createNvidiaProvider } from "./nvidia"; import type { AIProvider, CreateProviderOptions } from "../types"; import { AnthropicProvider } from "./anthropic"; @@ -49,6 +50,7 @@ import { LMStudioProvider } from "./lmstudio"; import { XAIGrokProvider } from "./xai-grok"; import { AzureOpenAIProvider } from "./azure-openai"; import { FireworksProvider } from "./fireworks-ai"; +import { NvidiaProvider } from "./nvidia"; /** * Provider factory - creates provider instances by ID @@ -73,7 +75,8 @@ export type ProviderConfig = | { type: "openai-codex"; apiKey: string } | { type: "github-models"; apiKey: string } | { type: "azure-openai"; apiKey: string; resourceName: string } - | { type: "fireworks"; apiKey: string }; + | { type: "fireworks"; apiKey: string } + | { type: "nvidia"; apiKey: string }; export function createProvider(config: ProviderConfig): AIProvider { switch (config.type) { @@ -120,6 +123,8 @@ export function createProvider(config: ProviderConfig): AIProvider { }); case "fireworks": return new FireworksProvider({ apiKey: config.apiKey }); + case "nvidia": + return new NvidiaProvider({ apiKey: config.apiKey }); default: throw new Error(`Unknown provider type: ${(config as any).type}`); } @@ -150,6 +155,7 @@ export function getAllProviderMetadata() { new GitHubModelsProvider({ apiKey: "dummy" }), new AzureOpenAIProvider({ apiKey: "dummy", baseUrl: "https://dummy.openai.azure.com" }), new FireworksProvider({ apiKey: "dummy" }), + new NvidiaProvider({ apiKey: "dummy" }), ].map((p) => ({ id: p.id, displayName: p.displayName, diff --git a/packages/ai-core/src/providers/nvidia.ts b/packages/ai-core/src/providers/nvidia.ts new file mode 100644 index 00000000..30283fda --- /dev/null +++ b/packages/ai-core/src/providers/nvidia.ts @@ -0,0 +1,70 @@ +/** + * NVIDIA NIM (build.nvidia.com) - OpenAI-compatible provider + * + * Hosted inference for open models via https://integrate.api.nvidia.com/v1. + * Get an API key (nvapi-...) at https://build.nvidia.com/. + */ + +import type { ModelInfo } from "../types"; +import { OpenAIProvider } from "./openai"; + +const MODELS: ModelInfo[] = [ + { + id: "meta/llama-3.3-70b-instruct", + displayName: "Llama 3.3 70B Instruct", + contextWindow: 128000, + supportsTools: true, + supportsVision: false, + }, + { + id: "nvidia/llama-3.3-nemotron-super-49b-v1.5", + displayName: "Nemotron Super 49B v1.5", + contextWindow: 128000, + supportsTools: true, + supportsVision: false, + }, + { + id: "qwen/qwen3-coder-480b-a35b-instruct", + displayName: "Qwen3 Coder 480B", + contextWindow: 256000, + supportsTools: true, + supportsVision: false, + }, + { + id: "openai/gpt-oss-20b", + displayName: "GPT-OSS 20B", + contextWindow: 128000, + supportsTools: true, + supportsVision: false, + }, + { + id: "moonshotai/kimi-k2.6", + displayName: "Kimi K2.6", + contextWindow: 256000, + supportsTools: true, + supportsVision: false, + }, + { + id: "meta/llama-3.1-8b-instruct", + displayName: "Llama 3.1 8B (Fast)", + contextWindow: 128000, + supportsTools: true, + supportsVision: false, + }, +]; + +export class NvidiaProvider extends OpenAIProvider { + constructor(options: { apiKey?: string } = {}) { + super({ + ...options, + id: "nvidia", + displayName: "NVIDIA NIM", + baseUrl: "https://integrate.api.nvidia.com/v1", + supportedModels: MODELS, + }); + } +} + +export function createNvidiaProvider(apiKey: string): NvidiaProvider { + return new NvidiaProvider({ apiKey }); +} diff --git a/packages/ai-core/src/runtime.ts b/packages/ai-core/src/runtime.ts index 8e6c3eb6..cd25e3f3 100644 --- a/packages/ai-core/src/runtime.ts +++ b/packages/ai-core/src/runtime.ts @@ -394,6 +394,8 @@ export function createProviderConfigFromEnv( }; case "fireworks": return { type: "fireworks", apiKey: getRequired(env, "FIREWORKS_API_KEY") }; + case "nvidia": + return { type: "nvidia", apiKey: getRequired(env, "NVIDIA_API_KEY") }; default: throw new Error(`Unsupported provider: ${provider as string}`); } From 5cfec8cd0ee785a9a26aadc2f94cd73127797401 Mon Sep 17 00:00:00 2001 From: UnoxyRich Date: Sat, 13 Jun 2026 10:55:52 +0800 Subject: [PATCH 03/12] feat: embedded browser with agent-control and IPC bridge --- apps/desktop/electron.vite.config.mjs | 12 +- apps/desktop/src/main/index.ts | 21 +- apps/desktop/src/main/ipc/browser.ts | 151 +++++++++ apps/desktop/src/renderer/src/App.vue | 37 +++ apps/desktop/src/renderer/src/agent-tools.ts | 298 ++++++++++++++++- .../src/renderer/src/agent/run-turn.ts | 12 + .../src/renderer/src/agent/subagent-runner.ts | 3 + .../browser/__tests__/snapshot-format.test.ts | 78 +++++ .../browser/__tests__/url-allowlist.test.ts | 23 ++ .../src/renderer/src/browser/controller.ts | 267 +++++++++++++++ .../renderer/src/browser/snapshot-format.ts | 57 ++++ .../renderer/src/browser/snapshot-script.ts | 311 ++++++++++++++++++ .../renderer/src/components/BrowserPanel.vue | 309 +++++++++++++++++ .../src/renderer/src/stores/browser.ts | 193 +++++++++++ apps/desktop/src/renderer/src/stores/chat.ts | 63 +++- 15 files changed, 1828 insertions(+), 7 deletions(-) create mode 100644 apps/desktop/src/main/ipc/browser.ts create mode 100644 apps/desktop/src/renderer/src/browser/__tests__/snapshot-format.test.ts create mode 100644 apps/desktop/src/renderer/src/browser/__tests__/url-allowlist.test.ts create mode 100644 apps/desktop/src/renderer/src/browser/controller.ts create mode 100644 apps/desktop/src/renderer/src/browser/snapshot-format.ts create mode 100644 apps/desktop/src/renderer/src/browser/snapshot-script.ts create mode 100644 apps/desktop/src/renderer/src/components/BrowserPanel.vue create mode 100644 apps/desktop/src/renderer/src/stores/browser.ts diff --git a/apps/desktop/electron.vite.config.mjs b/apps/desktop/electron.vite.config.mjs index 48a8178a..dd184e2f 100644 --- a/apps/desktop/electron.vite.config.mjs +++ b/apps/desktop/electron.vite.config.mjs @@ -72,7 +72,17 @@ export default defineConfig({ '@': path.resolve(__dirname, './src/renderer/src') } }, - plugins: [vue(), UnoCSS()], + plugins: [ + vue({ + template: { + compilerOptions: { + // Electron's tag (embedded browser panel) is not a Vue component. + isCustomElement: (tag) => tag === 'webview' + } + } + }), + UnoCSS() + ], optimizeDeps: { exclude: ['expo-secure-store', 'expo', 'keytar', 'better-sqlite3'] }, diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 450e8ab2..a5fff5dd 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -64,6 +64,7 @@ import { registerRepositoryIpc } from './ipc/repository' import { registerSkillsIpc } from './ipc/skills' import { registerPendingPluginsIpc } from './ipc/pending-plugins' import { registerWechatIpc, autoResumeBot as autoResumeWechatBot } from './ipc/wechat' +import { registerBrowserIpc, isAllowedBrowserUrl, BROWSER_PARTITION } from './ipc/browser' import { startWsServer } from './ws-server' import { mainLogger } from './logger' @@ -416,10 +417,27 @@ function createWindow(forcedMode?: WindowMode): void { // renderer. The sandbox would block require() in the preload layer. sandbox: false, nodeIntegration: false, - contextIsolation: true + contextIsolation: true, + // Embedded browser panel uses tags (see BrowserPanel.vue). + webviewTag: true } }) mainWindow = createdWindow + + // Lock down every the renderer attaches: sandboxed guest, no + // node, no preload, http(s) only, and only our persistent browser session. + createdWindow.webContents.on('will-attach-webview', (event: any, webPreferences: any, params: any) => { + delete webPreferences.preload + delete webPreferences.preloadURL + webPreferences.nodeIntegration = false + webPreferences.contextIsolation = true + webPreferences.sandbox = true + const src = String(params.src || '') + const srcAllowed = src === '' || src === 'about:blank' || isAllowedBrowserUrl(src) + if (!srcAllowed || params.partition !== BROWSER_PARTITION) { + event.preventDefault() + } + }) currentWindowFrameless = shouldUseFramelessWindow(mode) if (isDev) { @@ -626,6 +644,7 @@ app.whenReady().then(() => { registerSkillsIpc() registerPendingPluginsIpc() registerWechatIpc() + registerBrowserIpc() startWsServer().catch((err) => mainLogger.error({ err }, 'ws-server failed to start')) autoResumeWechatBot().catch((err) => mainLogger.warn({ err }, 'wechat auto-resume failed')) diff --git a/apps/desktop/src/main/ipc/browser.ts b/apps/desktop/src/main/ipc/browser.ts new file mode 100644 index 00000000..ed305eb4 --- /dev/null +++ b/apps/desktop/src/main/ipc/browser.ts @@ -0,0 +1,151 @@ +const { ipcMain, app, session, BrowserWindow, webContents: webContentsModule } = require('electron') +const path = require('path') + +let registered = false + +export const BROWSER_PARTITION = 'persist:browser' + +/** Only regular web pages are allowed inside the embedded browser. */ +export function isAllowedBrowserUrl(rawUrl: string): boolean { + try { + const parsed = new URL(String(rawUrl || '')) + return parsed.protocol === 'http:' || parsed.protocol === 'https:' + } catch { + return false + } +} + +function broadcast(channel: string, payload: any) { + for (const win of BrowserWindow.getAllWindows()) { + if (!win.isDestroyed()) win.webContents.send(channel, payload) + } +} + +type PendingDownload = { + item: any + filename: string +} + +const pendingDownloads = new Map() +let downloadSeq = 0 + +function setupBrowserSession() { + const ses = session.fromPartition(BROWSER_PARTITION) + + // Sites inside the embedded browser get no device/system permissions. + ses.setPermissionRequestHandler((_wc: any, _permission: string, callback: (granted: boolean) => void) => { + callback(false) + }) + + // Downloads pause until the user confirms via the renderer toast. + ses.on('will-download', (_event: any, item: any) => { + const id = `dl-${Date.now()}-${++downloadSeq}` + const filename = item.getFilename() || 'download' + try { item.pause() } catch { /* best effort */ } + pendingDownloads.set(id, { item, filename }) + + item.on('updated', () => { + broadcast('browser:download:progress', { + id, + filename, + receivedBytes: item.getReceivedBytes(), + totalBytes: item.getTotalBytes(), + state: item.getState(), + }) + }) + item.once('done', (_e: any, state: string) => { + pendingDownloads.delete(id) + broadcast('browser:download:done', { + id, + filename, + state, + savePath: state === 'completed' ? item.getSavePath() : undefined, + }) + }) + + broadcast('browser:download:request', { + id, + filename, + url: item.getURL(), + totalBytes: item.getTotalBytes(), + }) + }) +} + +function hardenGuestContents(contents: any) { + // Popups / target=_blank become new tabs in the renderer instead of windows. + contents.setWindowOpenHandler(({ url }: { url: string }) => { + if (isAllowedBrowserUrl(url)) broadcast('browser:openUrl', { url }) + return { action: 'deny' } + }) + + contents.on('will-navigate', (event: any, url: string) => { + if (!isAllowedBrowserUrl(url)) event.preventDefault() + }) +} + +export function registerBrowserIpc() { + if (registered) return + registered = true + + setupBrowserSession() + + app.on('web-contents-created', (_event: any, contents: any) => { + if (contents.getType() !== 'webview') return + hardenGuestContents(contents) + }) + + ipcMain.handle('browser:download:respond', async (_event: any, payload: { id: string; allow: boolean }) => { + try { + const pending = pendingDownloads.get(String(payload?.id)) + if (!pending) return { success: false, error: 'download not found (may have expired)' } + if (payload?.allow) { + const downloadsDir = app.getPath('downloads') + // Avoid clobbering an existing file with the same name. + let target = path.join(downloadsDir, pending.filename) + let n = 1 + const fsSync = require('fs') + const ext = path.extname(pending.filename) + const base = path.basename(pending.filename, ext) + while (fsSync.existsSync(target)) { + target = path.join(downloadsDir, `${base} (${n})${ext}`) + n += 1 + } + pending.item.setSavePath(target) + pending.item.resume() + return { success: true, savePath: target } + } + pending.item.cancel() + pendingDownloads.delete(String(payload?.id)) + return { success: true, cancelled: true } + } catch (err: any) { + return { success: false, error: err?.message || String(err) } + } + }) + + ipcMain.handle('browser:capturePage', async (_event: any, webContentsId: number) => { + try { + const contents = webContentsModule.fromId(Number(webContentsId)) + if (!contents || contents.isDestroyed()) { + return { success: false, error: 'browser tab not found' } + } + if (contents.getType() !== 'webview') { + return { success: false, error: 'capture is only allowed for browser tabs' } + } + const image = await contents.capturePage() + // Cap width to keep the payload reasonable for vision models. + const size = image.getSize() + const maxWidth = 1280 + const resized = size.width > maxWidth + ? image.resize({ width: maxWidth }) + : image + return { success: true, dataUrl: resized.toDataURL() } + } catch (err: any) { + return { success: false, error: err?.message || String(err) } + } + }) +} + +module.exports = { registerBrowserIpc, isAllowedBrowserUrl, BROWSER_PARTITION } + +export {} diff --git a/apps/desktop/src/renderer/src/App.vue b/apps/desktop/src/renderer/src/App.vue index 8e653074..24853569 100644 --- a/apps/desktop/src/renderer/src/App.vue +++ b/apps/desktop/src/renderer/src/App.vue @@ -18,9 +18,12 @@ import MessageSkeleton from './components/MessageSkeleton.vue' import QrPairModal from './components/QrPairModal.vue' import RepositoryPickerModal from './components/RepositoryPickerModal.vue' import SakuraPetals from './components/SakuraPetals.vue' +import BrowserPanel from './components/BrowserPanel.vue' +import { useBrowserStore } from './stores/browser' import type { ActiveCodingRepo } from './types/coding-session' const store = useChatStore() +const browser = useBrowserStore() const { invoke, on } = useIpc() const { theme, currentRainbowHue, hslToHex, resetTheme, setColor, setRainbow, setUI, DEFAULT_THEME } = useTheme() const { t, locale, setLocale, localeOptions } = useI18n() @@ -3809,6 +3812,28 @@ async function handleImportData() {
+
+
+
+
AI browser control
+

+ Lets the waifu drive the embedded browser panel (🌐) — navigate, click, type, and read pages. You share the same browser and can take over anytime. Passwords are never typed by the AI. +

+
+ +
+
+