diff --git a/.changeset/media-memory-feature.md b/.changeset/media-memory-feature.md new file mode 100644 index 0000000..41cda17 --- /dev/null +++ b/.changeset/media-memory-feature.md @@ -0,0 +1,13 @@ +--- +"think-app": minor +--- + +feat(app): add media memory support for voice memos, audio, and video + +- Add voice memo recording from system tray and main app +- Add audio file upload with drag-and-drop support +- Add video file upload with FFmpeg audio extraction and thumbnail generation +- Add local AI transcription using faster-whisper with timestamp segments +- Add new memory card components for voice memos, audio, and video +- Add transcription settings for Whisper model selection +- Add file size limits and input validation for security diff --git a/.gitignore b/.gitignore index cb68b61..70bcda6 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,7 @@ release/ extension/extension.pem # Compiled native messaging stub -backend/native_host/think-native-stub \ No newline at end of file +backend/native_host/think-native-stub + +# FFmpeg (downloaded on first use, cached locally) +app/public/ffmpeg/ \ No newline at end of file diff --git a/app/electron/main.js b/app/electron/main.js index 242f459..31bed15 100644 --- a/app/electron/main.js +++ b/app/electron/main.js @@ -6,6 +6,8 @@ const path = require('path'); const fs = require('fs'); const https = require('https'); const { installNativeHost } = require('./install-native-host'); +const { createTray, destroyTray, closeRecordingWindow } = require('./tray'); +const videoProcessor = require('./video-processor'); /** * Download a file with progress reporting. @@ -295,7 +297,7 @@ function createWindow() { webPreferences: { preload: path.join(__dirname, 'preload.js'), contextIsolation: true, - nodeIntegration: false + nodeIntegration: false, } }); @@ -357,6 +359,10 @@ app.whenReady().then(async () => { if (mainWindow && !mainWindow.isDestroyed()) { mainWindow.webContents.send('backend-ready', { token: APP_TOKEN }); } + + // Initialize system tray after backend is ready + const isDev = process.env.NODE_ENV === 'development' || !app.isPackaged; + createTray(mainWindow, APP_TOKEN, isDev); } catch (err) { console.error('Backend startup failed:', err); if (mainWindow && !mainWindow.isDestroyed()) { @@ -374,6 +380,8 @@ app.on('window-all-closed', () => { }); app.on('before-quit', () => { + closeRecordingWindow(); + destroyTray(); if (pythonProcess) { pythonProcess.kill(); } @@ -549,3 +557,85 @@ ipcMain.handle('pull-model', async (_event, modelName) => { ipcMain.handle('install-update', () => { autoUpdater.quitAndInstall(); }); + +// Open main window (called from recording popup when app is locked) +ipcMain.handle('open-main-window', () => { + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.show(); + mainWindow.focus(); + } else { + // No window exists - create one + createWindow(); + if (backendReady && mainWindow) { + mainWindow.webContents.once('did-finish-load', () => { + mainWindow.webContents.send('backend-ready', { token: APP_TOKEN }); + }); + } + } +}); + +// Open recording window (called from renderer to start voice recording) +ipcMain.handle('open-recording-window', () => { + const { openRecordingWindow } = require('./tray.js'); + const isDev = process.env.NODE_ENV === 'development' || !app.isPackaged; + openRecordingWindow(mainWindow, APP_TOKEN, isDev); +}); + +// Video processing IPC handlers + +// Write video data to a temporary file for FFmpeg processing +ipcMain.handle('write-temp-file', async (_event, data, filename) => { + try { + const buffer = Buffer.from(data); + const tempPath = await videoProcessor.writeTempFile(buffer, filename); + return { success: true, path: tempPath }; + } catch (error) { + console.error('[IPC] write-temp-file error:', error); + return { success: false, error: error.message }; + } +}); + +// Process video: extract audio and generate thumbnail +ipcMain.handle('process-video', async (event, videoPath) => { + try { + const result = await videoProcessor.processVideo(videoPath, (progress) => { + // Send progress updates to renderer + if (mainWindow && !mainWindow.isDestroyed()) { + event.sender.send('video-process-progress', progress); + } + }); + + // Read the audio and thumbnail files as buffers for upload + const audioBuffer = await videoProcessor.readFile(result.audioPath); + let thumbnailBuffer = null; + if (result.thumbnailPath) { + thumbnailBuffer = await videoProcessor.readFile(result.thumbnailPath); + } + + // Clean up temp files (audio and thumbnail) + await videoProcessor.deleteTempFile(result.audioPath); + if (result.thumbnailPath) { + await videoProcessor.deleteTempFile(result.thumbnailPath); + } + + return { + success: true, + audio: audioBuffer, + thumbnail: thumbnailBuffer, + }; + } catch (error) { + console.error('[IPC] process-video error:', error); + return { success: false, error: error.message }; + } +}); + +// Delete a temporary file +ipcMain.handle('delete-temp-file', async (_event, filePath) => { + try { + await videoProcessor.deleteTempFile(filePath); + return { success: true }; + } catch (error) { + console.error('[IPC] delete-temp-file error:', error); + return { success: false, error: error.message }; + } +}); diff --git a/app/electron/preload.js b/app/electron/preload.js index 7357078..80a8759 100644 --- a/app/electron/preload.js +++ b/app/electron/preload.js @@ -46,4 +46,20 @@ contextBridge.exposeInMainWorld('electronAPI', { ipcRenderer.removeAllListeners('update-downloaded'); }, installUpdate: () => ipcRenderer.invoke('install-update'), + // Open main window (for recording popup to trigger unlock) + openMainWindow: () => ipcRenderer.invoke('open-main-window'), + // Open recording window (for voice memo from main app) + openRecordingWindow: () => ipcRenderer.invoke('open-recording-window'), + // Send recording state to main process (for blur handler) + setRecordingState: (isRecording) => ipcRenderer.send('recording-state-changed', isRecording), + // Video processing APIs (native FFmpeg) + writeTempFile: (data, filename) => ipcRenderer.invoke('write-temp-file', data, filename), + processVideo: (videoPath) => ipcRenderer.invoke('process-video', videoPath), + deleteTempFile: (path) => ipcRenderer.invoke('delete-temp-file', path), + onVideoProcessProgress: (callback) => { + ipcRenderer.on('video-process-progress', (_, data) => callback(data)); + }, + removeVideoProcessListeners: () => { + ipcRenderer.removeAllListeners('video-process-progress'); + }, }); diff --git a/app/electron/tray.js b/app/electron/tray.js new file mode 100644 index 0000000..d0e3d1c --- /dev/null +++ b/app/electron/tray.js @@ -0,0 +1,191 @@ +const { Tray, Menu, BrowserWindow, nativeImage, app, ipcMain } = require('electron'); +const path = require('path'); + +let tray = null; +let recordingWindow = null; +let isRecording = false; + +// Listen for recording state changes from renderer +ipcMain.on('recording-state-changed', (event, recording) => { + isRecording = recording; +}); + +/** + * Create the system tray with context menu + */ +function createTray(mainWindow, appToken, isDev) { + let icon; + + try { + if (process.platform === 'darwin') { + // macOS: use template icon (auto-adapts to light/dark menu bar) + const templatePath = path.join(__dirname, '../public/icons/tray-iconTemplate.png'); + icon = nativeImage.createFromPath(templatePath); + icon.setTemplateImage(true); + } else { + // Windows/Linux: use colored icon at 32x32 + const iconPath = path.join(__dirname, '../public/icons/think-os-agent.png'); + icon = nativeImage.createFromPath(iconPath).resize({ width: 32, height: 32 }); + } + } catch (err) { + console.error('Failed to load tray icon:', err); + icon = nativeImage.createEmpty(); + } + + tray = new Tray(icon); + + const isMac = process.platform === 'darwin'; + + const contextMenu = Menu.buildFromTemplate([ + { + label: 'Record Voice Note', + accelerator: isMac ? 'Cmd+Shift+R' : 'Ctrl+Shift+R', + click: () => openRecordingWindow(mainWindow, appToken, isDev), + }, + { + label: 'Open Think', + accelerator: isMac ? 'Cmd+O' : 'Ctrl+O', + click: () => { + const win = BrowserWindow.getAllWindows().find(w => w !== recordingWindow); + if (win) { + win.show(); + win.focus(); + } else { + app.emit('activate'); + } + }, + }, + { type: 'separator' }, + { + label: isMac ? 'Quit Think' : 'Exit', + accelerator: isMac ? 'Cmd+Q' : undefined, + click: () => app.quit(), + }, + ]); + + tray.setToolTip('Think'); + tray.setContextMenu(contextMenu); + + // On macOS, clicking the tray icon opens the context menu + // On Windows, single-click opens recording, double-click opens app + if (process.platform !== 'darwin') { + tray.on('click', () => openRecordingWindow(mainWindow, appToken, isDev)); + tray.on('double-click', () => { + const win = BrowserWindow.getAllWindows().find(w => w !== recordingWindow); + if (win) { + win.show(); + win.focus(); + } else { + app.emit('activate'); + } + }); + } +} + +/** + * Open the compact recording popup window + */ +function openRecordingWindow(mainWindow, appToken, isDev) { + // If recording window already exists and is visible, focus it + if (recordingWindow && !recordingWindow.isDestroyed()) { + recordingWindow.focus(); + return; + } + + // Get the position for the popup (near tray on macOS, center on Windows) + let x, y; + if (tray) { + const trayBounds = tray.getBounds(); + const windowWidth = 320; + const windowHeight = 240; + + if (process.platform === 'darwin') { + // Position below the tray icon on macOS + x = Math.round(trayBounds.x + (trayBounds.width / 2) - (windowWidth / 2)); + y = Math.round(trayBounds.y + trayBounds.height + 4); + } else { + // Position above the tray icon on Windows (taskbar is at bottom) + x = Math.round(trayBounds.x + (trayBounds.width / 2) - (windowWidth / 2)); + y = Math.round(trayBounds.y - windowHeight - 4); + } + } + + recordingWindow = new BrowserWindow({ + width: 320, + height: 240, + x, + y, + frame: false, + transparent: false, + backgroundColor: '#1c1c1e', + resizable: false, + skipTaskbar: true, + alwaysOnTop: true, + webPreferences: { + preload: path.join(__dirname, 'preload.js'), + contextIsolation: true, + nodeIntegration: false, + }, + }); + + // Load the recording page + if (isDev) { + recordingWindow.loadURL('http://localhost:5173/#/recording'); + } else { + recordingWindow.loadFile(path.join(__dirname, '../dist/index.html'), { + hash: '/recording', + }); + } + + // Send the app token to the recording window once loaded + recordingWindow.webContents.once('did-finish-load', () => { + if (appToken) { + recordingWindow.webContents.send('backend-ready', { token: appToken }); + } + }); + + // Close when clicking outside (blur event) + recordingWindow.on('blur', () => { + // Small delay to allow for click handling + setTimeout(() => { + if (recordingWindow && !recordingWindow.isDestroyed()) { + // Don't close if user is actively recording (state tracked via IPC) + if (!isRecording) { + recordingWindow.close(); + } + } + }, 100); + }); + + recordingWindow.on('closed', () => { + recordingWindow = null; + isRecording = false; // Reset state when window closes + }); +} + +/** + * Close the recording window + */ +function closeRecordingWindow() { + if (recordingWindow && !recordingWindow.isDestroyed()) { + recordingWindow.close(); + recordingWindow = null; + } +} + +/** + * Destroy the tray icon + */ +function destroyTray() { + if (tray) { + tray.destroy(); + tray = null; + } +} + +module.exports = { + createTray, + openRecordingWindow, + closeRecordingWindow, + destroyTray, +}; diff --git a/app/electron/video-processor.js b/app/electron/video-processor.js new file mode 100644 index 0000000..99e78d9 --- /dev/null +++ b/app/electron/video-processor.js @@ -0,0 +1,369 @@ +const { spawn } = require('child_process'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); +const { app } = require('electron'); + +// Default timeout for FFmpeg operations (5 minutes) +const FFMPEG_TIMEOUT_MS = 5 * 60 * 1000; + +/** + * Validate that a video path is safe for processing. + * Prevents path traversal and ensures the file exists within allowed directories. + * + * @param {string} videoPath - Path to validate + * @returns {{ valid: boolean, error?: string }} - Validation result + */ +function validateVideoPath(videoPath) { + if (!videoPath || typeof videoPath !== 'string') { + return { valid: false, error: 'Invalid video path' }; + } + + // Resolve to absolute path + const absolutePath = path.resolve(videoPath); + + // Check that the path exists + if (!fs.existsSync(absolutePath)) { + return { valid: false, error: 'Video file does not exist' }; + } + + // Check that it's a file (not a directory or symlink to directory) + const stats = fs.lstatSync(absolutePath); + if (!stats.isFile()) { + return { valid: false, error: 'Path is not a file' }; + } + + // Get allowed directories (temp and user downloads) + const tempDir = app.getPath('temp'); + const downloadsDir = app.getPath('downloads'); + const homeDir = app.getPath('home'); + + // Use realpathSync to resolve symlinks (important for macOS where /var -> /private/var) + const realPath = fs.realpathSync(absolutePath); + const realTemp = fs.realpathSync(tempDir); + const realDownloads = fs.realpathSync(downloadsDir); + const realHome = fs.realpathSync(homeDir); + + // Allow files from temp directory, downloads, or anywhere under home directory + const isInAllowedDir = + realPath.startsWith(realTemp + path.sep) || + realPath.startsWith(realDownloads + path.sep) || + realPath.startsWith(realHome + path.sep); + + if (!isInAllowedDir) { + return { valid: false, error: 'Video file is outside allowed directories' }; + } + + return { valid: true }; +} + +/** + * Get the path to the FFmpeg binary. + * In development, uses ffmpeg-static from node_modules. + * In production, uses the bundled binary in resources. + */ +function getFFmpegPath() { + const platform = process.platform; + const ext = platform === 'win32' ? '.exe' : ''; + + if (app.isPackaged) { + // Production: bundled binary + return path.join(process.resourcesPath, 'ffmpeg', `ffmpeg${ext}`); + } else { + // Development: use ffmpeg-static package + try { + const ffmpegStatic = require('ffmpeg-static'); + return ffmpegStatic; + } catch { + // Fallback: system ffmpeg + return 'ffmpeg'; + } + } +} + +/** + * Parse FFmpeg progress from stderr output. + * Extracts time progress like "time=00:00:05.00" + * + * @param {string} stderr - FFmpeg stderr output chunk + * @param {number} duration - Total duration in seconds + * @returns {number|null} - Progress 0-100 or null if not parseable + */ +function parseProgress(stderr, duration) { + if (!duration || duration <= 0) return null; + + const timeMatch = stderr.match(/time=(\d{2}):(\d{2}):(\d{2})\.(\d{2})/); + if (timeMatch) { + const hours = parseInt(timeMatch[1], 10); + const minutes = parseInt(timeMatch[2], 10); + const seconds = parseInt(timeMatch[3], 10); + const centiseconds = parseInt(timeMatch[4], 10); + const currentTime = hours * 3600 + minutes * 60 + seconds + centiseconds / 100; + return Math.min(100, Math.round((currentTime / duration) * 100)); + } + return null; +} + +/** + * Get the duration of a video file using FFmpeg. + * + * @param {string} videoPath - Path to the video file (must be validated first) + * @returns {Promise} - Duration in seconds + */ +function getVideoDuration(videoPath) { + return new Promise((resolve, reject) => { + const ffmpegPath = getFFmpegPath(); + const args = ['-i', videoPath, '-f', 'null', '-']; + + const ffmpeg = spawn(ffmpegPath, args); + let stderr = ''; + let killed = false; + + // Set timeout for duration extraction (30 seconds should be enough) + const timeout = setTimeout(() => { + killed = true; + ffmpeg.kill('SIGKILL'); + console.warn('[VideoProcessor] Duration extraction timed out'); + resolve(0); + }, 30000); + + ffmpeg.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + ffmpeg.on('close', () => { + clearTimeout(timeout); + if (killed) return; + + // Parse duration from output like "Duration: 00:01:30.50" + const match = stderr.match(/Duration: (\d{2}):(\d{2}):(\d{2})\.(\d{2})/); + if (match) { + const hours = parseInt(match[1], 10); + const minutes = parseInt(match[2], 10); + const seconds = parseInt(match[3], 10); + const centiseconds = parseInt(match[4], 10); + resolve(hours * 3600 + minutes * 60 + seconds + centiseconds / 100); + } else { + // If duration parsing fails, use a default (will affect progress accuracy) + console.warn('[VideoProcessor] Could not parse video duration'); + resolve(0); + } + }); + + ffmpeg.on('error', (err) => { + clearTimeout(timeout); + reject(err); + }); + }); +} + +/** + * Extract audio from a video file. + * + * @param {string} videoPath - Path to the input video file (must be validated first) + * @param {function} onProgress - Progress callback (0-100) + * @returns {Promise} - Path to the extracted audio file (m4a) + */ +async function extractAudio(videoPath, onProgress) { + const ffmpegPath = getFFmpegPath(); + const outputPath = videoPath.replace(/\.[^.]+$/, '') + '_audio.m4a'; + + // Get duration for progress calculation + const duration = await getVideoDuration(videoPath); + + return new Promise((resolve, reject) => { + const args = [ + '-i', videoPath, + '-vn', // No video + '-acodec', 'aac', // AAC codec + '-b:a', '128k', // 128kbps bitrate + '-y', // Overwrite output + outputPath + ]; + + console.log('[VideoProcessor] Extracting audio:', ffmpegPath, args.join(' ')); + const ffmpeg = spawn(ffmpegPath, args); + let killed = false; + + // Set timeout (5 minutes for audio extraction) + const timeout = setTimeout(() => { + killed = true; + ffmpeg.kill('SIGKILL'); + reject(new Error('Audio extraction timed out')); + }, FFMPEG_TIMEOUT_MS); + + ffmpeg.stderr.on('data', (data) => { + const stderr = data.toString(); + const progress = parseProgress(stderr, duration); + if (progress !== null && onProgress) { + onProgress(progress); + } + }); + + ffmpeg.on('close', (code) => { + clearTimeout(timeout); + if (killed) return; + + if (code === 0) { + console.log('[VideoProcessor] Audio extraction complete:', outputPath); + resolve(outputPath); + } else { + reject(new Error(`FFmpeg exited with code ${code}`)); + } + }); + + ffmpeg.on('error', (err) => { + clearTimeout(timeout); + console.error('[VideoProcessor] FFmpeg error:', err); + reject(err); + }); + }); +} + +/** + * Generate a thumbnail from a video file. + * + * @param {string} videoPath - Path to the input video file (must be validated first) + * @param {number} timestamp - Timestamp in seconds (default: 1) + * @returns {Promise} - Path to the thumbnail file (jpg) or null on failure + */ +function generateThumbnail(videoPath, timestamp = 1) { + const ffmpegPath = getFFmpegPath(); + const outputPath = videoPath.replace(/\.[^.]+$/, '') + '_thumb.jpg'; + + return new Promise((resolve) => { + const args = [ + '-ss', String(timestamp), // Seek to timestamp + '-i', videoPath, + '-frames:v', '1', // Single frame + '-q:v', '2', // High quality JPEG + '-y', // Overwrite output + outputPath + ]; + + console.log('[VideoProcessor] Generating thumbnail:', ffmpegPath, args.join(' ')); + const ffmpeg = spawn(ffmpegPath, args); + let killed = false; + + // Set timeout (30 seconds for thumbnail generation) + const timeout = setTimeout(() => { + killed = true; + ffmpeg.kill('SIGKILL'); + console.warn('[VideoProcessor] Thumbnail generation timed out'); + resolve(null); + }, 30000); + + ffmpeg.on('close', (code) => { + clearTimeout(timeout); + if (killed) return; + + if (code === 0) { + console.log('[VideoProcessor] Thumbnail generation complete:', outputPath); + resolve(outputPath); + } else { + // Thumbnail failure is non-critical, resolve with null + console.warn('[VideoProcessor] Thumbnail generation failed, code:', code); + resolve(null); + } + }); + + ffmpeg.on('error', (err) => { + clearTimeout(timeout); + console.warn('[VideoProcessor] FFmpeg thumbnail error:', err); + // Non-critical, resolve with null + resolve(null); + }); + }); +} + +/** + * Process a video file: extract audio and generate thumbnail. + * + * @param {string} videoPath - Path to the video file + * @param {function} onProgress - Progress callback ({ progress: 0-100, stage: string }) + * @returns {Promise<{ audioPath: string, thumbnailPath: string|null }>} + */ +async function processVideo(videoPath, onProgress) { + console.log('[VideoProcessor] Processing video:', videoPath); + + // Validate the video path before processing + const validation = validateVideoPath(videoPath); + if (!validation.valid) { + throw new Error(validation.error || 'Invalid video path'); + } + + // Stage 1: Extract audio (0-80%) + onProgress?.({ progress: 0, stage: 'extracting_audio' }); + const audioPath = await extractAudio(videoPath, (p) => { + onProgress?.({ progress: Math.round(p * 0.8), stage: 'extracting_audio' }); + }); + + // Stage 2: Generate thumbnail (80-100%) + onProgress?.({ progress: 80, stage: 'generating_thumbnail' }); + const thumbnailPath = await generateThumbnail(videoPath, 1); + + onProgress?.({ progress: 100, stage: 'done' }); + + return { audioPath, thumbnailPath }; +} + +/** + * Write data to a temporary file. + * + * @param {Buffer} data - File data + * @param {string} filename - Original filename (for extension) + * @returns {Promise} - Path to the temp file + */ +function writeTempFile(data, filename) { + const tempDir = app.getPath('temp'); + const ext = path.extname(filename) || '.mp4'; + const tempPath = path.join(tempDir, `think_video_${Date.now()}${ext}`); + + return new Promise((resolve, reject) => { + fs.writeFile(tempPath, data, (err) => { + if (err) reject(err); + else resolve(tempPath); + }); + }); +} + +/** + * Delete a temporary file. + * + * @param {string} filePath - Path to the file to delete + */ +function deleteTempFile(filePath) { + return new Promise((resolve) => { + fs.unlink(filePath, (err) => { + if (err) { + console.warn('[VideoProcessor] Failed to delete temp file:', filePath, err); + } + resolve(); + }); + }); +} + +/** + * Read a file as a buffer. + * + * @param {string} filePath - Path to the file + * @returns {Promise} + */ +function readFile(filePath) { + return new Promise((resolve, reject) => { + fs.readFile(filePath, (err, data) => { + if (err) reject(err); + else resolve(data); + }); + }); +} + +module.exports = { + getFFmpegPath, + extractAudio, + generateThumbnail, + processVideo, + writeTempFile, + deleteTempFile, + readFile, +}; diff --git a/app/package.json b/app/package.json index 2f80b64..2887a33 100644 --- a/app/package.json +++ b/app/package.json @@ -33,6 +33,14 @@ "think-native-stub", "think-native-stub.exe" ] + }, + { + "from": "node_modules/ffmpeg-static/ffmpeg", + "to": "ffmpeg/ffmpeg" + }, + { + "from": "node_modules/ffmpeg-static/ffmpeg.exe", + "to": "ffmpeg/ffmpeg.exe" } ], "mac": { @@ -66,6 +74,7 @@ } }, "dependencies": { + "ffmpeg-static": "^5.2.0", "@microsoft/fetch-event-source": "^2.0.1", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-popover": "^1.1.15", @@ -97,6 +106,7 @@ "electron-builder": "^25.1.8", "png-to-ico": "^3.0.1", "postcss": "^8.4.49", + "sharp": "^0.34.5", "tailwindcss": "^3.4.15", "typescript": "^5.6.3", "vite": "^6.0.1", diff --git a/app/public/icons/tray-iconTemplate.png b/app/public/icons/tray-iconTemplate.png new file mode 100644 index 0000000..b71ecd6 Binary files /dev/null and b/app/public/icons/tray-iconTemplate.png differ diff --git a/app/public/icons/tray-iconTemplate@2x.png b/app/public/icons/tray-iconTemplate@2x.png new file mode 100644 index 0000000..750cefa Binary files /dev/null and b/app/public/icons/tray-iconTemplate@2x.png differ diff --git a/app/src/App.tsx b/app/src/App.tsx index 8bc1cce..90cb9e1 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -1,5 +1,5 @@ import { useState, useEffect } from "react"; -import { MemoryRouter, Routes, Route } from "react-router-dom"; +import { HashRouter, Routes, Route } from "react-router-dom"; import SetupWizard from "./SetupWizard"; import LockScreen from "./LockScreen"; import MainLayout from "./layouts/MainLayout"; @@ -7,8 +7,10 @@ import HomePage from "./pages/HomePage"; import ChatPage from "./pages/ChatPage"; import MemoriesPage from "./pages/MemoriesPage"; import SettingsPage from "./pages/SettingsPage"; +import RecordingPage from "./pages/RecordingPage"; import { NamePromptDialog } from "./components/NamePromptDialog"; import { Button } from "@/components/ui/button"; +import { Lock } from "lucide-react"; import { Toaster } from "@/components/ui/sonner"; import { toast } from "sonner"; import { useSystemTheme } from "@/hooks/useSystemTheme"; @@ -112,11 +114,12 @@ function App() { const modelsRes = await apiFetch("/api/settings/models?provider=ollama"); const modelsData = await modelsRes.json(); - // Check if current_model exists in the available models list + // Check if current_model exists AND is downloaded const hasConfiguredModel = modelsData.models?.some( - (m: { name: string }) => - m.name === modelsData.current_model || - m.name.startsWith(modelsData.current_model + ":") + (m: { name: string; is_downloaded: boolean }) => + m.is_downloaded && + (m.name === modelsData.current_model || + m.name.startsWith(modelsData.current_model + ":")) ); if (!hasConfiguredModel) { @@ -242,6 +245,25 @@ function App() { } if (appState === "locked") { + // Check if we're in the recording popup window + const isRecordingRoute = window.location.hash === '#/recording'; + + if (isRecordingRoute) { + // Compact message for the small recording popup + return ( +
+ +

Think is locked

+ +
+ ); + } + return ; } @@ -251,8 +273,12 @@ function App() { return ( <> - + + {/* Recording popup - standalone window without layout */} + } /> + + {/* Main app routes */} }> } /> } /> @@ -263,7 +289,7 @@ function App() { /> - + void; } -declare global { - interface Window { - electronAPI?: { - checkOllama: () => Promise<{ installed: boolean; running: boolean }>; - downloadOllama: () => Promise<{ success: boolean; error?: string }>; - pullModel: (model: string) => Promise<{ success: boolean; error?: string }>; - onOllamaDownloadProgress: (callback: (data: { progress: number; stage: string }) => void) => void; - onModelPullProgress: (callback: (data: { progress?: number; status: string }) => void) => void; - removeOllamaDownloadProgress: () => void; - removeModelPullProgress: () => void; - // Backend status handlers - onBackendReady: (callback: (data?: { token?: string }) => void) => void; - onBackendError: (callback: (data: { message: string }) => void) => void; - removeBackendListeners: () => void; - // App token for API authentication - getAppToken: () => string | null; - // Auto-update handlers - onUpdateDownloaded: (callback: (version: string) => void) => void; - removeUpdateListeners: () => void; - installUpdate: () => Promise; - }; - } -} - function WizardLayout({ title, children }: { title?: string; children: React.ReactNode }) { return (
diff --git a/app/src/components/AudioCard.tsx b/app/src/components/AudioCard.tsx new file mode 100644 index 0000000..b216888 --- /dev/null +++ b/app/src/components/AudioCard.tsx @@ -0,0 +1,257 @@ +import { useState, useRef, useEffect } from "react"; +import { Button } from "@/components/ui/button"; +import { + FileAudio, + X, + PanelRight, + Play, + Pause, + Loader2, +} from "lucide-react"; +import { cn } from "@/lib/utils"; +import { getAppToken } from "@/lib/api"; +import { API_BASE_URL } from "@/constants"; +import type { TranscriptionStatus } from "@/types/chat"; + +interface MemoryTag { + id: number; + name: string; + source: "ai" | "manual"; +} + +interface AudioMemory { + id: number; + type: "audio"; + title: string; + summary: string | null; + tags: MemoryTag[]; + created_at: string; + audio_duration?: number; + transcription_status?: TranscriptionStatus; +} + +interface AudioCardProps { + memory: AudioMemory; + onRemoveTag: (memoryId: number, tagId: number) => void; + onExpand: (id: number) => void; + formatDate: (date: string) => string; +} + +function formatDuration(seconds: number | undefined): string { + if (seconds === undefined || seconds === null) return "0:00"; + const totalSeconds = Math.floor(seconds); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const secs = totalSeconds % 60; + + if (hours > 0) { + return `${hours}:${minutes.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; + } + return `${minutes}:${secs.toString().padStart(2, "0")}`; +} + +function TranscriptionStatusBadge({ status }: { status?: TranscriptionStatus }) { + if (!status || status === "completed") return null; + + const statusConfig = { + pending: { label: "Pending", className: "bg-muted text-muted-foreground" }, + processing: { label: "Transcribing...", className: "bg-muted text-muted-foreground" }, + failed: { label: "Failed", className: "bg-red-100 text-red-700 dark:bg-red-900/30 dark:text-red-400" }, + }; + + const config = statusConfig[status] || statusConfig.pending; + + return ( + + {status === "processing" && } + {config.label} + + ); +} + +export function AudioCard({ + memory, + onRemoveTag, + onExpand, + formatDate, +}: AudioCardProps) { + const [isPlaying, setIsPlaying] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const audioRef = useRef(null); + const blobUrlRef = useRef(null); + + // Cleanup audio element and blob URL on unmount to prevent memory leaks + useEffect(() => { + return () => { + // Pause and clean up audio element + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current.src = ""; + audioRef.current = null; + } + // Revoke blob URL + if (blobUrlRef.current) { + URL.revokeObjectURL(blobUrlRef.current); + blobUrlRef.current = null; + } + }; + }, []); + + const handlePlayPause = async (e: React.MouseEvent) => { + e.stopPropagation(); + + if (!audioRef.current) { + // Create audio element on first play + setIsLoading(true); + const audio = new Audio(); + + // Fetch audio with auth token + const token = getAppToken(); + const response = await fetch(`${API_BASE_URL}/api/media/${memory.id}/stream`, { + headers: token ? { "X-App-Token": token } : {}, + }); + + if (!response.ok) { + console.error("Failed to load audio"); + setIsLoading(false); + return; + } + + const blob = await response.blob(); + const blobUrl = URL.createObjectURL(blob); + blobUrlRef.current = blobUrl; + audio.src = blobUrl; + audio.onended = () => setIsPlaying(false); + audio.oncanplay = () => setIsLoading(false); + audioRef.current = audio; + } + + if (isPlaying) { + audioRef.current?.pause(); + setIsPlaying(false); + } else { + audioRef.current?.play(); + setIsPlaying(true); + } + }; + + return ( +
+ {/* Hover actions - top right */} +
+ +
+ + {/* Header: Audio icon + Title */} +
+
+ +
+

+ {memory.title || "Audio File"} +

+
+ + {/* Audio Player */} +
+ +
+
+ {formatDuration(memory.audio_duration)} +
+ +
+
+ + {/* Summary */} + {memory.summary ? ( +

+ {memory.summary} +

+ ) : memory.transcription_status === "completed" ? ( +

+ Generating summary... +

+ ) : null} + + {/* Tags */} + {memory.tags.length > 0 && ( +
+ {memory.tags.map((tag) => ( + + {tag.name} + {tag.source === "manual" && ( + + )} + + ))} +
+ )} + + {/* Date */} +

+ {formatDate(memory.created_at)} +

+
+ ); +} diff --git a/app/src/components/AudioDropOverlay.tsx b/app/src/components/AudioDropOverlay.tsx new file mode 100644 index 0000000..781d05e --- /dev/null +++ b/app/src/components/AudioDropOverlay.tsx @@ -0,0 +1,235 @@ +import { useState, useEffect, useCallback } from "react"; +import { createPortal } from "react-dom"; +import { Mic, Upload } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { apiFetch } from "@/lib/api"; +import { toast } from "sonner"; + +const SUPPORTED_AUDIO_TYPES = [ + "audio/mpeg", + "audio/mp3", + "audio/wav", + "audio/wave", + "audio/x-wav", + "audio/mp4", + "audio/x-m4a", + "audio/m4a", + "audio/webm", + "audio/ogg", + "audio/flac", + "audio/x-flac", +]; + +const SUPPORTED_EXTENSIONS = ["mp3", "wav", "m4a", "webm", "ogg", "flac"]; + +interface AudioDropOverlayProps { + onUploadComplete?: () => void; +} + +export function AudioDropOverlay({ onUploadComplete }: AudioDropOverlayProps) { + const [isDragging, setIsDragging] = useState(false); + const [isUploading, setIsUploading] = useState(false); + + const isAudioFile = useCallback((file: File): boolean => { + // Check MIME type + if (SUPPORTED_AUDIO_TYPES.includes(file.type)) { + return true; + } + // Check extension as fallback + const ext = file.name.split(".").pop()?.toLowerCase(); + return ext ? SUPPORTED_EXTENSIONS.includes(ext) : false; + }, []); + + const hasAudioFiles = useCallback((dataTransfer: DataTransfer): boolean => { + if (dataTransfer.types.includes("Files")) { + // Check items if available + if (dataTransfer.items) { + for (let i = 0; i < dataTransfer.items.length; i++) { + const item = dataTransfer.items[i]; + if (item.kind === "file") { + const type = item.type; + if (SUPPORTED_AUDIO_TYPES.includes(type)) { + return true; + } + } + } + } + // Fall back to checking files + if (dataTransfer.files.length > 0) { + for (let i = 0; i < dataTransfer.files.length; i++) { + if (isAudioFile(dataTransfer.files[i])) { + return true; + } + } + } + // If we have files but can't determine type, show the overlay + return true; + } + return false; + }, [isAudioFile]); + + const uploadFile = async (file: File) => { + const formData = new FormData(); + formData.append("file", file); + + const response = await apiFetch("/api/media/upload", { + method: "POST", + body: formData, + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({ detail: "Upload failed" })); + throw new Error(error.detail || "Upload failed"); + } + + return response.json(); + }; + + const handleDrop = useCallback(async (e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragging(false); + + if (!e.dataTransfer?.files.length) return; + + const audioFiles = Array.from(e.dataTransfer.files).filter(isAudioFile); + + if (audioFiles.length === 0) { + toast.error("No audio files detected", { + description: `Supported formats: ${SUPPORTED_EXTENSIONS.join(", ")}`, + }); + return; + } + + setIsUploading(true); + + try { + // Upload files in parallel + const uploadPromises = audioFiles.map(async (file) => { + try { + await uploadFile(file); + return { success: true, name: file.name }; + } catch (error) { + return { success: false, name: file.name, error }; + } + }); + + const results = await Promise.all(uploadPromises); + + const successful = results.filter((r) => r.success); + const failed = results.filter((r) => !r.success); + + if (successful.length > 0) { + toast.success( + successful.length === 1 + ? "Audio file uploaded" + : `${successful.length} audio files uploaded`, + { + description: "Transcription will begin shortly", + } + ); + onUploadComplete?.(); + } + + if (failed.length > 0) { + toast.error( + `Failed to upload ${failed.length} file${failed.length > 1 ? "s" : ""}`, + { + description: failed.map((f) => f.name).join(", "), + } + ); + } + } catch (error) { + toast.error("Upload failed", { + description: error instanceof Error ? error.message : "Unknown error", + }); + } finally { + setIsUploading(false); + } + }, [isAudioFile, onUploadComplete]); + + const handleDragOver = useCallback((e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + }, []); + + const handleDragEnter = useCallback((e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + + if (e.dataTransfer && hasAudioFiles(e.dataTransfer)) { + setIsDragging(true); + } + }, [hasAudioFiles]); + + const handleDragLeave = useCallback((e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + + // Only hide if leaving the window entirely + if (e.relatedTarget === null || !document.body.contains(e.relatedTarget as Node)) { + setIsDragging(false); + } + }, []); + + useEffect(() => { + const handleWindowDragEnter = (e: DragEvent) => handleDragEnter(e); + const handleWindowDragOver = (e: DragEvent) => handleDragOver(e); + const handleWindowDragLeave = (e: DragEvent) => handleDragLeave(e); + const handleWindowDrop = (e: DragEvent) => handleDrop(e); + + window.addEventListener("dragenter", handleWindowDragEnter); + window.addEventListener("dragover", handleWindowDragOver); + window.addEventListener("dragleave", handleWindowDragLeave); + window.addEventListener("drop", handleWindowDrop); + + return () => { + window.removeEventListener("dragenter", handleWindowDragEnter); + window.removeEventListener("dragover", handleWindowDragOver); + window.removeEventListener("dragleave", handleWindowDragLeave); + window.removeEventListener("drop", handleWindowDrop); + }; + }, [handleDragEnter, handleDragOver, handleDragLeave, handleDrop]); + + if (!isDragging && !isUploading) return null; + + return createPortal( +
+
+
+ {isUploading ? ( + + ) : ( + + )} +
+
+

+ {isUploading ? "Uploading..." : "Drop audio files"} +

+

+ {isUploading + ? "Creating voice memories" + : `Supported: ${SUPPORTED_EXTENSIONS.join(", ")}`} +

+
+
+
, + document.body + ); +} diff --git a/app/src/components/MemoryDetailPanel.tsx b/app/src/components/MemoryDetailPanel.tsx index 733398d..a1bfb63 100644 --- a/app/src/components/MemoryDetailPanel.tsx +++ b/app/src/components/MemoryDetailPanel.tsx @@ -7,6 +7,9 @@ import { X, Globe, FileText, + Mic, + FileAudio, + Video, Loader2, Pencil, Check, @@ -17,11 +20,22 @@ import { Sparkles, Lightbulb, Network, + Play, + Pause, + ChevronDown, + ChevronUp, + RotateCcw, + Copy, + SkipBack, + SkipForward, } from "lucide-react"; +import { toast } from "sonner"; import { cn } from "@/lib/utils"; -import { apiFetch } from "@/lib/api"; +import { apiFetch, getAppToken } from "@/lib/api"; +import { API_BASE_URL } from "@/constants"; import { useMemoryEvents } from "../hooks/useMemoryEvents"; import { useConversation } from "../contexts/ConversationContext"; +import type { TranscriptionStatus, TranscriptSegment, VideoProcessingStatus } from "@/types/chat"; interface MemoryTag { id: number; @@ -31,13 +45,25 @@ interface MemoryTag { interface Memory { id: number; - type: "web" | "note"; + type: "web" | "note" | "voice_memo" | "audio" | "video" | "voice"; // "voice" for backwards compat url: string | null; title: string; content?: string; summary: string | null; tags: MemoryTag[]; created_at: string; + // Media-specific fields (voice memos and audio uploads) + audio_duration?: number; + transcript?: string; + transcription_status?: TranscriptionStatus; + transcript_segments?: TranscriptSegment[]; + media_source?: "recording" | "upload"; + // Video-specific fields + video_duration?: number; + video_width?: number; + video_height?: number; + thumbnail_path?: string; + video_processing_status?: VideoProcessingStatus; } interface Tag { @@ -80,6 +106,30 @@ export function MemoryDetailPanel({ const [tagSuggestions, setTagSuggestions] = useState([]); const [showTagSuggestions, setShowTagSuggestions] = useState(false); + // Voice memory state + const [isPlaying, setIsPlaying] = useState(false); + const [isAudioLoading, setIsAudioLoading] = useState(false); + const [showFullTranscript, setShowFullTranscript] = useState(false); + const [isRetrying, setIsRetrying] = useState(false); + const [copiedTranscript, setCopiedTranscript] = useState(false); + const [currentTime, setCurrentTime] = useState(0); + const [duration, setDuration] = useState(0); + const [playbackRate, setPlaybackRate] = useState(1); + const audioRef = useRef(null); + const blobUrlRef = useRef(null); + + // Video player state + const [isVideoLoading, setIsVideoLoading] = useState(false); + const [videoCurrentTime, setVideoCurrentTime] = useState(0); + const [videoDuration, setVideoDuration] = useState(0); + const [isVideoPlaying, setIsVideoPlaying] = useState(false); + const [videoPlaybackRate, setVideoPlaybackRate] = useState(1); + const [videoSrc, setVideoSrc] = useState(null); + const [videoThumbnailUrl, setVideoThumbnailUrl] = useState(null); + const videoRef = useRef(null); + const videoBlobUrlRef = useRef(null); + const videoThumbnailUrlRef = useRef(null); + const titleInputRef = useRef(null); const navigate = useNavigate(); const { startNewChat, addAttachedMemory } = useConversation(); @@ -107,6 +157,64 @@ export function MemoryDetailPanel({ } }, [memoryId, isOpen]); + // Fetch video thumbnail when memory has one + useEffect(() => { + if (!memory || memory.type !== "video" || !memory.thumbnail_path) { + return; + } + + let isCancelled = false; + let blobUrl: string | null = null; + + const fetchThumbnail = async () => { + try { + const token = getAppToken(); + const response = await fetch(`${API_BASE_URL}/api/video/${memory.id}/thumbnail`, { + headers: token ? { "X-App-Token": token } : {}, + }); + + if (!response.ok) { + throw new Error("Failed to load thumbnail"); + } + + const blob = await response.blob(); + + // Check if component was unmounted during fetch + if (isCancelled) { + return; + } + + blobUrl = URL.createObjectURL(blob); + + // Revoke previous URL if exists + if (videoThumbnailUrlRef.current) { + URL.revokeObjectURL(videoThumbnailUrlRef.current); + } + + videoThumbnailUrlRef.current = blobUrl; + setVideoThumbnailUrl(blobUrl); + } catch (error) { + if (!isCancelled) { + console.error("Failed to load video thumbnail:", error); + } + } + }; + + fetchThumbnail(); + + return () => { + isCancelled = true; + // Revoke both the ref URL and any URL created during this effect + if (videoThumbnailUrlRef.current) { + URL.revokeObjectURL(videoThumbnailUrlRef.current); + videoThumbnailUrlRef.current = null; + } + if (blobUrl && blobUrl !== videoThumbnailUrlRef.current) { + URL.revokeObjectURL(blobUrl); + } + }; + }, [memory?.id, memory?.type, memory?.thumbnail_path]); + // Reset editing state when panel closes useEffect(() => { if (!isOpen) { @@ -327,6 +435,349 @@ export function MemoryDetailPanel({ } }; + // Voice/Audio memory functions + const handlePlayPause = async () => { + if (!memory || (memory.type !== "voice_memo" && memory.type !== "audio" && memory.type !== "voice")) return; + + if (!audioRef.current) { + setIsAudioLoading(true); + let blobUrl: string | null = null; + + try { + const token = getAppToken(); + const response = await fetch(`${API_BASE_URL}/api/media/${memory.id}/stream`, { + headers: token ? { "X-App-Token": token } : {}, + }); + + if (!response.ok) { + throw new Error("Failed to load audio"); + } + + const blob = await response.blob(); + blobUrl = URL.createObjectURL(blob); + + const audio = new Audio(); + blobUrlRef.current = blobUrl; + audio.src = blobUrl; + audio.onended = () => setIsPlaying(false); + audio.oncanplay = () => { + setIsAudioLoading(false); + setDuration(audio.duration); + }; + audio.ontimeupdate = () => setCurrentTime(audio.currentTime); + audio.onloadedmetadata = () => setDuration(audio.duration); + audioRef.current = audio; + } catch (error) { + console.error("Failed to load audio:", error); + // Clean up blob URL if it was created but setup failed + if (blobUrl && !blobUrlRef.current) { + URL.revokeObjectURL(blobUrl); + } + setIsAudioLoading(false); + return; + } + } + + if (isPlaying) { + audioRef.current?.pause(); + setIsPlaying(false); + } else { + audioRef.current?.play(); + setIsPlaying(true); + } + }; + + // Video playback functions + const handleVideoPlayPause = async () => { + if (!memory || memory.type !== "video") return; + + // If video not loaded yet, fetch it + if (!videoSrc) { + setIsVideoLoading(true); + let blobUrl: string | null = null; + + try { + const token = getAppToken(); + const response = await fetch(`${API_BASE_URL}/api/video/${memory.id}/stream`, { + headers: token ? { "X-App-Token": token } : {}, + }); + + if (!response.ok) { + throw new Error("Failed to load video"); + } + + const blob = await response.blob(); + blobUrl = URL.createObjectURL(blob); + videoBlobUrlRef.current = blobUrl; + setVideoSrc(blobUrl); + setIsVideoLoading(false); + // Video will auto-play when loaded via onCanPlay handler + return; + } catch (error) { + console.error("Failed to load video:", error); + // Clean up blob URL if it was created but setup failed + if (blobUrl && !videoBlobUrlRef.current) { + URL.revokeObjectURL(blobUrl); + } + setIsVideoLoading(false); + return; + } + } + + // Toggle play/pause + if (isVideoPlaying) { + videoRef.current?.pause(); + setIsVideoPlaying(false); + } else { + videoRef.current?.play(); + setIsVideoPlaying(true); + } + }; + + const handleVideoSkip = (seconds: number) => { + if (videoRef.current) { + videoRef.current.currentTime = Math.max( + 0, + Math.min(videoRef.current.currentTime + seconds, videoRef.current.duration) + ); + } + }; + + const handleVideoSeek = (e: React.MouseEvent) => { + if (!videoRef.current || !videoDuration) return; + const rect = e.currentTarget.getBoundingClientRect(); + const percent = (e.clientX - rect.left) / rect.width; + videoRef.current.currentTime = percent * videoDuration; + }; + + const handleVideoPlaybackRateChange = () => { + const rates = [1, 1.25, 1.5, 1.75, 2]; + const currentIndex = rates.indexOf(videoPlaybackRate); + const nextRate = rates[(currentIndex + 1) % rates.length]; + setVideoPlaybackRate(nextRate); + if (videoRef.current) { + videoRef.current.playbackRate = nextRate; + } + }; + + const handleVideoSeekToTime = async (time: number) => { + if (!memory || memory.type !== "video") return; + + if (!videoRef.current) { + setIsVideoLoading(true); + const video = document.createElement("video"); + + const token = getAppToken(); + const response = await fetch(`${API_BASE_URL}/api/video/${memory.id}/stream`, { + headers: token ? { "X-App-Token": token } : {}, + }); + + if (!response.ok) { + console.error("Failed to load video"); + setIsVideoLoading(false); + return; + } + + const blob = await response.blob(); + const blobUrl = URL.createObjectURL(blob); + videoBlobUrlRef.current = blobUrl; + video.src = blobUrl; + video.onended = () => setIsVideoPlaying(false); + video.oncanplay = () => { + setIsVideoLoading(false); + setVideoDuration(video.duration); + video.currentTime = time; + video.play(); + setIsVideoPlaying(true); + }; + video.ontimeupdate = () => setVideoCurrentTime(video.currentTime); + video.onloadedmetadata = () => setVideoDuration(video.duration); + videoRef.current = video; + } else { + videoRef.current.currentTime = time; + if (!isVideoPlaying) { + videoRef.current.play(); + setIsVideoPlaying(true); + } + } + }; + + const handleSkip = (seconds: number) => { + if (audioRef.current) { + audioRef.current.currentTime = Math.max( + 0, + Math.min(audioRef.current.currentTime + seconds, audioRef.current.duration) + ); + } + }; + + const handleSeek = (e: React.MouseEvent) => { + if (!audioRef.current || !duration) return; + const rect = e.currentTarget.getBoundingClientRect(); + const percent = (e.clientX - rect.left) / rect.width; + audioRef.current.currentTime = percent * duration; + }; + + const handlePlaybackRateChange = () => { + const rates = [1, 1.25, 1.5, 1.75, 2]; + const currentIndex = rates.indexOf(playbackRate); + const nextRate = rates[(currentIndex + 1) % rates.length]; + setPlaybackRate(nextRate); + if (audioRef.current) { + audioRef.current.playbackRate = nextRate; + } + }; + + const handleCopyTranscript = async () => { + if (!memory?.transcript) return; + try { + await navigator.clipboard.writeText(memory.transcript); + setCopiedTranscript(true); + toast.success("Transcript copied to clipboard"); + setTimeout(() => setCopiedTranscript(false), 2000); + } catch { + toast.error("Failed to copy transcript"); + } + }; + + const handleSeekToTime = async (time: number) => { + // For video type, use video seek + if (memory?.type === "video") { + handleVideoSeekToTime(time); + return; + } + + // Initialize audio if not already loaded + if (!audioRef.current && (memory?.type === "voice_memo" || memory?.type === "audio" || memory?.type === "voice")) { + setIsAudioLoading(true); + const audio = new Audio(); + + const token = getAppToken(); + const response = await fetch( + `${API_BASE_URL}/api/media/${memory.id}/stream`, + { + headers: token ? { "X-App-Token": token } : {}, + } + ); + + if (!response.ok) { + console.error("Failed to load audio"); + setIsAudioLoading(false); + return; + } + + const blob = await response.blob(); + const blobUrl = URL.createObjectURL(blob); + blobUrlRef.current = blobUrl; + audio.src = blobUrl; + audio.onended = () => setIsPlaying(false); + audio.oncanplay = () => { + setIsAudioLoading(false); + setDuration(audio.duration); + // Seek after audio is ready + audio.currentTime = time; + audio.play(); + setIsPlaying(true); + }; + audio.ontimeupdate = () => setCurrentTime(audio.currentTime); + audio.onloadedmetadata = () => setDuration(audio.duration); + audioRef.current = audio; + } else if (audioRef.current) { + audioRef.current.currentTime = time; + if (!isPlaying) { + audioRef.current.play(); + setIsPlaying(true); + } + } + }; + + const handleRetryTranscription = async () => { + if (!memory || (memory.type !== "voice_memo" && memory.type !== "audio" && memory.type !== "voice" && memory.type !== "video")) return; + setIsRetrying(true); + + try { + // Use video endpoint for video type + const endpoint = memory.type === "video" + ? `/api/video/${memory.id}/retry` + : `/api/media/${memory.id}/retry`; + + const res = await apiFetch(endpoint, { + method: "POST", + }); + + if (res.ok) { + setMemory((prev) => + prev ? { ...prev, transcription_status: "pending" } : null + ); + } + } catch (err) { + console.error("Failed to retry transcription:", err); + } finally { + setIsRetrying(false); + } + }; + + const formatDuration = (seconds: number | undefined): string => { + if (seconds === undefined || seconds === null || !isFinite(seconds)) return "0:00"; + const totalSeconds = Math.floor(seconds); + const minutes = Math.floor(totalSeconds / 60); + const secs = totalSeconds % 60; + return `${minutes}:${secs.toString().padStart(2, "0")}`; + }; + + // Reset audio/video player when memory changes and cleanup blob URLs + useEffect(() => { + // Audio cleanup + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current = null; + } + if (blobUrlRef.current) { + URL.revokeObjectURL(blobUrlRef.current); + blobUrlRef.current = null; + } + setIsPlaying(false); + setCurrentTime(0); + setDuration(0); + setPlaybackRate(1); + setShowFullTranscript(false); + + // Video cleanup + if (videoRef.current) { + videoRef.current.pause(); + videoRef.current = null; + } + if (videoBlobUrlRef.current) { + URL.revokeObjectURL(videoBlobUrlRef.current); + videoBlobUrlRef.current = null; + } + if (videoThumbnailUrlRef.current) { + URL.revokeObjectURL(videoThumbnailUrlRef.current); + videoThumbnailUrlRef.current = null; + } + setVideoSrc(null); + setVideoThumbnailUrl(null); + setIsVideoPlaying(false); + setVideoCurrentTime(0); + setVideoDuration(0); + setVideoPlaybackRate(1); + }, [memoryId]); + + // Cleanup blob URLs on unmount + useEffect(() => { + return () => { + if (blobUrlRef.current) { + URL.revokeObjectURL(blobUrlRef.current); + } + if (videoBlobUrlRef.current) { + URL.revokeObjectURL(videoBlobUrlRef.current); + } + if (videoThumbnailUrlRef.current) { + URL.revokeObjectURL(videoThumbnailUrlRef.current); + } + }; + }, []); + if (!isOpen) return null; return createPortal( @@ -383,11 +834,17 @@ export function MemoryDetailPanel({
{memory.type === "web" ? ( + ) : memory.type === "voice_memo" || memory.type === "voice" ? ( + + ) : memory.type === "audio" ? ( + + ) : memory.type === "video" ? ( +
+ {/* Voice Memory: Enhanced Audio Player */} + {(memory.type === "voice_memo" || memory.type === "audio" || memory.type === "voice") && ( +
+
+ {/* Controls Row */} +
+ {/* Skip Back */} + + + {/* Play/Pause */} + + + {/* Skip Forward */} + +
+ + {/* Progress Bar */} +
+
+
+
+ + {/* Time Display & Playback Speed */} +
+ + {formatDuration(currentTime)} /{" "} + {formatDuration( + duration || + memory.audio_duration || + (memory.transcript_segments?.length + ? memory.transcript_segments[ + memory.transcript_segments.length - 1 + ].end + : 0) + )} + + +
+ + {/* Transcription Status */} + {memory.transcription_status && + memory.transcription_status !== "completed" && + (memory.transcription_status === "failed" || !memory.transcript) && ( +
+ + {memory.transcription_status === "processing" && ( + + )} + {memory.transcription_status === "pending" + ? "Pending" + : memory.transcription_status === "processing" + ? "Transcribing..." + : "Failed"} + + {memory.transcription_status === "failed" && ( + + )} +
+ )} +
+
+ )} + + {/* Video Player */} + {memory.type === "video" && ( +
+
+ {/* Video Processing Status */} + {memory.video_processing_status && memory.video_processing_status !== "ready" && ( +
+ + {(memory.video_processing_status === "pending_extraction" || memory.video_processing_status === "extracting") && ( + + )} + {memory.video_processing_status === "pending_extraction" + ? "Processing video..." + : memory.video_processing_status === "extracting" + ? "Extracting audio..." + : "Processing failed"} + +
+ )} + + {/* Video Controls (only when ready) */} + {memory.video_processing_status === "ready" && ( + <> + {/* Video/Thumbnail Display */} +
+ {videoSrc ? ( + /* Loaded video */ +
+ + {/* Controls (only show after video loaded) */} + {videoSrc && ( + <> + {/* Controls Row */} +
+ {/* Skip Back */} + + + {/* Play/Pause */} + + + {/* Skip Forward */} + +
+ + {/* Progress Bar */} +
+
+
+
+ + {/* Time Display & Playback Speed */} +
+ + {formatDuration(videoCurrentTime)} / {formatDuration(videoDuration || memory.video_duration || 0)} + + +
+ + )} + + )} + + {/* Transcription Status */} + {memory.video_processing_status === "ready" && memory.transcription_status && memory.transcription_status !== "completed" && (memory.transcription_status === "failed" || !memory.transcript) && ( +
+ + {memory.transcription_status === "processing" && ( + + )} + {memory.transcription_status === "pending" + ? "Pending" + : memory.transcription_status === "processing" + ? "Transcribing..." + : "Failed"} + + {memory.transcription_status === "failed" && ( + + )} +
+ )} +
+
+ )} + + {/* Transcript Section (for audio and video types) */} + {(memory.type === "voice_memo" || memory.type === "audio" || memory.type === "voice" || memory.type === "video") && memory.transcript && ( +
+
+ + +
+ {memory.transcript_segments && + memory.transcript_segments.length > 0 ? ( + // Transcript with timestamps - show in both collapsed/expanded +
+
+ {memory.transcript_segments.map((segment, index) => { + const playTime = memory.type === "video" ? videoCurrentTime : currentTime; + const isActive = + playTime >= segment.start && + playTime < segment.end; + return ( +
+ + + {segment.text} + +
+ ); + })} +
+ {/* Fade gradient when collapsed */} + {!showFullTranscript && ( +
+ )} +
+ ) : ( + // Fallback: plain text for memories without segments +
+
+ {memory.transcript} +
+ {!showFullTranscript && ( +
+ )} +
+ )} +
+ )} + {/* Summary Section */}
diff --git a/app/src/components/VideoCard.tsx b/app/src/components/VideoCard.tsx new file mode 100644 index 0000000..3ab8d0d --- /dev/null +++ b/app/src/components/VideoCard.tsx @@ -0,0 +1,309 @@ +import { useState, useRef, useEffect } from "react"; +import { Button } from "@/components/ui/button"; +import { + Video, + X, + PanelRight, + Play, + Loader2, +} from "lucide-react"; +import { cn } from "@/lib/utils"; +import { getAppToken } from "@/lib/api"; +import { API_BASE_URL } from "@/constants"; +import type { TranscriptionStatus, VideoProcessingStatus } from "@/types/chat"; + +interface MemoryTag { + id: number; + name: string; + source: "ai" | "manual"; +} + +interface VideoMemory { + id: number; + type: "video"; + title: string; + summary: string | null; + tags: MemoryTag[]; + created_at: string; + video_duration?: number; + video_width?: number; + video_height?: number; + thumbnail_path?: string; + video_processing_status?: VideoProcessingStatus; + transcription_status?: TranscriptionStatus; +} + +interface VideoCardProps { + memory: VideoMemory; + onRemoveTag: (memoryId: number, tagId: number) => void; + onExpand: (id: number) => void; + formatDate: (date: string) => string; +} + +function formatDuration(seconds: number | undefined): string { + if (seconds === undefined || seconds === null) return "0:00"; + const totalSeconds = Math.floor(seconds); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const secs = totalSeconds % 60; + + if (hours > 0) { + return `${hours}:${minutes.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; + } + return `${minutes}:${secs.toString().padStart(2, "0")}`; +} + +function ProcessingStatusBadge({ + videoStatus, + transcriptionStatus, +}: { + videoStatus?: VideoProcessingStatus; + transcriptionStatus?: TranscriptionStatus; +}) { + // Show video processing status if not ready + if (videoStatus && videoStatus !== "ready") { + const statusConfig = { + pending_extraction: { + label: "Processing...", + className: "bg-muted text-muted-foreground", + }, + extracting: { + label: "Extracting Audio...", + className: "bg-muted text-muted-foreground", + }, + failed: { + label: "Processing Failed", + className: "bg-red-100 text-red-700 dark:bg-red-900/30 dark:text-red-400", + }, + }; + const config = statusConfig[videoStatus] || statusConfig.pending_extraction; + + return ( + + {(videoStatus === "pending_extraction" || videoStatus === "extracting") && ( + + )} + {config.label} + + ); + } + + // Show transcription status if video is ready + if (!transcriptionStatus || transcriptionStatus === "completed") return null; + + const statusConfig = { + pending: { + label: "Pending Transcription", + className: "bg-muted text-muted-foreground", + }, + processing: { + label: "Transcribing...", + className: "bg-muted text-muted-foreground", + }, + failed: { + label: "Transcription Failed", + className: "bg-red-100 text-red-700 dark:bg-red-900/30 dark:text-red-400", + }, + }; + + const config = statusConfig[transcriptionStatus] || statusConfig.pending; + + return ( + + {transcriptionStatus === "processing" && } + {config.label} + + ); +} + +export function VideoCard({ memory, onRemoveTag, onExpand, formatDate }: VideoCardProps) { + const [thumbnailUrl, setThumbnailUrl] = useState(null); + const [isLoadingThumbnail, setIsLoadingThumbnail] = useState(false); + const thumbnailLoadedRef = useRef(false); + + // Load thumbnail on mount + useEffect(() => { + if (!memory.thumbnail_path || thumbnailLoadedRef.current) return; + + thumbnailLoadedRef.current = true; + setIsLoadingThumbnail(true); + + const token = getAppToken(); + fetch(`${API_BASE_URL}/api/video/${memory.id}/thumbnail`, { + headers: token ? { "X-App-Token": token } : {}, + }) + .then((response) => { + if (response.ok) return response.blob(); + throw new Error("Failed to load thumbnail"); + }) + .then((blob) => { + const url = URL.createObjectURL(blob); + setThumbnailUrl(url); + }) + .catch((error) => { + console.error("Failed to load thumbnail:", error); + }) + .finally(() => { + setIsLoadingThumbnail(false); + }); + + return () => { + if (thumbnailUrl) { + URL.revokeObjectURL(thumbnailUrl); + } + }; + }, [memory.id, memory.thumbnail_path]); + + // Cleanup blob URL on unmount + useEffect(() => { + return () => { + if (thumbnailUrl) { + URL.revokeObjectURL(thumbnailUrl); + } + }; + }, [thumbnailUrl]); + + return ( +
+ {/* Hover actions - top right */} +
+ +
+ + {/* Thumbnail */} +
onExpand(memory.id)} + > + {thumbnailUrl ? ( + {memory.title} + ) : isLoadingThumbnail ? ( +
+ +
+ ) : ( +
+
+ )} + + {/* Play overlay */} +
+
+ +
+
+ + {/* Duration badge */} + {memory.video_duration !== undefined && ( +
+ {formatDuration(memory.video_duration)} +
+ )} +
+ + {/* Header: Video icon + Title */} +
+
+
+

+ {memory.title || "Video"} +

+
+ + {/* Processing status */} +
+ +
+ + {/* Summary */} + {memory.summary ? ( +

+ {memory.summary} +

+ ) : memory.transcription_status === "completed" ? ( +

+ Generating summary... +

+ ) : null} + + {/* Tags */} + {memory.tags.length > 0 && ( +
+ {memory.tags.map((tag) => ( + + {tag.name} + {tag.source === "manual" && ( + + )} + + ))} +
+ )} + + {/* Date */} +

{formatDate(memory.created_at)}

+
+ ); +} diff --git a/app/src/components/VoiceMemoCard.tsx b/app/src/components/VoiceMemoCard.tsx new file mode 100644 index 0000000..1131591 --- /dev/null +++ b/app/src/components/VoiceMemoCard.tsx @@ -0,0 +1,252 @@ +import { useState, useRef, useEffect } from "react"; +import { Button } from "@/components/ui/button"; +import { + Mic, + X, + PanelRight, + Play, + Pause, + Loader2, +} from "lucide-react"; +import { cn } from "@/lib/utils"; +import { getAppToken } from "@/lib/api"; +import { API_BASE_URL } from "@/constants"; +import type { TranscriptionStatus } from "@/types/chat"; + +interface MemoryTag { + id: number; + name: string; + source: "ai" | "manual"; +} + +interface VoiceMemo { + id: number; + type: "voice_memo" | "voice"; // "voice" for backwards compat + title: string; + summary: string | null; + tags: MemoryTag[]; + created_at: string; + audio_duration?: number; + transcription_status?: TranscriptionStatus; +} + +interface VoiceMemoCardProps { + memory: VoiceMemo; + onRemoveTag: (memoryId: number, tagId: number) => void; + onExpand: (id: number) => void; + formatDate: (date: string) => string; +} + +function formatDuration(seconds: number | undefined): string { + if (seconds === undefined || seconds === null) return "0:00"; + const totalSeconds = Math.floor(seconds); + const minutes = Math.floor(totalSeconds / 60); + const secs = totalSeconds % 60; + return `${minutes}:${secs.toString().padStart(2, "0")}`; +} + +function TranscriptionStatusBadge({ status }: { status?: TranscriptionStatus }) { + if (!status || status === "completed") return null; + + const statusConfig = { + pending: { label: "Pending", className: "bg-muted text-muted-foreground" }, + processing: { label: "Transcribing...", className: "bg-muted text-muted-foreground" }, + failed: { label: "Failed", className: "bg-red-100 text-red-700 dark:bg-red-900/30 dark:text-red-400" }, + }; + + const config = statusConfig[status] || statusConfig.pending; + + return ( + + {status === "processing" && } + {config.label} + + ); +} + +export function VoiceMemoCard({ + memory, + onRemoveTag, + onExpand, + formatDate, +}: VoiceMemoCardProps) { + const [isPlaying, setIsPlaying] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const audioRef = useRef(null); + const blobUrlRef = useRef(null); + + // Cleanup audio element and blob URL on unmount to prevent memory leaks + useEffect(() => { + return () => { + // Pause and clean up audio element + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current.src = ""; + audioRef.current = null; + } + // Revoke blob URL + if (blobUrlRef.current) { + URL.revokeObjectURL(blobUrlRef.current); + blobUrlRef.current = null; + } + }; + }, []); + + const handlePlayPause = async (e: React.MouseEvent) => { + e.stopPropagation(); + + if (!audioRef.current) { + // Create audio element on first play + setIsLoading(true); + const audio = new Audio(); + + // Fetch audio with auth token + const token = getAppToken(); + const response = await fetch(`${API_BASE_URL}/api/media/${memory.id}/stream`, { + headers: token ? { "X-App-Token": token } : {}, + }); + + if (!response.ok) { + console.error("Failed to load audio"); + setIsLoading(false); + return; + } + + const blob = await response.blob(); + const blobUrl = URL.createObjectURL(blob); + blobUrlRef.current = blobUrl; + audio.src = blobUrl; + audio.onended = () => setIsPlaying(false); + audio.oncanplay = () => setIsLoading(false); + audioRef.current = audio; + } + + if (isPlaying) { + audioRef.current?.pause(); + setIsPlaying(false); + } else { + audioRef.current?.play(); + setIsPlaying(true); + } + }; + + return ( +
+ {/* Hover actions - top right */} +
+ +
+ + {/* Header: Voice icon + Title */} +
+
+ +
+

+ {memory.title || "Voice Memo"} +

+
+ + {/* Audio Player */} +
+ +
+
+ {formatDuration(memory.audio_duration)} +
+ +
+
+ + {/* Summary */} + {memory.summary ? ( +

+ {memory.summary} +

+ ) : memory.transcription_status === "completed" ? ( +

+ Generating summary... +

+ ) : null} + + {/* Tags */} + {memory.tags.length > 0 && ( +
+ {memory.tags.map((tag) => ( + + {tag.name} + {tag.source === "manual" && ( + + )} + + ))} +
+ )} + + {/* Date */} +

+ {formatDate(memory.created_at)} +

+
+ ); +} diff --git a/app/src/global.d.ts b/app/src/global.d.ts new file mode 100644 index 0000000..916402d --- /dev/null +++ b/app/src/global.d.ts @@ -0,0 +1,42 @@ +/** + * Global TypeScript declarations for Electron IPC APIs. + */ + +export {}; + +declare global { + interface Window { + electronAPI?: { + // Ollama management + checkOllama: () => Promise<{ installed: boolean; running: boolean }>; + downloadOllama: () => Promise<{ success: boolean; error?: string }>; + pullModel: (model: string) => Promise<{ success: boolean; error?: string }>; + onOllamaDownloadProgress: (callback: (data: { progress: number; stage: string }) => void) => void; + onModelPullProgress: (callback: (data: { progress?: number; status: string }) => void) => void; + removeOllamaDownloadProgress: () => void; + removeModelPullProgress: () => void; + // Backend status handlers + onBackendReady: (callback: (data?: { token?: string }) => void) => void; + onBackendError: (callback: (data: { message: string }) => void) => void; + removeBackendListeners: () => void; + // App token for API authentication + getAppToken: () => string | null; + // Auto-update handlers + onUpdateDownloaded: (callback: (version: string) => void) => void; + removeUpdateListeners: () => void; + installUpdate: () => Promise; + // Main window management + openMainWindow: () => Promise; + // Recording window management + openRecordingWindow: () => Promise; + setRecordingState: (isRecording: boolean) => void; + // Video processing (native FFmpeg) + // Note: Buffer objects from Node.js serialize to Uint8Array over IPC + writeTempFile: (data: ArrayBuffer, filename: string) => Promise<{ success: boolean; path?: string; error?: string }>; + processVideo: (videoPath: string) => Promise<{ success: boolean; audio?: Uint8Array; thumbnail?: Uint8Array | null; error?: string }>; + deleteTempFile: (path: string) => Promise<{ success: boolean; error?: string }>; + onVideoProcessProgress: (callback: (data: { progress: number; stage: string }) => void) => void; + removeVideoProcessListeners: () => void; + }; + } +} diff --git a/app/src/hooks/useVideoUpload.ts b/app/src/hooks/useVideoUpload.ts new file mode 100644 index 0000000..22fde1f --- /dev/null +++ b/app/src/hooks/useVideoUpload.ts @@ -0,0 +1,229 @@ +/** + * Hook for handling video upload with native FFmpeg audio extraction. + * + * Pipeline: + * 1. Get metadata (duration, dimensions) via HTML5 video + * 2. Write video to temp file via IPC (for native FFmpeg) + * 3. Upload video to POST /api/video/upload (in parallel with processing) + * 4. Process video via native FFmpeg (extract audio + thumbnail) + * 5. Upload audio to POST /api/video/{id}/audio + * 6. Upload thumbnail to POST /api/video/{id}/thumbnail + * 7. Cleanup temp file + */ + +import { useState, useCallback, useEffect } from "react"; +import { apiFetch } from "@/lib/api"; +import { getVideoMetadata, isVideoFile } from "@/lib/ffmpeg"; + +export type VideoUploadStatus = + | "idle" + | "getting_metadata" + | "uploading_video" + | "processing_video" + | "extracting_audio" + | "generating_thumbnail" + | "uploading_audio" + | "uploading_thumbnail" + | "done" + | "error"; + +export interface VideoUploadProgress { + status: VideoUploadStatus; + progress: number; // 0-100 + message: string; + error?: string; +} + +export interface VideoUploadResult { + memoryId: number; + success: boolean; + error?: string; +} + +export function useVideoUpload() { + const [uploadProgress, setUploadProgress] = useState({ + status: "idle", + progress: 0, + message: "", + }); + + const updateProgress = useCallback( + (status: VideoUploadStatus, progress: number, message: string, error?: string) => { + setUploadProgress({ status, progress, message, error }); + }, + [] + ); + + // Set up progress listener for native FFmpeg processing + useEffect(() => { + const electronAPI = window.electronAPI; + if (!electronAPI) return; + + electronAPI.onVideoProcessProgress((data) => { + const stageMessages: Record = { + extracting_audio: "Extracting audio...", + generating_thumbnail: "Generating thumbnail...", + done: "Processing complete", + }; + + const message = stageMessages[data.stage] || "Processing video..."; + const status = data.stage === "extracting_audio" ? "extracting_audio" : + data.stage === "generating_thumbnail" ? "generating_thumbnail" : + "processing_video"; + + // Map FFmpeg progress (0-100) to overall progress (30-70%) + const overallProgress = 30 + Math.round(data.progress * 0.4); + setUploadProgress(prev => { + // Only update if we're in a processing state + if (prev.status === "processing_video" || prev.status === "extracting_audio" || prev.status === "generating_thumbnail") { + return { status, progress: overallProgress, message }; + } + return prev; + }); + }); + + return () => { + electronAPI.removeVideoProcessListeners(); + }; + }, []); + + const uploadVideo = useCallback( + async (file: File): Promise => { + const electronAPI = window.electronAPI; + if (!electronAPI) { + const error = "Video upload requires Electron environment"; + updateProgress("error", 0, error, error); + return { memoryId: 0, success: false, error }; + } + + if (!isVideoFile(file)) { + const error = "Unsupported video format"; + updateProgress("error", 0, error, error); + return { memoryId: 0, success: false, error }; + } + + let tempPath: string | null = null; + + try { + // 1. Read file and get metadata + updateProgress("getting_metadata", 5, "Reading video file..."); + const videoArrayBuffer = await file.arrayBuffer(); + const videoBlob = new Blob([videoArrayBuffer], { type: file.type }); + const metadata = await getVideoMetadata(videoBlob); + + // 2. Write video to temp file for native FFmpeg processing + updateProgress("getting_metadata", 10, "Preparing video for processing..."); + const writeResult = await electronAPI.writeTempFile(videoArrayBuffer, file.name); + if (!writeResult.success || !writeResult.path) { + throw new Error(writeResult.error || "Failed to write temp file"); + } + tempPath = writeResult.path; + + // 3. Upload video to backend (run this while processing starts) + updateProgress("uploading_video", 15, "Uploading video..."); + const videoFormData = new FormData(); + videoFormData.append("file", new File([videoArrayBuffer], file.name, { type: file.type })); + videoFormData.append("duration", String(metadata.duration)); + videoFormData.append("width", String(metadata.width)); + videoFormData.append("height", String(metadata.height)); + + const videoResponse = await apiFetch("/api/video/upload", { + method: "POST", + body: videoFormData, + }); + + if (!videoResponse.ok) { + const errorData = await videoResponse.json().catch(() => ({})); + throw new Error(errorData.detail || "Failed to upload video"); + } + + const videoData = await videoResponse.json(); + const memoryId = videoData.id; + + // 4. Process video with native FFmpeg (extract audio + thumbnail) + updateProgress("processing_video", 30, "Processing video with FFmpeg..."); + const processResult = await electronAPI.processVideo(tempPath); + + if (!processResult.success) { + throw new Error(processResult.error || "Failed to process video"); + } + + // 5. Upload audio + updateProgress("uploading_audio", 75, "Uploading audio..."); + if (processResult.audio) { + const audioFormData = new FormData(); + audioFormData.append("file", new File([processResult.audio], "audio.m4a", { type: "audio/mp4" })); + + const audioResponse = await apiFetch(`/api/video/${memoryId}/audio`, { + method: "POST", + body: audioFormData, + }); + + if (!audioResponse.ok) { + const errorData = await audioResponse.json().catch(() => ({})); + throw new Error(errorData.detail || "Failed to upload audio"); + } + } + + // 6. Upload thumbnail (only if generation succeeded) + if (processResult.thumbnail) { + updateProgress("uploading_thumbnail", 90, "Uploading thumbnail..."); + const thumbnailFormData = new FormData(); + thumbnailFormData.append( + "file", + new File([processResult.thumbnail], "thumbnail.jpg", { type: "image/jpeg" }) + ); + + const thumbnailResponse = await apiFetch(`/api/video/${memoryId}/thumbnail`, { + method: "POST", + body: thumbnailFormData, + }); + + if (!thumbnailResponse.ok) { + // Thumbnail upload failure is non-critical + console.warn("Failed to upload thumbnail:", await thumbnailResponse.text()); + } + } else { + console.warn("Skipping thumbnail upload - generation failed"); + } + + // 7. Cleanup temp file + if (tempPath) { + await electronAPI.deleteTempFile(tempPath).catch(console.warn); + } + + // Done! + updateProgress("done", 100, "Video uploaded successfully"); + return { memoryId, success: true }; + } catch (error) { + // Cleanup on error + if (tempPath && electronAPI) { + await electronAPI.deleteTempFile(tempPath).catch(console.warn); + } + + const errorMessage = error instanceof Error ? error.message : "Upload failed"; + updateProgress("error", 0, errorMessage, errorMessage); + return { memoryId: 0, success: false, error: errorMessage }; + } + }, + [updateProgress] + ); + + const reset = useCallback(() => { + setUploadProgress({ + status: "idle", + progress: 0, + message: "", + }); + }, []); + + return { + uploadVideo, + uploadProgress, + reset, + isUploading: + uploadProgress.status !== "idle" && + uploadProgress.status !== "done" && + uploadProgress.status !== "error", + }; +} diff --git a/app/src/lib/ffmpeg.ts b/app/src/lib/ffmpeg.ts new file mode 100644 index 0000000..30dd646 --- /dev/null +++ b/app/src/lib/ffmpeg.ts @@ -0,0 +1,97 @@ +/** + * Video utility functions for client-side metadata extraction and validation. + * + * Note: Audio extraction and thumbnail generation are now handled by native FFmpeg + * in the Electron main process via IPC (see useVideoUpload hook). + */ + +/** + * Video metadata from HTML5 video element. + */ +export interface VideoMetadata { + duration: number; + width: number; + height: number; +} + +// Timeout for video metadata extraction (30 seconds) +const METADATA_TIMEOUT_MS = 30000; + +/** + * Get video metadata using HTML5 video element. + * This is faster than using FFmpeg for basic metadata. + * + * @param videoData - The video file or blob to get metadata from + * @returns Video metadata (duration, width, height) + */ +export function getVideoMetadata(videoData: File | Blob): Promise { + return new Promise((resolve, reject) => { + const video = document.createElement("video"); + const objectUrl = URL.createObjectURL(videoData); + let settled = false; + + const cleanup = () => { + URL.revokeObjectURL(objectUrl); + video.src = ""; + video.load(); // Release resources + }; + + // Timeout to prevent hanging indefinitely + const timeoutId = setTimeout(() => { + if (!settled) { + settled = true; + cleanup(); + reject(new Error("Video metadata extraction timed out")); + } + }, METADATA_TIMEOUT_MS); + + video.preload = "metadata"; + + video.onloadedmetadata = () => { + if (!settled) { + settled = true; + clearTimeout(timeoutId); + const metadata = { + duration: video.duration, + width: video.videoWidth, + height: video.videoHeight, + }; + cleanup(); + resolve(metadata); + } + }; + + video.onerror = () => { + if (!settled) { + settled = true; + clearTimeout(timeoutId); + cleanup(); + reject(new Error("Failed to load video metadata")); + } + }; + + video.src = objectUrl; + }); +} + +/** + * Check if a file is a supported video format. + */ +export function isVideoFile(file: File): boolean { + const supportedTypes = [ + "video/mp4", + "video/webm", + "video/quicktime", + "video/x-matroska", + "video/x-msvideo", + ]; + + if (supportedTypes.includes(file.type)) { + return true; + } + + // Check extension as fallback + const supportedExtensions = ["mp4", "webm", "mov", "mkv", "avi"]; + const ext = file.name.split(".").pop()?.toLowerCase(); + return ext ? supportedExtensions.includes(ext) : false; +} diff --git a/app/src/pages/HomePage.tsx b/app/src/pages/HomePage.tsx index 76fbb50..d1ff80a 100644 --- a/app/src/pages/HomePage.tsx +++ b/app/src/pages/HomePage.tsx @@ -1,21 +1,49 @@ -import { useState, useEffect } from "react"; +import { useState, useEffect, useRef } from "react"; import { Link, useNavigate } from "react-router-dom"; import { Button } from "@/components/ui/button"; import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card"; -import { Plus, Brain, MessageSquare } from "lucide-react"; +import { + Brain, + MessageSquare, + Globe, + FileText, + Mic, + FileAudio, + Video, + Upload, +} from "lucide-react"; +import { cn } from "@/lib/utils"; +import { toast } from "sonner"; import { useMemoryEvents } from "../hooks/useMemoryEvents"; import { apiFetch } from "@/lib/api"; import { ChatInput } from "@/components/ChatInput"; import { useConversation } from "@/contexts/ConversationContext"; import { useConversations } from "@/hooks/useConversations"; +import { useVideoUpload } from "@/hooks/useVideoUpload"; +import { ArrowRight } from "lucide-react"; + +type MemoryType = "web" | "note" | "voice_memo" | "audio" | "video" | "voice"; interface Memory { id: number; + type: MemoryType; url: string; title: string; created_at: string; } +const MEMORY_TYPE_CONFIG: Record< + MemoryType, + { icon: typeof Globe; colorClass: string } +> = { + web: { icon: Globe, colorClass: "text-muted-foreground" }, + note: { icon: FileText, colorClass: "text-amber-600" }, + voice_memo: { icon: Mic, colorClass: "text-orange-600" }, + voice: { icon: Mic, colorClass: "text-orange-600" }, // backwards compat + audio: { icon: FileAudio, colorClass: "text-blue-600" }, + video: { icon: Video, colorClass: "text-purple-600" }, +}; + interface HomePageProps { userName?: string | null; } @@ -33,14 +61,23 @@ function getGreeting(name?: string | null): string { return name ? `${greeting}, ${name}` : greeting; } +const AUDIO_EXTENSIONS = ["mp3", "wav", "m4a", "webm", "ogg", "flac"]; +const VIDEO_EXTENSIONS = ["mp4", "webm", "mov", "mkv", "avi"]; +const MAX_AUDIO_SIZE = 100 * 1024 * 1024; // 100 MB +const MAX_VIDEO_SIZE = 500 * 1024 * 1024; // 500 MB + export default function HomePage({ userName }: HomePageProps) { const [message, setMessage] = useState(""); const [recentMemories, setRecentMemories] = useState([]); + const [isUploading, setIsUploading] = useState(false); const navigate = useNavigate(); + const uploadInputRef = useRef(null); + const { selectConversation, setPendingMessage } = useConversation(); const { conversations } = useConversations(); + const { uploadVideo, isUploading: isUploadingVideo } = useVideoUpload(); useEffect(() => { fetchRecentMemories(); @@ -86,6 +123,82 @@ export default function HomePage({ userName }: HomePageProps) { navigate("/chat"); }; + const handleUpload = async (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + + const ext = file.name.split(".").pop()?.toLowerCase(); + if (!ext) { + toast.error("Could not determine file type"); + return; + } + + const isAudio = AUDIO_EXTENSIONS.includes(ext); + const isVideo = VIDEO_EXTENSIONS.includes(ext); + + if (!isAudio && !isVideo) { + toast.error(`Unsupported format. Use: ${[...AUDIO_EXTENSIONS, ...VIDEO_EXTENSIONS].join(", ")}`); + if (uploadInputRef.current) uploadInputRef.current.value = ""; + return; + } + + const maxSize = isVideo ? MAX_VIDEO_SIZE : MAX_AUDIO_SIZE; + const maxLabel = isVideo ? "500 MB" : "100 MB"; + if (file.size > maxSize) { + toast.error(`File too large. Maximum size is ${maxLabel}.`); + if (uploadInputRef.current) uploadInputRef.current.value = ""; + return; + } + + if (isVideo) { + // Use the video upload hook for proper FFmpeg processing + toast.info("Processing video...", { description: "Extracting audio for transcription" }); + const result = await uploadVideo(file); + if (result.success) { + toast.success("Video uploaded successfully"); + navigate("/memories"); + } else { + toast.error(result.error || "Failed to upload video"); + } + } else { + // Direct upload for audio files + setIsUploading(true); + const formData = new FormData(); + formData.append("file", file); + + try { + const res = await apiFetch("/api/media/upload", { + method: "POST", + body: formData, + }); + + if (res.ok) { + toast.success("Audio uploaded successfully"); + navigate("/memories"); + } else { + const data = await res.json(); + toast.error(data.detail || "Failed to upload audio"); + } + } catch { + toast.error("Failed to upload audio"); + } finally { + setIsUploading(false); + } + } + + if (uploadInputRef.current) uploadInputRef.current.value = ""; + }; + + const handleVoiceMemo = () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const electronAPI = (window as any).electronAPI; + if (electronAPI?.openRecordingWindow) { + electronAPI.openRecordingWindow(); + } else { + toast.error("Voice recording is only available in the desktop app"); + } + }; + // Get recent chats (first 5) const recentChats = conversations.slice(0, 5); @@ -96,7 +209,7 @@ export default function HomePage({ userName }: HomePageProps) { {getGreeting(userName)} -
+
+ + {/* Action Chips */} +
+ + + + +
+ + {/* Hidden file input for uploads */} + `.${e}`).join(",")} + onChange={handleUpload} + className="hidden" + />
-
+
{/* Recent Memories */} - - + +
+ Recent Memories - + {recentMemories.length === 0 ? (

No memories yet

) : ( -
    - {recentMemories.map((memory) => ( -
  • - - {memory.title || "Untitled"} - -
    • - ))} +
      + {recentMemories.map((memory) => { + const config = MEMORY_TYPE_CONFIG[memory.type] || MEMORY_TYPE_CONFIG.web; + const TypeIcon = config.icon; + return ( +
    • + + + {memory.title || "Untitled"} + +
      • + ); + })}
    )} - - + + {/* Recent Chats */} - - + +
    + Recent Chats - + {recentChats.length === 0 ? (

    No chats yet

    ) : ( -
      +
        {recentChats.map((chat) => (
      • {chat.title || "New conversation"}
        • ))}
      )} - - + + - {/* Quick Actions */} - - - - - Quick Actions - - - - - - - - -
diff --git a/app/src/pages/MemoriesPage.tsx b/app/src/pages/MemoriesPage.tsx index e873217..7014ce2 100644 --- a/app/src/pages/MemoriesPage.tsx +++ b/app/src/pages/MemoriesPage.tsx @@ -3,11 +3,38 @@ import { useSearchParams } from "react-router-dom"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { MemoryCard } from "@/components/MemoryCard"; +import { VoiceMemoCard } from "@/components/VoiceMemoCard"; +import { AudioCard } from "@/components/AudioCard"; +import { VideoCard } from "@/components/VideoCard"; import { MemoryDetailPanel } from "@/components/MemoryDetailPanel"; +import { AudioDropOverlay } from "@/components/AudioDropOverlay"; import { NoteEditor } from "@/components/editor"; -import { Plus, Search, Loader2, ChevronDown } from "lucide-react"; +import { + Plus, + Search, + Loader2, + ChevronDown, + Upload, + Globe, + FileText, + Mic, + FileAudio, + Video, + LayoutGrid, + Check, + Calendar, +} from "lucide-react"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { cn } from "@/lib/utils"; +import { toast } from "sonner"; import { useMemoryEvents } from "../hooks/useMemoryEvents"; +import { useVideoUpload } from "../hooks/useVideoUpload"; import { apiFetch } from "@/lib/api"; +import type { TranscriptionStatus, VideoProcessingStatus } from "@/types/chat"; interface MemoryTag { id: number; @@ -17,16 +44,27 @@ interface MemoryTag { interface Memory { id: number; - type: "web" | "note"; + type: "web" | "note" | "voice_memo" | "audio" | "video" | "voice"; // "voice" for backwards compat url: string | null; title: string; summary: string | null; tags: MemoryTag[]; created_at: string; + // Media-specific fields (voice memos and audio uploads) + audio_duration?: number; + transcription_status?: TranscriptionStatus; + media_source?: "recording" | "upload"; + // Video-specific fields + video_duration?: number; + video_width?: number; + video_height?: number; + thumbnail_path?: string; + video_processing_status?: VideoProcessingStatus; } interface MemoryWithContent extends Memory { content?: string; + transcript?: string; } interface Tag { @@ -35,9 +73,25 @@ interface Tag { usage_count: number; } -type TypeFilter = "all" | "web" | "note"; +type TypeFilter = "all" | "web" | "note" | "voice_memo" | "audio" | "video"; type DateFilter = "all" | "today" | "week" | "month"; +const TYPE_FILTER_OPTIONS = [ + { value: "all", label: "All", icon: LayoutGrid, iconColor: "text-muted-foreground" }, + { value: "web", label: "Web", icon: Globe, iconColor: "text-muted-foreground" }, + { value: "note", label: "Notes", icon: FileText, iconColor: "text-amber-600" }, + { value: "voice_memo", label: "Voice Memos", icon: Mic, iconColor: "text-orange-600" }, + { value: "audio", label: "Audio", icon: FileAudio, iconColor: "text-blue-600" }, + { value: "video", label: "Video", icon: Video, iconColor: "text-purple-600" }, +] as const; + +const DATE_FILTER_OPTIONS = [ + { value: "all", label: "All Time" }, + { value: "today", label: "Today" }, + { value: "week", label: "This Week" }, + { value: "month", label: "This Month" }, +] as const; + export default function MemoriesPage() { const [searchParams, setSearchParams] = useSearchParams(); @@ -50,7 +104,9 @@ export default function MemoriesPage() { // Filter state const [typeFilter, setTypeFilter] = useState("all"); + const [typeFilterOpen, setTypeFilterOpen] = useState(false); const [dateFilter, setDateFilter] = useState("all"); + const [dateFilterOpen, setDateFilterOpen] = useState(false); const [searchQuery, setSearchQuery] = useState(""); // Semantic search state @@ -73,8 +129,148 @@ export default function MemoriesPage() { const observerRef = useRef(null); const loadMoreRef = useRef(null); + // File upload refs + const audioInputRef = useRef(null); + const videoInputRef = useRef(null); + const [isUploading, setIsUploading] = useState(false); + + // New menu state + const [newMenuOpen, setNewMenuOpen] = useState(false); + + // Video upload hook + const { uploadVideo, uploadProgress, isUploading: isUploadingVideo } = useVideoUpload(); + const LIMIT = 20; + // Supported file extensions + const AUDIO_EXTENSIONS = ["mp3", "wav", "m4a", "webm", "ogg", "flac"]; + const VIDEO_EXTENSIONS = ["mp4", "webm", "mov", "mkv", "avi"]; + + // File size limits (in bytes) + const MAX_AUDIO_SIZE = 100 * 1024 * 1024; // 100 MB + const MAX_VIDEO_SIZE = 500 * 1024 * 1024; // 500 MB + + // File upload handler (audio and video) + const handleFileUpload = async (files: FileList | null) => { + if (!files || files.length === 0) return; + + const fileArray = Array.from(files); + + // Separate audio and video files + const audioFiles = fileArray.filter((file) => { + const ext = file.name.split(".").pop()?.toLowerCase(); + return ext && AUDIO_EXTENSIONS.includes(ext); + }); + + const videoFiles = fileArray.filter((file) => { + const ext = file.name.split(".").pop()?.toLowerCase(); + return ext && VIDEO_EXTENSIONS.includes(ext); + }); + + if (audioFiles.length === 0 && videoFiles.length === 0) { + toast.error("No supported files detected", { + description: `Audio: ${AUDIO_EXTENSIONS.join(", ")} | Video: ${VIDEO_EXTENSIONS.join(", ")}`, + }); + return; + } + + // Validate file sizes + const oversizedAudio = audioFiles.filter((f) => f.size > MAX_AUDIO_SIZE); + const oversizedVideo = videoFiles.filter((f) => f.size > MAX_VIDEO_SIZE); + + if (oversizedAudio.length > 0) { + toast.error(`${oversizedAudio.length} audio file(s) exceed 100 MB limit`, { + description: oversizedAudio.map((f) => f.name).join(", "), + }); + } + + if (oversizedVideo.length > 0) { + toast.error(`${oversizedVideo.length} video file(s) exceed 500 MB limit`, { + description: oversizedVideo.map((f) => f.name).join(", "), + }); + } + + // Filter out oversized files + const validAudioFiles = audioFiles.filter((f) => f.size <= MAX_AUDIO_SIZE); + const validVideoFiles = videoFiles.filter((f) => f.size <= MAX_VIDEO_SIZE); + + if (validAudioFiles.length === 0 && validVideoFiles.length === 0) { + return; + } + + // Upload audio files + if (validAudioFiles.length > 0) { + setIsUploading(true); + try { + const uploadPromises = validAudioFiles.map(async (file) => { + const formData = new FormData(); + formData.append("file", file); + const response = await apiFetch("/api/media/upload", { + method: "POST", + body: formData, + }); + if (!response.ok) { + throw new Error("Upload failed"); + } + return { success: true, name: file.name }; + }); + + const results = await Promise.all( + uploadPromises.map((p) => p.catch((e) => ({ success: false, error: e }))) + ); + const successful = results.filter((r) => r.success); + const failed = results.filter((r) => !r.success); + + if (successful.length > 0) { + toast.success( + successful.length === 1 + ? "Audio file uploaded" + : `${successful.length} audio files uploaded`, + { description: "Transcription will begin shortly" } + ); + fetchMemories(true); + } + + if (failed.length > 0) { + toast.error(`Failed to upload ${failed.length} audio file(s)`); + } + } catch (error) { + toast.error("Audio upload failed"); + } finally { + setIsUploading(false); + } + } + + // Upload video files (one at a time due to FFmpeg processing) + for (const videoFile of validVideoFiles) { + toast.info(`Processing ${videoFile.name}...`, { + description: "Extracting audio for transcription", + duration: 5000, + }); + + const result = await uploadVideo(videoFile); + + if (result.success) { + toast.success(`Video uploaded: ${videoFile.name}`, { + description: "Transcription will begin shortly", + }); + fetchMemories(true); + } else { + toast.error(`Failed to upload ${videoFile.name}`, { + description: result.error, + }); + } + } + + // Reset file inputs + if (audioInputRef.current) { + audioInputRef.current.value = ""; + } + if (videoInputRef.current) { + videoInputRef.current.value = ""; + } + }; + // Helper to check if a memory matches current filters const matchesFilters = useCallback( (memory: Memory) => { @@ -120,8 +316,8 @@ export default function MemoriesPage() { } }, onMemoryUpdated: (memoryId, data) => { - const memory = data as Memory; - setMemories((prev) => prev.map((m) => (m.id === memoryId ? memory : m))); + const update = data as Partial; + setMemories((prev) => prev.map((m) => (m.id === memoryId ? { ...m, ...update } : m))); }, onMemoryDeleted: (memoryId) => { setMemories((prev) => prev.filter((m) => m.id !== memoryId)); @@ -419,10 +615,80 @@ export default function MemoriesPage() {

Memories

{total} total

- + {/* Hidden file inputs */} + handleFileUpload(e.target.files)} + /> + handleFileUpload(e.target.files)} + /> + + {/* New dropdown */} + + + + + + + + + + +
{/* Memory Detail Panel */} @@ -456,57 +722,115 @@ export default function MemoriesPage() { } /> + {/* Audio Drop Overlay */} + fetchMemories(true)} /> + {/* Filters */}
- {/* Type filter */} -
- {(["all", "web", "note"] as TypeFilter[]).map((type) => ( - - ))} -
- - {/* Date filter */} -
- - -
+ + {(() => { + const option = TYPE_FILTER_OPTIONS.find( + (o) => o.value === typeFilter + ); + const Icon = option?.icon || LayoutGrid; + return ( + <> + + {option?.label || "All"} + + ); + })()} + + + + + + {TYPE_FILTER_OPTIONS.map((option) => { + const Icon = option.icon; + const isSelected = typeFilter === option.value; + return ( + + ); + })} + + + + {/* Date filter dropdown */} + + + + + + {DATE_FILTER_OPTIONS.map((option) => { + const isSelected = dateFilter === option.value; + return ( + + ); + })} + + {/* Search */}
@@ -536,13 +860,36 @@ export default function MemoriesPage() {
{displayMemories.map((memory) => (
- + {memory.type === "voice_memo" || memory.type === "voice" ? ( + + ) : memory.type === "audio" ? ( + + ) : memory.type === "video" ? ( + + ) : ( + + )}
))}
diff --git a/app/src/pages/RecordingPage.tsx b/app/src/pages/RecordingPage.tsx new file mode 100644 index 0000000..9291fe9 --- /dev/null +++ b/app/src/pages/RecordingPage.tsx @@ -0,0 +1,299 @@ +import { useState, useRef, useEffect } from "react"; +import { Mic, Square, Loader2, X, Check, ChevronDown } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { apiFetch } from "@/lib/api"; + +type RecordingState = "idle" | "recording" | "uploading" | "done" | "error"; + +export default function RecordingPage() { + const [state, setState] = useState("idle"); + const [duration, setDuration] = useState(0); + const [error, setError] = useState(null); + const [audioDevices, setAudioDevices] = useState([]); + const [selectedDeviceId, setSelectedDeviceId] = useState(""); + + const mediaRecorderRef = useRef(null); + const chunksRef = useRef([]); + const timerRef = useRef | null>(null); + const streamRef = useRef(null); + + // Enumerate audio devices on mount and after permission granted + const enumerateDevices = async () => { + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const mics = devices.filter((d) => d.kind === "audioinput"); + setAudioDevices(mics); + + // Restore saved preference if still available + const saved = localStorage.getItem("think_preferred_mic"); + if (saved && mics.some((m) => m.deviceId === saved)) { + setSelectedDeviceId(saved); + } + } catch (err) { + console.error("Failed to enumerate devices:", err); + } + }; + + useEffect(() => { + enumerateDevices(); + // Re-enumerate when devices change (e.g., plugging in a mic) + navigator.mediaDevices.addEventListener("devicechange", enumerateDevices); + return () => { + navigator.mediaDevices.removeEventListener("devicechange", enumerateDevices); + }; + }, []); + + // Notify main process of recording state changes (for blur handler) + useEffect(() => { + const isRecording = state === "recording" || state === "uploading"; + window.electronAPI?.setRecordingState?.(isRecording); + return () => { + window.electronAPI?.setRecordingState?.(false); + }; + }, [state]); + + // Cleanup on unmount + useEffect(() => { + return () => { + stopTimer(); + cleanupStream(); + }; + }, []); + + const stopTimer = () => { + if (timerRef.current) { + clearInterval(timerRef.current); + timerRef.current = null; + } + }; + + const cleanupStream = () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + } + }; + + const formatDuration = (seconds: number): string => { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins}:${secs.toString().padStart(2, "0")}`; + }; + + const startRecording = async () => { + try { + setError(null); + const audioConstraints = selectedDeviceId + ? { deviceId: { exact: selectedDeviceId } } + : true; + const stream = await navigator.mediaDevices.getUserMedia({ audio: audioConstraints }); + streamRef.current = stream; + + // Re-enumerate devices after permission granted (to get labels) + enumerateDevices(); + + // Use webm/opus for good compression and broad support + const mimeType = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") + ? "audio/webm;codecs=opus" + : "audio/webm"; + + const mediaRecorder = new MediaRecorder(stream, { mimeType }); + mediaRecorderRef.current = mediaRecorder; + chunksRef.current = []; + + mediaRecorder.ondataavailable = (e) => { + if (e.data.size > 0) { + chunksRef.current.push(e.data); + } + }; + + mediaRecorder.onstop = async () => { + stopTimer(); + cleanupStream(); + + if (chunksRef.current.length === 0) { + setState("idle"); + return; + } + + setState("uploading"); + await uploadRecording(); + }; + + mediaRecorder.onerror = (e) => { + console.error("MediaRecorder error:", e); + setError("Recording failed"); + setState("error"); + cleanupStream(); + }; + + mediaRecorder.start(1000); // Collect data every second + setState("recording"); + setDuration(0); + + // Start duration timer + timerRef.current = setInterval(() => { + setDuration((prev) => prev + 1); + }, 1000); + } catch (err) { + console.error("Failed to start recording:", err); + setError("Microphone access denied"); + setState("error"); + } + }; + + const stopRecording = () => { + if (mediaRecorderRef.current && state === "recording") { + mediaRecorderRef.current.stop(); + } + }; + + const uploadRecording = async () => { + try { + const blob = new Blob(chunksRef.current, { type: "audio/webm" }); + + // Create a file with a timestamp name + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const file = new File([blob], `voice-note-${timestamp}.webm`, { + type: "audio/webm", + }); + + const formData = new FormData(); + formData.append("file", file); + + const response = await apiFetch("/api/media/record", { + method: "POST", + body: formData, + }); + + if (!response.ok) { + throw new Error("Upload failed"); + } + + setState("done"); + + // Close the window after a brief success indication + setTimeout(() => { + window.close(); + }, 1000); + } catch (err) { + console.error("Failed to upload recording:", err); + setError("Upload failed"); + setState("error"); + } + }; + + const handleClose = () => { + if (state === "recording") { + stopRecording(); + } else { + window.close(); + } + }; + + return ( +
+ {/* Close button */} + + + {/* Title */} +

+ {state === "idle" && "Click to record"} + {state === "recording" && "Recording..."} + {state === "uploading" && "Saving..."} + {state === "done" && "Saved!"} + {state === "error" && (error || "Error")} +

+ + {/* Microphone selector - only show when idle or error */} + {(state === "idle" || state === "error") && audioDevices.length > 1 && ( +
+ + +
+ )} + + {/* Record button */} + + + {/* Duration */} + {state === "recording" && ( +

+ {formatDuration(duration)} +

+ )} + + {/* Hint */} + {state === "idle" && ( +

+ Press to start recording +

+ )} + {state === "recording" && ( +

+ Press to stop +

+ )} +
+ ); +} diff --git a/app/src/types/chat.ts b/app/src/types/chat.ts index ba3ac56..6ce034b 100644 --- a/app/src/types/chat.ts +++ b/app/src/types/chat.ts @@ -9,10 +9,43 @@ export interface SourceMemory { export interface AttachedMemory { id: number; title: string; - type: "web" | "note"; + type: "web" | "note" | "voice_memo" | "audio" | "video" | "voice"; // "voice" for backwards compat url?: string; } +export type MemoryType = "web" | "note" | "voice_memo" | "audio" | "video" | "voice"; // "voice" for backwards compat + +export type MediaSource = "recording" | "upload"; + +export type TranscriptionStatus = "pending" | "processing" | "completed" | "failed"; + +export interface TranscriptSegment { + start: number; + end: number; + text: string; +} + +export interface MediaMemoryFields { + audio_duration?: number; + transcription_status?: TranscriptionStatus; + transcript?: string; + transcript_segments?: TranscriptSegment[]; + media_source?: MediaSource; +} + +// Alias for backwards compatibility +export type VoiceMemoryFields = MediaMemoryFields; + +export type VideoProcessingStatus = "pending_extraction" | "extracting" | "ready" | "failed"; + +export interface VideoMemoryFields extends MediaMemoryFields { + video_duration?: number; + video_width?: number; + video_height?: number; + thumbnail_path?: string; + video_processing_status?: VideoProcessingStatus; +} + export interface TokenUsage { prompt_tokens: number; completion_tokens: number; diff --git a/backend/app/db/crud.py b/backend/app/db/crud.py index cc63d94..a9d5257 100644 --- a/backend/app/db/crud.py +++ b/backend/app/db/crud.py @@ -1,7 +1,11 @@ +import json +import logging from contextlib import contextmanager from datetime import datetime, timedelta from sqlalchemy import select, func +logger = logging.getLogger(__name__) + from ..models import Memory, Setting, Tag, MemoryTag, Conversation, Message, MessageSource from .core import get_session_maker, run_sync, serialize_embedding @@ -129,7 +133,7 @@ def _get(): result = [] for m in memories: - result.append({ + memory_dict = { "id": m.id, "type": m.type, "url": m.url, @@ -137,7 +141,26 @@ def _get(): "summary": m.summary, "tags": tags_by_memory.get(m.id, []), "created_at": m.created_at.isoformat(), - }) + } + # Add media-specific fields for voice memos and audio + if m.type in ("voice_memo", "audio"): + memory_dict.update({ + "audio_duration": m.audio_duration, + "transcription_status": m.transcription_status, + "media_source": m.media_source, + }) + # Add video-specific fields + elif m.type == "video": + memory_dict.update({ + "video_duration": m.video_duration, + "video_width": m.video_width, + "video_height": m.video_height, + "thumbnail_path": m.thumbnail_path, + "video_processing_status": m.video_processing_status, + "transcription_status": m.transcription_status, + "media_source": m.media_source, + }) + result.append(memory_dict) return result, total @@ -162,7 +185,7 @@ def _get(): for mt, tag in memory_tags ] - return { + result = { "id": memory.id, "type": memory.type, "url": memory.url, @@ -174,6 +197,45 @@ def _get(): "created_at": memory.created_at.isoformat(), } + # Add media-specific fields for voice memos and audio + if memory.type in ("voice_memo", "audio"): + result.update({ + "audio_path": memory.audio_path, + "audio_format": memory.audio_format, + "audio_duration": memory.audio_duration, + "transcript": memory.transcript, + "transcription_status": memory.transcription_status, + "media_source": memory.media_source, + "transcript_segments": ( + json.loads(memory.transcript_segments) + if memory.transcript_segments + else None + ), + }) + # Add video-specific fields + elif memory.type == "video": + result.update({ + "video_path": memory.video_path, + "video_format": memory.video_format, + "video_duration": memory.video_duration, + "video_width": memory.video_width, + "video_height": memory.video_height, + "thumbnail_path": memory.thumbnail_path, + "video_processing_status": memory.video_processing_status, + "audio_path": memory.audio_path, + "audio_format": memory.audio_format, + "transcript": memory.transcript, + "transcription_status": memory.transcription_status, + "media_source": memory.media_source, + "transcript_segments": ( + json.loads(memory.transcript_segments) + if memory.transcript_segments + else None + ), + }) + + return result + return await run_sync(_get) @@ -865,3 +927,385 @@ def _add(): } return await run_sync(_add) + + +# Media memory functions (voice memos and audio uploads) + +async def create_media_memory( + title: str, + audio_path: str, + audio_format: str, + memory_type: str = "voice_memo", + media_source: str = "recording", + audio_duration: float | None = None, +) -> dict: + """Create a new media memory record (voice memo or audio upload). + + Args: + title: Initial title (may be updated after transcription) + audio_path: Relative path to encrypted audio file + audio_format: Audio format (e.g., "mp3", "wav") + memory_type: "voice_memo" for recordings, "audio" for uploads + media_source: "recording" or "upload" + audio_duration: Duration in seconds + + Returns: + Dict with memory info + """ + def _create(): + with get_session_maker()() as session: + memory = Memory( + type=memory_type, + title=title, + audio_path=audio_path, + audio_format=audio_format, + audio_duration=audio_duration, + media_source=media_source, + transcription_status="pending", + ) + session.add(memory) + session.commit() + session.refresh(memory) + return { + "id": memory.id, + "type": memory.type, + "title": memory.title, + "audio_path": memory.audio_path, + "audio_format": memory.audio_format, + "audio_duration": memory.audio_duration, + "media_source": memory.media_source, + "transcription_status": memory.transcription_status, + "created_at": memory.created_at.isoformat(), + } + + return await run_sync(_create) + + +async def update_memory_transcript( + memory_id: int, transcript: str, segments: list[dict] | None = None +) -> bool: + """Update the transcript and segments for a voice memory.""" + def _update(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory: + return False + memory.transcript = transcript + # Also store transcript as content for search/embedding + memory.content = transcript + # Store segments with timestamps as JSON + if segments: + memory.transcript_segments = json.dumps(segments) + # Set audio_duration from last segment if not already set + # (WebM recordings may not have duration extracted by mutagen) + if not memory.audio_duration and len(segments) > 0: + memory.audio_duration = segments[-1]["end"] + logger.info( + f"Set audio duration from transcript segments for memory {memory_id}: {memory.audio_duration}s" + ) + session.commit() + return True + + return await run_sync(_update) + + +async def update_transcription_status(memory_id: int, status: str) -> bool: + """Update the transcription status for a voice memory. + + Args: + memory_id: Memory ID + status: One of "pending", "processing", "completed", "failed" + """ + def _update(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory: + return False + memory.transcription_status = status + session.commit() + return True + + return await run_sync(_update) + + +async def reset_transcription_status_if_not_processing(memory_id: int) -> bool: + """Atomically reset transcription status to 'pending' only if not currently processing. + + This prevents race conditions when multiple retry requests come in simultaneously. + + Args: + memory_id: Memory ID + + Returns: + True if status was reset (was not processing), False if already processing or not found + """ + def _update(): + with get_session_maker()() as session: + from sqlalchemy import update + # Atomic update: only update if status is not 'processing' + result = session.execute( + update(Memory) + .where(Memory.id == memory_id) + .where(Memory.transcription_status != "processing") + .values(transcription_status="pending") + ) + session.commit() + # rowcount tells us if any row was actually updated + return result.rowcount > 0 + + return await run_sync(_update) + + +async def get_media_memory(memory_id: int) -> dict | None: + """Get a media memory (voice memo or audio) with audio details.""" + def _get(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory or memory.type not in ("voice_memo", "audio"): + return None + + # Get tags + memory_tags = session.execute( + select(MemoryTag, Tag) + .join(Tag, MemoryTag.tag_id == Tag.id) + .where(MemoryTag.memory_id == memory_id) + ).all() + tags = [ + {"id": tag.id, "name": tag.name, "source": mt.source} + for mt, tag in memory_tags + ] + + return { + "id": memory.id, + "type": memory.type, + "title": memory.title, + "content": memory.content, + "summary": memory.summary, + "transcript": memory.transcript, + "audio_path": memory.audio_path, + "audio_format": memory.audio_format, + "audio_duration": memory.audio_duration, + "media_source": memory.media_source, + "transcription_status": memory.transcription_status, + "tags": tags, + "created_at": memory.created_at.isoformat(), + } + + return await run_sync(_get) + + +async def delete_media_memory(memory_id: int) -> str | None: + """Delete a media memory and return the audio path for cleanup. + + Returns: + The audio_path if memory was deleted (for file cleanup), None if not found + """ + def _delete(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory: + return None + audio_path = memory.audio_path + session.delete(memory) + session.commit() + return audio_path + + return await run_sync(_delete) + + +# Video memory functions + +async def create_video_memory( + title: str, + video_path: str, + video_format: str, + video_duration: float | None = None, + video_width: int | None = None, + video_height: int | None = None, + media_source: str = "upload", +) -> dict: + """Create a new video memory record. + + Args: + title: Initial title (may be updated after transcription) + video_path: Relative path to encrypted video file + video_format: Video format (e.g., "mp4", "webm", "mov") + video_duration: Duration in seconds + video_width: Video width in pixels + video_height: Video height in pixels + media_source: Source of the video (usually "upload") + + Returns: + Dict with memory info + """ + def _create(): + with get_session_maker()() as session: + memory = Memory( + type="video", + title=title, + video_path=video_path, + video_format=video_format, + video_duration=video_duration, + video_width=video_width, + video_height=video_height, + media_source=media_source, + video_processing_status="pending_extraction", + ) + session.add(memory) + session.commit() + session.refresh(memory) + return { + "id": memory.id, + "type": memory.type, + "title": memory.title, + "video_path": memory.video_path, + "video_format": memory.video_format, + "video_duration": memory.video_duration, + "video_width": memory.video_width, + "video_height": memory.video_height, + "media_source": memory.media_source, + "video_processing_status": memory.video_processing_status, + "created_at": memory.created_at.isoformat(), + } + + return await run_sync(_create) + + +async def update_video_processing_status(memory_id: int, status: str) -> bool: + """Update the video processing status. + + Args: + memory_id: Memory ID + status: One of "pending_extraction", "extracting", "ready", "failed" + """ + def _update(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory or memory.type != "video": + return False + memory.video_processing_status = status + session.commit() + return True + + return await run_sync(_update) + + +async def update_video_audio( + memory_id: int, + audio_path: str, + audio_format: str, +) -> bool: + """Update the extracted audio info for a video memory. + + Called after frontend extracts audio and uploads it. + + Args: + memory_id: Memory ID + audio_path: Relative path to encrypted audio file + audio_format: Audio format (e.g., "m4a", "mp3") + """ + def _update(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory or memory.type != "video": + return False + memory.audio_path = audio_path + memory.audio_format = audio_format + memory.transcription_status = "pending" + memory.video_processing_status = "ready" + session.commit() + return True + + return await run_sync(_update) + + +async def update_video_thumbnail(memory_id: int, thumbnail_path: str) -> bool: + """Update the thumbnail path for a video memory. + + Args: + memory_id: Memory ID + thumbnail_path: Relative path to encrypted thumbnail file + """ + def _update(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory or memory.type != "video": + return False + memory.thumbnail_path = thumbnail_path + session.commit() + return True + + return await run_sync(_update) + + +async def get_video_memory(memory_id: int) -> dict | None: + """Get a video memory with all details.""" + def _get(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory or memory.type != "video": + return None + + # Get tags + memory_tags = session.execute( + select(MemoryTag, Tag) + .join(Tag, MemoryTag.tag_id == Tag.id) + .where(MemoryTag.memory_id == memory_id) + ).all() + tags = [ + {"id": tag.id, "name": tag.name, "source": mt.source} + for mt, tag in memory_tags + ] + + return { + "id": memory.id, + "type": memory.type, + "title": memory.title, + "content": memory.content, + "summary": memory.summary, + "video_path": memory.video_path, + "video_format": memory.video_format, + "video_duration": memory.video_duration, + "video_width": memory.video_width, + "video_height": memory.video_height, + "thumbnail_path": memory.thumbnail_path, + "video_processing_status": memory.video_processing_status, + "audio_path": memory.audio_path, + "audio_format": memory.audio_format, + "transcript": memory.transcript, + "transcription_status": memory.transcription_status, + "media_source": memory.media_source, + "transcript_segments": ( + json.loads(memory.transcript_segments) + if memory.transcript_segments + else None + ), + "tags": tags, + "created_at": memory.created_at.isoformat(), + } + + return await run_sync(_get) + + +async def delete_video_memory(memory_id: int) -> dict | None: + """Delete a video memory and return paths for cleanup. + + Returns: + Dict with video_path, audio_path, and thumbnail_path for file cleanup, + or None if not found + """ + def _delete(): + with get_session_maker()() as session: + memory = session.get(Memory, memory_id) + if not memory or memory.type != "video": + return None + paths = { + "video_path": memory.video_path, + "audio_path": memory.audio_path, + "thumbnail_path": memory.thumbnail_path, + } + session.delete(memory) + session.commit() + return paths + + return await run_sync(_delete) diff --git a/backend/app/db/migrations.py b/backend/app/db/migrations.py index 3b731e6..4a02efa 100644 --- a/backend/app/db/migrations.py +++ b/backend/app/db/migrations.py @@ -507,6 +507,52 @@ def migration_016(conn: Connection) -> None: ), {"key": f"api_key_{new_provider}", "value": old_key_result[0]}) +@migration(17, "Add media memory columns for voice, audio, and video") +def migration_017(conn: Connection) -> None: + """Add all columns for media memory support (voice memos, audio uploads, video). + + Columns added: + - Audio: audio_path, audio_format, audio_duration, transcript, transcription_status, + transcript_segments, media_source + - Video: video_path, video_format, video_duration, thumbnail_path, video_width, + video_height, video_processing_status + """ + result = conn.execute(text("PRAGMA table_info(memories)")).fetchall() + columns = [row[1] for row in result] + + # Audio/voice columns + if "audio_path" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN audio_path VARCHAR(500)")) + if "audio_format" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN audio_format VARCHAR(20)")) + if "audio_duration" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN audio_duration REAL")) + if "transcript" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN transcript TEXT")) + if "transcription_status" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN transcription_status VARCHAR(20)")) + if "transcript_segments" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN transcript_segments TEXT")) + if "media_source" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN media_source VARCHAR(20)")) + + # Video columns + if "video_path" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN video_path VARCHAR(500)")) + if "video_format" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN video_format VARCHAR(20)")) + if "video_duration" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN video_duration REAL")) + if "thumbnail_path" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN thumbnail_path VARCHAR(500)")) + if "video_width" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN video_width INTEGER")) + if "video_height" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN video_height INTEGER")) + if "video_processing_status" not in columns: + conn.execute(text("ALTER TABLE memories ADD COLUMN video_processing_status VARCHAR(20)")) + + # --- Migration runner --- def run_migrations(conn: Connection) -> list[tuple[int, str]]: diff --git a/backend/app/main.py b/backend/app/main.py index a721446..da02e83 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -5,9 +5,11 @@ from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse +import httpx from .db import is_db_initialized from .routes import router +from . import config # Configure logging for all app modules logging.basicConfig( @@ -33,9 +35,41 @@ def filter(self, record: logging.LogRecord) -> bool: # Apply filter to uvicorn access logger logging.getLogger("uvicorn.access").addFilter(EndpointFilter()) +logger = logging.getLogger(__name__) + + +async def check_ollama_model(): + """Log warning if configured Ollama model is not available.""" + if config.settings.ai_provider != "ollama": + return + + model = config.settings.ollama_model + base_url = config.settings.ollama_base_url.replace("/v1", "") + + try: + async with httpx.AsyncClient(timeout=5.0) as client: + response = await client.get(f"{base_url}/api/tags") + if response.status_code == 200: + models = response.json().get("models", []) + model_names = [m.get("name", "").split(":")[0] for m in models] + if model.split(":")[0] in model_names: + logger.info(f"Ollama model '{model}' is available") + else: + logger.warning( + f"Ollama model '{model}' not found - " + "AI features will fail until model is downloaded via Setup Wizard" + ) + except httpx.ConnectError: + logger.warning("Ollama is not running") + except Exception as e: + logger.warning(f"Could not check Ollama: {e}") + @asynccontextmanager async def lifespan(app: FastAPI): + # Check Ollama model availability (logs warning if missing) + await check_ollama_model() + # Start native messaging socket server for secure extension communication from .native_messaging import start_native_messaging_server, stop_native_messaging_server diff --git a/backend/app/models.py b/backend/app/models.py index 89b0bc1..7311788 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,5 +1,5 @@ from datetime import datetime -from sqlalchemy import String, Text, DateTime, LargeBinary, ForeignKey, Integer +from sqlalchemy import String, Text, DateTime, LargeBinary, ForeignKey, Integer, Float from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship @@ -11,8 +11,9 @@ class Memory(Base): __tablename__ = "memories" id: Mapped[int] = mapped_column(primary_key=True) - type: Mapped[str] = mapped_column(String(20), default="web") + type: Mapped[str] = mapped_column(String(20), default="web") # "web" | "note" | "voice_memo" | "audio" | "video" url: Mapped[str | None] = mapped_column(String(2048), nullable=True) + media_source: Mapped[str | None] = mapped_column(String(20), nullable=True) # "recording" | "upload" title: Mapped[str | None] = mapped_column(String(500), nullable=True) original_title: Mapped[str | None] = mapped_column(String(500), nullable=True) content: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -23,6 +24,23 @@ class Memory(Base): processing_attempts: Mapped[int] = mapped_column(Integer, default=0) created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + # Voice memory fields + audio_path: Mapped[str | None] = mapped_column(String(500), nullable=True) + audio_format: Mapped[str | None] = mapped_column(String(20), nullable=True) + audio_duration: Mapped[float | None] = mapped_column(Float, nullable=True) + transcript: Mapped[str | None] = mapped_column(Text, nullable=True) + transcription_status: Mapped[str | None] = mapped_column(String(20), nullable=True) # "pending" | "processing" | "completed" | "failed" + transcript_segments: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array of {start, end, text} + + # Video memory fields + video_path: Mapped[str | None] = mapped_column(String(500), nullable=True) + video_format: Mapped[str | None] = mapped_column(String(20), nullable=True) # mp4, webm, mov, mkv, avi + video_duration: Mapped[float | None] = mapped_column(Float, nullable=True) + thumbnail_path: Mapped[str | None] = mapped_column(String(500), nullable=True) + video_width: Mapped[int | None] = mapped_column(Integer, nullable=True) + video_height: Mapped[int | None] = mapped_column(Integer, nullable=True) + video_processing_status: Mapped[str | None] = mapped_column(String(20), nullable=True) # "pending_extraction" | "extracting" | "ready" | "failed" + tags: Mapped[list["MemoryTag"]] = relationship(back_populates="memory", cascade="all, delete-orphan") diff --git a/backend/app/routes/__init__.py b/backend/app/routes/__init__.py index 999f6f8..d70970d 100644 --- a/backend/app/routes/__init__.py +++ b/backend/app/routes/__init__.py @@ -6,6 +6,8 @@ from .settings import router as settings_router from .conversations import router as conversations_router from .jobs import router as jobs_router +from .media import router as media_router +from .video import router as video_router router = APIRouter() router.include_router(auth_router) @@ -14,3 +16,5 @@ router.include_router(settings_router) router.include_router(conversations_router) router.include_router(jobs_router) +router.include_router(media_router) +router.include_router(video_router) diff --git a/backend/app/routes/auth.py b/backend/app/routes/auth.py index 4b0ede2..a17f783 100644 --- a/backend/app/routes/auth.py +++ b/backend/app/routes/auth.py @@ -3,6 +3,8 @@ from ..config import reload_settings from ..db import init_db, is_db_initialized, db_exists, reset_db_connection from ..services.secrets import derive_db_key, set_api_key, get_api_key, delete_api_key +from ..services.audio_storage import set_encryption_key as set_audio_encryption_key, clear_encryption_key as clear_audio_encryption_key +from ..services.video_storage import set_encryption_key as set_video_encryption_key, clear_encryption_key as clear_video_encryption_key from ..schemas import SetPasswordRequest, UnlockRequest, ApiKeyRequest @@ -28,6 +30,10 @@ async def setup_password(request: SetPasswordRequest): await init_db(db_key) reload_settings() # Load settings from newly created DB + # Initialize encryption keys for media storage + set_audio_encryption_key(request.password) + set_video_encryption_key(request.password) + return {"success": True} @@ -44,6 +50,11 @@ async def unlock(request: UnlockRequest): raise HTTPException(status_code=401, detail="Invalid password") reload_settings() # Load settings from unlocked DB + + # Initialize encryption keys for media storage + set_audio_encryption_key(request.password) + set_video_encryption_key(request.password) + return {"success": True} @@ -51,6 +62,9 @@ async def unlock(request: UnlockRequest): async def logout(): """Lock the database (logout).""" reset_db_connection() + # Clear media encryption keys + clear_audio_encryption_key() + clear_video_encryption_key() return {"success": True} diff --git a/backend/app/routes/media.py b/backend/app/routes/media.py new file mode 100644 index 0000000..7179af8 --- /dev/null +++ b/backend/app/routes/media.py @@ -0,0 +1,279 @@ +"""Media memory API routes (voice memos and audio uploads).""" +import asyncio +import logging + +from fastapi import APIRouter, HTTPException, UploadFile, File +from fastapi.responses import StreamingResponse + +from ..db.crud import ( + create_media_memory, + get_media_memory, + get_memory, + delete_media_memory, + reset_transcription_status_if_not_processing, +) +from ..services.audio_storage import save_audio_file, read_audio_file, delete_audio_file +from ..services.audio_utils import ( + validate_audio_format, + get_format_from_mime, + get_audio_duration, + SUPPORTED_FORMATS, +) +from ..services.ai_processing import process_voice_memory_async +from ..events import event_manager, MemoryEvent, EventType + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/media", tags=["media"]) + +# File size limits (in bytes) +MAX_AUDIO_SIZE = 100 * 1024 * 1024 # 100 MB + + +@router.post("/record") +async def record_voice_memo(file: UploadFile = File(...)): + """Upload a voice memo from a quick recording. + + Accepts multipart form data with an audio file. + Supported formats: mp3, wav, m4a, webm, ogg, flac + Max size: 100 MB + """ + if not file.filename: + raise HTTPException(status_code=400, detail="No filename provided") + + # Validate format from filename or content type + audio_format = validate_audio_format(file.filename) + if not audio_format and file.content_type: + audio_format = get_format_from_mime(file.content_type) + + if not audio_format: + raise HTTPException( + status_code=400, + detail=f"Unsupported audio format. Supported: {', '.join(SUPPORTED_FORMATS)}" + ) + + # Read file content + audio_data = await file.read() + if not audio_data: + raise HTTPException(status_code=400, detail="Empty audio file") + + # Check file size + if len(audio_data) > MAX_AUDIO_SIZE: + raise HTTPException( + status_code=413, + detail=f"File too large. Maximum size is {MAX_AUDIO_SIZE // (1024 * 1024)} MB" + ) + + # Extract duration + duration = get_audio_duration(audio_data, audio_format) + + # Save encrypted audio file + audio_path = save_audio_file(audio_data, audio_format) + + # Create voice memo record + initial_title = "Voice Memo" + memory = await create_media_memory( + title=initial_title, + audio_path=audio_path, + audio_format=audio_format, + memory_type="voice_memo", + media_source="recording", + audio_duration=duration, + ) + + memory_id = memory["id"] + + # Emit creation event + full_memory = await get_memory(memory_id) + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_CREATED, + memory_id=memory_id, + data=full_memory, + ) + ) + + # Start background processing (transcription + AI) + asyncio.create_task(process_voice_memory_async(memory_id)) + + logger.info(f"Created voice memo {memory_id}") + return memory + + +@router.post("/upload") +async def upload_audio(file: UploadFile = File(...)): + """Upload an audio file for transcription. + + Accepts multipart form data with an audio file. + Supported formats: mp3, wav, m4a, webm, ogg, flac + Max size: 100 MB + """ + if not file.filename: + raise HTTPException(status_code=400, detail="No filename provided") + + # Validate format from filename or content type + audio_format = validate_audio_format(file.filename) + if not audio_format and file.content_type: + audio_format = get_format_from_mime(file.content_type) + + if not audio_format: + raise HTTPException( + status_code=400, + detail=f"Unsupported audio format. Supported: {', '.join(SUPPORTED_FORMATS)}" + ) + + # Read file content + audio_data = await file.read() + if not audio_data: + raise HTTPException(status_code=400, detail="Empty audio file") + + # Check file size + if len(audio_data) > MAX_AUDIO_SIZE: + raise HTTPException( + status_code=413, + detail=f"File too large. Maximum size is {MAX_AUDIO_SIZE // (1024 * 1024)} MB" + ) + + # Extract duration + duration = get_audio_duration(audio_data, audio_format) + + # Save encrypted audio file + audio_path = save_audio_file(audio_data, audio_format) + + # Create audio memory record with original filename as title + initial_title = file.filename.rsplit(".", 1)[0] if "." in file.filename else file.filename + memory = await create_media_memory( + title=initial_title, + audio_path=audio_path, + audio_format=audio_format, + memory_type="audio", + media_source="upload", + audio_duration=duration, + ) + + memory_id = memory["id"] + + # Emit creation event + full_memory = await get_memory(memory_id) + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_CREATED, + memory_id=memory_id, + data=full_memory, + ) + ) + + # Start background processing (transcription + AI) + asyncio.create_task(process_voice_memory_async(memory_id)) + + logger.info(f"Created audio memory {memory_id} from {file.filename}") + return memory + + +@router.get("/{memory_id}/stream") +async def stream_audio(memory_id: int): + """Stream the audio file for playback. + + Returns the decrypted audio with appropriate content type. + """ + memory = await get_media_memory(memory_id) + if not memory: + raise HTTPException(status_code=404, detail="Media memory not found") + + audio_path = memory.get("audio_path") + if not audio_path: + raise HTTPException(status_code=404, detail="Audio file not found") + + audio_format = memory.get("audio_format", "mp3") + + try: + audio_data = read_audio_file(audio_path) + except FileNotFoundError: + raise HTTPException(status_code=404, detail="Audio file not found on disk") + + # Determine content type + content_type_map = { + "mp3": "audio/mpeg", + "wav": "audio/wav", + "m4a": "audio/mp4", + "webm": "audio/webm", + "ogg": "audio/ogg", + "flac": "audio/flac", + } + content_type = content_type_map.get(audio_format, "audio/mpeg") + + # Stream the audio data + async def audio_generator(): + # Yield in chunks for efficient streaming + chunk_size = 64 * 1024 # 64KB chunks + for i in range(0, len(audio_data), chunk_size): + yield audio_data[i:i + chunk_size] + + return StreamingResponse( + audio_generator(), + media_type=content_type, + headers={ + "Content-Length": str(len(audio_data)), + "Accept-Ranges": "bytes", + } + ) + + +@router.post("/{memory_id}/retry") +async def retry_transcription(memory_id: int): + """Retry transcription for a failed media memory.""" + memory = await get_media_memory(memory_id) + if not memory: + raise HTTPException(status_code=404, detail="Media memory not found") + + # Atomically reset status to pending only if not already processing + # This prevents race conditions when multiple retry requests come in + status_reset = await reset_transcription_status_if_not_processing(memory_id) + if not status_reset: + raise HTTPException( + status_code=400, + detail="Transcription is already in progress" + ) + + # Fetch complete memory for SSE event + updated_memory = await get_media_memory(memory_id) + + # Emit update event with full memory data + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data=updated_memory, + ) + ) + + # Start background processing + asyncio.create_task(process_voice_memory_async(memory_id)) + + logger.info(f"Retrying transcription for media memory {memory_id}") + return {"success": True, "message": "Transcription retry started"} + + +@router.delete("/{memory_id}") +async def delete_media_memory_endpoint(memory_id: int): + """Delete a media memory and its audio file.""" + # Delete from database and get audio path + audio_path = await delete_media_memory(memory_id) + + if audio_path is None: + raise HTTPException(status_code=404, detail="Media memory not found") + + # Delete the audio file + delete_audio_file(audio_path) + + # Emit deletion event + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_DELETED, + memory_id=memory_id, + data=None, + ) + ) + + logger.info(f"Deleted media memory {memory_id}") + return {"deleted": True} diff --git a/backend/app/routes/settings.py b/backend/app/routes/settings.py index 91d8283..e673668 100644 --- a/backend/app/routes/settings.py +++ b/backend/app/routes/settings.py @@ -610,3 +610,58 @@ async def get_embedding_model_impact() -> EmbeddingModelImpact: affected_count=affected_count, current_model=current_model, ) + + +# Transcription (Whisper) settings + +class TranscriptionSettings(BaseModel): + whisper_model: str + + +class TranscriptionModelInfo(BaseModel): + name: str + description: str + size_mb: int + + +# Whisper model options with descriptions +WHISPER_MODEL_INFO = [ + {"name": "tiny", "description": "Fastest, least accurate", "size_mb": 75}, + {"name": "base", "description": "Good balance of speed and accuracy (recommended)", "size_mb": 145}, + {"name": "small", "description": "More accurate, slower", "size_mb": 465}, + {"name": "medium", "description": "Most accurate, slowest", "size_mb": 1500}, +] + + +@router.get("/settings/transcription") +async def get_transcription_settings() -> TranscriptionSettings: + """Get current transcription settings.""" + from ..services.transcription import get_whisper_model_setting + + whisper_model = await get_whisper_model_setting() + return TranscriptionSettings(whisper_model=whisper_model) + + +@router.post("/settings/transcription") +async def update_transcription_settings(settings: TranscriptionSettings): + """Update transcription settings.""" + from ..services.transcription import WHISPER_MODELS, unload_whisper_model + + if settings.whisper_model not in WHISPER_MODELS: + raise HTTPException( + status_code=400, + detail=f"Invalid model. Choose from: {', '.join(WHISPER_MODELS)}" + ) + + await set_setting("whisper_model", settings.whisper_model) + + # Unload the current model so it will be reloaded with new settings + unload_whisper_model() + + return {"success": True, "whisper_model": settings.whisper_model} + + +@router.get("/settings/transcription-models") +async def get_transcription_models() -> list[TranscriptionModelInfo]: + """Get available Whisper models with descriptions.""" + return [TranscriptionModelInfo(**m) for m in WHISPER_MODEL_INFO] diff --git a/backend/app/routes/video.py b/backend/app/routes/video.py new file mode 100644 index 0000000..adea36f --- /dev/null +++ b/backend/app/routes/video.py @@ -0,0 +1,409 @@ +"""Video memory API routes.""" +import asyncio +import logging + +from fastapi import APIRouter, HTTPException, UploadFile, File, Form +from fastapi.responses import StreamingResponse + +from ..db.crud import ( + create_video_memory, + get_video_memory, + get_memory, + delete_video_memory, + update_video_audio, + update_video_thumbnail, + update_video_processing_status, + reset_transcription_status_if_not_processing, +) +from ..services.video_storage import ( + save_video_file, + read_video_file, + delete_video_file, + save_thumbnail, + read_thumbnail, + delete_thumbnail, +) +from ..services.audio_storage import ( + save_audio_file, + read_audio_file, + delete_audio_file, +) +from ..services.ai_processing import process_voice_memory_async +from ..events import event_manager, MemoryEvent, EventType + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/video", tags=["video"]) + +# File size limits (in bytes) +MAX_VIDEO_SIZE = 500 * 1024 * 1024 # 500 MB +MAX_AUDIO_SIZE = 100 * 1024 * 1024 # 100 MB for extracted audio +MAX_THUMBNAIL_SIZE = 10 * 1024 * 1024 # 10 MB for thumbnails + +# Supported video formats +SUPPORTED_VIDEO_FORMATS = {"mp4", "webm", "mov", "mkv", "avi"} +SUPPORTED_VIDEO_MIMES = { + "video/mp4": "mp4", + "video/webm": "webm", + "video/quicktime": "mov", + "video/x-matroska": "mkv", + "video/x-msvideo": "avi", +} + + +def validate_video_format(filename: str) -> str | None: + """Extract and validate video format from filename.""" + if "." not in filename: + return None + ext = filename.rsplit(".", 1)[1].lower() + return ext if ext in SUPPORTED_VIDEO_FORMATS else None + + +def get_format_from_mime(mime_type: str) -> str | None: + """Get video format from MIME type.""" + return SUPPORTED_VIDEO_MIMES.get(mime_type) + + +@router.post("/upload") +async def upload_video( + file: UploadFile = File(...), + duration: float = Form(None), + width: int = Form(None), + height: int = Form(None), +): + """Upload a video file. + + Accepts multipart form data with: + - file: The video file + - duration: Video duration in seconds (from frontend metadata) + - width: Video width in pixels + - height: Video height in pixels + + Supported formats: mp4, webm, mov, mkv, avi + Max size: 500 MB + """ + if not file.filename: + raise HTTPException(status_code=400, detail="No filename provided") + + # Validate format from filename or content type + video_format = validate_video_format(file.filename) + if not video_format and file.content_type: + video_format = get_format_from_mime(file.content_type) + + if not video_format: + raise HTTPException( + status_code=400, + detail=f"Unsupported video format. Supported: {', '.join(SUPPORTED_VIDEO_FORMATS)}" + ) + + # Validate dimensions (if provided) + if duration is not None and (duration < 0 or duration > 86400): # Max 24 hours + raise HTTPException(status_code=400, detail="Invalid video duration") + if width is not None and (width < 1 or width > 7680): # Max 8K + raise HTTPException(status_code=400, detail="Invalid video width") + if height is not None and (height < 1 or height > 4320): # Max 8K + raise HTTPException(status_code=400, detail="Invalid video height") + + # Read file content + video_data = await file.read() + if not video_data: + raise HTTPException(status_code=400, detail="Empty video file") + + # Check file size + if len(video_data) > MAX_VIDEO_SIZE: + raise HTTPException( + status_code=413, + detail=f"File too large. Maximum size is {MAX_VIDEO_SIZE // (1024 * 1024)} MB" + ) + + # Save encrypted video file + video_path = save_video_file(video_data, video_format) + + # Create video memory record with original filename as title + initial_title = file.filename.rsplit(".", 1)[0] if "." in file.filename else file.filename + memory = await create_video_memory( + title=initial_title, + video_path=video_path, + video_format=video_format, + video_duration=duration, + video_width=width, + video_height=height, + media_source="upload", + ) + + memory_id = memory["id"] + + # Emit creation event + full_memory = await get_memory(memory_id) + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_CREATED, + memory_id=memory_id, + data=full_memory, + ) + ) + + logger.info(f"Created video memory {memory_id} from {file.filename}") + return memory + + +@router.post("/{memory_id}/audio") +async def upload_extracted_audio( + memory_id: int, + file: UploadFile = File(...), +): + """Upload extracted audio from a video. + + Called by the frontend after extracting audio with ffmpeg.wasm. + This starts the transcription pipeline. + """ + memory = await get_video_memory(memory_id) + if not memory: + raise HTTPException(status_code=404, detail="Video memory not found") + + if not file.filename: + raise HTTPException(status_code=400, detail="No filename provided") + + # Get audio format from filename + audio_format = "m4a" # Default + if "." in file.filename: + ext = file.filename.rsplit(".", 1)[1].lower() + if ext in {"m4a", "mp3", "wav", "webm", "ogg", "aac"}: + audio_format = ext + + # Read and save audio file + audio_data = await file.read() + if not audio_data: + raise HTTPException(status_code=400, detail="Empty audio file") + + # Check file size + if len(audio_data) > MAX_AUDIO_SIZE: + raise HTTPException( + status_code=413, + detail=f"Audio file too large. Maximum size is {MAX_AUDIO_SIZE // (1024 * 1024)} MB" + ) + + audio_path = save_audio_file(audio_data, audio_format) + + # Update video memory with audio info + await update_video_audio(memory_id, audio_path, audio_format) + + # Fetch complete memory for SSE event + updated_memory = await get_video_memory(memory_id) + + # Emit update event with full memory data + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data=updated_memory, + ) + ) + + # Start transcription pipeline + asyncio.create_task(process_voice_memory_async(memory_id)) + + logger.info(f"Added extracted audio to video memory {memory_id}") + return {"success": True, "audio_path": audio_path} + + +@router.post("/{memory_id}/thumbnail") +async def upload_thumbnail( + memory_id: int, + file: UploadFile = File(...), +): + """Upload a thumbnail for a video. + + Called by the frontend after generating thumbnail with ffmpeg.wasm. + """ + memory = await get_video_memory(memory_id) + if not memory: + raise HTTPException(status_code=404, detail="Video memory not found") + + # Read thumbnail data + thumbnail_data = await file.read() + if not thumbnail_data: + raise HTTPException(status_code=400, detail="Empty thumbnail file") + + # Check file size + if len(thumbnail_data) > MAX_THUMBNAIL_SIZE: + raise HTTPException( + status_code=413, + detail=f"Thumbnail too large. Maximum size is {MAX_THUMBNAIL_SIZE // (1024 * 1024)} MB" + ) + + # Get image format (default to jpg) + image_format = "jpg" + if file.filename and "." in file.filename: + ext = file.filename.rsplit(".", 1)[1].lower() + if ext in {"jpg", "jpeg", "png", "webp"}: + image_format = ext + + # Save encrypted thumbnail + thumbnail_path = save_thumbnail(thumbnail_data, image_format) + + # Update video memory + await update_video_thumbnail(memory_id, thumbnail_path) + + # Fetch complete memory for SSE event + updated_memory = await get_video_memory(memory_id) + + # Emit update event with full memory data + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data=updated_memory, + ) + ) + + logger.info(f"Added thumbnail to video memory {memory_id}") + return {"success": True, "thumbnail_path": thumbnail_path} + + +@router.get("/{memory_id}/stream") +async def stream_video(memory_id: int): + """Stream the video file for playback. + + Returns the decrypted video with appropriate content type. + """ + memory = await get_video_memory(memory_id) + if not memory: + raise HTTPException(status_code=404, detail="Video memory not found") + + video_path = memory.get("video_path") + if not video_path: + raise HTTPException(status_code=404, detail="Video file not found") + + video_format = memory.get("video_format", "mp4") + + try: + video_data = read_video_file(video_path) + except FileNotFoundError: + raise HTTPException(status_code=404, detail="Video file not found on disk") + + # Determine content type + content_type_map = { + "mp4": "video/mp4", + "webm": "video/webm", + "mov": "video/quicktime", + "mkv": "video/x-matroska", + "avi": "video/x-msvideo", + } + content_type = content_type_map.get(video_format, "video/mp4") + + # Stream the video data + async def video_generator(): + # Yield in chunks for efficient streaming + chunk_size = 256 * 1024 # 256KB chunks for video + for i in range(0, len(video_data), chunk_size): + yield video_data[i:i + chunk_size] + + return StreamingResponse( + video_generator(), + media_type=content_type, + headers={ + "Content-Length": str(len(video_data)), + "Accept-Ranges": "bytes", + } + ) + + +@router.get("/{memory_id}/thumbnail") +async def get_thumbnail(memory_id: int): + """Get the thumbnail image for a video.""" + memory = await get_video_memory(memory_id) + if not memory: + raise HTTPException(status_code=404, detail="Video memory not found") + + thumbnail_path = memory.get("thumbnail_path") + if not thumbnail_path: + raise HTTPException(status_code=404, detail="Thumbnail not found") + + try: + thumbnail_data = read_thumbnail(thumbnail_path) + except FileNotFoundError: + raise HTTPException(status_code=404, detail="Thumbnail file not found on disk") + + # Determine content type from path + content_type = "image/jpeg" + if thumbnail_path.endswith(".png.enc"): + content_type = "image/png" + elif thumbnail_path.endswith(".webp.enc"): + content_type = "image/webp" + + return StreamingResponse( + iter([thumbnail_data]), + media_type=content_type, + headers={"Content-Length": str(len(thumbnail_data))}, + ) + + +@router.post("/{memory_id}/retry") +async def retry_transcription(memory_id: int): + """Retry transcription for a failed video memory.""" + memory = await get_video_memory(memory_id) + if not memory: + raise HTTPException(status_code=404, detail="Video memory not found") + + if not memory.get("audio_path"): + raise HTTPException( + status_code=400, + detail="Audio not yet extracted. Please retry audio extraction first." + ) + + # Atomically reset status to pending only if not already processing + status_reset = await reset_transcription_status_if_not_processing(memory_id) + if not status_reset: + raise HTTPException( + status_code=400, + detail="Transcription is already in progress" + ) + + # Fetch complete memory for SSE event + updated_memory = await get_video_memory(memory_id) + + # Emit update event with full memory data + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data=updated_memory, + ) + ) + + # Start background processing + asyncio.create_task(process_voice_memory_async(memory_id)) + + logger.info(f"Retrying transcription for video memory {memory_id}") + return {"success": True, "message": "Transcription retry started"} + + +@router.delete("/{memory_id}") +async def delete_video_memory_endpoint(memory_id: int): + """Delete a video memory and all its files.""" + # Delete from database and get paths + paths = await delete_video_memory(memory_id) + + if paths is None: + raise HTTPException(status_code=404, detail="Video memory not found") + + # Delete all associated files + if paths.get("video_path"): + delete_video_file(paths["video_path"]) + if paths.get("audio_path"): + delete_audio_file(paths["audio_path"]) + if paths.get("thumbnail_path"): + delete_thumbnail(paths["thumbnail_path"]) + + # Emit deletion event + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_DELETED, + memory_id=memory_id, + data=None, + ) + ) + + logger.info(f"Deleted video memory {memory_id}") + return {"deleted": True} diff --git a/backend/app/services/ai_processing.py b/backend/app/services/ai_processing.py index e841a1d..abac6d8 100644 --- a/backend/app/services/ai_processing.py +++ b/backend/app/services/ai_processing.py @@ -5,12 +5,15 @@ from .ai import get_client, get_model from .embeddings import get_embedding, get_current_embedding_model +from .transcription import transcribe_audio from ..db.crud import ( get_memory, update_memory_summary, update_memory_embedding_summary, update_memory_title, update_memory_embedding, + update_memory_transcript, + update_transcription_status, add_tags_to_memory, get_all_tags, update_conversation_title, @@ -350,3 +353,195 @@ async def process_conversation_title_async(conversation_id: int, message: str) - except Exception as e: logger.error(f"Failed to process conversation title {conversation_id}: {e}") + + +async def generate_voice_title(transcript: str) -> str: + """Generate a concise title from a voice transcript.""" + client = await get_client() + model = get_model() + + prompt = f"""Generate a concise, descriptive title for this voice note transcript. + +Transcript: {transcript[:1000]} + +Requirements: +- 5-10 words maximum +- Capture the main topic or key point +- Be informative and scannable + +Title:""" + + try: + response = await client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": "You are a helpful assistant that creates concise, descriptive titles for voice notes. Respond with only the title, no quotes or extra formatting."}, + {"role": "user", "content": prompt} + ], + max_tokens=50, + ) + title = response.choices[0].message.content.strip() if response.choices[0].message.content else "" + # Remove quotes if the model wrapped the title + if title.startswith('"') and title.endswith('"'): + title = title[1:-1] + return title + except Exception as e: + logger.error(f"Failed to generate voice title: {e}") + return "" + + +async def process_voice_memory_async(memory_id: int) -> None: + """Background task to process a media memory (voice memo or audio upload). + + Pipeline: + 1. Set status to "processing" + 2. Transcribe audio via faster-whisper + 3. Store transcript + 4. Generate title from transcript + 5. Generate summary, embedding_summary, tags + 6. Create embedding from embedding_summary + 7. Set status to "completed" + 8. Emit MEMORY_UPDATED event + """ + try: + # Get the memory + memory = await get_memory(memory_id) + if not memory: + logger.error(f"Media memory {memory_id} not found for processing") + return + + memory_type = memory.get("type") + if memory_type not in ("voice_memo", "audio", "voice", "video"): + logger.error(f"Memory {memory_id} is not a media memory (type={memory_type})") + return + + audio_path = memory.get("audio_path") + if not audio_path: + logger.error(f"Voice memory {memory_id} has no audio_path") + await update_transcription_status(memory_id, "failed") + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data={"transcription_status": "failed"}, + ) + ) + return + + # 1. Set status to processing + await update_transcription_status(memory_id, "processing") + + # Emit update event for processing status + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data={"transcription_status": "processing"}, + ) + ) + + # 2. Transcribe the audio + logger.info(f"Starting transcription for voice memory {memory_id}") + transcript, segments = await transcribe_audio(audio_path) + + if not transcript or not transcript.strip(): + logger.warning(f"Transcription produced no text for memory {memory_id}") + await update_transcription_status(memory_id, "failed") + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data={"transcription_status": "failed"}, + ) + ) + return + + # 3. Store transcript and segments + await update_memory_transcript(memory_id, transcript, segments) + logger.info( + f"Stored transcript for voice memory {memory_id}: " + f"{len(transcript)} chars, {len(segments)} segments" + ) + + # Get existing tags for context + all_tags = await get_all_tags() + existing_tag_names = [t["name"] for t in all_tags] + + # 4-5. Generate title, summary, embedding_summary, and tags in parallel + tasks = [ + generate_voice_title(transcript), + generate_summary(transcript, ""), + generate_embedding_summary(transcript, ""), + generate_tags(transcript, "", existing_tag_names), + ] + results = await asyncio.gather(*tasks) + + title = results[0] + summary = results[1] + embedding_summary = results[2] + tags = results[3] + + updated = False + + # Update title + if title: + await update_memory_title(memory_id, title) + logger.info(f"Updated voice memory {memory_id} title: '{title}'") + updated = True + + # Update summary + if summary: + await update_memory_summary(memory_id, summary) + logger.info(f"Updated voice memory {memory_id} with summary") + updated = True + + # Update embedding summary and create embedding + if embedding_summary: + await update_memory_embedding_summary(memory_id, embedding_summary) + logger.info(f"Updated voice memory {memory_id} with embedding summary") + updated = True + + # 6. Create embedding from embedding_summary + try: + if embedding_summary.strip(): + embedding = await get_embedding(embedding_summary) + embedding_model = get_current_embedding_model() + await update_memory_embedding(memory_id, embedding, embedding_model) + logger.info(f"Created embedding for voice memory {memory_id}") + except Exception as e: + logger.error(f"Failed to create embedding for voice memory {memory_id}: {e}") + + # Add AI-generated tags + if tags: + await add_tags_to_memory(memory_id, tags, source="ai") + logger.info(f"Added {len(tags)} AI tags to voice memory {memory_id}: {tags}") + updated = True + + # 7. Set status to completed + await update_transcription_status(memory_id, "completed") + + # 8. Emit update event (always emit, even if AI generation failed) + updated_memory = await get_memory(memory_id) + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data=updated_memory, + ) + ) + logger.info(f"Emitted update event for voice memory {memory_id}") + + except Exception as e: + logger.error(f"Failed to process voice memory {memory_id}: {e}") + # Set status to failed and notify frontend + try: + await update_transcription_status(memory_id, "failed") + await event_manager.publish( + MemoryEvent( + type=EventType.MEMORY_UPDATED, + memory_id=memory_id, + data={"transcription_status": "failed"}, + ) + ) + except Exception: + pass diff --git a/backend/app/services/audio_storage.py b/backend/app/services/audio_storage.py new file mode 100644 index 0000000..9fe76bb --- /dev/null +++ b/backend/app/services/audio_storage.py @@ -0,0 +1,180 @@ +"""Encrypted audio file storage service for voice memories.""" +import hashlib +import logging +import uuid +from pathlib import Path +import platform +import os + +from cryptography.fernet import Fernet, InvalidToken +import base64 + +from .secrets import get_or_create_salt + +logger = logging.getLogger(__name__) + + +def _validate_path_within_directory(file_path: Path, base_dir: Path) -> None: + """Validate that file_path resolves within base_dir to prevent path traversal attacks.""" + try: + resolved_path = file_path.resolve() + resolved_base = base_dir.resolve() + if not resolved_path.is_relative_to(resolved_base): + raise ValueError(f"Path traversal attempt detected: {file_path}") + except (ValueError, RuntimeError) as e: + raise ValueError(f"Invalid file path: {file_path}") from e + +# In-memory cache for the derived encryption key +_encryption_key: bytes | None = None + + +def _get_audio_dir() -> Path: + """Get the audio storage directory based on platform.""" + system = platform.system() + if system == "Darwin": + data_dir = Path.home() / "Library" / "Application Support" / "Think" / "audio" + elif system == "Windows": + data_dir = Path(os.environ.get("LOCALAPPDATA", Path.home())) / "Think" / "audio" + else: + data_dir = Path.home() / ".local" / "share" / "Think" / "audio" + data_dir.mkdir(parents=True, exist_ok=True) + return data_dir + + +def derive_encryption_key(master_password: str) -> bytes: + """Derive a Fernet encryption key from the master password. + + Uses the same salt as the database key derivation for consistency. + """ + salt = get_or_create_salt() + # Use PBKDF2 to derive a key, but with a different context than DB key + key_material = hashlib.pbkdf2_hmac( + 'sha256', + (master_password + "_audio").encode(), # Add context to differentiate from DB key + salt.encode(), + 100000, + dklen=32 # Fernet requires 32 bytes + ) + # Fernet requires base64-encoded 32-byte key + return base64.urlsafe_b64encode(key_material) + + +def set_encryption_key(master_password: str) -> None: + """Set the encryption key from the master password. + + Called after successful database unlock. + """ + global _encryption_key + _encryption_key = derive_encryption_key(master_password) + + +def clear_encryption_key() -> None: + """Clear the encryption key (on logout).""" + global _encryption_key + _encryption_key = None + + +def _get_fernet() -> Fernet: + """Get the Fernet instance for encryption/decryption.""" + if _encryption_key is None: + raise RuntimeError("Encryption key not set. Database must be unlocked first.") + return Fernet(_encryption_key) + + +def save_audio_file(audio_data: bytes, audio_format: str) -> str: + """Save an audio file with encryption. + + Args: + audio_data: Raw audio file bytes + audio_format: File format (e.g., "mp3", "wav", "webm") + + Returns: + Relative path to the encrypted file (e.g., "abc123.mp3.enc") + """ + fernet = _get_fernet() + + # Generate unique filename + filename = f"{uuid.uuid4()}.{audio_format}.enc" + file_path = _get_audio_dir() / filename + + # Encrypt and save + encrypted_data = fernet.encrypt(audio_data) + file_path.write_bytes(encrypted_data) + + logger.info(f"Saved encrypted audio file: {filename}") + return filename + + +def read_audio_file(relative_path: str) -> bytes: + """Read and decrypt an audio file. + + Args: + relative_path: Relative path returned by save_audio_file + + Returns: + Decrypted audio file bytes + + Raises: + ValueError: If path traversal attempt detected + FileNotFoundError: If audio file doesn't exist + RuntimeError: If decryption fails + """ + fernet = _get_fernet() + base_dir = _get_audio_dir() + file_path = base_dir / relative_path + + _validate_path_within_directory(file_path, base_dir) + + if not file_path.exists(): + raise FileNotFoundError(f"Audio file not found: {relative_path}") + + encrypted_data = file_path.read_bytes() + try: + return fernet.decrypt(encrypted_data) + except InvalidToken: + logger.error(f"Decryption failed for audio file: {relative_path}") + raise RuntimeError("Failed to decrypt audio file. The file may be corrupted or the encryption key may have changed.") + + +def delete_audio_file(relative_path: str) -> bool: + """Delete an encrypted audio file. + + Args: + relative_path: Relative path returned by save_audio_file + + Returns: + True if file was deleted, False if it didn't exist + + Raises: + ValueError: If path traversal attempt detected + """ + base_dir = _get_audio_dir() + file_path = base_dir / relative_path + + _validate_path_within_directory(file_path, base_dir) + + if file_path.exists(): + file_path.unlink() + logger.info(f"Deleted audio file: {relative_path}") + return True + return False + + +def get_audio_file_path(relative_path: str) -> Path: + """Get the full path to an audio file. + + Args: + relative_path: Relative path returned by save_audio_file + + Returns: + Full Path object to the file + + Raises: + ValueError: If path traversal attempt detected + """ + base_dir = _get_audio_dir() + file_path = base_dir / relative_path + + _validate_path_within_directory(file_path, base_dir) + + return file_path diff --git a/backend/app/services/audio_utils.py b/backend/app/services/audio_utils.py new file mode 100644 index 0000000..b8b6e4d --- /dev/null +++ b/backend/app/services/audio_utils.py @@ -0,0 +1,136 @@ +"""Audio utilities for format validation and metadata extraction.""" +import io +import logging +from pathlib import Path + +from mutagen import File as MutagenFile +from mutagen.mp3 import MP3 +from mutagen.wave import WAVE +from mutagen.mp4 import MP4 +from mutagen.oggopus import OggOpus +from mutagen.oggvorbis import OggVorbis +from mutagen.flac import FLAC + +logger = logging.getLogger(__name__) + +# Supported audio formats +SUPPORTED_FORMATS = {"mp3", "wav", "m4a", "webm", "ogg", "flac"} + +# MIME type to format mapping +MIME_TO_FORMAT = { + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/wav": "wav", + "audio/wave": "wav", + "audio/x-wav": "wav", + "audio/mp4": "m4a", + "audio/x-m4a": "m4a", + "audio/m4a": "m4a", + "audio/webm": "webm", + "audio/ogg": "ogg", + "audio/flac": "flac", + "audio/x-flac": "flac", +} + + +def validate_audio_format(filename: str) -> str | None: + """Validate that a filename has a supported audio extension. + + Args: + filename: The filename to validate + + Returns: + The lowercase format extension if valid, None otherwise + """ + ext = Path(filename).suffix.lower().lstrip(".") + if ext in SUPPORTED_FORMATS: + return ext + return None + + +def get_format_from_mime(mime_type: str) -> str | None: + """Get audio format from MIME type. + + Args: + mime_type: The MIME type (e.g., "audio/mpeg") + + Returns: + The format string if recognized, None otherwise + """ + return MIME_TO_FORMAT.get(mime_type.lower()) + + +def get_audio_duration(audio_data: bytes, audio_format: str) -> float | None: + """Extract duration in seconds from audio data. + + Args: + audio_data: Raw audio file bytes + audio_format: File format (e.g., "mp3", "wav") + + Returns: + Duration in seconds, or None if extraction fails + """ + try: + # Create a file-like object from bytes + audio_file = io.BytesIO(audio_data) + + # Use the appropriate mutagen class based on format + if audio_format == "mp3": + audio = MP3(audio_file) + elif audio_format == "wav": + audio = WAVE(audio_file) + elif audio_format == "m4a": + audio = MP4(audio_file) + elif audio_format == "ogg": + # Try Opus first, then Vorbis + try: + audio = OggOpus(audio_file) + except Exception: + audio_file.seek(0) + audio = OggVorbis(audio_file) + elif audio_format == "flac": + audio = FLAC(audio_file) + elif audio_format == "webm": + # WebM is typically Opus in an EBML container + # mutagen doesn't directly support WebM, so we try to handle it + # by using the generic File approach + audio_file.seek(0) + audio = MutagenFile(audio_file) + if audio is None: + logger.warning(f"Could not parse WebM audio file") + return None + else: + # Try generic mutagen detection + audio = MutagenFile(audio_file) + if audio is None: + logger.warning(f"Unsupported audio format for duration: {audio_format}") + return None + + duration = audio.info.length if audio and audio.info else None + return float(duration) if duration else None + + except Exception as e: + logger.warning(f"Failed to extract audio duration: {e}") + return None + + +def format_duration(seconds: float | None) -> str: + """Format duration in seconds to human-readable string (M:SS or H:MM:SS). + + Args: + seconds: Duration in seconds + + Returns: + Formatted string like "2:34" or "1:05:30" + """ + if seconds is None: + return "0:00" + + total_seconds = int(seconds) + hours = total_seconds // 3600 + minutes = (total_seconds % 3600) // 60 + secs = total_seconds % 60 + + if hours > 0: + return f"{hours}:{minutes:02d}:{secs:02d}" + return f"{minutes}:{secs:02d}" diff --git a/backend/app/services/transcription.py b/backend/app/services/transcription.py new file mode 100644 index 0000000..4578a58 --- /dev/null +++ b/backend/app/services/transcription.py @@ -0,0 +1,204 @@ +"""Transcription service using faster-whisper for local speech-to-text.""" +import asyncio +import logging +import tempfile +import os +from pathlib import Path + +from faster_whisper import WhisperModel + +from .audio_storage import read_audio_file +from ..db.crud import get_setting + +logger = logging.getLogger(__name__) + +# Supported Whisper models (smallest to largest) +WHISPER_MODELS = ["tiny", "base", "small", "medium"] +DEFAULT_WHISPER_MODEL = "base" + +# Cached model instance +_whisper_model: WhisperModel | None = None +_loaded_model_name: str | None = None + + +async def get_whisper_model_setting() -> str: + """Get the configured Whisper model from settings.""" + model = await get_setting("whisper_model") + if model and model in WHISPER_MODELS: + return model + return DEFAULT_WHISPER_MODEL + + +def _get_whisper_model(model_name: str) -> WhisperModel: + """Get or create the Whisper model instance. + + Models are cached to avoid reloading on every transcription. + """ + global _whisper_model, _loaded_model_name + + if _whisper_model is not None and _loaded_model_name == model_name: + return _whisper_model + + logger.info(f"Loading Whisper model: {model_name}") + + # Use CPU by default for broader compatibility + # faster-whisper will use CTranslate2 which is efficient on CPU + _whisper_model = WhisperModel( + model_name, + device="cpu", + compute_type="int8", # Use int8 quantization for faster CPU inference + ) + _loaded_model_name = model_name + + logger.info(f"Whisper model '{model_name}' loaded successfully") + return _whisper_model + + +def unload_whisper_model() -> None: + """Unload the Whisper model to free memory.""" + global _whisper_model, _loaded_model_name + _whisper_model = None + _loaded_model_name = None + logger.info("Whisper model unloaded") + + +def _transcribe_audio_sync(audio_path: str, model_name: str) -> tuple[str, list[dict]]: + """Synchronous transcription - runs in thread pool. + + This function contains all blocking operations and should be called + via asyncio.to_thread() to avoid blocking the event loop. + """ + # Read and decrypt the audio file + audio_data = read_audio_file(audio_path) + + # Extract format from filename (e.g., "abc123.mp3.enc" -> "mp3") + parts = audio_path.split(".") + if len(parts) >= 3: + audio_format = parts[-2] # Get format before .enc + else: + audio_format = "wav" # Default fallback + + # Write to a temp file (Whisper needs a file path) + temp_suffix = f".{audio_format}" + with tempfile.NamedTemporaryFile(suffix=temp_suffix, delete=False) as temp_file: + temp_path = temp_file.name + temp_file.write(audio_data) + + try: + # Load model and transcribe + model = _get_whisper_model(model_name) + + # Transcribe with automatic language detection + segments, info = model.transcribe( + temp_path, + beam_size=5, + language=None, # Auto-detect language + vad_filter=True, # Filter out non-speech + ) + + # Collect segments with timestamps + transcript_parts = [] + segment_list = [] + for segment in segments: + text = segment.text.strip() + transcript_parts.append(text) + segment_list.append({ + "start": round(segment.start, 2), + "end": round(segment.end, 2), + "text": text, + }) + + transcript = " ".join(transcript_parts) + + logger.info( + f"Transcription complete: {len(transcript)} chars, " + f"{len(segment_list)} segments, " + f"language={info.language}, probability={info.language_probability:.2f}" + ) + + return transcript, segment_list + + finally: + # Clean up temp file + try: + os.unlink(temp_path) + except Exception as e: + logger.warning(f"Failed to delete temp file: {e}") + + +async def transcribe_audio(audio_path: str) -> tuple[str, list[dict]]: + """Transcribe an encrypted audio file. + + Args: + audio_path: Relative path to the encrypted audio file + + Returns: + Tuple of (full transcript text, list of segment dicts with start/end/text) + + Raises: + Exception: If transcription fails + """ + # Get configured model (async DB call) + model_name = await get_whisper_model_setting() + + # Run blocking transcription in thread pool + return await asyncio.to_thread(_transcribe_audio_sync, audio_path, model_name) + + +def _transcribe_audio_bytes_sync(audio_data: bytes, audio_format: str, model_name: str) -> tuple[str, list[dict]]: + """Synchronous transcription from bytes - runs in thread pool. + + This function contains all blocking operations and should be called + via asyncio.to_thread() to avoid blocking the event loop. + """ + # Write to temp file + temp_suffix = f".{audio_format}" + with tempfile.NamedTemporaryFile(suffix=temp_suffix, delete=False) as temp_file: + temp_path = temp_file.name + temp_file.write(audio_data) + + try: + model = _get_whisper_model(model_name) + + segments, info = model.transcribe( + temp_path, + beam_size=5, + language=None, + vad_filter=True, + ) + + transcript_parts = [] + segment_list = [] + for segment in segments: + text = segment.text.strip() + transcript_parts.append(text) + segment_list.append({ + "start": round(segment.start, 2), + "end": round(segment.end, 2), + "text": text, + }) + + return " ".join(transcript_parts), segment_list + + finally: + try: + os.unlink(temp_path) + except Exception as e: + logger.warning(f"Failed to delete temp file: {e}") + + +async def transcribe_audio_bytes(audio_data: bytes, audio_format: str) -> tuple[str, list[dict]]: + """Transcribe audio data directly from bytes. + + Args: + audio_data: Raw audio bytes + audio_format: Audio format (e.g., "mp3", "wav") + + Returns: + Tuple of (full transcript text, list of segment dicts with start/end/text) + """ + # Get configured model (async DB call) + model_name = await get_whisper_model_setting() + + # Run blocking transcription in thread pool + return await asyncio.to_thread(_transcribe_audio_bytes_sync, audio_data, audio_format, model_name) diff --git a/backend/app/services/video_storage.py b/backend/app/services/video_storage.py new file mode 100644 index 0000000..b5aa79c --- /dev/null +++ b/backend/app/services/video_storage.py @@ -0,0 +1,272 @@ +"""Encrypted video file storage service for video memories.""" +import hashlib +import logging +import uuid +from pathlib import Path +import platform +import os + +from cryptography.fernet import Fernet, InvalidToken +import base64 + +from .secrets import get_or_create_salt + +logger = logging.getLogger(__name__) + + +def _validate_path_within_directory(file_path: Path, base_dir: Path) -> None: + """Validate that file_path resolves within base_dir to prevent path traversal attacks.""" + try: + resolved_path = file_path.resolve() + resolved_base = base_dir.resolve() + if not resolved_path.is_relative_to(resolved_base): + raise ValueError(f"Path traversal attempt detected: {file_path}") + except (ValueError, RuntimeError) as e: + raise ValueError(f"Invalid file path: {file_path}") from e + +# In-memory cache for the derived encryption key +_encryption_key: bytes | None = None + + +def _get_video_dir() -> Path: + """Get the video storage directory based on platform.""" + system = platform.system() + if system == "Darwin": + data_dir = Path.home() / "Library" / "Application Support" / "Think" / "video" + elif system == "Windows": + data_dir = Path(os.environ.get("LOCALAPPDATA", Path.home())) / "Think" / "video" + else: + data_dir = Path.home() / ".local" / "share" / "Think" / "video" + data_dir.mkdir(parents=True, exist_ok=True) + return data_dir + + +def _get_thumbnail_dir() -> Path: + """Get the thumbnail storage directory based on platform.""" + system = platform.system() + if system == "Darwin": + data_dir = Path.home() / "Library" / "Application Support" / "Think" / "thumbnails" + elif system == "Windows": + data_dir = Path(os.environ.get("LOCALAPPDATA", Path.home())) / "Think" / "thumbnails" + else: + data_dir = Path.home() / ".local" / "share" / "Think" / "thumbnails" + data_dir.mkdir(parents=True, exist_ok=True) + return data_dir + + +def derive_encryption_key(master_password: str) -> bytes: + """Derive a Fernet encryption key from the master password. + + Uses the same salt as the database key derivation for consistency. + """ + salt = get_or_create_salt() + # Use PBKDF2 to derive a key, but with a different context than DB key + key_material = hashlib.pbkdf2_hmac( + 'sha256', + (master_password + "_video").encode(), # Add context to differentiate from DB key + salt.encode(), + 100000, + dklen=32 # Fernet requires 32 bytes + ) + # Fernet requires base64-encoded 32-byte key + return base64.urlsafe_b64encode(key_material) + + +def set_encryption_key(master_password: str) -> None: + """Set the encryption key from the master password. + + Called after successful database unlock. + """ + global _encryption_key + _encryption_key = derive_encryption_key(master_password) + + +def clear_encryption_key() -> None: + """Clear the encryption key (on logout).""" + global _encryption_key + _encryption_key = None + + +def _get_fernet() -> Fernet: + """Get the Fernet instance for encryption/decryption.""" + if _encryption_key is None: + raise RuntimeError("Encryption key not set. Database must be unlocked first.") + return Fernet(_encryption_key) + + +def save_video_file(video_data: bytes, video_format: str) -> str: + """Save a video file with encryption. + + Args: + video_data: Raw video file bytes + video_format: File format (e.g., "mp4", "webm", "mov") + + Returns: + Relative path to the encrypted file (e.g., "abc123.mp4.enc") + """ + fernet = _get_fernet() + + # Generate unique filename + filename = f"{uuid.uuid4()}.{video_format}.enc" + file_path = _get_video_dir() / filename + + # Encrypt and save + encrypted_data = fernet.encrypt(video_data) + file_path.write_bytes(encrypted_data) + + logger.info(f"Saved encrypted video file: {filename}") + return filename + + +def read_video_file(relative_path: str) -> bytes: + """Read and decrypt a video file. + + Args: + relative_path: Relative path returned by save_video_file + + Returns: + Decrypted video file bytes + + Raises: + ValueError: If path traversal attempt detected + FileNotFoundError: If video file doesn't exist + RuntimeError: If decryption fails + """ + fernet = _get_fernet() + base_dir = _get_video_dir() + file_path = base_dir / relative_path + + _validate_path_within_directory(file_path, base_dir) + + if not file_path.exists(): + raise FileNotFoundError(f"Video file not found: {relative_path}") + + encrypted_data = file_path.read_bytes() + try: + return fernet.decrypt(encrypted_data) + except InvalidToken: + logger.error(f"Decryption failed for video file: {relative_path}") + raise RuntimeError("Failed to decrypt video file. The file may be corrupted or the encryption key may have changed.") + + +def delete_video_file(relative_path: str) -> bool: + """Delete an encrypted video file. + + Args: + relative_path: Relative path returned by save_video_file + + Returns: + True if file was deleted, False if it didn't exist + + Raises: + ValueError: If path traversal attempt detected + """ + base_dir = _get_video_dir() + file_path = base_dir / relative_path + + _validate_path_within_directory(file_path, base_dir) + + if file_path.exists(): + file_path.unlink() + logger.info(f"Deleted video file: {relative_path}") + return True + return False + + +def get_video_file_path(relative_path: str) -> Path: + """Get the full path to a video file. + + Args: + relative_path: Relative path returned by save_video_file + + Returns: + Full Path object to the file + + Raises: + ValueError: If path traversal attempt detected + """ + base_dir = _get_video_dir() + file_path = base_dir / relative_path + + _validate_path_within_directory(file_path, base_dir) + + return file_path + + +def save_thumbnail(thumbnail_data: bytes, image_format: str = "jpg") -> str: + """Save a thumbnail image with encryption. + + Args: + thumbnail_data: Raw image file bytes + image_format: File format (e.g., "jpg", "png") + + Returns: + Relative path to the encrypted file (e.g., "abc123.jpg.enc") + """ + fernet = _get_fernet() + + # Generate unique filename + filename = f"{uuid.uuid4()}.{image_format}.enc" + file_path = _get_thumbnail_dir() / filename + + # Encrypt and save + encrypted_data = fernet.encrypt(thumbnail_data) + file_path.write_bytes(encrypted_data) + + logger.info(f"Saved encrypted thumbnail: {filename}") + return filename + + +def read_thumbnail(relative_path: str) -> bytes: + """Read and decrypt a thumbnail image. + + Args: + relative_path: Relative path returned by save_thumbnail + + Returns: + Decrypted image file bytes + + Raises: + ValueError: If path traversal attempt detected + FileNotFoundError: If thumbnail doesn't exist + RuntimeError: If decryption fails + """ + fernet = _get_fernet() + base_dir = _get_thumbnail_dir() + file_path = base_dir / relative_path + + _validate_path_within_directory(file_path, base_dir) + + if not file_path.exists(): + raise FileNotFoundError(f"Thumbnail not found: {relative_path}") + + encrypted_data = file_path.read_bytes() + try: + return fernet.decrypt(encrypted_data) + except InvalidToken: + logger.error(f"Decryption failed for thumbnail: {relative_path}") + raise RuntimeError("Failed to decrypt thumbnail. The file may be corrupted or the encryption key may have changed.") + + +def delete_thumbnail(relative_path: str) -> bool: + """Delete an encrypted thumbnail file. + + Args: + relative_path: Relative path returned by save_thumbnail + + Returns: + True if file was deleted, False if it didn't exist + + Raises: + ValueError: If path traversal attempt detected + """ + base_dir = _get_thumbnail_dir() + file_path = base_dir / relative_path + + _validate_path_within_directory(file_path, base_dir) + + if file_path.exists(): + file_path.unlink() + logger.info(f"Deleted thumbnail: {relative_path}") + return True + return False diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 0af422f..1c42ad8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -18,8 +18,13 @@ rotki-pysqlcipher3 = "^2024.10.1" sqlalchemy = {extras = ["asyncio"], version = "^2.0.0"} sqlite-vec = "^0.1.0" numpy = "^2.0.0" +# Voice memory support +faster-whisper = "^1.0.0" +mutagen = "^1.47.0" +cryptography = "^43.0.0" # Windows-only: pywin32 for native messaging stub pywin32 = {version = "^306", markers = "sys_platform == 'win32'"} +python-multipart = "^0.0.21" [tool.poetry.group.dev.dependencies] pyinstaller = "^6.0.0" diff --git a/package.json b/package.json index f864819..e415a2a 100644 --- a/package.json +++ b/package.json @@ -21,5 +21,8 @@ "@commitlint/cli": "^20.2.0", "@commitlint/config-conventional": "^20.2.0", "husky": "^9.1.7" + }, + "pnpm": { + "onlyBuiltDependencies": ["ffmpeg-static"] } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e3f80e2..761b0ec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: electron-updater: specifier: ^6.6.2 version: 6.6.2 + ffmpeg-static: + specifier: ^5.2.0 + version: 5.3.0 lucide-react: specifier: ^0.460.0 version: 0.460.0(react@18.3.1) @@ -111,6 +114,9 @@ importers: postcss: specifier: ^8.4.49 version: 8.5.6 + sharp: + specifier: ^0.34.5 + version: 0.34.5 tailwindcss: specifier: ^3.4.15 version: 3.4.18 @@ -399,6 +405,10 @@ packages: resolution: {integrity: sha512-KTy0OqRDLR5y/zZMnizyx09z/rPlPC/zKhYgH8o/q6PuAjoQAKlRfY4zzv0M64yybQ//6//4H1n14pxaLZfUnA==} engines: {node: '>=v18'} + '@derhuerst/http-basic@8.2.4': + resolution: {integrity: sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw==} + engines: {node: '>=6.0.0'} + '@develar/schema-utils@2.6.5': resolution: {integrity: sha512-0cp4PsWQ/9avqTVMCtZ+GirikIA36ikvjtHweU4/j8yLtgObI0+JUPhYFScgwlteveGB1rt3Cm8UhN04XayDig==} engines: {node: '>= 8.9.0'} @@ -434,6 +444,9 @@ packages: resolution: {integrity: sha512-fKpv9kg4SPmt+hY7SVBnIYULE9QJl8L3sCfcBsnqbJwwBwAeTLokJ9TRt9y7bK0JAzIW2y78TVVjvnQEms/yyA==} engines: {node: '>=16.4'} + '@emnapi/runtime@1.8.1': + resolution: {integrity: sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==} + '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} engines: {node: '>=18'} @@ -628,6 +641,143 @@ packages: '@hapi/topo@6.0.2': resolution: {integrity: sha512-KR3rD5inZbGMrHmgPxsJ9dbi6zEK+C3ZwUwTa+eMwWLz7oijWUTWD2pMSNNYJAU6Qq+65NkxXjqHr/7LM2Xkqg==} + '@img/colour@1.0.0': + resolution: {integrity: sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-ppc64@1.2.4': + resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} + cpu: [ppc64] + os: [linux] + + '@img/sharp-libvips-linux-riscv64@1.2.4': + resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} + cpu: [riscv64] + os: [linux] + + '@img/sharp-libvips-linux-s390x@1.2.4': + resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} + cpu: [s390x] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-ppc64@0.34.5': + resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ppc64] + os: [linux] + + '@img/sharp-linux-riscv64@0.34.5': + resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [riscv64] + os: [linux] + + '@img/sharp-linux-s390x@0.34.5': + resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-wasm32@0.34.5': + resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.34.5': + resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + '@inquirer/external-editor@1.0.3': resolution: {integrity: sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==} engines: {node: '>=18'} @@ -1317,6 +1467,9 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/node@10.17.60': + resolution: {integrity: sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==} + '@types/node@12.20.55': resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} @@ -1610,6 +1763,9 @@ packages: caniuse-lite@1.0.30001756: resolution: {integrity: sha512-4HnCNKbMLkLdhJz3TToeVWHSnfJvPaq6vu/eRP0Ahub/07n484XHhBF5AJoSGHdVrS8tKFauUQz8Bp9P7LVx7A==} + caseless@0.12.0: + resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} + ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -1731,6 +1887,10 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + concat-stream@2.0.0: + resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} + engines: {'0': node >= 6.0} + concurrently@9.2.1: resolution: {integrity: sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng==} engines: {node: '>=18'} @@ -2067,6 +2227,10 @@ packages: picomatch: optional: true + ffmpeg-static@5.3.0: + resolution: {integrity: sha512-H+K6sW6TiIX6VGend0KQwthe+kaceeH/luE8dIZyOP35ik7ahYojDuqlTV1bOrtEwl01sy2HFNGQfi5IDJvotg==} + engines: {node: '>=16'} + filelist@1.0.4: resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} @@ -2268,6 +2432,9 @@ packages: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} + http-response-object@3.0.2: + resolution: {integrity: sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==} + http2-wrapper@1.0.3: resolution: {integrity: sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==} engines: {node: '>=10.19.0'} @@ -2956,6 +3123,9 @@ packages: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} + parse-cache-control@1.0.1: + resolution: {integrity: sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg==} + parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} @@ -3392,6 +3562,10 @@ packages: set-cookie-parser@2.7.2: resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + sharp@0.34.5: + resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -3614,6 +3788,9 @@ packages: resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} engines: {node: '>=10'} + typedarray@0.0.6: + resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -4197,6 +4374,13 @@ snapshots: '@types/conventional-commits-parser': 5.0.2 chalk: 5.6.2 + '@derhuerst/http-basic@8.2.4': + dependencies: + caseless: 0.12.0 + concat-stream: 2.0.0 + http-response-object: 3.0.2 + parse-cache-control: 1.0.1 + '@develar/schema-utils@2.6.5': dependencies: ajv: 6.12.6 @@ -4280,6 +4464,11 @@ snapshots: transitivePeerDependencies: - supports-color + '@emnapi/runtime@1.8.1': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.25.12': optional: true @@ -4393,6 +4582,102 @@ snapshots: dependencies: '@hapi/hoek': 11.0.7 + '@img/colour@1.0.0': {} + + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-s390x@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-ppc64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.2.4 + optional: true + + '@img/sharp-linux-riscv64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.2.4 + optional: true + + '@img/sharp-linux-s390x@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-wasm32@0.34.5': + dependencies: + '@emnapi/runtime': 1.8.1 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-ia32@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + '@inquirer/external-editor@1.0.3(@types/node@22.19.3)': dependencies: chardet: 2.1.1 @@ -5064,6 +5349,8 @@ snapshots: '@types/ms@2.1.0': {} + '@types/node@10.17.60': {} + '@types/node@12.20.55': {} '@types/node@20.19.25': @@ -5471,6 +5758,8 @@ snapshots: caniuse-lite@1.0.30001756: {} + caseless@0.12.0: {} + ccount@2.0.1: {} chalk@4.1.2: @@ -5586,6 +5875,13 @@ snapshots: concat-map@0.0.1: {} + concat-stream@2.0.0: + dependencies: + buffer-from: 1.1.2 + inherits: 2.0.4 + readable-stream: 3.6.2 + typedarray: 0.0.6 + concurrently@9.2.1: dependencies: chalk: 4.1.2 @@ -5976,6 +6272,15 @@ snapshots: optionalDependencies: picomatch: 4.0.3 + ffmpeg-static@5.3.0: + dependencies: + '@derhuerst/http-basic': 8.2.4 + env-paths: 2.2.1 + https-proxy-agent: 5.0.1 + progress: 2.0.3 + transitivePeerDependencies: + - supports-color + filelist@1.0.4: dependencies: minimatch: 5.1.6 @@ -6248,6 +6553,10 @@ snapshots: transitivePeerDependencies: - supports-color + http-response-object@3.0.2: + dependencies: + '@types/node': 10.17.60 + http2-wrapper@1.0.3: dependencies: quick-lru: 5.1.1 @@ -7013,6 +7322,8 @@ snapshots: dependencies: callsites: 3.1.0 + parse-cache-control@1.0.1: {} + parse-entities@4.0.2: dependencies: '@types/unist': 2.0.11 @@ -7495,6 +7806,37 @@ snapshots: set-cookie-parser@2.7.2: {} + sharp@0.34.5: + dependencies: + '@img/colour': 1.0.0 + detect-libc: 2.1.2 + semver: 7.7.3 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-libvips-darwin-arm64': 1.2.4 + '@img/sharp-libvips-darwin-x64': 1.2.4 + '@img/sharp-libvips-linux-arm': 1.2.4 + '@img/sharp-libvips-linux-arm64': 1.2.4 + '@img/sharp-libvips-linux-ppc64': 1.2.4 + '@img/sharp-libvips-linux-riscv64': 1.2.4 + '@img/sharp-libvips-linux-s390x': 1.2.4 + '@img/sharp-libvips-linux-x64': 1.2.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-ppc64': 0.34.5 + '@img/sharp-linux-riscv64': 0.34.5 + '@img/sharp-linux-s390x': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-wasm32': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-ia32': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -7740,6 +8082,8 @@ snapshots: type-fest@0.13.1: optional: true + typedarray@0.0.6: {} + typescript@5.9.3: {} uc.micro@2.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c9105a1..18524aa 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,3 +5,4 @@ packages: onlyBuiltDependencies: - electron - esbuild + - sharp diff --git a/screenshot.png b/screenshot.png index 8e2a67a..99f5df2 100644 Binary files a/screenshot.png and b/screenshot.png differ