feat(app): add media memory support for voice memos, audio, and video

.changeset/media-memory-feature.md

@@ -0,0 +1,13 @@
+---
+"think-app": minor
+---
+
+feat(app): add media memory support for voice memos, audio, and video
+
+- Add voice memo recording from system tray and main app
+- Add audio file upload with drag-and-drop support
+- Add video file upload with FFmpeg audio extraction and thumbnail generation
+- Add local AI transcription using faster-whisper with timestamp segments
+- Add new memory card components for voice memos, audio, and video
+- Add transcription settings for Whisper model selection
+- Add file size limits and input validation for security

.gitignore

@@ -39,4 +39,7 @@ release/
 extension/extension.pem
 
 # Compiled native messaging stub
-backend/native_host/think-native-stub
+backend/native_host/think-native-stub
+
+# FFmpeg (downloaded on first use, cached locally)
+app/public/ffmpeg/

app/electron/main.js

@@ -6,6 +6,8 @@ const path = require('path');
 const fs = require('fs');
 const https = require('https');
 const { installNativeHost } = require('./install-native-host');
+const { createTray, destroyTray, closeRecordingWindow } = require('./tray');
+const videoProcessor = require('./video-processor');
 
 /**
  * Download a file with progress reporting.
@@ -295,7 +297,7 @@ function createWindow() {
     webPreferences: {
       preload: path.join(__dirname, 'preload.js'),
       contextIsolation: true,
-      nodeIntegration: false
+      nodeIntegration: false,
     }
   });
 
@@ -357,6 +359,10 @@ app.whenReady().then(async () => {
     if (mainWindow && !mainWindow.isDestroyed()) {
       mainWindow.webContents.send('backend-ready', { token: APP_TOKEN });
     }
+
+    // Initialize system tray after backend is ready
+    const isDev = process.env.NODE_ENV === 'development' || !app.isPackaged;
+    createTray(mainWindow, APP_TOKEN, isDev);
   } catch (err) {
     console.error('Backend startup failed:', err);
     if (mainWindow && !mainWindow.isDestroyed()) {
@@ -374,6 +380,8 @@ app.on('window-all-closed', () => {
 });
 
 app.on('before-quit', () => {
+  closeRecordingWindow();
+  destroyTray();
   if (pythonProcess) {
     pythonProcess.kill();
   }
@@ -549,3 +557,85 @@ ipcMain.handle('pull-model', async (_event, modelName) => {
 ipcMain.handle('install-update', () => {
   autoUpdater.quitAndInstall();
 });
+
+// Open main window (called from recording popup when app is locked)
+ipcMain.handle('open-main-window', () => {
+  if (mainWindow && !mainWindow.isDestroyed()) {
+    mainWindow.show();
+    mainWindow.focus();
+  } else {
+    // No window exists - create one
+    createWindow();
+    if (backendReady && mainWindow) {
+      mainWindow.webContents.once('did-finish-load', () => {
+        mainWindow.webContents.send('backend-ready', { token: APP_TOKEN });
+      });
+    }
+  }
+});
+
+// Open recording window (called from renderer to start voice recording)
+ipcMain.handle('open-recording-window', () => {
+  const { openRecordingWindow } = require('./tray.js');
+  const isDev = process.env.NODE_ENV === 'development' || !app.isPackaged;
+  openRecordingWindow(mainWindow, APP_TOKEN, isDev);
+});
+
+// Video processing IPC handlers
+
+// Write video data to a temporary file for FFmpeg processing
+ipcMain.handle('write-temp-file', async (_event, data, filename) => {
+  try {
+    const buffer = Buffer.from(data);
+    const tempPath = await videoProcessor.writeTempFile(buffer, filename);
+    return { success: true, path: tempPath };
+  } catch (error) {
+    console.error('[IPC] write-temp-file error:', error);
+    return { success: false, error: error.message };
+  }
+});
+
+// Process video: extract audio and generate thumbnail
+ipcMain.handle('process-video', async (event, videoPath) => {
+  try {
+    const result = await videoProcessor.processVideo(videoPath, (progress) => {
+      // Send progress updates to renderer
+      if (mainWindow && !mainWindow.isDestroyed()) {
+        event.sender.send('video-process-progress', progress);
+      }
+    });
+
+    // Read the audio and thumbnail files as buffers for upload
+    const audioBuffer = await videoProcessor.readFile(result.audioPath);
+    let thumbnailBuffer = null;
+    if (result.thumbnailPath) {
+      thumbnailBuffer = await videoProcessor.readFile(result.thumbnailPath);
+    }
+
+    // Clean up temp files (audio and thumbnail)
+    await videoProcessor.deleteTempFile(result.audioPath);
+    if (result.thumbnailPath) {
+      await videoProcessor.deleteTempFile(result.thumbnailPath);
+    }
+
+    return {
+      success: true,
+      audio: audioBuffer,
+      thumbnail: thumbnailBuffer,
+    };
+  } catch (error) {
+    console.error('[IPC] process-video error:', error);
+    return { success: false, error: error.message };
+  }
+});
+
+// Delete a temporary file
+ipcMain.handle('delete-temp-file', async (_event, filePath) => {
+  try {
+    await videoProcessor.deleteTempFile(filePath);
+    return { success: true };
+  } catch (error) {
+    console.error('[IPC] delete-temp-file error:', error);
+    return { success: false, error: error.message };
+  }
+});

app/electron/preload.js

@@ -46,4 +46,20 @@ contextBridge.exposeInMainWorld('electronAPI', {
     ipcRenderer.removeAllListeners('update-downloaded');
   },
   installUpdate: () => ipcRenderer.invoke('install-update'),
+  // Open main window (for recording popup to trigger unlock)
+  openMainWindow: () => ipcRenderer.invoke('open-main-window'),
+  // Open recording window (for voice memo from main app)
+  openRecordingWindow: () => ipcRenderer.invoke('open-recording-window'),
+  // Send recording state to main process (for blur handler)
+  setRecordingState: (isRecording) => ipcRenderer.send('recording-state-changed', isRecording),
+  // Video processing APIs (native FFmpeg)
+  writeTempFile: (data, filename) => ipcRenderer.invoke('write-temp-file', data, filename),
+  processVideo: (videoPath) => ipcRenderer.invoke('process-video', videoPath),
+  deleteTempFile: (path) => ipcRenderer.invoke('delete-temp-file', path),
+  onVideoProcessProgress: (callback) => {
+    ipcRenderer.on('video-process-progress', (_, data) => callback(data));
+  },
+  removeVideoProcessListeners: () => {
+    ipcRenderer.removeAllListeners('video-process-progress');
+  },
 });

app/electron/tray.js

@@ -0,0 +1,191 @@
+const { Tray, Menu, BrowserWindow, nativeImage, app, ipcMain } = require('electron');
+const path = require('path');
+
+let tray = null;
+let recordingWindow = null;
+let isRecording = false;
+
+// Listen for recording state changes from renderer
+ipcMain.on('recording-state-changed', (event, recording) => {
+  isRecording = recording;
+});
+
+/**
+ * Create the system tray with context menu
+ */
+function createTray(mainWindow, appToken, isDev) {
+  let icon;
+
+  try {
+    if (process.platform === 'darwin') {
+      // macOS: use template icon (auto-adapts to light/dark menu bar)
+      const templatePath = path.join(__dirname, '../public/icons/tray-iconTemplate.png');
+      icon = nativeImage.createFromPath(templatePath);
+      icon.setTemplateImage(true);
+    } else {
+      // Windows/Linux: use colored icon at 32x32
+      const iconPath = path.join(__dirname, '../public/icons/think-os-agent.png');
+      icon = nativeImage.createFromPath(iconPath).resize({ width: 32, height: 32 });
+    }
+  } catch (err) {
+    console.error('Failed to load tray icon:', err);
+    icon = nativeImage.createEmpty();
+  }
+
+  tray = new Tray(icon);
+
+  const isMac = process.platform === 'darwin';
+
+  const contextMenu = Menu.buildFromTemplate([
+    {
+      label: 'Record Voice Note',
+      accelerator: isMac ? 'Cmd+Shift+R' : 'Ctrl+Shift+R',
+      click: () => openRecordingWindow(mainWindow, appToken, isDev),
+    },
+    {
+      label: 'Open Think',
+      accelerator: isMac ? 'Cmd+O' : 'Ctrl+O',
+      click: () => {
+        const win = BrowserWindow.getAllWindows().find(w => w !== recordingWindow);
+        if (win) {
+          win.show();
+          win.focus();
+        } else {
+          app.emit('activate');
+        }
+      },
+    },
+    { type: 'separator' },
+    {
+      label: isMac ? 'Quit Think' : 'Exit',
+      accelerator: isMac ? 'Cmd+Q' : undefined,
+      click: () => app.quit(),
+    },
+  ]);
+
+  tray.setToolTip('Think');
+  tray.setContextMenu(contextMenu);
+
+  // On macOS, clicking the tray icon opens the context menu
+  // On Windows, single-click opens recording, double-click opens app
+  if (process.platform !== 'darwin') {
+    tray.on('click', () => openRecordingWindow(mainWindow, appToken, isDev));
+    tray.on('double-click', () => {
+      const win = BrowserWindow.getAllWindows().find(w => w !== recordingWindow);
+      if (win) {
+        win.show();
+        win.focus();
+      } else {
+        app.emit('activate');
+      }
+    });
+  }
+}
+
+/**
+ * Open the compact recording popup window
+ */
+function openRecordingWindow(mainWindow, appToken, isDev) {
+  // If recording window already exists and is visible, focus it
+  if (recordingWindow && !recordingWindow.isDestroyed()) {
+    recordingWindow.focus();
+    return;
+  }
+
+  // Get the position for the popup (near tray on macOS, center on Windows)
+  let x, y;
+  if (tray) {
+    const trayBounds = tray.getBounds();
+    const windowWidth = 320;
+    const windowHeight = 240;
+
+    if (process.platform === 'darwin') {
+      // Position below the tray icon on macOS
+      x = Math.round(trayBounds.x + (trayBounds.width / 2) - (windowWidth / 2));
+      y = Math.round(trayBounds.y + trayBounds.height + 4);
+    } else {
+      // Position above the tray icon on Windows (taskbar is at bottom)
+      x = Math.round(trayBounds.x + (trayBounds.width / 2) - (windowWidth / 2));
+      y = Math.round(trayBounds.y - windowHeight - 4);
+    }
+  }
+
+  recordingWindow = new BrowserWindow({
+    width: 320,
+    height: 240,
+    x,
+    y,
+    frame: false,
+    transparent: false,
+    backgroundColor: '#1c1c1e',
+    resizable: false,
+    skipTaskbar: true,
+    alwaysOnTop: true,
+    webPreferences: {
+      preload: path.join(__dirname, 'preload.js'),
+      contextIsolation: true,
+      nodeIntegration: false,
+    },
+  });
+
+  // Load the recording page
+  if (isDev) {
+    recordingWindow.loadURL('http://localhost:5173/#/recording');
+  } else {
+    recordingWindow.loadFile(path.join(__dirname, '../dist/index.html'), {
+      hash: '/recording',
+    });
+  }
+
+  // Send the app token to the recording window once loaded
+  recordingWindow.webContents.once('did-finish-load', () => {
+    if (appToken) {
+      recordingWindow.webContents.send('backend-ready', { token: appToken });
+    }
+  });
+
+  // Close when clicking outside (blur event)
+  recordingWindow.on('blur', () => {
+    // Small delay to allow for click handling
+    setTimeout(() => {
+      if (recordingWindow && !recordingWindow.isDestroyed()) {
+        // Don't close if user is actively recording (state tracked via IPC)
+        if (!isRecording) {
+          recordingWindow.close();
+        }
+      }
+    }, 100);
+  });
+
+  recordingWindow.on('closed', () => {
+    recordingWindow = null;
+    isRecording = false; // Reset state when window closes
+  });
+}
+
+/**
+ * Close the recording window
+ */
+function closeRecordingWindow() {
+  if (recordingWindow && !recordingWindow.isDestroyed()) {
+    recordingWindow.close();
+    recordingWindow = null;
+  }
+}
+
+/**
+ * Destroy the tray icon
+ */
+function destroyTray() {
+  if (tray) {
+    tray.destroy();
+    tray = null;
+  }
+}
+
+module.exports = {
+  createTray,
+  openRecordingWindow,
+  closeRecordingWindow,
+  destroyTray,
+};