diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef0363c7..cacaa2de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,7 @@ name: Build on: push: tags: ['v*'] - branches: ['*'] + branches: ['**'] workflow_dispatch: env: @@ -157,6 +157,12 @@ jobs: working-directory: app env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Azure Trusted Signing (Windows). Consumed by electron-builder's + # win.azureSignOptions via the Entra ID EnvironmentCredential. + # Unused on Linux/macOS builds. + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} run: yarn run ${{ matrix.build_cmd }} --publish never ${{ matrix.arch == 'arm64' && '--arm64' || '' }} - name: Upload Electron build diff --git a/.gitignore b/.gitignore index 289bd787..5e0be942 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ db/ **/.venv** **/venv** **/db/* +# ...but app/src/main/db is source (SQLite schema/migrations), not a data dir. +!app/src/main/db/ +!app/src/main/db/** **/.yarn/* !**/.yarn/releases !**/.yarn/patches @@ -42,4 +45,8 @@ db/ !**/.yarn/versions **/resources/ **/qdrant_storage -.qdrant-initialized \ No newline at end of file +.qdrant-initialized +**/__pycache__/ +*.pyc +*.pyo +**/.claude/** \ No newline at end of file diff --git a/app/.env.example b/app/.env.example index 83665c93..78922bcf 100644 --- a/app/.env.example +++ b/app/.env.example @@ -8,9 +8,25 @@ VITE_GRAPH_API_KEY= VITE_ARBITRUM_SEPOLIA_RPC_URL= +# ── Network / wallet (see src/renderer/src/lib/network.ts) ──────────────────── +# Active chain + USDC contracts. Defaults to arbitrumSepolia (testnet) on every +# build until we deploy on Arbitrum One. Set to override (e.g. arbitrum). +# VITE_NETWORK=arbitrumSepolia # or: arbitrum +# VITE_ARBITRUM_RPC_URL= # mainnet RPC (defaults to public arb1 endpoint) + # ── Agent ──────────────────────────────────────────────────────────────────── VITE_USE_AGENT=true +# ── Privy (auth / embedded wallets) ────────────────────────────────────────── +# App ID from the Privy dashboard (used in src/renderer/src/main.tsx). +VITE_PRIVY_APP_ID= + +# ── Bug reports (in-app "report a problem") ────────────────────────────────── +# URL of the serverless proxy that files GitHub issues (it holds the GH token so +# we never ship it in the app — see tools/bug-report-worker/). Leave blank and +# the reporter falls back to opening a prefilled GitHub issue in the browser. +VITE_BUGREPORT_ENDPOINT= + # ── Spotify (playback / OAuth PKCE) ────────────────────────────────────────── VITE_SPOTIFY_CLIENT_ID= VITE_SPOTIFY_CLIENT_SECRET= diff --git a/app/README.md b/app/README.md index e56a8a19..e13b3c96 100644 --- a/app/README.md +++ b/app/README.md @@ -13,11 +13,12 @@ An Electron application with React and TypeScript Requirements: **Node 22** (Corepack enables Yarn 4), **Python 3.12+**, and `tar` on your PATH. `ffmpeg` is optional (audio extraction). ```bash -yarn install # deps + native-module rebuild -yarn dev # first run auto-provisions the dev backend, then launches +yarn install +yarn setup +yarn dev ``` -That's the whole flow. The first `yarn dev` runs a one-time setup (`scripts/setup.mjs`, also available as `yarn setup`) that provisions everything a fresh clone is missing, **per-OS**: +That's the whole flow. The first `yarn dev` runs a one-time setup that provisions everything a fresh clone is missing, **per-OS**: - **`.env`** — copied from `.env.example`. Fill in real values to enable auth / Firebase / Spotify / catalog features; the app still launches with blanks. - **yt-dlp** → `resources/bin//`. diff --git a/app/electron-builder.yml b/app/electron-builder.yml index 10292e06..42594dfc 100644 --- a/app/electron-builder.yml +++ b/app/electron-builder.yml @@ -36,9 +36,19 @@ extraResources: win: executableName: SOND3R target: - - target: portable + - target: nsis arch: [x64] - artifactName: ${name}.${ext} + # Azure Trusted Signing. electron-builder installs the `TrustedSigning` + # PowerShell module on the runner and signs via `Invoke-TrustedSigning`, + # authenticating through Microsoft Entra ID env vars set in CI: + # AZURE_TENANT_ID / AZURE_CLIENT_ID / AZURE_CLIENT_SECRET (a service principal + # holding the "Trusted Signing Certificate Profile Signer" role). + # The four values below are NOT secrets and belong in source control. + azureSignOptions: + endpoint: https://scus.codesigning.azure.net/ + codeSigningAccountName: SOND3R + certificateProfileName: prod-signing + publisherName: Fangorn mac: executableName: SOND3R @@ -53,10 +63,16 @@ linux: appImage: artifactName: ${name}.${ext} nsis: + oneClick: false + allowToChangeInstallationDirectory: true artifactName: ${name}-${version}-setup.${ext} shortcutName: ${productName} uninstallDisplayName: ${productName} createDesktopShortcut: always + # Keep wallet/account + vector index data when the app is uninstalled. + # (false is the electron-builder default; set explicitly so a future edit + # doesn't silently start wiping users' crypto account data.) + deleteAppDataOnUninstall: false npmRebuild: false nodeGypRebuild: false detectUpdateChannel: false diff --git a/app/electron.vite.config.ts b/app/electron.vite.config.ts index 4ca3f7bc..36c4c8d2 100644 --- a/app/electron.vite.config.ts +++ b/app/electron.vite.config.ts @@ -6,7 +6,10 @@ export default defineConfig({ main: { build: { rollupOptions: { - external: ['dbus-final', 'x11'] + // better-sqlite3 ships a native .node binary (can't be bundled); + // music-metadata is ESM-only and loaded via dynamic import(). Keep both + // external so they're required from node_modules at runtime. + external: ['dbus-final', 'x11', 'better-sqlite3', 'music-metadata'] } } }, diff --git a/app/package.json b/app/package.json index b5efc166..a3a2ef14 100644 --- a/app/package.json +++ b/app/package.json @@ -12,7 +12,8 @@ "typecheck:web": "tsc --noEmit -p tsconfig.web.json --composite false", "typecheck": "yarn typecheck:node && npm run typecheck:web", "start": "electron-vite preview", - "dev": "electron-vite dev", + "dev": "node scripts/dev.mjs", + "clean": "node scripts/clean.mjs", "build": "yarn typecheck && electron-vite build", "setup": "node scripts/setup.mjs", "predev": "node scripts/setup.mjs", @@ -36,6 +37,7 @@ "@semaphore-protocol/identity": "^4.14.2", "@tanstack/react-query": "^5", "@whatwg-node/fetch": "^0.10.13", + "better-sqlite3": "^12.11.1", "boring-avatars": "^2.0.4", "d3": "^7.9.0", "electron-updater": "^6.3.9", @@ -44,6 +46,7 @@ "graphql": "^16.13.2", "mime": "^4.1.0", "mime-types": "^3.0.2", + "music-metadata": "^11.13.0", "react": "^19.2.1", "react-dom": "^19.2.1", "viem": "^2.47.1", @@ -56,6 +59,7 @@ "@electron/rebuild": "^4.0.4", "@eslint/js": "^9.39.1", "@graphprotocol/client-cli": "^3.0.7", + "@types/better-sqlite3": "^7.6.13", "@types/jsmediatags": "^3.9.6", "@types/mime-types": "^3.0.1", "@types/node": "^24.10.1", diff --git a/app/scripts/build-snapshot-manifest.mjs b/app/scripts/build-snapshot-manifest.mjs new file mode 100644 index 00000000..f9aa437a --- /dev/null +++ b/app/scripts/build-snapshot-manifest.mjs @@ -0,0 +1,112 @@ +#!/usr/bin/env node +/** + * build-snapshot-manifest.mjs — regenerate src/main/snapshot-manifest.json + * + * The catalog snapshot .gz (~14 GB) is too large to pin to Pinata as one object, + * so it's uploaded in parts (`pinata upload-split`), each part pinned separately + * and named `.partNNN`. A public IPFS gateway can only fetch by CID, and + * mapping those part NAMES to their CIDs needs Pinata's authenticated API — which + * the shipped app must NOT carry. So we resolve name->CID ONCE, here, and commit + * the resulting CID list. At runtime the app just reads that list and streams the + * parts back-to-back from the gateway (see loadManifest/downloadGz in index.ts). + * + * Usage: + * PINATA_JWT=... node scripts/build-snapshot-manifest.mjs + * # or put PINATA_JWT in app/.env (loaded automatically) + * + * The JWT must belong to the account that holds the parts. Writes the manifest + * to src/main/snapshot-manifest.json next to index.ts. + */ +import 'dotenv/config' +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' + +const prefix = process.argv[2] +if (!prefix) { + console.error('Usage: node scripts/build-snapshot-manifest.mjs ') + process.exit(1) +} +const jwt = process.env.PINATA_JWT +if (!jwt) { + console.error('Error: PINATA_JWT not set (export it or add it to app/.env).') + process.exit(1) +} + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const outPath = path.join(__dirname, '..', 'src', 'main', 'snapshot-manifest.json') + +// Page through every file whose name starts with `prefix`. +async function* listAll(namePrefix) { + let token + for (;;) { + const url = new URL('https://api.pinata.cloud/v3/files/public') + url.searchParams.set('name', namePrefix) + url.searchParams.set('limit', '100') + if (token) url.searchParams.set('pageToken', token) + const res = await fetch(url, { headers: { Authorization: `Bearer ${jwt}` } }) + if (!res.ok) throw new Error(`Pinata list failed: ${res.status} ${await res.text()}`) + const { data } = await res.json() + for (const f of data?.files ?? []) yield f + if (!data?.next_page_token) break + token = data.next_page_token + } +} + +// Parse the `.partNNN` suffix. The zero-pad WIDTH matters: one prefix can collect +// parts from more than one upload run (e.g. an aborted run that padded +// differently), so we key by width and keep only the complete contiguous set. +function parsePart(name) { + const m = /\.part(\d+)$/.exec(name ?? '') + return m ? { index: parseInt(m[1], 10), width: m[1].length } : null +} +function isComplete(arr) { + const seen = new Set(arr.map((p) => p.index)) + if (seen.size !== arr.length) return false + for (let i = 0; i < arr.length; i++) if (!seen.has(i)) return false + return true +} + +const byWidth = new Map() +let scanned = 0 +for await (const f of listAll(prefix)) { + const p = parsePart(f.name) + if (!p) continue + if (!byWidth.has(p.width)) byWidth.set(p.width, []) + byWidth.get(p.width).push({ index: p.index, name: f.name, id: f.id, cid: f.cid, size: f.size }) + process.stdout.write(`\r scanned ${++scanned} part object(s)`) +} +process.stdout.write('\n') + +if (byWidth.size > 1) { + console.log(`Note: parts span ${byWidth.size} padding widths (overlapping upload runs):`) + for (const [w, arr] of byWidth) { + console.log(` width ${w}: ${arr.length} part(s)${isComplete(arr) ? ' (complete)' : ' (incomplete — ignored)'}`) + } +} +const complete = [...byWidth.values()].filter(isComplete) +if (!complete.length) { + console.error('❌ No complete contiguous part set (0..n-1) found for this prefix.') + process.exit(1) +} +// Prefer the largest complete set (the real full upload, not a short aborted one). +complete.sort((a, b) => b.length - a.length) +const parts = complete[0].slice().sort((a, b) => a.index - b.index) + +const missing = parts.filter((p) => !p.cid) +if (missing.length) { + console.error(`❌ ${missing.length} part(s) have no cid — aborting.`) + process.exit(1) +} + +const manifest = { + name: prefix, + total_size: parts.reduce((s, p) => s + (p.size || 0), 0), + part_size: parts.reduce((m, p) => Math.max(m, p.size || 0), 0), + parts, + rebuilt: new Date().toISOString(), +} +fs.writeFileSync(outPath, JSON.stringify(manifest, null, 2) + '\n') +console.log(`✅ Wrote ${parts.length} part(s) → ${path.relative(process.cwd(), outPath)}`) +console.log(` total_size = ${(manifest.total_size / 1024 ** 3).toFixed(2)} GB`) +console.log(' Remember to update sha256 in resolveSnapshot() if the catalog changed.') diff --git a/app/scripts/clean.mjs b/app/scripts/clean.mjs new file mode 100644 index 00000000..13dfe884 --- /dev/null +++ b/app/scripts/clean.mjs @@ -0,0 +1,48 @@ +#!/usr/bin/env node +// `yarn clean` — wipe the app's persisted state so the next `yarn dev` behaves +// like a first-ever launch: re-downloads + recovers the Qdrant catalog snapshot, +// rebuilds the warmup indexes, starts with an empty local DB, and a fresh login. +// +// This removes only Electron's userData directory (per-OS app-data location). +// Dev build assets — the Qdrant engine binary, yt-dlp, the Python venv, .env — +// live in resources/ and vectordb/ and are provisioned by setup.mjs, NOT here, +// so a clean stays fast and doesn't re-download the toolchain. +// +// Close the app (and its Qdrant/python sidecars) before running this. +import { rmSync, existsSync, readFileSync } from 'node:fs' +import { join, dirname } from 'node:path' +import { homedir } from 'node:os' +import { fileURLToPath } from 'node:url' + +const APP = join(dirname(fileURLToPath(import.meta.url)), '..') +// Electron names userData after app.getName(), which defaults to package.json name. +const appName = JSON.parse(readFileSync(join(APP, 'package.json'), 'utf8')).name + +if (!appName) { + console.error('✗ could not read "name" from package.json — refusing to guess the userData path.') + process.exit(1) +} + +function userDataDir(name) { + const home = homedir() + switch (process.platform) { + case 'win32': + return join(process.env.APPDATA || join(home, 'AppData', 'Roaming'), name) + case 'darwin': + return join(home, 'Library', 'Application Support', name) + default: + return join(process.env.XDG_CONFIG_HOME || join(home, '.config'), name) + } +} + +const dir = userDataDir(appName) + +if (existsSync(dir)) { + rmSync(dir, { recursive: true, force: true }) + console.log(`✓ removed app state → ${dir}`) + console.log( + ' next `yarn dev` runs the first-launch flow: snapshot download + warmup, empty local DB, fresh login.' + ) +} else { + console.log(`• nothing to clean — no app state at ${dir}`) +} diff --git a/app/scripts/dev.mjs b/app/scripts/dev.mjs new file mode 100644 index 00000000..26664cb3 --- /dev/null +++ b/app/scripts/dev.mjs @@ -0,0 +1,49 @@ +// Dev launcher with a Linux-only sandbox fallback. +// +// On Ubuntu 24.04 and *-lowlatency kernels, AppArmor restricts unprivileged +// user namespaces (kernel.apparmor_restrict_unprivileged_userns=1). Electron's +// Chromium sandbox needs such a namespace to start; when it's blocked the +// sandboxed syscalls fail and Electron dies before DevTools attaches, with a +// misleading fatal error like: +// FATAL:platform_shared_memory_region_posix.cc ... /dev/shm ... No such process (3) +// +// When we detect that restriction we disable the sandbox for `yarn dev` only. +// Production builds never run this script, so they keep the sandbox. On macOS, +// Windows, or a Linux box without the restriction, nothing changes. +import { spawn } from 'node:child_process' +import { readFileSync } from 'node:fs' + +const env = { ...process.env } + +function unprivilegedUsernsBlocked() { + if (process.platform !== 'linux') return false + try { + const flag = readFileSync( + '/proc/sys/kernel/apparmor_restrict_unprivileged_userns', + 'utf8' + ).trim() + return flag === '1' + } catch { + return false + } +} + +if (unprivilegedUsernsBlocked() && !('ELECTRON_DISABLE_SANDBOX' in env)) { + env.ELECTRON_DISABLE_SANDBOX = '1' + console.log( + '[dev] AppArmor restricts unprivileged user namespaces; disabling the ' + + 'Electron sandbox for dev (ELECTRON_DISABLE_SANDBOX=1). Set ' + + 'ELECTRON_DISABLE_SANDBOX explicitly to override.' + ) +} + +const child = spawn('yarn', ['electron-vite', 'dev', ...process.argv.slice(2)], { + stdio: 'inherit', + env, + shell: process.platform === 'win32' +}) + +child.on('exit', (code, signal) => { + if (signal) process.kill(process.pid, signal) + else process.exit(code ?? 0) +}) diff --git a/app/src/main/bug-report.ts b/app/src/main/bug-report.ts new file mode 100644 index 00000000..2164ca7e --- /dev/null +++ b/app/src/main/bug-report.ts @@ -0,0 +1,216 @@ +// ─── Bug reports ────────────────────────────────────────────────────────────── +// In-app "report a problem" plumbing for the early-access preview. The renderer +// collects a short description; main attaches diagnostics (app version, OS, +// recent logs) and files a GitHub issue. +// +// Delivery is deliberately keyless. `.env` ships inside the packaged app and is +// user-editable (see .env.example), so we can't embed a GitHub token. Instead we +// POST to a small serverless proxy that holds the token (VITE_BUGREPORT_ENDPOINT +// → see tools/bug-report-worker). If that endpoint isn't configured or the POST +// fails, we fall back to opening a prefilled GitHub "New issue" page in the +// browser, so the user is never left at a dead end. + +import { app, ipcMain, net, shell } from 'electron' +import os from 'os' +import path from 'path' +import fs from 'fs' + +const REPO = 'fangorn-network/sonder' +const LABELS = ['bug', 'early-access'] + +// VITE_* env is injected into the main process too (electron-vite); same pattern +// as the RPC URL in index.ts. Undefined in dev / before the proxy is deployed. +const ENDPOINT = (import.meta as unknown as { env?: Record }) + .env?.VITE_BUGREPORT_ENDPOINT + +// ── Ring buffer of recent main-process console output ────────────────────────── +// The app already logs richly with `[tag]` prefixes ([boot], [qdrant], [py], …). +// We tee those into a bounded buffer so a report can carry the last moments +// before something went wrong, without writing anything to disk or phoning home. +const MAX_LOG_LINES = 300 +const ring: string[] = [] +let captureInstalled = false + +export function installLogCapture(): void { + if (captureInstalled) return + captureInstalled = true + for (const method of ['log', 'info', 'warn', 'error'] as const) { + const original = console[method].bind(console) + console[method] = (...args: unknown[]) => { + try { + const line = args.map(a => (typeof a === 'string' ? a : safeStr(a))).join(' ') + ring.push(`${new Date().toISOString()} [${method}] ${line}`) + if (ring.length > MAX_LOG_LINES) ring.shift() + } catch { + /* logging must never throw */ + } + original(...args) + } + } +} + +function safeStr(v: unknown): string { + try { + return typeof v === 'object' ? JSON.stringify(v) : String(v) + } catch { + return String(v) + } +} + +// ── Diagnostics ──────────────────────────────────────────────────────────────── +export interface Diagnostics { + appVersion: string + platform: string + arch: string + os: string + electron: string + chrome: string + node: string +} + +export function collectDiagnostics(): Diagnostics { + return { + appVersion: app.getVersion(), + platform: process.platform, + arch: process.arch, + os: `${os.type()} ${os.release()}`, + electron: process.versions.electron, + chrome: process.versions.chrome, + node: process.versions.node, + } +} + +// Last ~8KB of the Python backend's stderr (the one log we already persist). +function pyStderrTail(maxBytes = 8000): string { + try { + const p = path.join(app.getPath('userData'), 'py-stderr.log') + const { size } = fs.statSync(p) + const start = Math.max(0, size - maxBytes) + const fd = fs.openSync(p, 'r') + try { + const buf = Buffer.alloc(size - start) + fs.readSync(fd, buf, 0, buf.length, start) + return buf.toString('utf8') + } finally { + fs.closeSync(fd) + } + } catch { + return '' + } +} + +function collectLogTail(): string { + return [ + '── main process ──', + ring.join('\n') || '(no captured logs)', + '', + '── python backend (py-stderr.log tail) ──', + pyStderrTail() || '(none)', + ].join('\n') +} + +// ── Report assembly ────────────────────────────────────────────────────────────── +export interface BugReportInput { + description: string + expected?: string + email?: string + userId?: string +} + +export interface BugReportResult { + ok: boolean + via?: 'api' | 'browser' + url?: string + error?: string +} + +function buildTitle(description: string): string { + const first = description.trim().split('\n')[0].slice(0, 80) + return first ? `[bug] ${first}` : '[bug] (no description)' +} + +function buildBody(input: BugReportInput, diag: Diagnostics, logTail: string): string { + const lines = [ + '### What happened', + input.description.trim() || '_(none given)_', + '', + ] + if (input.expected?.trim()) { + lines.push('### What I expected', input.expected.trim(), '') + } + lines.push( + '### Environment', + `- App: v${diag.appVersion}`, + `- OS: ${diag.os} (${diag.platform}/${diag.arch})`, + `- Electron ${diag.electron} · Chrome ${diag.chrome} · Node ${diag.node}`, + '', + '### Reporter', + `- Contact: ${input.email?.trim() || '_(not provided)_'}`, + `- Account: ${input.userId || '_(signed out)_'}`, + '', + '### Recent logs', + '```', + logTail, + '```', + '', + '_Filed from the in-app reporter (early access preview)._', + ) + return lines.join('\n') +} + +async function fileViaProxy( + title: string, + body: string, +): Promise { + if (!ENDPOINT) return null + try { + const res = await net.fetch(ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ title, body, labels: LABELS }), + }) + if (!res.ok) { + console.error('[bug-report] proxy responded', res.status) + return null + } + const data = (await res.json().catch(() => ({}))) as Record + const url = (data.url ?? data.html_url) as string | undefined + return { ok: true, via: 'api', url } + } catch (e) { + console.error('[bug-report] proxy request failed:', e) + return null + } +} + +// Browser fallback — open GitHub's prefilled "New issue" page. URLs are length +// bounded (encodeURIComponent inflates the payload), so the log tail is trimmed +// hard here; the full tail only travels over the proxy path. +function fileViaBrowser(title: string, body: string): BugReportResult { + const trimmed = + body.length > 4000 ? `${body.slice(0, 4000)}\n\n… (logs truncated — see app for full output)` : body + const url = + `https://github.com/${REPO}/issues/new` + + `?title=${encodeURIComponent(title)}` + + `&body=${encodeURIComponent(trimmed)}` + + `&labels=${encodeURIComponent(LABELS.join(','))}` + shell.openExternal(url) + return { ok: true, via: 'browser' } +} + +export async function submitBugReport(input: BugReportInput): Promise { + if (!input?.description?.trim()) { + return { ok: false, error: 'A description is required.' } + } + const diag = collectDiagnostics() + const body = buildBody(input, diag, collectLogTail()) + const title = buildTitle(input.description) + + const viaProxy = await fileViaProxy(title, body) + if (viaProxy) return viaProxy + return fileViaBrowser(title, body) +} + +export function registerBugReportIpc(): void { + ipcMain.handle('bug:diagnostics', () => collectDiagnostics()) + ipcMain.handle('bug:submit', async (_e, input: BugReportInput) => submitBugReport(input)) +} diff --git a/app/src/main/db/index.ts b/app/src/main/db/index.ts new file mode 100644 index 00000000..394829cc --- /dev/null +++ b/app/src/main/db/index.ts @@ -0,0 +1,110 @@ +/** + * main/db/index.ts + * + * Process-wide SQLite handle (better-sqlite3) for local app data. Opened lazily + * on first use and migrated idempotently, so importing this module is cheap and + * the DB file isn't created until something actually reads/writes. Currently + * backs the local-music metadata library (see main/local/catalog.ts). + * + * better-sqlite3 ships a native binary; it's kept external from the main bundle + * (electron.vite.config.ts) and rebuilt against Electron's ABI by the + * `electron-builder install-app-deps` postinstall hook. + */ + +import Database from 'better-sqlite3' +import { app } from 'electron' +import path from 'path' + +let db: Database.Database | null = null + +/** Open (once) and return the shared database handle. */ +export function getDb(): Database.Database { + if (db) return db + const file = path.join(app.getPath('userData'), 'sond3r.db') + db = new Database(file) + db.pragma('journal_mode = WAL') + migrate(db) + return db +} + +/** Idempotent schema setup — safe to run on every open. */ +function migrate(d: Database.Database): void { + d.exec(` + CREATE TABLE IF NOT EXISTS local_track_meta ( + local_id TEXT PRIMARY KEY, + path TEXT NOT NULL, + schema_version INTEGER NOT NULL DEFAULT 1, + track_id TEXT, + isrc_code TEXT, + title TEXT NOT NULL, + by_artist TEXT NOT NULL, + album_name TEXT, + date_published TEXT, + duration_ms INTEGER, + contributors TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_ltm_by_artist ON local_track_meta(by_artist); + CREATE INDEX IF NOT EXISTS idx_ltm_album ON local_track_meta(album_name); + + -- Semantic taxonomy tags for a local track (genres / moods / themes / + -- contexts), shaped to schemas/TrackTaxonomySchema.json. Stored locally only; + -- this is the source that would feed embedding generation once that's enabled + -- for users, but nothing here is published or embedded today. Keyed by the + -- local file id (same as local_track_meta); track_id mirrors the labeled + -- track's canonical id so the row is embedding-ready. See + -- main/local/taxonomy.ts. + CREATE TABLE IF NOT EXISTS local_track_tags ( + local_id TEXT PRIMARY KEY, + schema_version INTEGER NOT NULL DEFAULT 1, + track_id TEXT, + genres TEXT NOT NULL DEFAULT '[]', + moods TEXT NOT NULL DEFAULT '[]', + themes TEXT NOT NULL DEFAULT '[]', + contexts TEXT NOT NULL DEFAULT '[]', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + + -- User-supplied album / artist artwork, stored inline. Keyed by scope + -- ('album' | 'artist') + a normalized key (see main/local/art.ts). + CREATE TABLE IF NOT EXISTS local_art ( + scope TEXT NOT NULL, + art_key TEXT NOT NULL, + mime TEXT NOT NULL, + data BLOB NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope, art_key) + ); + + -- "Favorited" artists, albums, and songs. kind is 'track' | 'artist' | + -- 'album'; ref is the renderer-built reference for that thing (a local file + -- id for tracks, an artist name for artists, "artist album" for albums). + -- See main/local/favorites.ts. + CREATE TABLE IF NOT EXISTS local_favorites ( + kind TEXT NOT NULL, + ref TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (kind, ref) + ); + + -- User playlists and their ordered tracks. Tracks reference local files by + -- their file id (local_id); rows whose file is no longer present are simply + -- skipped when the renderer resolves them. See main/local/playlists.ts. + CREATE TABLE IF NOT EXISTS local_playlists ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS local_playlist_tracks ( + playlist_id TEXT NOT NULL, + local_id TEXT NOT NULL, + position INTEGER NOT NULL, + added_at INTEGER NOT NULL, + PRIMARY KEY (playlist_id, local_id) + ); + CREATE INDEX IF NOT EXISTS idx_lpt_playlist ON local_playlist_tracks(playlist_id, position); + `) +} diff --git a/app/src/main/index.ts b/app/src/main/index.ts index c37960a8..74aa0cfb 100644 --- a/app/src/main/index.ts +++ b/app/src/main/index.ts @@ -1,10 +1,12 @@ import { app, shell, BrowserWindow, ipcMain, net, session } from 'electron' import { join } from 'path' import { electronApp, optimizer, is } from '@electron-toolkit/utils' -import { spawn } from 'child_process' +import { spawn, execSync } from 'child_process' +import { Socket as NetSocket } from 'net' import path from 'path' import fs, { createWriteStream, createReadStream } from 'fs' import http from 'http' +import os from 'os' import zlib from 'zlib' import crypto from 'crypto' import { Transform } from 'stream' @@ -20,6 +22,10 @@ import { ToolboxConfigManager } from './agent/toolbox-config-manager' import { registerSpotifyAuth, handleSpotifyCallback } from './spotify/SpotifyAuth' import { registerPlaybackIpc } from './playback/ipc' import { registerLocalMusicIpc } from './local/ipc' +import { trackCover } from './local/deezer' +import { installLogCapture, registerBugReportIpc } from './bug-report' +import snapshotManifest from './snapshot-manifest.json' +// import snapshotManifest from './small.json' // ─── Stream cache ───────────────────────────────────────────────────────────── interface StreamEntry { @@ -242,7 +248,19 @@ function resolveYt(query: string): Promise { let qdrantProc: ReturnType | null = null let pyProcess: ReturnType | null = null +// Liveness signals for the query server's slow cold start, watched by +// waitForHealth. `pyLastOutput` is bumped on every stdout/stderr chunk (the +// model download + collection load stream progress), so a slow-but-working boot +// keeps the wait alive; `pyExitCode` flips non-null if the process dies, so the +// wait can fail fast instead of sitting out the full timeout. +let pyLastOutput = 0 +let pyExitCode: number | null = null +let pyExited = false let rendererServer: http.Server | null = null +// Set once we're shutting down so the qdrant exit handler doesn't try to respawn +// a sidecar we just deliberately killed. +let quitting = false +let qdrantRetries = 0 if (app.isPackaged) { dotenv.config({ path: path.join(process.resourcesPath, '.env') }) @@ -254,16 +272,74 @@ const SNAPSHOT_GATEWAY = "https://green-reasonable-heron-957.mypinata.cloud/ipfs // (import.meta as any).env?.VITE_PINATA_GATEWAY ?? 'https://gateway.pinata.cloud/ipfs' // For now a pinned constant. Later: read the latest sond3r.embeddings.snapshot -// record off Fangorn and return its cid + checksum, so a new catalog ships -// without an app update. Keep this the single source of truth for the artifact. -async function resolveSnapshot(): Promise<{ cid: string; sha256: string }> { +// record off Fangorn and return its checksum, so a new catalog ships without an +// app update. Keep this the single source of truth for the artifact identity. +// +// The .gz is too big for a single Pinata object (14 GB), so it was split into N +// parts with `pinata upload-split`, each part pinned separately. The reassembled +// .gz exceeds Pinata's limits too, so the MANIFEST listing the part CIDs is NOT +// pinned — it ships as a bundled local resource (`resources/snapshot.manifest.json`, +// rebuilt with `embeddings/src/rebuild-manifest.mjs`). downloadGz reads that local +// manifest, then streams each part back-to-back into one .gz (the runtime +// equivalent of `cat ...part* > ...gz`). +// +// `sha256` (of the DECOMPRESSED .snapshot) is the catalog's identity: it gates +// the snapshot marker and keys the query server's cache, so a new catalog = a new +// sha256. ('' skips verification while testing.) +async function resolveSnapshot(): Promise<{ sha256: string }> { return { - cid: 'bafybeifn2ddof4aasmvg2mf3ufc67tkxphf2w5jqucmkev4newmf7okg3u', - // sha256 of the DECOMPRESSED .snapshot ('' skips verification while testing) - sha256: '957de00bfc8df0ab1388b7556889ef924c7a7136164389ce0db2d312b8f083dd', + sha256: '', } } +// One entry from the `parts` array of a `pinata upload-split` manifest. Only +// `index` (ordering) and `cid` (where to fetch the bytes) are load-bearing here; +// `size` lets us report accurate aggregate download progress. +interface SnapshotPart { + index: number + name: string + id: string + cid: string + size: number +} +interface SnapshotManifest { + name: string + total_size: number + part_size: number + parts: SnapshotPart[] + // Optional: the size of the reassembled .gz once decompressed. When the + // manifest builder records it we report it exactly; otherwise computeSnapshotInfo + // falls back to a ratio estimate (see SNAPSHOT_UNCOMPRESSED_RATIO). + uncompressed_size?: number +} + +// Rough multiplier from compressed (.gz) → uncompressed snapshot size, used only +// when the manifest has no `uncompressed_size`. The snapshot is mostly float32 +// vectors (high entropy, compresses poorly), so the inflation is modest. This is +// a deliberately conservative estimate — its job is to warn users it's a big +// download, not to be exact. +const SNAPSHOT_UNCOMPRESSED_RATIO = 1.25 + +// Load and validate the committed split-upload manifest (the part-CID list). +// `snapshot-manifest.json` is checked into this repo next to this file and bundled +// into the build automatically, so the app ships with the CIDs and never needs a +// Pinata key at runtime — it fetches each part from the public gateway by CID. +// When a new catalog ships, regenerate this file from the part names with +// `node scripts/build-snapshot-manifest.mjs ` (needs PINATA_JWT). +// Returns the parts sorted into index order and sanity-checked for completeness, +// so a partial or reordered manifest fails loudly here rather than producing a +// corrupt .gz. +function loadManifest(): SnapshotPart[] { + const manifest = snapshotManifest as SnapshotManifest + const parts = [...(manifest.parts ?? [])].sort((a, b) => a.index - b.index) + if (!parts.length) throw new Error('snapshot manifest has no parts') + parts.forEach((p, i) => { + if (p.index !== i) throw new Error(`snapshot manifest missing part ${i} (got index ${p.index})`) + if (!p.cid) throw new Error(`snapshot manifest part ${i} has no cid`) + }) + return parts +} + function qdrantBin(): string { const ext = process.platform === 'win32' ? '.exe' : '' if (app.isPackaged) return path.join(process.resourcesPath, 'qdrant', `qdrant${ext}`) @@ -271,27 +347,122 @@ function qdrantBin(): string { return path.join(app.getAppPath(), 'resources', 'qdrant', dir, `qdrant${ext}`) } +// Localhost ports owned exclusively by our sidecars: Qdrant HTTP/gRPC and the +// Python query server. If one is busy at boot it's an orphan from a previous run +// that didn't exit cleanly (a crash, a SIGKILL, or a `yarn dev` restart that +// raced its own teardown). A busy 6333 makes a freshly-spawned Qdrant die +// instantly with "Address already in use", which wedges the whole boot. +const SIDECAR_PORTS = [6333, 6334, 8080] + +const delay = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) + +// Is anything accepting connections on this localhost port? +function probePortInUse(port: number): Promise { + return new Promise((resolve) => { + const sock = new NetSocket() + const finish = (inUse: boolean): void => { sock.destroy(); resolve(inUse) } + sock.setTimeout(500) + sock.once('connect', () => finish(true)) + sock.once('timeout', () => finish(false)) + sock.once('error', () => finish(false)) // ECONNREFUSED ⇒ free + sock.connect(port, '127.0.0.1') + }) +} + +// Kill whatever is listening on `port` (best-effort, cross-platform). Only ever +// used to reclaim our own orphaned sidecars at boot. +function killPort(port: number): void { + try { + if (process.platform === 'win32') { + const out = execSync('netstat -ano -p tcp', { encoding: 'utf8' }) + const pids = new Set() + for (const line of out.split('\n')) { + if (line.includes(`:${port} `) && /LISTENING/i.test(line)) { + const pid = line.trim().split(/\s+/).pop() + if (pid && pid !== '0') pids.add(pid) + } + } + for (const pid of pids) { try { execSync(`taskkill /PID ${pid} /F /T`) } catch { /* gone */ } } + } else { + let pids: string[] = [] + try { + pids = execSync(`lsof -ti tcp:${port}`, { encoding: 'utf8' }).split('\n').map((s) => s.trim()).filter(Boolean) + } catch { /* lsof missing, or nothing on the port */ } + if (pids.length) { + for (const pid of pids) { try { process.kill(Number(pid), 'SIGKILL') } catch { /* gone */ } } + } else { + try { execSync(`fuser -k ${port}/tcp`) } catch { /* fuser missing, or nothing on the port */ } + } + } + } catch { /* best effort — startQdrant's retry is the backstop */ } +} + +// Pre-flight before spawning Qdrant: guarantee our sidecar ports are free. Give a +// still-tearing-down orphan a brief grace period to release the port on its own +// (the common restart-race case), then kill it, then wait for the OS to actually +// release the socket before we let Qdrant try to bind. +async function reclaimSidecarPorts(): Promise { + for (const port of SIDECAR_PORTS) { + if (!(await probePortInUse(port))) continue + console.warn(`[boot] port ${port} busy — a previous sidecar didn't exit cleanly`) + for (let i = 0; i < 5 && (await probePortInUse(port)); i++) await delay(200) // grace + if (await probePortInUse(port)) { + console.warn(`[boot] reclaiming port ${port}`) + killPort(port) + } + for (let i = 0; i < 25 && (await probePortInUse(port)); i++) await delay(200) // wait for release + if (await probePortInUse(port)) console.error(`[boot] port ${port} still busy after reclaim — Qdrant may fail to start`) + } +} + function startQdrant() { const storage = path.join(app.getPath('userData'), 'qdrant_storage') const snapshotsDir = path.join(app.getPath('userData'), 'qdrant_snapshots') fs.mkdirSync(storage, { recursive: true }) fs.mkdirSync(snapshotsDir, { recursive: true }) + // Cap the background optimizer/HNSW indexing that fires once after a snapshot + // recover. On first launch Qdrant rebuilds the index over the 10M-point + // collection, which otherwise pins every core at ~100% before the user has + // even searched (a relaunch is quiet + fast because the built index is already + // persisted in qdrant_storage — no recover, no re-optimize). server.py already + // sleeps idle OpenMP threads and caps its native pools to half the cores for + // exactly this reason; Qdrant is a separate Rust process those env vars don't + // touch, so it was the one thing left free to lock up the machine. A positive + // optimizer_cpu_budget caps the indexing job to that many CPUs — half the box, + // leaving the rest responsive. The index still builds, just without melting + // everything; it's a one-time cost per snapshot. + const optimizerCpus = String(Math.max(1, Math.floor(os.cpus().length / 2))) + qdrantProc = spawn(qdrantBin(), [], { + detached: process.platform !== 'win32', env: { ...process.env, QDRANT__STORAGE__STORAGE_PATH: storage, QDRANT__STORAGE__SNAPSHOTS_PATH: snapshotsDir, + QDRANT__STORAGE__PERFORMANCE__OPTIMIZER_CPU_BUDGET: optimizerCpus, QDRANT__SERVICE__HTTP_PORT: '6333', QDRANT__SERVICE__GRPC_PORT: '6334', QDRANT__SERVICE__HOST: '127.0.0.1', // localhost only, never exposed QDRANT__TELEMETRY_DISABLED: 'true', // no phone-home; matches the ethos }, }) + console.log(`[qdrant] starting (storage=${storage}, optimizer capped to ${optimizerCpus} cpus)`) + const spawnedAt = Date.now() qdrantProc.stdout?.on('data', (d) => console.log('[qdrant]', d.toString().trimEnd())) qdrantProc.stderr?.on('data', (d) => console.error('[qdrant]', d.toString().trimEnd())) qdrantProc.on('error', (err) => console.error('[qdrant] failed to start:', err)) - qdrantProc.on('exit', (code) => console.log('[qdrant] exited with code', code)) + qdrantProc.on('exit', (code) => { + console.log('[qdrant] exited with code', code) + // Died almost immediately and we're not shutting down → something grabbed the + // port between our pre-flight and bind (e.g. an orphan that outraced reclaim). + // Reclaim and respawn a few times before letting the boot surface the failure. + if (!quitting && code !== 0 && Date.now() - spawnedAt < 5000 && qdrantRetries < 3) { + qdrantRetries++ + console.warn(`[qdrant] died on startup; reclaiming ports and retrying (${qdrantRetries}/3)`) + void reclaimSidecarPorts().then(() => { if (!quitting) startQdrant() }) + } + }) } async function waitForReady(url: string, timeoutMs = 60000): Promise { @@ -306,23 +477,177 @@ async function waitForReady(url: string, timeoutMs = 60000): Promise { throw new Error(`timed out waiting for ${url}`) } +// Wait for the query server's /health, gated on process liveness rather than a +// fixed wall clock. The server doesn't bind /health until it has pulled the +// embedding model and mmapped the (10M-point) collection from the freshly +// recovered 15 GB snapshot — minutes on a cold disk. A plain waitForReady either +// aborts that healthy-but-slow boot (the old 60s default did exactly this) or, +// if its timeout is bumped blindly, hangs the splash for minutes when the server +// actually crashes. So instead: +// • fail fast the moment the process exits — a dead backend won't ever answer; +// • keep waiting as long as it's alive AND still emitting output (the model +// download / collection load stream progress), so slow boots aren't capped; +// • give up only if it goes quiet for `idleMs` (genuinely wedged), with a +// generous absolute `maxMs` backstop so we can't wait forever. +async function waitForHealth( + url: string, + { idleMs = 180000, maxMs = 900000 }: { idleMs?: number; maxMs?: number } = {}, +): Promise { + const start = Date.now() + while (Date.now() - start < maxMs) { + if (pyExited) { + throw new Error(`backend process exited (code ${pyExitCode}) before ${url} came up`) + } + try { + const r = await net.fetch(url) + if (r.ok) return + } catch { /* not up yet */ } + const idle = Date.now() - pyLastOutput + if (idle > idleMs) { + throw new Error(`timed out waiting for ${url}: backend silent for ${Math.round(idle / 1000)}s`) + } + await new Promise((r) => setTimeout(r, 300)) + } + throw new Error(`timed out waiting for ${url} after ${Math.round((Date.now() - start) / 1000)}s`) +} + +// Like waitForReady, but forwards the /ready 503 body to the splash so it can +// show real warmup progress (records scanned during the lexical build) instead +// of a blind spinner. Resolves when /ready flips to 200. +async function waitForWarmup(url: string, win: BrowserWindow, timeoutMs = 300000): Promise { + const start = Date.now() + while (Date.now() - start < timeoutMs) { + try { + const r = await net.fetch(url) + if (r.ok) return + if (r.status === 503) { + const body = await r.json().catch(() => null) + if (body && !win.isDestroyed()) { + win.webContents.send('snapshot:warmup', { + phase: typeof body.phase === 'string' ? body.phase : null, + pct: typeof body.pct === 'number' ? body.pct : null, + indexed: typeof body.indexed === 'number' ? body.indexed : 0, + total: typeof body.total === 'number' ? body.total : 0, + }) + } + } + } catch { /* not up yet */ } + await new Promise((r) => setTimeout(r, 300)) + } + throw new Error(`timed out waiting for ${url}`) +} + +// ─── Boot step logging ─────────────────────────────────────────────────────── +// The data backend comes up in distinct phases (Qdrant ready → snapshot fetch → +// decompress → recover → query server → warmup → background HNSW indexing) and a +// slow or stuck boot used to show up as a silent gap with no way to tell which +// phase owned the time. These helpers stamp every phase with its elapsed time so +// the logs read as a timeline: `[boot] +12.4s ✓ recover snapshot (8.1s)`. +let bootStartedAt = 0 +// Set once the main window + resolved snapshot exist, so the snapshot:* IPC +// handlers (download/delete, registered earlier in bootstrap) can drive the +// install on demand. `installing` guards against re-entrant download clicks. +let bootWindow: BrowserWindow | null = null +let currentSnapshot: { sha256: string } = { sha256: '' } +let installing = false +function bootLog(msg: string): void { + const since = bootStartedAt ? `+${((Date.now() - bootStartedAt) / 1000).toFixed(1)}s ` : '' + console.log(`[boot] ${since}${msg}`) +} +async function bootStep(label: string, fn: () => Promise): Promise { + const t0 = Date.now() + bootLog(`▶ ${label}…`) + try { + const out = await fn() + bootLog(`✓ ${label} (${((Date.now() - t0) / 1000).toFixed(1)}s)`) + return out + } catch (err) { + bootLog(`✗ ${label} failed after ${((Date.now() - t0) / 1000).toFixed(1)}s`) + throw err + } +} + +// Post-recover, Qdrant builds the HNSW vector index over the collection in the +// background — the silent, all-core CPU spike users hit before the index lands. +// It logs nothing on its own, so poll the collection and report indexed/total +// until the optimizer goes green: this both makes "is indexing actually done +// yet?" answerable from the logs (correlatable with the CPU dropping) AND streams +// the same progress to the renderer (`snapshot:indexing`) so the loading bar can +// stay up — and the user browse the app — until the search index is really hot. +function logIndexingProgress(win: BrowserWindow): void { + const t0 = Date.now() + let lastIndexed = -1 + const send = (indexed: number, total: number, done: boolean): void => { + if (win.isDestroyed()) return + win.webContents.send('snapshot:indexing', { + indexed, total, + pct: total > 0 ? Math.min(100, (indexed / total) * 100) : null, + done, + }) + } + const tick = async (): Promise => { + try { + const r = await net.fetch('http://127.0.0.1:6333/collections/fangorn') + if (r.ok) { + const res = (await r.json())?.result ?? {} + const total = res.points_count ?? 0 + const indexed = res.indexed_vectors_count ?? 0 + const status = res.status ?? 'unknown' // green = no pending optimizations + if (indexed !== lastIndexed) { + const pct = total ? ((indexed / total) * 100).toFixed(1) : '0.0' + bootLog(`[index] ${indexed}/${total} vectors indexed (${pct}%) — status=${status}`) + lastIndexed = indexed + } + if (status === 'green') { + bootLog(`[index] ✓ HNSW indexing settled — ${indexed}/${total} vectors in ${((Date.now() - t0) / 1000).toFixed(0)}s`) + send(indexed, total, true) + return + } + send(indexed, total, false) + } + } catch { /* qdrant momentarily busy mid-optimization — retry */ } + if (Date.now() - t0 < 30 * 60 * 1000) setTimeout(() => void tick(), 5000) + else { bootLog('[index] stopped tracking indexing after 30m (still not green)'); send(lastIndexed, lastIndexed, true) } + } + void tick() +} + // Download the gzipped snapshot from IPFS, streaming progress to the window. -async function downloadGz(cid: string, dest: string, win: BrowserWindow): Promise { - const res = await fetch(`${SNAPSHOT_GATEWAY}/${cid}`) // global Node fetch - if (!res.ok || !res.body) throw new Error(`snapshot download failed: ${res.status}`) +// Reads the bundled split-upload manifest, then streams every part back-to-back +// into one .gz — the runtime equivalent of `cat ...part* > ...gz`. Aggregate +// progress is reported against the manifest's known total size, so the bar +// reflects the whole reassembled file rather than restarting per part. +async function downloadGz(dest: string, win: BrowserWindow): Promise { + const parts = loadManifest() + const total = parts.reduce((sum, p) => sum + (p.size || 0), 0) - const total = Number(res.headers.get('content-length')) || 0 const out = createWriteStream(dest) let received = 0 - const reader = (res.body as any).getReader() - for (; ;) { - const { done, value } = await reader.read() - if (done) break - received += value.length - out.write(Buffer.from(value)) - win.webContents.send('snapshot:progress', { received, total }) + try { + for (const part of parts) { + const res = await fetch(`${SNAPSHOT_GATEWAY}/${part.cid}`) // global Node fetch + if (!res.ok || !res.body) { + throw new Error(`snapshot part ${part.index} download failed: ${res.status}`) + } + + const reader = (res.body as any).getReader() + for (; ;) { + const { done, value } = await reader.read() + if (done) break + received += value.length + // Respect stream backpressure so a fast gateway can't outrun the disk. + if (!out.write(Buffer.from(value))) { + await new Promise((r) => out.once('drain', () => r())) + } + win.webContents.send('snapshot:progress', { received, total }) + } + } + } catch (e) { + out.destroy() + throw e } + out.end() await new Promise((r) => out.on('finish', () => r())) } @@ -340,13 +665,162 @@ async function gunzipVerify(gzPath: string, outPath: string, sha256: string): Pr } } -// First run: fetch + decompress + recover. Later runs: collection exists → skip. -async function ensureCollection(win: BrowserWindow): Promise { +// Records which snapshot is currently recovered into qdrant_storage, so a later +// launch can tell "same snapshot" (reuse everything, including the query server's +// on-disk lexical-index cache → no re-warm) from "new snapshot" (wipe + +// re-download + re-warm). The identity is the decompressed snapshot's sha256. +// Lives in userData alongside qdrant_storage. (`cid` is read for backward compat +// with markers written before the manifest stopped being pinned.) +function snapshotMarkerPath(): string { + return path.join(app.getPath('userData'), 'catalog-snapshot.json') +} +function readSnapshotMarker(): { sha256?: string; cid?: string } | null { + try { + const m = JSON.parse(fs.readFileSync(snapshotMarkerPath(), 'utf8')) + return m && typeof m === 'object' ? m : null + } catch { + return null + } +} +function writeSnapshotMarker(sha256: string): void { + try { + fs.writeFileSync(snapshotMarkerPath(), JSON.stringify({ sha256 })) + } catch (e) { + console.error('[snapshot] marker write failed:', e) + } +} + +// "Installed" = the catalog collection actually exists in Qdrant with data. +// Ground truth (a stray marker or an empty server-created collection doesn't +// count), so the opt-in gate and the Settings "delete" state stay honest. +async function isSnapshotInstalled(): Promise { + try { + const r = await net.fetch('http://127.0.0.1:6333/collections/fangorn') + if (!r.ok) return false + const res = (await r.json())?.result ?? {} + return (res.points_count ?? 0) > 0 + } catch { + return false + } +} + +// Best-effort recursive byte size of a directory (the on-disk qdrant_storage +// footprint). The collection is a handful of large mmap files per segment, not a +// deep tree, so this stays fast. +async function dirSize(dir: string): Promise { + let total = 0 + let entries: fs.Dirent[] + try { entries = await fs.promises.readdir(dir, { withFileTypes: true }) } catch { return 0 } + for (const e of entries) { + const p = path.join(dir, e.name) + try { + if (e.isDirectory()) total += await dirSize(p) + else total += (await fs.promises.stat(p)).size + } catch { /* file vanished mid-walk — skip */ } + } + return total +} + +export interface SnapshotInfo { + installed: boolean + compressedBytes: number // the .gz download size (X) + uncompressedBytes: number // unpacked snapshot size (Y) — exact if the manifest has it, else estimated + requiredBytes: number // peak free disk needed to install (Z): .gz + unpacked coexist mid-decompress + freeBytes: number // free disk on the userData volume right now + installedBytes: number // current on-disk footprint when installed (else 0) +} + +// Numbers behind the download disclosure + the Settings data panel. +async function computeSnapshotInfo(): Promise { + const m = snapshotManifest as SnapshotManifest + const compressed = m.total_size || (m.parts ?? []).reduce((s, p) => s + (p.size || 0), 0) + const uncompressed = m.uncompressed_size || Math.round(compressed * SNAPSHOT_UNCOMPRESSED_RATIO) + const userData = app.getPath('userData') + + let freeBytes = 0 + try { const st = await fs.promises.statfs(userData); freeBytes = st.bavail * st.bsize } catch { /* unknown */ } + + const installed = await isSnapshotInstalled() + const installedBytes = installed ? await dirSize(path.join(userData, 'qdrant_storage')) : 0 + + return { + installed, + compressedBytes: compressed, + uncompressedBytes: uncompressed, + requiredBytes: compressed + uncompressed, + freeBytes, + installedBytes, + } +} + +// The download → recover → serve → warm tail of boot, factored out so it runs +// either automatically (already installed) or on the user's opt-in click. Streams +// the same snapshot:status / backend:ready / snapshot:indexing events either way. +async function installBackend(win: BrowserWindow): Promise { + await bootStep('ensure collection (download/recover or cached)', () => ensureCollection(win, currentSnapshot)) + bootLog('▶ start query server') + startQueryServer() + await bootStep('wait for query server /health', () => waitForHealth('http://127.0.0.1:8080/health')) + win.webContents.send('snapshot:status', 'warming') + await bootStep('warmup (embeddings + vector index hot)', () => + waitForWarmup('http://127.0.0.1:8080/ready', win, 300000)) + bootLog('✓ data backend ready — search is live') + win.webContents.send('backend:ready') + logIndexingProgress(win) +} + +// Free the disk and return to the "not installed" state: stop the query server +// (it holds the collection open), drop the Qdrant collection (which removes its +// segment files), and clear leftover temp files + the marker. Then re-surface the +// consent prompt so the user can re-download later. +async function deleteSnapshot(): Promise<{ ok: boolean; error?: string }> { + try { + killQueryServer() + await net.fetch('http://127.0.0.1:6333/collections/fangorn', { method: 'DELETE' }).catch(() => {}) + const userData = app.getPath('userData') + try { fs.unlinkSync(path.join(userData, 'fangorn.snapshot.gz')) } catch { /* none */ } + try { fs.rmSync(path.join(userData, 'qdrant_snapshots', 'fangorn'), { recursive: true, force: true }) } catch { /* none */ } + try { fs.unlinkSync(snapshotMarkerPath()) } catch { /* none */ } + bootLog('[snapshot] deleted — catalog uninstalled') + if (bootWindow && !bootWindow.isDestroyed()) { + bootWindow.webContents.send('snapshot:needs-consent', await computeSnapshotInfo()) + } + return { ok: true } + } catch (e) { + return { ok: false, error: String(e) } + } +} + +// Ensure the catalog for `snapshot` is recovered. Unchanged snapshot → no-op (the +// query server reuses its cached lexical index, so warmup is near-instant). New +// snapshot → remove the stale collection first, then fetch + decompress + recover +// the new one and stamp the marker so the next launch knows it's current. +async function ensureCollection(win: BrowserWindow, snapshot: { sha256: string }): Promise { + const { sha256 } = snapshot const exists = await net.fetch('http://127.0.0.1:6333/collections/fangorn') .then((r) => r.ok).catch(() => false) - if (exists) return + const marker = readSnapshotMarker() + // Old markers (pre-unpinned-manifest) keyed on cid and have no sha256; treat + // such a collection as current rather than forcing a needless 14 GB re-fetch. + const markedId = marker?.sha256 + + // Already on this snapshot — or a pre-existing/legacy collection we adopt as + // current. Either way nothing to fetch. + if (exists && (!marker || !markedId || markedId === sha256)) { + if (!markedId) writeSnapshotMarker(sha256) + bootLog('collection already present and current — skipping download/recover (cached)') + return + } + bootLog(exists ? 'collection present but stale — refreshing' : 'no collection yet — first-run fetch') + + // A different snapshot is live → drop the old collection's data before pulling + // the new one, so stale vectors don't linger in qdrant_storage. + if (exists && markedId && markedId !== sha256) { + console.log(`[snapshot] new catalog ${sha256} (was ${markedId}) — removing old collection`) + await net.fetch('http://127.0.0.1:6333/collections/fangorn', { method: 'DELETE' }) + .catch((e) => console.error('[snapshot] failed to delete old collection:', e)) + } - const { cid, sha256 } = await resolveSnapshot() const userData = app.getPath('userData') const gzPath = path.join(userData, 'fangorn.snapshot.gz') @@ -356,21 +830,26 @@ async function ensureCollection(win: BrowserWindow): Promise { const snapPath = path.join(snapDir, 'fangorn.snapshot') win.webContents.send('snapshot:status', 'downloading') - await downloadGz(cid, gzPath, win) + await bootStep('download snapshot (~15 GB compressed)', () => downloadGz(gzPath, win)) win.webContents.send('snapshot:status', 'decompressing') - await gunzipVerify(gzPath, snapPath, sha256) + await bootStep('decompress + verify snapshot', () => gunzipVerify(gzPath, snapPath, sha256)) try { fs.unlinkSync(gzPath) } catch { } win.webContents.send('snapshot:status', 'recovering') - const res = await net.fetch('http://127.0.0.1:6333/collections/fangorn/snapshots/recover', { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ location: `file://${snapPath}` }), + await bootStep('recover snapshot into Qdrant', async () => { + const res = await net.fetch('http://127.0.0.1:6333/collections/fangorn/snapshots/recover', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ location: `file://${snapPath}` }), + }) + if (!res.ok) throw new Error(`recover failed: ${res.status} ${await res.text()}`) }) - if (!res.ok) throw new Error(`recover failed: ${res.status} ${await res.text()}`) try { fs.unlinkSync(snapPath) } catch { } + + // Stamp only after a successful recover, so an interrupted upgrade retries. + writeSnapshotMarker(sha256) } function startQueryServer() { @@ -381,6 +860,8 @@ function startQueryServer() { // no chroma path, no checkpoint, no graph key — that all moved to the builder. // The vector dim is read from the recovered snapshot's collection (and query // embeddings are Matryoshka-truncated to match), so nothing is hardcoded here. + // Text search is served by Qdrant full-text payload indexes (built on first + // launch, persisted in qdrant_storage), so there's no lexical-index cache to key. const [bin, args, cwd]: [string, string[], string] = app.isPackaged ? [ path.join(root, serverName), @@ -396,18 +877,45 @@ function startQueryServer() { path.join(root, 'vectordb'), ] - pyProcess = spawn(bin, args, { cwd, env: { ...process.env } }) + pyProcess = spawn(bin, args, { cwd, detached: process.platform !== 'win32', env: { ...process.env } }) + console.log(`[py] query server spawned (pid=${pyProcess.pid}, ${app.isPackaged ? 'packaged binary' : 'dev venv'})`) - pyProcess.stdout?.on('data', (d) => console.log('[py]', d.toString().trimEnd())) - pyProcess.stderr?.on('data', (d) => console.error('[py]', d.toString().trimEnd())) + // Reset liveness tracking for this boot. Seed last-output to "now" so the + // initial process-spawn → first-log gap doesn't read as a silent hang. + pyLastOutput = Date.now() + pyExitCode = null + pyExited = false + + const markOutput = (): void => { pyLastOutput = Date.now() } + pyProcess.stdout?.on('data', (d) => { markOutput(); console.log('[py]', d.toString().trimEnd()) }) + pyProcess.stderr?.on('data', (d) => { markOutput(); console.error('[py]', d.toString().trimEnd()) }) pyProcess.on('error', (err) => console.error('[py] failed to start:', err)) - pyProcess.on('exit', (code) => console.log('[py] exited with code', code)) + pyProcess.on('exit', (code) => { + pyExited = true + pyExitCode = code + console.log('[py] exited with code', code) + }) const pyLog = createWriteStream(path.join(app.getPath('userData'), 'py-stderr.log')) pyProcess.stderr?.on('data', (d) => pyLog.write(d)) pyProcess.stdout?.on('data', (d) => pyLog.write(d)) } +// Stop just the query server (Qdrant stays up). Used by deleteSnapshot, which +// needs the server's hold on the collection released before dropping it, but +// keeps Qdrant alive to serve the DELETE. Mirrors killSidecars' group-kill. +function killQueryServer(): void { + const proc = pyProcess + pyProcess = null + if (!proc || proc.pid == null || proc.killed) return + try { + if (process.platform === 'win32') spawn('taskkill', ['/PID', String(proc.pid), '/F', '/T']) + else process.kill(-proc.pid, 'SIGKILL') + } catch { + try { proc.kill('SIGKILL') } catch { /* already gone */ } + } +} + // ─── Local renderer server ──────────────────────────────────────────────────── function startRendererServer(): Promise { @@ -442,6 +950,14 @@ function startRendererServer(): Promise { // ─── Browser window ─────────────────────────────────────────────────────────── +// Native window-controls-overlay tint per theme. `color` is the overlay strip +// background (matches the header --bg1), `symbolColor` the min/max/close glyphs +// (matches --fg). Keep these in sync with the palettes in renderer App.css. +const TITLEBAR_OVERLAY = { + light: { color: '#f0ece5', symbolColor: '#1a1714' }, + dark: { color: '#1b1714', symbolColor: '#f0ece5' }, +} as const + function createWindow(): BrowserWindow { const mainWindow = new BrowserWindow({ width: 1200, height: 800, @@ -451,7 +967,7 @@ function createWindow(): BrowserWindow { // Native window-controls overlay tinted to match the light editorial header // (App.tsx BG1 background / FG symbols) so min/max/close read as part of the // app instead of black boxes. Height matches the 40px header. - titleBarOverlay: { color: '#f0ece5', symbolColor: '#1a1714', height: 40 }, + titleBarOverlay: { ...TITLEBAR_OVERLAY.light, height: 40 }, backgroundColor: '#1a1a1a', autoHideMenuBar: true, webPreferences: { preload: join(__dirname, '../preload/index.js'), @@ -461,8 +977,12 @@ function createWindow(): BrowserWindow { }, }) - mainWindow.webContents.openDevTools({ mode: 'detach' }) - mainWindow.on('ready-to-show', () => mainWindow.show()) + mainWindow.on('ready-to-show', () => { + mainWindow.show() + // Auto-open DevTools in development only (`yarn dev`); never in packaged + // builds. Docked (not detached) — detached can silently no-op on Linux. + if (is.dev) mainWindow.webContents.openDevTools() + }) mainWindow.webContents.setWindowOpenHandler((details) => { shell.openExternal(details.url) return { action: 'deny' } @@ -480,6 +1000,30 @@ function createWindow(): BrowserWindow { // ─── IPC handlers ───────────────────────────────────────────────────────────── function registerIpcHandlers() { + // Escape hatch for a wedged client-side auth state (e.g. the Privy user was + // deleted server-side but this device still holds their session, blocking a + // clean re-login). Wipes all storage — cookies, localStorage, IndexedDB, + // cache — for the renderer's `persist:main` partition, then reloads. The + // account + embedded wallet live server-side and are restored on next login. + ipcMain.handle('session:reset', async (event) => { + await session.fromPartition('persist:main').clearStorageData() + BrowserWindow.fromWebContents(event.sender)?.webContents.reload() + return { success: true } + }) + + // Re-tint the native window-controls overlay (min/max/close) to match the + // active light/dark theme. Windows/Linux only — macOS draws traffic lights + // the OS themes itself, and calling setTitleBarOverlay there throws. + ipcMain.handle('window:set-theme', (event, theme: 'light' | 'dark') => { + if (process.platform === 'darwin') return + const overlay = TITLEBAR_OVERLAY[theme] ?? TITLEBAR_OVERLAY.light + try { + BrowserWindow.fromWebContents(event.sender)?.setTitleBarOverlay({ ...overlay, height: 40 }) + } catch (e) { + console.warn('[main] setTitleBarOverlay failed:', e) + } + }) + ipcMain.handle('shell:open-external', (_event, url: string) => { if (typeof url === 'string' && (url.startsWith('https://') || url.startsWith('http://'))) { shell.openExternal(url) @@ -491,9 +1035,8 @@ function registerIpcHandlers() { return { status: res.status, body: await res.text() } }) - ipcMain.handle('deezer:api', async (_event, { url }) => { - const res = await fetch(url, { headers: { 'Accept': 'application/json' } }) - return { status: res.status, body: await res.text() } + ipcMain.handle('deezer:track-cover', async (_event, { artist, title }) => { + return trackCover(artist, title) }) ipcMain.handle('spotify:api', async (_event, { url, method, token, body }) => { @@ -570,8 +1113,34 @@ function registerIpcHandlers() { }) ipcMain.handle('backend:is-ready', async () => { - return net.fetch('http://127.0.0.1:8080/health').then(r => r.ok).catch(() => false) + return net.fetch('http://127.0.0.1:8080/ready').then(r => r.ok).catch(() => false) }) + + // ── Opt-in catalog snapshot: disclosure / download / delete ────────────── + // Sizes + install state for the startup disclosure and the Settings panel. + ipcMain.handle('snapshot:info', async () => computeSnapshotInfo()) + // User consented to the (large) download — run the install tail on demand. + ipcMain.handle('snapshot:download', async () => { + if (installing) return { ok: false, error: 'already installing' } + if (!bootWindow) return { ok: false, error: 'window not ready' } + installing = true + bootStartedAt = Date.now() + bootLog('user consented — installing catalog') + try { + await installBackend(bootWindow) + return { ok: true } + } catch (e) { + bootLog(`✗ catalog install failed: ${String(e)}`) + if (!bootWindow.isDestroyed()) bootWindow.webContents.send('backend:error', String(e)) + return { ok: false, error: String(e) } + } finally { + installing = false + } + }) + // Delete the catalog, free the disk, return to "not installed". + ipcMain.handle('snapshot:delete', async () => deleteSnapshot()) + + registerBugReportIpc() // bug:submit / bug:diagnostics (in-app problem reporter) } // ─── Arbitrum RPC proxy ─────────────────────────────────────────────────────── @@ -666,8 +1235,12 @@ if (!gotLock) { // brought up afterwards, behind a boot screen the renderer renders. app.whenReady().then(async () => { + installLogCapture() // tee console output into the bug-report ring buffer first electronApp.setAppUserModelId('com.electron') + // Reclaim any sidecar ports an unclean previous exit left orphaned, so the new + // Qdrant doesn't die on "Address already in use" and wedge the boot. + await reclaimSidecarPorts() startQdrant() startYtStreamProxy() updateYtDlp() @@ -708,17 +1281,34 @@ app.whenReady().then(async () => { // Wait until the renderer is mounted so it can actually receive boot events. await new Promise((r) => mainWindow.webContents.once('did-finish-load', () => r())) - // Backend comes up behind the window: Qdrant ready → collection present - // (download + decompress + recover on first run, skip if cached) → query server. + // Backend comes up behind the window. The catalog download is opt-in: if it's + // already installed we bring it straight up; if not, we stop at Qdrant and hand + // the renderer a size disclosure so the user can choose to download (or skip and + // use the app — local music — without a catalog). See installBackend / the + // snapshot:* IPC handlers for the on-demand path. + bootStartedAt = Date.now() + bootLog('starting data backend') try { - await waitForReady('http://127.0.0.1:6333/readyz') - await ensureCollection(mainWindow) - startQueryServer() - await waitForReady('http://127.0.0.1:8080/health') - console.log('[boot] data backend ready') - mainWindow.webContents.send('backend:ready') + const snapshot = await bootStep('resolve snapshot', () => resolveSnapshot()) + currentSnapshot = snapshot + bootWindow = mainWindow + // Generous timeout: on a cold start Qdrant mmaps a large (10M-point) collection + // and rebuilds any payload indexes baked into the config before it serves + // /readyz — that can take a couple of minutes. The default 60s aborts the boot + // mid-rebuild (and a stale full-text index makes this much worse until the query + // server's startup drops it). + await bootStep('wait for Qdrant /readyz', () => + waitForReady('http://127.0.0.1:6333/readyz', 300000)) + + if (await isSnapshotInstalled()) { + bootLog('catalog already installed — bringing it up') + await installBackend(mainWindow) + } else { + bootLog('catalog not installed — awaiting user consent to download') + mainWindow.webContents.send('snapshot:needs-consent', await computeSnapshotInfo()) + } } catch (err) { - console.error('[boot] backend failed to come up:', err) + bootLog(`✗ backend failed to come up: ${String(err)}`) mainWindow.webContents.send('backend:error', String(err)) } @@ -741,19 +1331,43 @@ app.on('window-all-closed', () => { if (process.platform !== 'darwin') app.quit() }) -app.on('before-quit', async () => { - // Kill both child processes; /T on Windows takes their trees with them. +// Tear down the sidecars (Qdrant + Python query server). Both are spawned +// `detached` on POSIX so each leads its own process group — killing the +// negative PID takes the whole group (and any grandchildren) with it. SIGKILL +// can't be trapped, so a busy or mid-graceful-shutdown uvicorn can't linger and +// keep pegging the CPU. Idempotent: the nulled refs make repeat calls +// (before-quit → exit) no-ops. /T on Windows takes the child's tree instead. +function killSidecars(): void { for (const proc of [pyProcess, qdrantProc]) { - if (proc && !proc.killed) { + if (!proc || proc.pid == null || proc.killed) continue + try { if (process.platform === 'win32') { spawn('taskkill', ['/PID', String(proc.pid), '/F', '/T']) } else { - proc.kill() + process.kill(-proc.pid, 'SIGKILL') } + } catch { + try { proc.kill('SIGKILL') } catch { /* already gone */ } } } + pyProcess = null + qdrantProc = null +} + +app.on('before-quit', async () => { + quitting = true + killSidecars() if (rendererServer) { await new Promise((resolve) => rendererServer?.close(() => resolve())) } await providerManager.shutdown() -}) \ No newline at end of file +}) + +// `before-quit` doesn't fire on every exit path: when `electron-vite dev` +// restarts the main process on a file change — or you Ctrl-C the dev server — +// Electron receives a raw signal instead. Without these, each restart orphans +// the previous Qdrant + Python pair, which keep running and pile up until they +// saturate the CPU. `exit` is the last-ditch synchronous safety net. +process.on('exit', killSidecars) +process.on('SIGINT', () => { quitting = true; killSidecars(); process.exit(0) }) +process.on('SIGTERM', () => { quitting = true; killSidecars(); process.exit(0) }) \ No newline at end of file diff --git a/app/src/main/local/LocalLibrary.ts b/app/src/main/local/LocalLibrary.ts index 61ad3eb0..f156f1e4 100644 --- a/app/src/main/local/LocalLibrary.ts +++ b/app/src/main/local/LocalLibrary.ts @@ -17,7 +17,11 @@ import fs from 'fs' import path from 'path' import crypto from 'crypto' import mime from 'mime-types' -import { AUDIO_EXTENSIONS, type LocalTrack } from './types' +import { + AUDIO_EXTENSIONS, + type ImportMode, type ImportProgress, type ImportSummary, type LibraryRoots, type LocalTrack, +} from './types' +import { listMeta } from './catalog' // ─── File server ──────────────────────────────────────────────────────────── @@ -88,6 +92,41 @@ export function startLocalFileServer(): Promise { }) } +/** Resolve a served id back to its on-disk path. Only ids from the last scan + * resolve, so metadata/tag IPC can't be pointed at arbitrary files. */ +export function pathForId(id: string): string | undefined { + return registry.get(id)?.path +} + +/** Every file from the last scan as { id, path } — used by the auto-organizer. */ +export function listRegistry(): { id: string; path: string }[] { + return [...registry.entries()].map(([id, e]) => ({ id, path: e.path })) +} + +/** + * Drop a track from the library. The on-disk file is permanently deleted ONLY + * when it lives inside the primary library folder (a copy made by import); files + * under a referenced-in-place root (the user's source folder/drive) are left + * untouched so the source stays intact. Either way the id is removed from the + * serve registry so it stops streaming immediately. Returns whether the file was + * deleted (false when it was a referenced source we left alone, or already gone). + * + * Caller is responsible for clearing any stored metadata/tags (see ipc). + */ +export function dropTrack(id: string): { deletedFile: boolean } { + const entry = registry.get(id) + registry.delete(id) + if (!entry) return { deletedFile: false } + if (!isInside(entry.path, getSavedDir())) return { deletedFile: false } + try { + fs.rmSync(entry.path) + return { deletedFile: true } + } catch (e) { + console.warn('[local-music] could not delete file:', entry.path, e) + return { deletedFile: false } + } +} + // ─── Music directory (default + persisted override) ─────────────────────────── /** OS-standard music location: ~/Music, %USERPROFILE%\Music, etc. */ @@ -103,25 +142,92 @@ function settingsPath(): string { return path.join(app.getPath('userData'), 'local-music.json') } -/** The last folder the user picked, falling back to the OS music dir. */ -export function getSavedDir(): string { +interface Settings { + /** Primary library folder — where copy-imports land and "Change folder" sets. */ + dir: string + /** Referenced-in-place folders (e.g. an external drive) scanned alongside dir. */ + extraRoots: string[] +} + +function readSettings(): Settings { try { - const saved = JSON.parse(fs.readFileSync(settingsPath(), 'utf-8')) - if (typeof saved.dir === 'string' && saved.dir) return saved.dir + const s = JSON.parse(fs.readFileSync(settingsPath(), 'utf-8')) + const dir = typeof s.dir === 'string' && s.dir ? s.dir : getDefaultMusicDir() + const extraRoots = Array.isArray(s.extraRoots) + ? s.extraRoots.filter((x: unknown): x is string => typeof x === 'string' && !!x) + : [] + return { dir, extraRoots } } catch { - /* no saved dir yet */ + return { dir: getDefaultMusicDir(), extraRoots: [] } } - return getDefaultMusicDir() } -export function saveDir(dir: string): void { +function writeSettings(s: Settings): void { try { - fs.writeFileSync(settingsPath(), JSON.stringify({ dir })) + fs.writeFileSync(settingsPath(), JSON.stringify(s)) } catch (e) { - console.warn('[local-music] could not persist dir:', e) + console.warn('[local-music] could not persist settings:', e) } } +/** True when `child` is `parent` or nested inside it (path-wise). */ +function isInside(child: string, parent: string): boolean { + const rel = path.relative(path.resolve(parent), path.resolve(child)) + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)) +} + +/** The primary library folder, falling back to the OS music dir. */ +export function getSavedDir(): string { + return readSettings().dir +} + +/** Persist the primary library folder, keeping any referenced roots. */ +export function saveDir(dir: string): void { + const s = readSettings() + s.dir = dir + writeSettings(s) +} + +/** Primary + referenced folders, for the renderer to display/manage. */ +export function getRoots(): LibraryRoots { + const s = readSettings() + return { primary: s.dir, extra: s.extraRoots } +} + +/** Every folder to scan (primary first), de-duped by resolved path. */ +function getAllRoots(): string[] { + const s = readSettings() + const seen = new Set() + const out: string[] = [] + for (const r of [s.dir, ...s.extraRoots]) { + const n = path.resolve(r) + if (!seen.has(n)) { seen.add(n); out.push(r) } + } + return out +} + +/** Add a referenced-in-place root. Returns false (no-op) if it overlaps the + * primary folder or an existing root, so the same tree isn't scanned twice. */ +export function addExtraRoot(p: string): boolean { + const s = readSettings() + if (isInside(p, s.dir) || isInside(s.dir, p)) return false + for (const r of s.extraRoots) { + if (isInside(p, r) || isInside(r, p)) return false + } + s.extraRoots.push(p) + writeSettings(s) + return true +} + +/** Drop a referenced-in-place root (its tracks stop appearing on next scan; + * their stored metadata rows are left intact). */ +export function removeExtraRoot(p: string): void { + const s = readSettings() + const target = path.resolve(p) + s.extraRoots = s.extraRoots.filter((r) => path.resolve(r) !== target) + writeSettings(s) +} + // ─── Scan ────────────────────────────────────────────────────────────────── function fileId(absPath: string): string { @@ -163,29 +269,128 @@ async function walk(dir: string, out: string[], depth = 0): Promise { } } -/** Recursively scan `dir`, (re)build the serve registry, return playable tracks. */ -export async function scan(dir: string): Promise { +/** + * Scan the library and (re)build the serve registry. With no `target`, scans + * every root (primary + referenced); with a `target`, first makes it the primary + * folder ("Change folder") and then scans. Returns the union of playable tracks, + * de-duped by path (a file reachable from two roots appears once). + */ +export async function scan(target?: string): Promise { await startLocalFileServer() + if (target) saveDir(target) const files: string[] = [] - await walk(dir, files) - files.sort((a, b) => a.localeCompare(b)) + for (const root of getAllRoots()) await walk(root, files) + const seen = new Set() + const unique = files.filter((f) => { + const n = path.resolve(f) + return seen.has(n) ? false : (seen.add(n), true) + }) + unique.sort((a, b) => a.localeCompare(b)) + + // Stored metadata, loaded once and merged in below. A track with a row is + // "labeled" (its stored title/artist/album override the filename guess) and + // carries the extra schema fields; everything else is Unlabeled. + const stored = listMeta() registry.clear() - return files.map((absPath) => { + return unique.map((absPath) => { const id = fileId(absPath) const ext = path.extname(absPath).toLowerCase() const contentType = (mime.lookup(absPath) || 'application/octet-stream') as string registry.set(id, { path: absPath, contentType }) - const meta = deriveMeta(absPath) + const streamUrl = `http://127.0.0.1:${port}/${id}` + + const meta = stored.get(id) + if (meta) { + return { + id, path: absPath, ext, streamUrl, + title: meta.title, + artist: meta.byArtist, + album: meta.albumName, + labeled: true, + trackId: meta.trackId, + isrcCode: meta.isrcCode, + datePublished: meta.datePublished, + durationMs: meta.durationMs, + contributors: meta.contributors, + } + } + + const derived = deriveMeta(absPath) return { - id, - path: absPath, - title: meta.title, - artist: meta.artist, - album: meta.album, - ext, - streamUrl: `http://127.0.0.1:${port}/${id}`, + id, path: absPath, ext, streamUrl, + title: derived.title, + artist: derived.artist, + album: derived.album, + labeled: false, } }) } + +// ─── Import ────────────────────────────────────────────────────────────────── + +/** Count the audio files under `dir` (for the import dialog's preview). */ +export async function countAudioFiles(dir: string): Promise { + const files: string[] = [] + await walk(dir, files) + return files.length +} + +/** + * Bulk-import a folder (e.g. an external drive) into the library. + * + * - 'reference' — add `source` as a referenced-in-place root (no copying). + * - 'copy' — copy every audio file into `/`, + * preserving the subfolder structure (so the bulk labeler still + * groups by folder) and skipping files already at the target. + * + * Either way the next scan surfaces the new tracks as Unlabeled. `onProgress` + * fires per file for copy imports. + */ +export async function importFolder( + source: string, + mode: ImportMode, + onProgress?: (p: ImportProgress) => void, +): Promise { + const src = path.resolve(source) + + if (mode === 'reference') { + const referenced = addExtraRoot(source) + return { mode, source, total: 0, copied: 0, skipped: 0, failed: 0, referenced } + } + + const dir = getSavedDir() + if (isInside(src, dir) || isInside(dir, src)) { + throw new Error('That folder overlaps your library folder — nothing to copy.') + } + + const files: string[] = [] + await walk(src, files) + const total = files.length + const destRoot = path.join(dir, path.basename(src) || 'Imported') + + let copied = 0 + let skipped = 0 + let failed = 0 + for (const file of files) { + const dest = path.join(destRoot, path.relative(src, file)) + try { + let exists = false + try { exists = fs.statSync(dest).isFile() } catch { /* not there yet */ } + if (exists) { + skipped++ + } else { + await fs.promises.mkdir(path.dirname(dest), { recursive: true }) + await fs.promises.copyFile(file, dest) + copied++ + } + } catch (e) { + console.warn('[local-music] import copy failed:', file, e) + failed++ + } + onProgress?.({ total, processed: copied + skipped + failed, file: path.basename(file) }) + } + + return { mode, source, dest: destRoot, total, copied, skipped, failed, referenced: false } +} diff --git a/app/src/main/local/art.ts b/app/src/main/local/art.ts new file mode 100644 index 00000000..d3a9a94f --- /dev/null +++ b/app/src/main/local/art.ts @@ -0,0 +1,201 @@ +/** + * main/local/art.ts + * + * User-supplied album / artist artwork. The user picks a local image; we read it + * and stash the bytes inline in SQLite (main/db, `local_art`). Stored inline + * rather than by path so artwork survives the source image being moved or + * deleted, and there's no file cache to manage. Returned to the renderer as a + * `data:` URL ready to drop into an . + * + * Art is keyed by a scope ('album' | 'artist') plus a normalized key the renderer + * builds (artist name, or artist + album). Normalizing here keeps "Artist" and + * "artist " pointing at the same artwork. + */ + +import fs from 'fs' +import mime from 'mime-types' +import { net } from 'electron' +import { getDb } from '../db' +import type { ArtSaveItem, ArtSaveResult, ArtScope } from './types' + +function normKey(key: string): string { + return key.trim().toLowerCase() +} + +interface ArtRow { mime: string; data: Buffer } + +function toDataUrl(row: ArtRow): string { + return `data:${row.mime};base64,${row.data.toString('base64')}` +} + +/** Read an image off disk and return it as a `data:` URL, without storing it. + * Used to preview a freshly picked image before it's committed under a key. */ +export function imageFileToDataUrl(filePath: string): string { + const data = fs.readFileSync(filePath) + const m = (mime.lookup(filePath) || 'image/png') as string + return toDataUrl({ mime: m, data }) +} + +/** The normalized keys that currently have stored artwork, grouped by scope. + * Lets the renderer flag which artists/albums still need art in one round-trip + * instead of probing each key. Keys are normalized (see normKey), so callers + * must compare against the same normalization. */ +export function listArtKeys(): { artist: string[]; album: string[] } { + const rows = getDb() + .prepare('SELECT scope, art_key FROM local_art') + .all() as { scope: ArtScope; art_key: string }[] + const out: { artist: string[]; album: string[] } = { artist: [], album: [] } + for (const r of rows) { + if (r.scope === 'artist' || r.scope === 'album') out[r.scope].push(r.art_key) + } + return out +} + +/** The stored artwork as a data URL, or null if none is set. */ +export function getArt(scope: ArtScope, key: string): string | null { + const row = getDb() + .prepare('SELECT mime, data FROM local_art WHERE scope = ? AND art_key = ?') + .get(scope, normKey(key)) as ArtRow | undefined + return row ? toDataUrl(row) : null +} + +/** Persist image bytes as the artwork for (scope, key); returns its data URL. */ +function storeArt(scope: ArtScope, key: string, m: string, data: Buffer): string { + getDb() + .prepare( + `INSERT INTO local_art (scope, art_key, mime, data, updated_at) + VALUES (@scope, @key, @mime, @data, @now) + ON CONFLICT(scope, art_key) DO UPDATE SET + mime = excluded.mime, data = excluded.data, updated_at = excluded.updated_at`, + ) + .run({ scope, key: normKey(key), mime: m, data, now: Date.now() }) + return toDataUrl({ mime: m, data }) +} + +/** Read an image off disk and store it as the artwork for (scope, key). */ +export function setArtFromFile(scope: ArtScope, key: string, filePath: string): string { + const data = fs.readFileSync(filePath) + const m = (mime.lookup(filePath) || 'image/png') as string + return storeArt(scope, key, m, data) +} + +/** Download an image over the network (via Electron's net stack, so it isn't + * subject to the renderer CSP) and return its bytes + content type. */ +async function fetchImage(url: string): Promise<{ mime: string; data: Buffer }> { + const res = await net.fetch(url) + if (!res.ok) throw new Error(`Image fetch failed (${res.status})`) + const buf = Buffer.from(await res.arrayBuffer()) + // Content-Type may be absent; when present, drop any `; charset=…` parameter. + const header = res.headers.get('content-type') + const ct = header ? header.split(';')[0].trim() : '' + const m = ct.startsWith('image/') ? ct : (mime.lookup(url) || 'image/jpeg') as string + return { mime: m, data: buf } +} + +/** Fetch a remote image and return it as a `data:` URL, without storing it. */ +export async function imageUrlToDataUrl(url: string): Promise { + const { mime: m, data } = await fetchImage(url) + return toDataUrl({ mime: m, data }) +} + +/** Download a remote image and store it as the artwork for (scope, key). */ +export async function setArtFromUrl(scope: ArtScope, key: string, url: string): Promise { + const { mime: m, data } = await fetchImage(url) + return storeArt(scope, key, m, data) +} + +/** Read an image off disk as bytes + content type (async sibling of fetchImage). */ +async function readImage(filePath: string): Promise<{ mime: string; data: Buffer }> { + const data = await fs.promises.readFile(filePath) + return { mime: (mime.lookup(filePath) || 'image/png') as string, data } +} + +/** How many images a bulk save resolves at once. The remote downloads are the + * slow part, so this parallelizes them without flooding the network. */ +const ART_SAVE_CONCURRENCY = 6 + +/** + * Commit many staged artworks in one call. Bytes are resolved concurrently + * (remote URLs downloaded, local files read), then every success is persisted in + * a single SQLite transaction. Replaces the renderer awaiting one IPC + one + * download/read per target. Failures are skipped; returns success/failure counts. + */ +export async function setArtMany(items: ArtSaveItem[]): Promise { + interface Resolved { scope: ArtScope; key: string; mime: string; data: Buffer } + const resolved: (Resolved | null)[] = new Array(items.length).fill(null) + + let next = 0 + const worker = async (): Promise => { + while (next < items.length) { + const i = next++ + const it = items[i] + try { + const img = it.source === 'remote' ? await fetchImage(it.src) : await readImage(it.src) + resolved[i] = { scope: it.scope, key: it.key, mime: img.mime, data: img.data } + } catch { + /* skip — tallied as failed below */ + } + } + } + await Promise.all(Array.from({ length: Math.min(ART_SAVE_CONCURRENCY, items.length) }, worker)) + + const ok = resolved.filter((r): r is Resolved => r !== null) + getDb().transaction((rows: Resolved[]) => { + for (const r of rows) storeArt(r.scope, r.key, r.mime, r.data) + })(ok) + + return { saved: ok.length, failed: items.length - ok.length } +} + +export function clearArt(scope: ArtScope, key: string): void { + getDb().prepare('DELETE FROM local_art WHERE scope = ? AND art_key = ?').run(scope, normKey(key)) +} + +// Album art keys are `artist` + NUL + `album` (see renderer lib/artKeys.ts). Built +// via fromCharCode so there's no raw NUL byte in this source file. +// +// DEBUGGING THE STORED KEYS: the NUL separator is invisible in a terminal and trips +// up the line-oriented text tools — `grep` reports "binary file matches" and hides +// the row, and BSD/macOS `sed` can't match it. Do NOT pipe dumped keys through +// grep/sed/awk; you'll either lose rows or see `artistalbum` mashed together with no +// visible boundary. Inspect with SQL instead, which renders the byte safely: +// SELECT scope, replace(art_key, char(0), '|'), length(art_key) FROM local_art; +// SELECT scope, hex(art_key) FROM local_art; -- the NUL shows up as `00` +// `char(0)` is also how you write a NUL into a WHERE clause when matching a key. +const ART_KEY_SEP = String.fromCharCode(0) + +/** + * Re-home stored artwork after an artist merge: move the artist photo and every + * album cover keyed under an old artist spelling to the canonical artist. If the + * canonical key already has art, the canonical wins (the old row is dropped). Old + * spellings equal to the canonical (case-insensitively) are skipped. Album keys + * are matched by their `NUL` prefix via substr — avoids escaping a NUL in + * a LIKE pattern. + */ +export function renameArtistArt(oldNames: string[], canonical: string): void { + const db = getDb() + const canon = normKey(canonical) + const has = db.prepare('SELECT 1 FROM local_art WHERE scope = ? AND art_key = ?') + const del = db.prepare('DELETE FROM local_art WHERE scope = ? AND art_key = ?') + const upd = db.prepare('UPDATE local_art SET art_key = ?, updated_at = ? WHERE scope = ? AND art_key = ?') + const now = Date.now() + for (const name of oldNames) { + const old = normKey(name) + if (!old || old === canon) continue + + // Artist photo. + if (has.get('artist', canon)) del.run('artist', old) + else upd.run(canon, now, 'artist', old) + + // Album covers prefixed with this artist (`NUL…`). + const prefix = old + ART_KEY_SEP + const rows = db + .prepare("SELECT art_key FROM local_art WHERE scope = 'album' AND substr(art_key, 1, ?) = ?") + .all(prefix.length, prefix) as { art_key: string }[] + for (const r of rows) { + const newKey = canon + r.art_key.slice(old.length) + if (has.get('album', newKey)) del.run('album', r.art_key) + else upd.run(newKey, now, 'album', r.art_key) + } + } +} diff --git a/app/src/main/local/artSearch.ts b/app/src/main/local/artSearch.ts new file mode 100644 index 00000000..7c5758d5 --- /dev/null +++ b/app/src/main/local/artSearch.ts @@ -0,0 +1,118 @@ +/** + * main/local/artSearch.ts + * + * Third-party artwork lookup for the local-library editors. Runs in the main + * process so it isn't subject to the renderer's CSP (which blocks remote images + * and cross-origin fetches): main fetches the provider JSON, downloads small + * thumbnails, and hands the renderer inline `data:` URLs it can render directly. + * + * Backed by Deezer's public, key-free API, which covers BOTH album covers and + * artist photos. Kept provider-shaped (returns generic ArtCandidate[]) so another + * source — Cover Art Archive, iTunes, fanart.tv — could be slotted in later. + * + * Two entry points: searchArt() backs the manual picker and returns up to + * MAX_RESULTS candidates, each with a downloaded thumbnail. bestTryArt() backs the + * one-click "Best try" — it resolves many targets in a single call, downloading + * only the first match's thumbnail per target (since that's all Best try shows), + * so a big import is one IPC round-trip plus N light lookups rather than one full + * eight-thumbnail search per item. + */ + +import { imageUrlToDataUrl } from './art' +import { searchAlbums, searchArtists } from './deezer' +import type { ArtBestTryProgress, ArtCandidate, ArtQuery, ArtRequest, ArtScope } from './types' + +const SEARCH_LIMIT = 12 // raw hits to consider before de-duping +const MAX_RESULTS = 8 // candidates returned to the renderer +/** How many Deezer lookups a "Best try" batch runs at once — enough to feel quick + * on a big import without hammering the keyless API. */ +const BEST_TRY_CONCURRENCY = 4 + +/** A raw provider hit before any thumbnail is downloaded — the JSON lookup + * without the (expensive) image fetches. Shared by the full search and Best try. */ +type RawHit = { fullUrl?: string; thumbUrl?: string; label: string } + +/** Run the provider JSON search for one query and map it to raw hits, picking a + * sensible thumb + full URL for each. No images are downloaded here. */ +async function hitsFor(scope: ArtScope, q: ArtQuery): Promise { + const artist = q.artist.trim() + if (!artist) return [] + if (scope === 'album') { + const album = (q.album ?? '').trim() + if (!album) return [] + const albums = await searchAlbums(artist, album, SEARCH_LIMIT) + return albums.map((a) => ({ + fullUrl: a.cover_xl || a.cover_big, + thumbUrl: a.cover_medium || a.cover_big, + label: [a.artist?.name, a.title].filter(Boolean).join(' — ') || a.title, + })) + } + const artists = await searchArtists(artist, SEARCH_LIMIT) + return artists.map((a) => ({ + fullUrl: a.picture_xl || a.picture_big, + thumbUrl: a.picture_medium || a.picture_big, + label: a.name, + })) +} + +const usable = (r: RawHit): boolean => !!(r.fullUrl && r.thumbUrl) + +/** Build an ArtCandidate from a usable hit, downloading its thumbnail as a data + * URL. Returns null if the thumbnail fails to download. Caller must pre-check + * `usable(r)`. */ +async function toCandidate(r: RawHit): Promise { + try { + return { + source: 'Deezer', + label: r.label, + thumbDataUrl: await imageUrlToDataUrl(r.thumbUrl!), + fullUrl: r.fullUrl!, + } + } catch { + return null + } +} + +/** Search third-party artwork for an album cover or artist photo. Returns up to + * MAX_RESULTS candidates (de-duped by full URL, each with a downloaded thumbnail + * fetched concurrently), or [] on any failure — the caller surfaces "no results". */ +export async function searchArt(scope: ArtScope, q: ArtQuery): Promise { + const seen = new Set() + const picks = (await hitsFor(scope, q)) + .filter((r) => usable(r) && !seen.has(r.fullUrl!) && seen.add(r.fullUrl!)) + .slice(0, MAX_RESULTS) + const settled = await Promise.all(picks.map(toCandidate)) + return settled.filter((c): c is ArtCandidate => c !== null) +} + +/** The single best candidate for one query, downloading ONLY its thumbnail (not + * the up-to-MAX_RESULTS the full search fetches). Null when there's no usable hit + * or its thumbnail fails to download. */ +async function searchFirst(scope: ArtScope, q: ArtQuery): Promise { + const hit = (await hitsFor(scope, q)).find(usable) + return hit ? toCandidate(hit) : null +} + +/** + * "Best try" over many targets in a single call. Returns the first match for each + * request (or null) in the same order. Lookups run a few at a time so the keyless + * API isn't hammered; `onProgress` ticks once per request resolved. Replaces the + * renderer firing one IPC call + one full (eight-thumbnail) search per target. + */ +export async function bestTryArt( + reqs: ArtRequest[], + onProgress?: (p: ArtBestTryProgress) => void, +): Promise<(ArtCandidate | null)[]> { + const results: (ArtCandidate | null)[] = new Array(reqs.length).fill(null) + let next = 0 + let done = 0 + const worker = async (): Promise => { + while (next < reqs.length) { + const i = next++ + results[i] = await searchFirst(reqs[i].scope, reqs[i].query) + onProgress?.({ done: ++done, total: reqs.length }) + } + } + await Promise.all(Array.from({ length: Math.min(BEST_TRY_CONCURRENCY, reqs.length) }, worker)) + return results +} diff --git a/app/src/main/local/catalog.ts b/app/src/main/local/catalog.ts new file mode 100644 index 00000000..05f461e8 --- /dev/null +++ b/app/src/main/local/catalog.ts @@ -0,0 +1,212 @@ +/** + * main/local/catalog.ts + * + * Structured metadata for on-disk tracks, persisted in SQLite (main/db). A track + * is "labeled" once it has a title + artist; the renderer uses that to split the + * local library into Library vs Unlabeled. Metadata is keyed by the local file id + * (LocalLibrary.fileId — a hash of the absolute path). + * + * Three sources feed the editor: embedded file tags (readEmbeddedTags), an online + * lookup (renderer-side MusicBrainz), and manual entry. Only what the user saves + * lands here. + */ + +import crypto from 'crypto' +import path from 'path' +import { getDb } from '../db' +import type { Contributor, LocalTrackMeta } from './types' + +const SCHEMA_VERSION = 1 + +/** Stored row, shaped for merging into LocalTrack during a scan. */ +export interface StoredMeta extends LocalTrackMeta { + localId: string + path: string + trackId: string +} + +interface Row { + local_id: string + path: string + schema_version: number + track_id: string | null + isrc_code: string | null + title: string + by_artist: string + album_name: string | null + date_published: string | null + duration_ms: number | null + contributors: string | null + created_at: number + updated_at: number +} + +/** + * Canonical track id — SHA-256 of `${artist}:${title}`, first 24 hex chars. + * Must stay identical to the renderer's computeTrackId (src/renderer/src/lib/ + * trackId.ts) so a locally-labeled track resolves to the same id PublishModal uses. + */ +export function computeTrackId(artist: string, title: string): string { + return crypto.createHash('sha256').update(`${artist}:${title}`).digest('hex').slice(0, 24) +} + +function rowToStored(r: Row): StoredMeta { + return { + localId: r.local_id, + path: r.path, + trackId: r.track_id ?? computeTrackId(r.by_artist, r.title), + title: r.title, + byArtist: r.by_artist, + albumName: r.album_name, + datePublished: r.date_published, + isrcCode: r.isrc_code, + durationMs: r.duration_ms, + contributors: parseContributors(r.contributors), + } +} + +function parseContributors(json: string | null): Contributor[] { + if (!json) return [] + try { + const v = JSON.parse(json) + return Array.isArray(v) ? v : [] + } catch { + return [] + } +} + +// ─── CRUD ───────────────────────────────────────────────────────────────────── + +export function getMeta(localId: string): StoredMeta | null { + const row = getDb() + .prepare('SELECT * FROM local_track_meta WHERE local_id = ?') + .get(localId) as Row | undefined + return row ? rowToStored(row) : null +} + +/** All stored metadata, keyed by local id — loaded once per scan to merge in. */ +export function listMeta(): Map { + const rows = getDb().prepare('SELECT * FROM local_track_meta').all() as Row[] + return new Map(rows.map((r) => [r.local_id, rowToStored(r)])) +} + +/** + * Insert or update a track's metadata. Requires title + artist (the "labeled" + * threshold); returns the stored result (with computed trackId). + */ +export function upsertMeta(localId: string, filePath: string, meta: LocalTrackMeta): StoredMeta { + const title = meta.title.trim() + const byArtist = meta.byArtist.trim() + if (!title || !byArtist) throw new Error('title and artist are required') + + const trackId = computeTrackId(byArtist, title) + const now = Date.now() + getDb() + .prepare( + `INSERT INTO local_track_meta + (local_id, path, schema_version, track_id, isrc_code, title, by_artist, + album_name, date_published, duration_ms, contributors, created_at, updated_at) + VALUES + (@local_id, @path, @schema_version, @track_id, @isrc_code, @title, @by_artist, + @album_name, @date_published, @duration_ms, @contributors, @now, @now) + ON CONFLICT(local_id) DO UPDATE SET + path = excluded.path, + schema_version = excluded.schema_version, + track_id = excluded.track_id, + isrc_code = excluded.isrc_code, + title = excluded.title, + by_artist = excluded.by_artist, + album_name = excluded.album_name, + date_published = excluded.date_published, + duration_ms = excluded.duration_ms, + contributors = excluded.contributors, + updated_at = excluded.updated_at`, + ) + .run({ + local_id: localId, + path: filePath, + schema_version: SCHEMA_VERSION, + track_id: trackId, + isrc_code: meta.isrcCode || null, + title, + by_artist: byArtist, + album_name: meta.albumName || null, + date_published: meta.datePublished || null, + duration_ms: meta.durationMs ?? null, + contributors: meta.contributors?.length ? JSON.stringify(meta.contributors) : null, + now, + }) + + return { + localId, path: filePath, trackId, + title, byArtist, + albumName: meta.albumName || null, + datePublished: meta.datePublished || null, + isrcCode: meta.isrcCode || null, + durationMs: meta.durationMs ?? null, + contributors: meta.contributors ?? [], + } +} + +export function deleteMeta(localId: string): void { + getDb().prepare('DELETE FROM local_track_meta WHERE local_id = ?').run(localId) +} + +/** + * Rewrite a labeled track's artist — used when merging duplicate artist spellings + * (e.g. "God Smack" → "Godsmack"). Recomputes trackId (which is derived from + * artist + title) so the track still resolves to the right canonical id. No-op if + * the track has no stored row or the new name is blank. + */ +export function setArtistName(localId: string, byArtist: string): void { + const artist = byArtist.trim() + if (!artist) return + const row = getDb() + .prepare('SELECT title FROM local_track_meta WHERE local_id = ?') + .get(localId) as { title: string } | undefined + if (!row) return + getDb() + .prepare('UPDATE local_track_meta SET by_artist = ?, track_id = ?, updated_at = ? WHERE local_id = ?') + .run(artist, computeTrackId(artist, row.title), Date.now(), localId) +} + +// ─── Embedded tags ────────────────────────────────────────────────────────── + +/** + * Best-effort metadata from the file's embedded tags (ID3/Vorbis/MP4) via + * music-metadata. Used to prefill the editor and to power bulk auto-label. Falls + * back to the filename for the title so the form is never blank. music-metadata + * is ESM-only, hence the dynamic import. + */ +export async function readEmbeddedTags(absPath: string): Promise { + const fallbackTitle = path.basename(absPath, path.extname(absPath)) + try { + const mm = await import('music-metadata') + const { common, format } = await mm.parseFile(absPath, { duration: true }) + + const contributors: Contributor[] = (common.composer ?? []).map((name) => ({ + role: 'composer', name, id: null, + })) + + return { + title: (common.title || fallbackTitle).trim(), + byArtist: (common.artist || common.albumartist || '').trim(), + albumName: common.album?.trim() || null, + datePublished: common.date || (common.year ? String(common.year) : null), + isrcCode: common.isrc?.[0] || null, + durationMs: format.duration ? Math.round(format.duration * 1000) : null, + contributors, + } + } catch { + // Unreadable / no tags — hand back just the filename title. + return { + title: fallbackTitle, + byArtist: '', + albumName: null, + datePublished: null, + isrcCode: null, + durationMs: null, + contributors: [], + } + } +} diff --git a/app/src/main/local/deezer.ts b/app/src/main/local/deezer.ts new file mode 100644 index 00000000..c359f3fe --- /dev/null +++ b/app/src/main/local/deezer.ts @@ -0,0 +1,93 @@ +/** + * main/local/deezer.ts + * + * The single place that talks to Deezer's public, key-free API. Lives in the main + * process because the renderer's CSP blocks cross-origin fetches — anything in the + * renderer that needs Deezer goes through an IPC handler backed by this module. + * + * Holds the base URL, the JSON shapes we read, the advanced-query helper, the + * fetch+parse wrapper, and the typed search calls. Callers (artSearch, the + * `deezer:track-cover` IPC handler) stay free of raw URLs and JSON field names. + */ + +import { net } from 'electron' + +/** Base URL for every Deezer endpoint. */ +export const DEEZER_API = 'https://api.deezer.com' + +// ─── JSON shapes (only the fields we read) ────────────────────────────────────── + +export interface DeezerArtist { + id: number + name: string + picture_medium?: string + picture_xl?: string + picture_big?: string +} + +export interface DeezerAlbum { + id: number + title: string + cover_medium?: string + cover_big?: string + cover_xl?: string + artist?: { name?: string } +} + +export interface DeezerTrack { + id: number + title: string + album?: { cover_xl?: string; cover_big?: string; cover_medium?: string } +} + +/** Every Deezer search endpoint returns `{ data: [...] }`. */ +interface DeezerList { + data?: T[] +} + +// ─── Low-level helpers ────────────────────────────────────────────────────────── + +/** Advanced-query fragment: `field:"value"`, with embedded quotes stripped. */ +export const term = (field: string, value: string): string => + `${field}:"${value.replace(/"/g, '')}"` + +/** GET a Deezer URL and parse its JSON, or null on any network/parse failure. */ +export async function deezerJson(url: string): Promise { + try { + const res = await net.fetch(url) + if (!res.ok) return null + return (await res.json()) as T + } catch { + return null + } +} + +// ─── Typed searches ───────────────────────────────────────────────────────────── + +/** Search albums by artist + album title. */ +export function searchAlbums( + artist: string, + album: string, + limit: number, +): Promise { + const q = encodeURIComponent([term('artist', artist), term('album', album)].join(' ')) + return deezerJson>(`${DEEZER_API}/search/album?q=${q}&limit=${limit}`).then( + (d) => d?.data ?? [], + ) +} + +/** Search artists by name. */ +export function searchArtists(artist: string, limit: number): Promise { + const q = encodeURIComponent(artist) + return deezerJson>(`${DEEZER_API}/search/artist?q=${q}&limit=${limit}`).then( + (d) => d?.data ?? [], + ) +} + +/** Full-resolution album cover for the first track matching artist + title, or + * null. Backs the BrowseView album-art fallback. */ +export async function trackCover(artist: string, title: string): Promise { + const q = encodeURIComponent([term('artist', artist), term('track', title)].join(' ')) + const data = await deezerJson>(`${DEEZER_API}/search?q=${q}&limit=1`) + return data?.data?.[0]?.album?.cover_xl ?? null +} diff --git a/app/src/main/local/discovery.ts b/app/src/main/local/discovery.ts new file mode 100644 index 00000000..22ed368a --- /dev/null +++ b/app/src/main/local/discovery.ts @@ -0,0 +1,113 @@ +/** + * main/local/discovery.ts + * + * Read-only check of which locally-labeled tracks are present in the SOND3R + * catalog (the Qdrant `fangorn` collection), so the UI can flag them as + * "semantically discoverable". Nothing is written to Qdrant here. + * + * Matching is by the `fields.trackId` payload, NOT by point id: catalog points + * are stored under random UUIDs, so they can't be addressed deterministically. + * The catalog's trackId is `sha256(`${artist}:${title}`)[:24]` computed over the + * *lowercased* artist + title — note this differs from the local `computeTrackId` + * (./catalog), which preserves the original casing. We recompute the catalog form + * here and ask Qdrant which of those trackIds exist via a batched MatchAny scroll. + * (Verified against live catalog points, e.g. "Hey" by Pixies.) + */ + +import { createHash } from 'crypto' +import { net } from 'electron' +import * as catalog from './catalog' + +// Qdrant sidecar — localhost only, same host/port main/index.ts boots it on. +const QDRANT_URL = 'http://127.0.0.1:6333' +const COLLECTION = 'fangorn' +// MatchAny does a full collection scan per scroll (fields.trackId isn't indexed), +// so prefer bigger chunks — fewer scans — over many small ones. Each scan is +// ~0.4s over 860k points regardless of how many ids we test against. +const CHUNK = 1000 + +/** + * The catalog's canonical track id: first 24 hex of `sha256("{artist}:{title}")` + * over the trimmed + lowercased artist and title. This is the publish/ingest + * scheme, which lowercases — unlike the local computeTrackId in ./catalog. + */ +function catalogTrackId(artist: string, title: string): string { + const key = `${artist.trim().toLowerCase()}:${title.trim().toLowerCase()}` + return createHash('sha256').update(key).digest('hex').slice(0, 24) +} + +/** + * Ask Qdrant which of these catalog trackIds exist, batched. Filters on the + * `fields.trackId` payload with MatchAny and pages through each chunk's results. + * Resolves to the subset present; on any error (Qdrant down, collection missing) + * returns what it has so far rather than throwing — callers treat "can't tell" + * as "nothing to flag". + */ +async function existingTrackIds(trackIds: string[]): Promise> { + const present = new Set() + for (let i = 0; i < trackIds.length; i += CHUNK) { + const chunk = trackIds.slice(i, i + CHUNK) + let offset: unknown = undefined + do { + const ctrl = new AbortController() + const timer = setTimeout(() => ctrl.abort(), 8000) + try { + const res = await net.fetch(`${QDRANT_URL}/collections/${COLLECTION}/points/scroll`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + filter: { must: [{ key: 'fields.trackId', match: { any: chunk } }] }, + limit: chunk.length, + offset: offset ?? undefined, + with_payload: { include: ['fields.trackId'] }, + with_vector: false, + }), + signal: ctrl.signal, + }) + if (!res.ok) return present + const body = await res.json().catch(() => null) + const result = body?.result + const points = result && Array.isArray(result.points) ? result.points : [] + for (const pt of points) { + const tid = pt?.payload?.fields?.trackId + if (typeof tid === 'string') present.add(tid) + } + offset = result?.next_page_offset ?? null + } catch { + return present // Qdrant down / not ready — nothing discoverable for now. + } finally { + clearTimeout(timer) + } + } while (offset) + } + return present +} + +/** + * Local ids of labeled tracks that have a vector in the catalog. Derives each + * labeled track's catalog trackId from its artist + title, batch-checks which + * exist, and maps the hits back to local ids. Several local files can share one + * trackId (the same song twice), so a trackId maps to a list of local ids. + */ +export async function listDiscoverableLocalIds(): Promise { + const metas = catalog.listMeta() + if (metas.size === 0) return [] + + const tidToLocals = new Map() + for (const [localId, meta] of metas) { + if (!meta.title || !meta.byArtist) continue + const tid = catalogTrackId(meta.byArtist, meta.title) + const arr = tidToLocals.get(tid) + if (arr) arr.push(localId) + else tidToLocals.set(tid, [localId]) + } + if (tidToLocals.size === 0) return [] + + const existing = await existingTrackIds([...tidToLocals.keys()]) + const out: string[] = [] + for (const tid of existing) { + const locals = tidToLocals.get(tid) + if (locals) out.push(...locals) + } + return out +} diff --git a/app/src/main/local/favorites.ts b/app/src/main/local/favorites.ts new file mode 100644 index 00000000..a14d5653 --- /dev/null +++ b/app/src/main/local/favorites.ts @@ -0,0 +1,51 @@ +/** + * main/local/favorites.ts + * + * "Favorited" artists, albums, and songs for the local library, persisted in + * SQLite (main/db, `local_favorites`). A favorite is just a (kind, ref) pair: + * - kind 'track' → ref is the local file id (LocalLibrary.fileId) + * - kind 'artist' → ref is the artist name + * - kind 'album' → ref is "\0" (see favAlbumRef in the renderer) + * + * Refs are built and kept consistent by the renderer (LocalMusicView), so they're + * stored verbatim here — membership is an exact-string match. + */ + +import { getDb } from '../db' + +export type FavKind = 'track' | 'artist' | 'album' + +/** Favorited refs grouped by kind — loaded once and held in the renderer. */ +export interface Favorites { + track: string[] + artist: string[] + album: string[] +} + +const KINDS: FavKind[] = ['track', 'artist', 'album'] + +export function listFavorites(): Favorites { + const rows = getDb() + .prepare('SELECT kind, ref FROM local_favorites') + .all() as { kind: string; ref: string }[] + const out: Favorites = { track: [], artist: [], album: [] } + for (const r of rows) { + if ((KINDS as string[]).includes(r.kind)) out[r.kind as FavKind].push(r.ref) + } + return out +} + +/** Flip a favorite on/off. Returns the new state (true = now favorited). */ +export function toggleFavorite(kind: FavKind, ref: string): boolean { + const db = getDb() + const existing = db + .prepare('SELECT 1 FROM local_favorites WHERE kind = ? AND ref = ?') + .get(kind, ref) + if (existing) { + db.prepare('DELETE FROM local_favorites WHERE kind = ? AND ref = ?').run(kind, ref) + return false + } + db.prepare('INSERT INTO local_favorites (kind, ref, created_at) VALUES (?, ?, ?)') + .run(kind, ref, Date.now()) + return true +} diff --git a/app/src/main/local/ipc.ts b/app/src/main/local/ipc.ts index 2f9cfcc2..4a517d16 100644 --- a/app/src/main/local/ipc.ts +++ b/app/src/main/local/ipc.ts @@ -9,6 +9,16 @@ import { ipcMain, dialog, BrowserWindow } from 'electron' import * as lib from './LocalLibrary' +import * as catalog from './catalog' +import * as taxonomy from './taxonomy' +import * as discovery from './discovery' +import * as art from './art' +import * as artSearch from './artSearch' +import { autoOrganize, mergeArtist } from './organize' +import * as favorites from './favorites' +import * as playlists from './playlists' +import type { ArtBestTryProgress, ArtQuery, ArtRequest, ArtSaveItem, ArtScope, ImportMode, ImportProgress, LocalTrackMeta, LocalTrackTags, OrganizeProgress } from './types' +import type { FavKind } from './favorites' export function registerLocalMusicIpc(): void { void lib.startLocalFileServer() @@ -31,13 +41,199 @@ export function registerLocalMusicIpc(): void { }) ipcMain.handle('local:scan', async (_e, dir?: string) => { - const target = dir || lib.getSavedDir() - lib.saveDir(target) + // lib.scan persists `dir` as the primary folder when given, then scans every + // root (primary + referenced). try { - return await lib.scan(target) + return await lib.scan(dir || undefined) } catch (err) { console.error('[local-music] scan failed:', err) throw err } }) + + // ── Bulk import (external folders / drives) ────────────────────────────────── + // Pick a source folder and report how many audio files it holds, so the import + // dialog can preview before the user commits to copy vs reference. + ipcMain.handle('local:import:pick-source', async () => { + const win = BrowserWindow.getFocusedWindow() ?? BrowserWindow.getAllWindows()[0] + const opts = { title: 'Choose a folder to import', properties: ['openDirectory' as const] } + const result = win ? await dialog.showOpenDialog(win, opts) : await dialog.showOpenDialog(opts) + const dir = result.filePaths[0] + if (result.canceled || !dir) return null + return { path: dir, audioCount: await lib.countAudioFiles(dir) } + }) + + // Run the import, streaming throttled progress back to the caller's window. + ipcMain.handle('local:import:run', async (e, source: string, mode: ImportMode) => { + let last = 0 + const onProgress = (p: ImportProgress) => { + const now = Date.now() + if (p.processed >= p.total || now - last > 120) { + last = now + if (!e.sender.isDestroyed()) e.sender.send('local:import:progress', p) + } + } + return lib.importFolder(source, mode, onProgress) + }) + + // ── Library roots (primary + referenced folders) ───────────────────────────── + ipcMain.handle('local:roots:list', () => lib.getRoots()) + ipcMain.handle('local:roots:remove', (_e, p: string) => { lib.removeExtraRoot(p) }) + + // ── Auto-organize (smart bulk labeling) ────────────────────────────────────── + // Resolve + merge + label every Unlabeled track, streaming throttled progress. + ipcMain.handle('local:auto-organize', (e) => { + let last = 0 + const onProgress = (p: OrganizeProgress) => { + const now = Date.now() + if (p.processed >= p.total || now - last > 120) { + last = now + if (!e.sender.isDestroyed()) e.sender.send('local:organize:progress', p) + } + } + return autoOrganize(onProgress) + }) + + // Resolve a duplicate-artist suggestion: merge the variant spellings into the + // canonical one the user picked. Returns how many tracks were renamed. + ipcMain.handle('local:artist:merge', (_e, canonical: string, variants: string[]) => + mergeArtist(canonical, variants)) + + // ── Track metadata library ────────────────────────────────────────────────── + // Tracks are addressed by their served id; the path is resolved from the last + // scan's registry so these can't touch files outside the scanned folder. + + ipcMain.handle('local:meta:get', (_e, localId: string) => catalog.getMeta(localId)) + + ipcMain.handle('local:meta:upsert', (_e, localId: string, meta: LocalTrackMeta) => { + const filePath = lib.pathForId(localId) + if (!filePath) throw new Error('unknown track id') + return catalog.upsertMeta(localId, filePath, meta) + }) + + ipcMain.handle('local:meta:delete', (_e, localId: string) => { + catalog.deleteMeta(localId) + }) + + // Drop a track from the library: delete the file if it's a library copy (leaves + // referenced-in-place source files intact), and clear its metadata + tags. + ipcMain.handle('local:track:drop', (_e, localId: string) => { + const res = lib.dropTrack(localId) + catalog.deleteMeta(localId) + taxonomy.deleteTags(localId) + return res + }) + + ipcMain.handle('local:read-tags', async (_e, localId: string) => { + const filePath = lib.pathForId(localId) + if (!filePath) throw new Error('unknown track id') + return catalog.readEmbeddedTags(filePath) + }) + + // ── Semantic taxonomy tags ──────────────────────────────────────────────────── + // Local-only enrichment (genres/moods/themes/contexts) per TrackTaxonomySchema. + // Stored, never published or embedded. The canonical trackId is taken from the + // track's saved metadata so it stays consistent with the labeled library. + + ipcMain.handle('local:tags:get', (_e, localId: string) => taxonomy.getTags(localId)) + + // All stored tag rows in one shot — lets the renderer flag which tracks carry + // taxonomy without a per-row round-trip. Maps don't survive IPC, so send values. + ipcMain.handle('local:tags:list', () => Array.from(taxonomy.listTags().values())) + + ipcMain.handle('local:tags:upsert', (_e, localId: string, tags: LocalTrackTags) => { + if (!lib.pathForId(localId)) throw new Error('unknown track id') + const trackId = catalog.getMeta(localId)?.trackId ?? null + return taxonomy.upsertTags(localId, trackId, tags) + }) + + ipcMain.handle('local:tags:delete', (_e, localId: string) => { + taxonomy.deleteTags(localId) + }) + + // ── Catalog discoverability ─────────────────────────────────────────────────── + // Which labeled tracks have a vector in the local Qdrant catalog (read-only). + // Drives the "semantically discoverable" indicator in the renderer. + ipcMain.handle('local:discoverable:list', () => discovery.listDiscoverableLocalIds()) + + // ── Album / artist artwork ────────────────────────────────────────────────── + // Normalized keys that already have stored art — lets the renderer surface only + // the artists/albums still missing artwork without probing each key. + ipcMain.handle('local:art:keys', () => art.listArtKeys()) + + ipcMain.handle('local:art:get', (_e, scope: ArtScope, key: string) => art.getArt(scope, key)) + + ipcMain.handle('local:art:clear', (_e, scope: ArtScope, key: string) => { art.clearArt(scope, key) }) + + // Opens the native image picker; resolves to the chosen path, or null if the + // user cancels. The dialog title is tailored to the scope being set. + const pickImageFile = async (scope: ArtScope): Promise => { + const win = BrowserWindow.getFocusedWindow() ?? BrowserWindow.getAllWindows()[0] + const opts = { + title: scope === 'artist' ? 'Choose artist image' : 'Choose album cover', + properties: ['openFile' as const], + filters: [{ name: 'Images', extensions: ['png', 'jpg', 'jpeg', 'webp', 'gif', 'bmp', 'avif'] }], + } + const result = win ? await dialog.showOpenDialog(win, opts) : await dialog.showOpenDialog(opts) + const file = result.filePaths[0] + return result.canceled || !file ? null : file + } + + // Opens the native image picker, stores the chosen image, and returns it as a + // data URL. Returns null if the user cancels. + ipcMain.handle('local:art:pick-set', async (_e, scope: ArtScope, key: string) => { + const file = await pickImageFile(scope) + return file ? art.setArtFromFile(scope, key, file) : null + }) + + // Pick-only: open the image picker and return { path, dataUrl } without storing + // anything. The editors use this to stage artwork, then commit it under the + // final artist/album key on save (so renaming before save doesn't orphan art). + ipcMain.handle('local:art:pick', async (_e, scope: ArtScope) => { + const file = await pickImageFile(scope) + return file ? { path: file, dataUrl: art.imageFileToDataUrl(file) } : null + }) + + // Commit a previously picked image (by path) as the artwork for (scope, key). + ipcMain.handle('local:art:set-file', (_e, scope: ArtScope, key: string, filePath: string) => + art.setArtFromFile(scope, key, filePath)) + + // Third-party artwork search (Deezer). Returns candidates with inline thumbnail + // data URLs (CSP-safe) plus a full-res URL to commit later. + ipcMain.handle('local:art:search', (_e, scope: ArtScope, q: ArtQuery) => artSearch.searchArt(scope, q)) + + // "Best try": resolve many missing-art targets in one call, streaming throttled + // progress. Only the first match's thumbnail is downloaded per target (vs. the + // full search's up-to-MAX_RESULTS), so a big import is one IPC + N light lookups. + ipcMain.handle('local:art:bestTry', (e, reqs: ArtRequest[]) => { + let last = 0 + const onProgress = (p: ArtBestTryProgress) => { + const now = Date.now() + if (p.done >= p.total || now - last > 120) { + last = now + if (!e.sender.isDestroyed()) e.sender.send('local:art:bestTry:progress', p) + } + } + return artSearch.bestTryArt(reqs, onProgress) + }) + + // Commit a third-party image (by URL) as the artwork for (scope, key). + ipcMain.handle('local:art:set-url', (_e, scope: ArtScope, key: string, url: string) => + art.setArtFromUrl(scope, key, url)) + + // Bulk-commit staged artwork in one call (downloads/reads run concurrently in + // main, then persist in a single transaction). Returns saved/failed counts. + ipcMain.handle('local:art:set-many', (_e, items: ArtSaveItem[]) => art.setArtMany(items)) + + // ── Favorites (artists / albums / songs) ───────────────────────────────────── + ipcMain.handle('local:fav:list', () => favorites.listFavorites()) + ipcMain.handle('local:fav:toggle', (_e, kind: FavKind, ref: string) => favorites.toggleFavorite(kind, ref)) + + // ── Playlists ──────────────────────────────────────────────────────────────── + ipcMain.handle('local:playlist:list', () => playlists.listPlaylists()) + ipcMain.handle('local:playlist:create', (_e, name: string) => playlists.createPlaylist(name)) + ipcMain.handle('local:playlist:rename', (_e, id: string, name: string) => { playlists.renamePlaylist(id, name) }) + ipcMain.handle('local:playlist:delete', (_e, id: string) => { playlists.deletePlaylist(id) }) + ipcMain.handle('local:playlist:add', (_e, id: string, localId: string) => { playlists.addToPlaylist(id, localId) }) + ipcMain.handle('local:playlist:remove', (_e, id: string, localId: string) => { playlists.removeFromPlaylist(id, localId) }) } diff --git a/app/src/main/local/organize.ts b/app/src/main/local/organize.ts new file mode 100644 index 00000000..fe8fda6c --- /dev/null +++ b/app/src/main/local/organize.ts @@ -0,0 +1,349 @@ +/** + * main/local/organize.ts + * + * Auto-organizer for messy imports. For every Unlabeled track it resolves + * artist / title / album through a HIERARCHY of sources, best first: + * + * artist: embedded tag → "Artist - Title" filename → containing folder name + * title: embedded tag → filename → nearest non-generic ancestor folder + * (so own-productions named "Mixdown.mp3" under …/PeaceMaker/Mixdown/ + * become "PeaceMaker") + * album: embedded tag → (none) + * + * Everything is cleaned (whitespace collapsed, "Last, First" un-flipped, diacritics + * folded for matching) and then MERGED: variant spellings of the same artist/album + * collapse to one canonical display name (seeded by the names already in the + * Library, which win), and tracks that resolve to the same artist+title are treated + * as duplicates and skipped. Names that look like the same artist but differ too + * much to merge safely (e.g. "AC DC" vs "ACDC") are returned as review suggestions + * rather than auto-merged. + * + * Resolvable tracks are written via catalog.upsertMeta (moving them into the + * Library); the rest are left Unlabeled for manual labeling. + */ + +import path from 'path' +import { listMeta, readEmbeddedTags, setArtistName, upsertMeta } from './catalog' +import { renameArtistArt } from './art' +import { listRegistry } from './LocalLibrary' +import type { MergeSuggestion, OrganizeProgress, OrganizeReport } from './types' + +// ─── Text helpers ───────────────────────────────────────────────────────────── + +const clean = (s: string): string => s.replace(/\s+/g, ' ').trim() + +const stripDiacritics = (s: string): string => s.normalize('NFKD').replace(/[̀-ͯ]/g, '') + +/** Matching key: lowercase, diacritics folded, punctuation → space, leading + * "the " dropped. Folds "AeroSmith"/"Aerosmith", "ACDC "/"ACDC", "The X"/"X". */ +function canonKey(name: string): string { + let s = stripDiacritics(name).toLowerCase() + s = s.replace(/&/g, ' and ').replace(/[^a-z0-9]+/g, ' ') + s = s.replace(/\s+/g, ' ').trim().replace(/^the /, '') + return s +} + +/** Tighter key (no spaces) — catches space-only variants like "AC DC" vs "ACDC" + * for review suggestions; too aggressive to auto-merge on. */ +const tightKey = (name: string): string => canonKey(name).replace(/\s+/g, '') + +/** "Aguilera, Christina" → "Christina Aguilera". Skips "Earth, Wind & Fire", + * "Tyler, The Creator", and many-word sides. */ +function unflipName(name: string): string { + const m = name.match(/^([^,]+),\s*([^,]+)$/) + if (!m) return name + const last = m[1].trim() + const first = m[2].trim() + if (/&|\band\b/i.test(name)) return name + if (/^the\b/i.test(first) || /^the\b/i.test(last)) return name + if (last.split(/\s+/).length > 3 || first.split(/\s+/).length > 2) return name + return `${first} ${last}` +} + +// Folder / file names that carry no real song info — skip them as artist/title. +const GENERIC = new Set([ + 'mixdown', 'master', 'mastered', 'final', 'bounce', 'export', 'render', 'audio', + 'track', 'untitled', 'recording', 'new recording', 'media', 'mp3', 'wav', 'flac', + 'song', 'songs', 'music', 'unknown', 'unknown artist', 'unknown album', 'cache', + 'history', 'images', 'goody', +]) +function isGeneric(name: string): boolean { + const k = canonKey(name) + if (!k) return true + const base = k.replace(/\s*\d+$/, '').trim() // "mixdown 2" → "mixdown" + return GENERIC.has(k) || GENERIC.has(base) +} + +// ─── Per-file resolution ──────────────────────────────────────────────────── + +interface Parsed { artist: string | null; title: string | null } + +/** Derive artist/title from a filename, handling the common messy shapes: + * "045. Artist - Title", "02 - Artist - Title", "Artist - 07 - Title", + * "08 Title", and scene-style "12-the_cure-like_cockatoos-tns". */ +function parseFilename(absPath: string): Parsed { + const raw = path.basename(absPath, path.extname(absPath)) + let base = raw.replace(/^\s*[([][^)\]]*[)\]]\s*/, '').trim() // strip leading "(mp3)" / "[..]" + base = base.replace(/^copy of\s+/i, '').trim() // "Copy of X" → "X" + base = base.replace(/\s*\(\d+\)\s*$/, '').trim() // strip "(1)" dup marker + base = base.replace(/\s*\(\s*\d{2,4}\s*k(?:bps)?\s*\)\s*$/i, '').trim() // strip "(192k)" bitrate + + const sceneStyle = !/\s/.test(base) && (base.match(/-/g)?.length ?? 0) >= 2 + base = base.replace(/_/g, ' ').replace(/\s+/g, ' ').trim() + + let artist: string | null = null + let title: string | null = null + + if (sceneStyle) { + const toks = base.split('-').map(clean).filter(Boolean) + if (toks.length && /^\d{1,3}$/.test(toks[0])) toks.shift() // leading track no. + if (toks.length > 2 && /^[a-z]{1,4}$/.test(toks[toks.length - 1])) toks.pop() // scene tag + if (toks.length === 1) title = toks[0] + else if (toks.length >= 2) { artist = toks[0]; title = toks.slice(1).join(' ') } + return { artist: artist && clean(artist), title: title && clean(title) } + } + + const tm = base.match(/^(\d{1,3})\s*[-._)]?\s+(.*)$/) // leading track number + if (tm) base = tm[2].trim() + + const parts = base.split(/\s+-\s+/).map((p) => p.trim()).filter(Boolean) + if (parts.length === 1) { + title = parts[0] + } else if (parts.length === 2) { + artist = parts[0]; title = parts[1] + } else { + artist = parts[0]; title = parts[parts.length - 1] // "Artist - 07 - Title" + } + if (title) { + const t2 = title.match(/^(\d{1,3})\s*[-._)]?\s+(.*)$/) // "07 Title" + if (t2) title = t2[2].trim() + } + return { artist: artist && clean(artist), title: title && clean(title) } +} + +/** Containing folder as artist, unless it's a generic dump folder. */ +function folderArtist(absPath: string): string | null { + const parent = path.basename(path.dirname(absPath)) + return parent && !isGeneric(parent) ? clean(unflipName(parent)) : null +} + +/** Nearest non-generic ancestor folder — the song name for own-productions whose + * file is generic ("Mixdown") and whose project folder holds the real title. */ +function folderTitle(absPath: string): string | null { + let dir = path.dirname(absPath) + for (let i = 0; i < 4; i++) { + const name = path.basename(dir) + if (!name || name === path.dirname(dir)) break + if (!isGeneric(name)) return clean(name) + dir = path.dirname(dir) + } + return null +} + +interface Resolved { + id: string + path: string + artist: string | null + title: string | null + album: string | null + date: string | null +} + +async function resolveOne(id: string, absPath: string): Promise { + const tags = await readEmbeddedTags(absPath) // title falls back to the basename + const parsed = parseFilename(absPath) + const rawBase = clean(path.basename(absPath, path.extname(absPath))).toLowerCase() + // readEmbeddedTags returns the basename as title when there's no real tag title. + const tagTitleReal = !!tags.title && clean(tags.title).toLowerCase() !== rawBase + + let artist = clean(tags.byArtist) || parsed.artist || folderArtist(absPath) + if (artist) artist = clean(unflipName(artist)) + + let title = (tagTitleReal && clean(tags.title)) || parsed.title || null + if (!title || isGeneric(title)) { + const ft = folderTitle(absPath) + if (ft && !isGeneric(ft)) title = ft + } + if (!title && tags.title) title = clean(tags.title) // last resort: filename basename + + return { + id, path: absPath, + artist: artist || null, + title: title || null, + album: tags.albumName ? clean(tags.albumName) : null, + date: tags.datePublished, + } +} + +// ─── Canonical-name grouping ────────────────────────────────────────────────── + +interface Group { variants: Map; labeledDisplay: string | null } + +function noteName(map: Map, name: string, fromLabeled: boolean): void { + const k = canonKey(name) + if (!k) return + let g = map.get(k) + if (!g) { g = { variants: new Map(), labeledDisplay: null }; map.set(k, g) } + g.variants.set(name, (g.variants.get(name) ?? 0) + 1) + if (fromLabeled && !g.labeledDisplay) g.labeledDisplay = name +} + +/** The display spelling for a group: an existing Library spelling wins, else the + * most frequently seen variant. */ +function groupDisplay(g: Group, fallback = ''): string { + if (g.labeledDisplay) return g.labeledDisplay + let best = fallback + let bestN = -1 + for (const [variant, n] of g.variants) if (n > bestN) { best = variant; bestN = n } + return best +} + +function displayFor(map: Map, name: string): string { + const g = map.get(canonKey(name)) + return g ? groupDisplay(g, name) : name +} + +/** Number of variant spellings folded away (sum of variants-1 per merged group). */ +function mergedCount(map: Map): number { + let n = 0 + for (const g of map.values()) if (g.variants.size > 1) n += g.variants.size - 1 + return n +} + +// ─── Orchestration ──────────────────────────────────────────────────────────── + +const RESOLVE_CONCURRENCY = 8 + +export async function autoOrganize(onProgress?: (p: OrganizeProgress) => void): Promise { + const labeled = listMeta() + const unlabeled = listRegistry().filter((e) => !labeled.has(e.id)) + const total = unlabeled.length + + // 1) Resolve every unlabeled track (bounded concurrency for tag reads). + const resolved: Resolved[] = [] + let processed = 0 + let next = 0 + const worker = async () => { + while (next < unlabeled.length) { + const e = unlabeled[next++] + resolved.push(await resolveOne(e.id, e.path)) + onProgress?.({ total, processed: ++processed }) + } + } + await Promise.all(Array.from({ length: Math.min(RESOLVE_CONCURRENCY, total) }, worker)) + + // 2) Canonical artist names — seed with the Library's existing names (authoritative). + const artists = new Map() + for (const m of labeled.values()) noteName(artists, m.byArtist, true) + for (const r of resolved) if (r.artist) noteName(artists, r.artist, false) + const artistOf = (name: string) => displayFor(artists, name) + + // 3) Canonical album names, keyed within their (canonical) artist. + const albumGroupKey = (artist: string, album: string) => `${canonKey(artist)}||${canonKey(album)}` + const albumGroups = new Map() + for (const m of labeled.values()) if (m.albumName) noteName2(albumGroups, albumGroupKey(m.byArtist, m.albumName), m.albumName, true) + for (const r of resolved) if (r.artist && r.album) noteName2(albumGroups, albumGroupKey(artistOf(r.artist), r.album), r.album, false) + const albumOf = (artist: string, album: string) => displayFor2(albumGroups, albumGroupKey(artist, album), album) + + // 4) Near-duplicate artist suggestions (same tight key, different canonical key). + const tight = new Map>() + for (const g of artists.values()) { + const display = groupDisplay(g) + const tk = tightKey(display) + if (!tk) continue + const s = tight.get(tk) ?? new Set() + s.add(display) + tight.set(tk, s) + } + const artistSuggestions: MergeSuggestion[] = [] + for (const set of tight.values()) { + if (set.size > 1) { + const variants = [...set].sort() + artistSuggestions.push({ canonical: variants[0], variants }) + } + } + + // 5) Write resolvable tracks, skipping duplicates (same canonical artist+title, + // including against the existing Library). + const songKey = (artist: string, title: string) => `${canonKey(artist)}||${canonKey(title)}` + const seenSongs = new Set() + for (const m of labeled.values()) seenSongs.add(songKey(m.byArtist, m.title)) + + let labeledCount = 0 + let duplicates = 0 + // Local ids of the tracks left Unlabeled because they couldn't be resolved — + // these are exactly the "needs review" tracks (duplicates are tracked + // separately, not here), so the renderer can surface the same set it counts. + const needsReviewIds: string[] = [] + for (const r of resolved) { + if (!r.artist || !r.title) { needsReviewIds.push(r.id); continue } + const artist = artistOf(r.artist) + const sk = songKey(artist, r.title) + if (seenSongs.has(sk)) { duplicates++; continue } + seenSongs.add(sk) + try { + upsertMeta(r.id, r.path, { + title: r.title, + byArtist: artist, + albumName: r.album ? albumOf(artist, r.album) : null, + datePublished: r.date, + isrcCode: null, + durationMs: null, + contributors: [], + }) + labeledCount++ + } catch { + needsReviewIds.push(r.id) + } + } + + return { + processed: total, + labeled: labeledCount, + duplicates, + needsReview: needsReviewIds.length, + needsReviewIds, + artistsMerged: mergedCount(artists), + albumsMerged: mergedCount(albumGroups), + artistSuggestions: artistSuggestions.slice(0, 20), + } +} + +/** + * Resolve a "possible duplicate artists" suggestion: rewrite every labeled track + * whose artist matches one of `variants` (by canonical key) to `canonical`, the + * spelling the user picked, and re-home that artist's stored artwork. Reuses the + * same canonKey the organizer groups by, so e.g. "God Smack", "god smack" and + * "Godsmack" all collapse. Returns how many tracks were changed. + */ +export function mergeArtist(canonical: string, variants: string[]): number { + const target = canonical.trim() + if (!target) return 0 + // Match against every variant's canonical key, including the canonical's own + // (so case-only twins of the chosen spelling normalize too). + const keys = new Set([target, ...variants].map(canonKey)) + + let changed = 0 + for (const m of listMeta().values()) { + if (m.byArtist !== target && keys.has(canonKey(m.byArtist))) { + setArtistName(m.localId, target) + changed++ + } + } + // Move artist photo + album covers off the old spellings onto the canonical. + renameArtistArt(variants, target) + return changed +} + +// Album groups are keyed by a composite (artist||album) string rather than a bare +// canonical name, so the generic group helpers take an explicit key here. +function noteName2(map: Map, key: string, display: string, fromLabeled: boolean): void { + let g = map.get(key) + if (!g) { g = { variants: new Map(), labeledDisplay: null }; map.set(key, g) } + g.variants.set(display, (g.variants.get(display) ?? 0) + 1) + if (fromLabeled && !g.labeledDisplay) g.labeledDisplay = display +} +function displayFor2(map: Map, key: string, fallback: string): string { + const g = map.get(key) + return g ? groupDisplay(g, fallback) : fallback +} diff --git a/app/src/main/local/playlists.ts b/app/src/main/local/playlists.ts new file mode 100644 index 00000000..b84bc1ba --- /dev/null +++ b/app/src/main/local/playlists.ts @@ -0,0 +1,109 @@ +/** + * main/local/playlists.ts + * + * User playlists for the local library, persisted in SQLite (main/db, + * `local_playlists` + `local_playlist_tracks`). A playlist is a named, ordered + * list of local file ids; the renderer resolves those ids against the current + * scan, skipping any whose file is no longer present. + */ + +import crypto from 'crypto' +import { getDb } from '../db' + +export interface LocalPlaylist { + id: string + name: string + /** Ordered local file ids (LocalLibrary.fileId). */ + trackIds: string[] + createdAt: number + updatedAt: number +} + +interface PlaylistRow { + id: string + name: string + created_at: number + updated_at: number +} + +/** All playlists with their ordered track ids. */ +export function listPlaylists(): LocalPlaylist[] { + const db = getDb() + const rows = db + .prepare('SELECT id, name, created_at, updated_at FROM local_playlists ORDER BY created_at') + .all() as PlaylistRow[] + + const trackRows = db + .prepare('SELECT playlist_id, local_id FROM local_playlist_tracks ORDER BY playlist_id, position') + .all() as { playlist_id: string; local_id: string }[] + const byPlaylist = new Map() + for (const r of trackRows) { + const arr = byPlaylist.get(r.playlist_id) + if (arr) arr.push(r.local_id) + else byPlaylist.set(r.playlist_id, [r.local_id]) + } + + return rows.map((p) => ({ + id: p.id, + name: p.name, + trackIds: byPlaylist.get(p.id) ?? [], + createdAt: p.created_at, + updatedAt: p.updated_at, + })) +} + +export function createPlaylist(name: string): LocalPlaylist { + const clean = name.trim() || 'Untitled playlist' + const id = crypto.randomUUID() + const now = Date.now() + getDb() + .prepare('INSERT INTO local_playlists (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)') + .run(id, clean, now, now) + return { id, name: clean, trackIds: [], createdAt: now, updatedAt: now } +} + +export function renamePlaylist(id: string, name: string): void { + const clean = name.trim() + if (!clean) throw new Error('playlist name is required') + getDb() + .prepare('UPDATE local_playlists SET name = ?, updated_at = ? WHERE id = ?') + .run(clean, Date.now(), id) +} + +export function deletePlaylist(id: string): void { + const db = getDb() + const tx = db.transaction(() => { + db.prepare('DELETE FROM local_playlist_tracks WHERE playlist_id = ?').run(id) + db.prepare('DELETE FROM local_playlists WHERE id = ?').run(id) + }) + tx() +} + +/** Append a track to a playlist (no-op if it's already there). */ +export function addToPlaylist(playlistId: string, localId: string): void { + const db = getDb() + const tx = db.transaction(() => { + const exists = db + .prepare('SELECT 1 FROM local_playlist_tracks WHERE playlist_id = ? AND local_id = ?') + .get(playlistId, localId) + if (exists) return + const { pos } = db + .prepare('SELECT COALESCE(MAX(position), -1) + 1 AS pos FROM local_playlist_tracks WHERE playlist_id = ?') + .get(playlistId) as { pos: number } + db.prepare( + 'INSERT INTO local_playlist_tracks (playlist_id, local_id, position, added_at) VALUES (?, ?, ?, ?)', + ).run(playlistId, localId, pos, Date.now()) + db.prepare('UPDATE local_playlists SET updated_at = ? WHERE id = ?').run(Date.now(), playlistId) + }) + tx() +} + +export function removeFromPlaylist(playlistId: string, localId: string): void { + const db = getDb() + const tx = db.transaction(() => { + db.prepare('DELETE FROM local_playlist_tracks WHERE playlist_id = ? AND local_id = ?') + .run(playlistId, localId) + db.prepare('UPDATE local_playlists SET updated_at = ? WHERE id = ?').run(Date.now(), playlistId) + }) + tx() +} diff --git a/app/src/main/local/taxonomy.ts b/app/src/main/local/taxonomy.ts new file mode 100644 index 00000000..ef73e003 --- /dev/null +++ b/app/src/main/local/taxonomy.ts @@ -0,0 +1,137 @@ +/** + * main/local/taxonomy.ts + * + * Semantic taxonomy tags for on-disk tracks (genres / moods / themes / contexts), + * persisted in SQLite (main/db). Shaped to schemas/TrackTaxonomySchema.json and + * keyed by the local file id (LocalLibrary.fileId), the same key as the metadata + * catalog (main/local/catalog.ts). + * + * This is the local-first replacement for the old network tagging flow + * (renderer TrackTag.tsx), which published tags to the network and triggered + * embedding reingest. Here we only store: nothing is published or embedded. The + * row carries schema_version + track_id so it's ready to feed embeddings once + * that capability is turned on for users. + */ + +import { getDb } from '../db' +import { computeTrackId } from './catalog' +import type { LocalTrackTags } from './types' + +const SCHEMA_VERSION = 1 + +/** Stored row, including the managed schema/track fields. */ +export interface StoredTags extends LocalTrackTags { + localId: string + schemaVersion: number + trackId: string | null +} + +interface Row { + local_id: string + schema_version: number + track_id: string | null + genres: string + moods: string + themes: string + contexts: string + created_at: number + updated_at: number +} + +function parseList(json: string | null): string[] { + if (!json) return [] + try { + const v = JSON.parse(json) + return Array.isArray(v) ? v.filter((x): x is string => typeof x === 'string') : [] + } catch { + return [] + } +} + +function rowToStored(r: Row): StoredTags { + return { + localId: r.local_id, + schemaVersion: r.schema_version, + trackId: r.track_id, + genres: parseList(r.genres), + moods: parseList(r.moods), + themes: parseList(r.themes), + contexts: parseList(r.contexts), + } +} + +// ─── CRUD ───────────────────────────────────────────────────────────────────── + +export function getTags(localId: string): StoredTags | null { + const row = getDb() + .prepare('SELECT * FROM local_track_tags WHERE local_id = ?') + .get(localId) as Row | undefined + return row ? rowToStored(row) : null +} + +/** All stored tags, keyed by local id — loaded once per scan to merge in. */ +export function listTags(): Map { + const rows = getDb().prepare('SELECT * FROM local_track_tags').all() as Row[] + return new Map(rows.map((r) => [r.local_id, rowToStored(r)])) +} + +/** + * Insert or update a track's taxonomy tags. `trackId` is the labeled track's + * canonical id (or null if it hasn't been labeled). Storing an all-empty set is + * allowed and simply persists an empty tag row; callers that want to clear should + * use deleteTags instead. + */ +export function upsertTags( + localId: string, + trackId: string | null, + tags: LocalTrackTags, +): StoredTags { + const now = Date.now() + const genres = JSON.stringify(tags.genres ?? []) + const moods = JSON.stringify(tags.moods ?? []) + const themes = JSON.stringify(tags.themes ?? []) + const contexts = JSON.stringify(tags.contexts ?? []) + + getDb() + .prepare( + `INSERT INTO local_track_tags + (local_id, schema_version, track_id, genres, moods, themes, contexts, created_at, updated_at) + VALUES + (@local_id, @schema_version, @track_id, @genres, @moods, @themes, @contexts, @now, @now) + ON CONFLICT(local_id) DO UPDATE SET + schema_version = excluded.schema_version, + track_id = excluded.track_id, + genres = excluded.genres, + moods = excluded.moods, + themes = excluded.themes, + contexts = excluded.contexts, + updated_at = excluded.updated_at`, + ) + .run({ + local_id: localId, + schema_version: SCHEMA_VERSION, + track_id: trackId, + genres, + moods, + themes, + contexts, + now, + }) + + return { + localId, + schemaVersion: SCHEMA_VERSION, + trackId, + genres: tags.genres ?? [], + moods: tags.moods ?? [], + themes: tags.themes ?? [], + contexts: tags.contexts ?? [], + } +} + +export function deleteTags(localId: string): void { + getDb().prepare('DELETE FROM local_track_tags WHERE local_id = ?').run(localId) +} + +// Re-export so callers can derive a track id without reaching into catalog.ts. +export { computeTrackId } diff --git a/app/src/main/local/types.ts b/app/src/main/local/types.ts index 4a09cb26..3308245e 100644 --- a/app/src/main/local/types.ts +++ b/app/src/main/local/types.ts @@ -20,6 +20,178 @@ export interface LocalTrack { ext: string /** http://127.0.0.1:/ — fed straight into