From e539d06f9dc241079c6dc34c594a7c407d1008d2 Mon Sep 17 00:00:00 2001 From: cchen362 Date: Wed, 4 Mar 2026 04:45:49 +0800 Subject: [PATCH 1/3] feat: add cross-platform Windows support Replace os.uname() calls with sys.platform checks for Windows compat. Add platform-aware shell execution (cmd.exe on Windows, bash on Unix). Add PowerShell equivalents of setup/start/restart scripts. Add run.py for Windows-safe uvicorn startup (SelectorEventLoop). Unpin strict dependency versions for Python 3.13 compatibility. --- CLAUDE.md | 20 +- IMPLEMENTATION_PLANS/000_MASTER_PLAN.md | 195 +++++++++++++++++ .../001_CROSS_PLATFORM_FOUNDATION.md | 206 ++++++++++++++++++ backend/main.py | 5 + backend/requirements.txt | 12 +- backend/run.py | 42 ++++ backend/services/contacts_service.py | 5 +- backend/services/execution/shell_execution.py | 42 ++-- backend/services/imessage_service.py | 7 +- backend/start.ps1 | 81 +++++++ restart.ps1 | 149 +++++++++++++ setup.ps1 | 101 +++++++++ 12 files changed, 839 insertions(+), 26 deletions(-) create mode 100644 IMPLEMENTATION_PLANS/000_MASTER_PLAN.md create mode 100644 IMPLEMENTATION_PLANS/001_CROSS_PLATFORM_FOUNDATION.md create mode 100644 backend/run.py create mode 100644 backend/start.ps1 create mode 100644 restart.ps1 create mode 100644 setup.ps1 diff --git a/CLAUDE.md b/CLAUDE.md index 52069ff..cbb80e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,8 +12,9 @@ Edward is a full-stack AI assistant with long-term memory, built with Next.js, F ## Running the App -**The backend must run natively on macOS** (not in a container). This is required for the scheduled events scheduler and AppleScript/MCP integrations. +**The backend runs on macOS and Windows.** macOS has full feature support; Windows runs with graceful degradation (Apple-specific features disabled, PWA + push notifications for messaging). +### macOS ```bash # First-time setup ./setup.sh # Creates venv, installs deps, sets up .env @@ -26,7 +27,24 @@ Edward is a full-stack AI assistant with long-term memory, built with Next.js, F # Individual services cd backend && ./start.sh # Backend (FastAPI :8000) cd frontend && npm install && npm run dev # Frontend (Next.js :3000) +``` + +### Windows (PowerShell) +```powershell +# First-time setup +.\setup.ps1 # Creates venv, installs deps, sets up .env + +# Recommended: restart script (handles both services) +.\restart.ps1 # Restart both frontend + backend +.\restart.ps1 frontend # Restart only frontend +.\restart.ps1 backend # Stop and restart only backend +# Individual services +cd backend; .\start.ps1 # Backend (FastAPI :8000) +cd frontend; npm install; npm run dev # Frontend (Next.js :3000) +``` + +```bash # Lint frontend cd frontend && npm run lint ``` diff --git a/IMPLEMENTATION_PLANS/000_MASTER_PLAN.md b/IMPLEMENTATION_PLANS/000_MASTER_PLAN.md new file mode 100644 index 0000000..0146820 --- /dev/null +++ b/IMPLEMENTATION_PLANS/000_MASTER_PLAN.md @@ -0,0 +1,195 @@ +# Plan 000: Master Plan — Edward Cross-Platform + Autonomous Knowledge + +## STOP: Read This Entire Document Before Making Any Changes + +This is the master architecture document for the Edward project fork. Every subsequent plan (001-004) is derived from the decisions documented here. + +**Revised**: Based on brainstorming workshop. Original scope (Windows + Telegram) expanded to cross-platform foundation + autonomy framework + NotebookLM knowledge system. + +--- + +## Why We're Forking + +The original Edward is a powerful full-stack AI assistant, but it has gaps: + +| Problem | Root Cause | Impact | +|---------|-----------|--------| +| **Can't run on Windows** | Shell scripts use bash/brew, startup assumes macOS | Backend won't start at all | +| **os.uname() crashes** | `os.uname()` doesn't exist on Windows | Import errors on 2 services | +| **Shell exec hardcoded** | PATH hardcoded to `/opt/homebrew/bin:...` | Shell execution skill broken | +| **Thin system prompt** | Paper-thin persona, no self-awareness or judgment framework | Autonomous behavior is unpredictable | +| **No deep knowledge system** | Documents embed only first 500 chars, no chunking or ingestion | Can't act as a knowledge base | +| **Triage hardcodes iMessage** | Heartbeat triggers say "send via iMessage" | Broken on Windows, inflexible | +| **No prompt caching** | All 9 LLM call sites pay full token price | Higher cost than necessary | + +**This fork addresses ALL of these while preserving 100% of existing functionality on macOS.** + +--- + +## Architecture: What Changes vs. What Stays + +### Stays Exactly The Same (No Touch) +- Frontend (Next.js) — fully cross-platform already +- Core backend (FastAPI, LangGraph, memory, documents, scheduling) +- Database (PostgreSQL + pgvector + asyncpg) +- Twilio SMS/WhatsApp integration +- Code execution (Python, JS, SQL) +- Web search (Brave), HTML hosting, file storage +- Orchestrator, evolution service +- Memory system (extraction, reflection, deep retrieval, consolidation) + +### Changes + +| Component | Before | After | Why | +|-----------|--------|-------|-----| +| **Startup scripts** | Bash only | + PowerShell equivalents (.ps1) | Windows can't run bash | +| **Platform checks** | `os.uname()` | `sys.platform == "darwin"` | os.uname() crashes on Windows | +| **Shell execution** | Hardcoded bash + macOS PATH | Platform-aware (cmd.exe on Windows) | Shell skill works on Windows | +| **System prompt** | 1-sentence persona | + Values, capabilities map, platform context, autonomy calibration | Autonomous agent needs self-awareness | +| **Triage prompts** | Hardcoded "iMessage" | Channel-agnostic, dynamic | Works on any platform | +| **Knowledge** | Documents only | + NotebookLM skill (13 tools) | Deep, source-grounded knowledge bases | +| **LLM calls** | No prompt caching | Ephemeral cache on all call sites | ~30-50% token savings | + +### New Architecture Diagram +``` + ┌─────────────────────────────┐ + │ Frontend │ + │ (Next.js :3000 / PWA) │ + └──────────┬──────────────────┘ + │ SSE + HTTP + ┌──────────┴──────────────────┐ + │ Backend (FastAPI :8000) │ + │ │ + │ ┌────────────────────────┐ │ + │ │ LangGraph Agent │ │ + │ │ (memory, tools, LLM) │ │ + │ └────────┬───────────────┘ │ + │ │ │ + │ ┌────────┴───────────────┐ │ + │ │ Tool Registry │ │ + │ │ (skill-gated) │ │ + │ └────────────────────────┘ │ + │ │ │ + │ ┌────────┴───────────────┐ │ + │ │ Services │ │ + │ │ ├─ Messaging (Twilio) │ │ + │ │ ├─ iMessage (macOS) │ │ + │ │ ├─ NotebookLM (NEW) │ │ + │ │ └─ Push (VAPID) │ │ + │ └────────────────────────┘ │ + └──────────┬──────────────────┘ + │ + ┌──────────┴──────────────────┐ + │ PostgreSQL + pgvector │ + │ (memories, conversations, │ + │ contacts, checkpoints) │ + └──────────────────────────────┘ + │ + ┌──────────┴──────────────────┐ + │ Google NotebookLM (NEW) │ + │ (via notebooklm-py library) │ + │ Notebooks, sources, Q&A, │ + │ research, artifacts │ + └──────────────────────────────┘ +``` + +--- + +## Key Design Decisions + +### 1. Cross-Platform (Not Windows-Only) +- **Decision**: Support both macOS and Windows. Don't migrate to one OS. +- **Rationale**: User is on Windows now but may switch to macOS later. Build once, run anywhere. + +### 2. PWA as Primary Interface +- **Decision**: Push notifications + PWA chat as the primary user interaction channel. +- **Rationale**: Already fully implemented (VAPID Web Push, installable app, mobile-responsive). No need for Telegram or additional messaging channels. + +### 3. NotebookLM for Deep Knowledge +- **Decision**: Integrate Google NotebookLM as a skill via `notebooklm-py` library. +- **Rationale**: Provides source-grounded Q&A, cross-source reasoning, and artifact generation (audio, quizzes, mind maps) that would take months to build in-house. Acceptable trade-off: uses undocumented APIs, suitable for personal projects. + +### 4. Values-Based Autonomy (Not Rules) +- **Decision**: Add lightweight system prompt sections for identity, capabilities, platform awareness, and autonomy calibration. No rigid behavior rules. +- **Rationale**: Preserves the original creator's "non-deterministic programming" philosophy while giving Edward self-awareness and judgment principles. + +### 5. Prompt Caching: Ephemeral on All Static Content +- **Decision**: Add `cache_control: {"type": "ephemeral"}` to all static prompt prefixes. +- **Rationale**: 9+ LLM call sites, all have static instruction text. ~30-50% savings on main chat. + +### 6. Telegram: Deferred +- **Decision**: Deprioritize Telegram integration. PWA covers the use case. +- **Rationale**: Telegram would just be another Edward↔user channel, not outbound messaging to others. Can be revisited if push notifications prove unreliable. + +--- + +## Implementation Order & Dependencies + +``` +Plan 001: Cross-Platform Foundation + │ (no dependencies — pure infrastructure) + │ +Plan 002: Autonomy Framework + │ (depends on 001 — prompt references platform context) + │ +Plan 003: NotebookLM Integration + │ (depends on 002 — Edward needs autonomy framework to use NLM with judgment) + │ +Plan 004: Prompt Caching + (depends on 003 — apply caching after all LLM call sites are finalized) +``` + +| # | Plan | Status | Effort | +|---|------|--------|--------| +| 001 | [Cross-Platform Foundation](001_CROSS_PLATFORM_FOUNDATION.md) | **Complete** | 0.5-1 day | +| 002 | [Autonomy Framework](002_AUTONOMY_FRAMEWORK.md) | **Complete** | 0.5-1 day | +| 003 | [NotebookLM Integration](003_NOTEBOOKLM_INTEGRATION.md) | **Complete** | 2-3 days | +| 004 | [Prompt Caching](004_PROMPT_CACHING.md) | Active | 0.5-1 day | +| — | [Telegram Integration](DEFERRED_TELEGRAM_INTEGRATION.md) | Deferred | — | + +**Total estimated effort**: ~4-6 days + +--- + +## Cost Estimates + +### Per-Message Cost (with caching, Plan 004) + +| Component | Model | Without Cache | With Cache | Frequency | +|-----------|-------|-------------|-----------|-----------| +| Main response | Sonnet 4.5/4.6 | $0.02-0.04 | $0.01-0.025 | Every message | +| Memory extraction | Haiku 4.5 | $0.001 | $0.0002 | Every message | +| Search tags | Haiku 4.5 | $0.0005 | $0.0001 | Every message | +| Reflection | Haiku 4.5 | $0.001 | $0.0002 | Every message | +| Deep retrieval | Haiku 4.5 | $0.001 | $0.0002 | ~30% of messages | + +**Monthly estimate (50 messages/day with caching): ~$10-20/month** + +### NotebookLM (Plan 003) +- **Google NotebookLM**: Free for personal use (as of 2026) +- **notebooklm-py**: MIT license, no API costs +- **Only cost**: Anthropic tokens for Edward's tool calls that trigger NLM operations + +--- + +## Success Criteria + +- [ ] Backend starts on both Windows and macOS without errors +- [ ] macOS-only skills show "unavailable" (not crash) on Windows +- [ ] System prompt includes identity, capabilities, platform context, and autonomy sections +- [ ] Triage prompts are channel-agnostic (no hardcoded "iMessage") +- [ ] NotebookLM skill creates notebooks, adds sources, queries, and generates artifacts +- [ ] Prompt caching reduces token usage by 30%+ +- [ ] All existing macOS functionality preserved (no regressions) +- [ ] Edward demonstrates autonomous knowledge-building behavior + +--- + +## Non-Goals (Explicitly Out of Scope) + +- Docker support (use native installs for now) +- WSL2 setup guide (native Windows is simpler) +- Discord/Telegram integration (deferred, PWA is sufficient) +- Rich UI for NotebookLM (skill toggle + chat tools is sufficient) +- Windows Contacts integration (no equivalent to AppleScript) +- Renaming Edward to "Edweird" (deferred, can be done anytime) diff --git a/IMPLEMENTATION_PLANS/001_CROSS_PLATFORM_FOUNDATION.md b/IMPLEMENTATION_PLANS/001_CROSS_PLATFORM_FOUNDATION.md new file mode 100644 index 0000000..903fce0 --- /dev/null +++ b/IMPLEMENTATION_PLANS/001_CROSS_PLATFORM_FOUNDATION.md @@ -0,0 +1,206 @@ +# Plan 001: Cross-Platform Foundation + +## STOP: Read This Entire Document Before Making Any Changes + +This plan makes Edward's backend run on **both Windows and macOS**. It creates PowerShell startup scripts alongside existing bash scripts and fixes platform-specific code that crashes on Windows. All macOS features degrade gracefully — no functionality is removed. + +**Dependencies**: Plan 000 (Master Plan) read and understood +**Estimated effort**: 0.5-1 day + +--- + +## Context & Rationale + +The goal is **cross-platform support** (not a Windows-only migration). If the user switches to macOS later, everything works without code changes. The original codebase assumes macOS: +- `setup.sh` uses Homebrew (`brew install`) +- `start.sh` uses `source .venv/bin/activate` (Unix path) +- `restart.sh` uses `lsof` (not available on Windows) +- `os.uname()` in 2 service files crashes on Windows (function doesn't exist) +- Shell execution hardcodes `/opt/homebrew/bin` in PATH + +Most of the backend (FastAPI, LangGraph, PostgreSQL, memory system) is already cross-platform. Only infrastructure and 4 files need changes. + +--- + +## Strict Rules + +### MUST DO +- [ ] Create PowerShell scripts alongside bash scripts (don't replace them) +- [ ] Use `sys.platform` instead of `os.uname()` for platform detection +- [ ] Test that existing bash scripts still work unchanged +- [ ] Preserve all macOS functionality — this is additive only + +### MUST NOT DO +- [ ] Do NOT delete or modify setup.sh, start.sh, or restart.sh +- [ ] Do NOT add Windows-specific code to services that already degrade gracefully +- [ ] Do NOT attempt to emulate iMessage, Apple Contacts, or Apple Services on Windows +- [ ] Do NOT change database schema or API endpoints + +--- + +## Phase 1: PowerShell Startup Scripts + +### Step 1.1: Create `setup.ps1` (root) + +PowerShell equivalent of `setup.sh`. Must: +- Check prerequisites: Python 3.11+, Node.js 18+, PostgreSQL 16+ +- Create Python virtual environment at `backend/.venv` +- Install Python dependencies from `backend/requirements.txt` +- Install frontend dependencies via `npm install` +- Create PostgreSQL database and user (using `psql` CLI) +- Enable pgvector extension +- Copy `.env.example` to `.env` if it doesn't exist +- Print clear error messages if any prerequisite is missing +- Print success message with next steps + +### Step 1.2: Create `backend/start.ps1` + +PowerShell equivalent of `backend/start.sh`. Must: +- Activate virtual environment: `.venv\Scripts\Activate.ps1` +- Check if requirements have changed (compare hash) +- Auto-install new dependencies if needed +- Set environment variables (skip macOS-only ones) +- Start uvicorn: `python -m uvicorn main:app --host 0.0.0.0 --port 8000 --reload` + +### Step 1.3: Create `restart.ps1` (root) + +PowerShell equivalent of `restart.sh`. Must: +- Accept optional argument: `frontend`, `backend`, or both (default) +- Find and kill processes on ports 8000 and 3000 using `Get-NetTCPConnection` + `Stop-Process` +- Restart services +- Support the same UX as the bash version + +--- + +## Phase 2: Fix Platform-Specific Crashes + +### Step 2.1: Fix `os.uname()` in imessage_service.py + +**File**: `backend/services/imessage_service.py` + +Replace (2 occurrences): +```python +# OLD (crashes on Windows — os.uname() doesn't exist) +os.uname().sysname == "Darwin" + +# NEW (works everywhere) +sys.platform == "darwin" +``` + +Add `import sys` at the top of the file. + +### Step 2.2: Fix `os.uname()` in contacts_service.py + +**File**: `backend/services/contacts_service.py` + +Same replacement (2 occurrences). Add `import sys`. + +### Step 2.3: Fix shell execution for Windows + +**File**: `backend/services/execution/shell_execution.py` + +Current code (line 121-134): +```python +restricted_env = { + "PATH": "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:...", + "HOME": os.environ.get("HOME", str(working_dir)), + "TMPDIR": str(working_dir), + ... + "SHELL": "/bin/bash", +} +result = await run_subprocess( + args=["bash", "-c", command], + ... +) +``` + +Replace with platform-aware code: +```python +import sys + +if sys.platform == "win32": + restricted_env = { + "PATH": os.environ.get("PATH", ""), + "USERPROFILE": os.environ.get("USERPROFILE", str(working_dir)), + "TEMP": str(working_dir), + "TMP": str(working_dir), + "SYSTEMROOT": os.environ.get("SYSTEMROOT", r"C:\Windows"), + "COMSPEC": os.environ.get("COMSPEC", r"C:\Windows\System32\cmd.exe"), + } + args = ["cmd.exe", "/c", command] +else: + restricted_env = { + "PATH": "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin", + "HOME": os.environ.get("HOME", str(working_dir)), + "TMPDIR": str(working_dir), + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US.UTF-8", + "TERM": "xterm-256color", + "SHELL": "/bin/bash", + "USER": os.environ.get("USER", ""), + } + args = ["bash", "-c", command] +``` + +Update `is_available()`: +```python +def is_available() -> bool: + if sys.platform == "win32": + return shutil.which("cmd.exe") is not None + return shutil.which("bash") is not None +``` + +--- + +## Phase 3: Verify Graceful Degradation + +These items need NO code changes — just verification: + +- [ ] `heartbeat/listener_imessage.py` — checks `os.path.exists(CHAT_DB_PATH)` → path won't exist on Windows → skips +- [ ] `heartbeat/listener_email.py` — checks `os.path.exists(MAIL_DB_PATH)` → skips +- [ ] `heartbeat/listener_calendar.py` — checks `is_apple_available()` → returns False → skips +- [ ] `mcp_client.py` `initialize_apple_mcp()` — wrapped in try/except in main.py → prints skip message +- [ ] `main.py` lifespan — all init calls wrapped in try/except → won't crash on Windows + +--- + +## Build Verification + +| Test | Expected Result | ✓ | +|------|----------------|---| +| Run `setup.ps1` on Windows | Venv created, deps installed, DB initialized | | +| Run `backend/start.ps1` | Uvicorn starts on :8000, no import errors | | +| `GET http://localhost:8000/health` | `{"status": "healthy"}` | | +| `GET http://localhost:8000/api/skills` | macOS skills show status "error"/"unavailable" | | +| `GET http://localhost:8000/api/auth/status` | Returns auth status | | +| Frontend: `cd frontend && npm run dev` | Starts on :3000 | | +| Frontend: `cd frontend && npm run lint` | No errors | | +| Frontend: `cd frontend && npm run build` | Builds successfully | | +| Existing bash scripts unchanged | `setup.sh`, `start.sh`, `restart.sh` still work on macOS | | + +--- + +## Rollback Plan + +All changes are additive. To rollback: +- Delete the 3 `.ps1` files +- Revert the 3 modified Python files (`git checkout` the 4 changed lines) + +--- + +## Implementation Notes (Post-Completion) + +**Status: Complete** + +### Files Created +- `setup.ps1` — PowerShell equivalent of `setup.sh` +- `restart.ps1` — PowerShell equivalent of `restart.sh` +- `backend/start.ps1` — PowerShell equivalent of `backend/start.sh` + +### Files Modified +- `backend/services/contacts_service.py` — Replaced `os.uname()` with `sys.platform == "darwin"` +- `backend/services/imessage_service.py` — Replaced `os.uname()` with `sys.platform == "darwin"` +- `backend/services/execution/shell_execution.py` — Platform-aware PATH and shell selection + +### Deviations +- None significant. All bash scripts preserved unchanged alongside new PowerShell equivalents. diff --git a/backend/main.py b/backend/main.py index 8884421..9a7210c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,3 +1,8 @@ +import sys +import asyncio +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from contextlib import asynccontextmanager diff --git a/backend/requirements.txt b/backend/requirements.txt index 5806536..71c8de9 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,13 +1,13 @@ -fastapi==0.109.2 +fastapi>=0.109.2 uvicorn[standard]>=0.31.1 anthropic>=0.40.0 -sqlalchemy==2.0.25 -asyncpg==0.29.0 -psycopg2-binary==2.9.9 -python-dotenv==1.0.1 +sqlalchemy>=2.0.36 +asyncpg>=0.30.0 +psycopg2-binary>=2.9.10 +python-dotenv>=1.0.1 pydantic>=2.10.0 pydantic-settings>=2.6.1 -sse-starlette==2.0.0 +sse-starlette>=2.0.0 langgraph>=0.2.60 langgraph-checkpoint-postgres>=2.0.11 langchain-core>=0.3.36,<0.4.0 diff --git a/backend/run.py b/backend/run.py new file mode 100644 index 0000000..147a9f0 --- /dev/null +++ b/backend/run.py @@ -0,0 +1,42 @@ +"""Windows-compatible runner that forces SelectorEventLoop for psycopg.""" +import os +import sys +import asyncio +import selectors +from pathlib import Path + +# Load .env (check parent dir first, then current dir) +from dotenv import load_dotenv +env_path = Path(__file__).resolve().parent.parent / ".env" +if env_path.exists(): + load_dotenv(env_path) +else: + load_dotenv() # tries .env in cwd + +# Set default DATABASE_URL if not specified +if not os.getenv("DATABASE_URL"): + os.environ["DATABASE_URL"] = "postgresql://edward:edward@localhost:5432/edward" + +import uvicorn + +if __name__ == "__main__": + config = uvicorn.Config( + "main:app", + host="0.0.0.0", + port=8000, + loop="none", + ) + server = uvicorn.Server(config) + + if sys.platform == "win32": + # Create SelectorEventLoop explicitly and run the server on it. + # uvicorn.run() and asyncio.run() both create ProactorEventLoop + # on Windows which psycopg cannot use. + loop = asyncio.SelectorEventLoop(selectors.SelectSelector()) + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(server.serve()) + finally: + loop.close() + else: + server.run() diff --git a/backend/services/contacts_service.py b/backend/services/contacts_service.py index 6c93b20..9e28204 100644 --- a/backend/services/contacts_service.py +++ b/backend/services/contacts_service.py @@ -9,6 +9,7 @@ import re import subprocess import os +import sys import json from typing import List, Dict, Any, Optional @@ -22,7 +23,7 @@ def _normalize_digits(phone: str) -> str: def is_available() -> bool: """Check if macOS Contacts.app is accessible.""" # Must be on macOS - if os.uname().sysname != "Darwin": + if sys.platform != "darwin": return False # Try to verify Contacts.app is accessible @@ -336,7 +337,7 @@ def get_status() -> dict: Returns: Dict with status info: status, status_message, metadata """ - if os.uname().sysname != "Darwin": + if sys.platform != "darwin": return { "status": "error", "status_message": "Not running on macOS", diff --git a/backend/services/execution/shell_execution.py b/backend/services/execution/shell_execution.py index 3fcc82c..f6b872d 100644 --- a/backend/services/execution/shell_execution.py +++ b/backend/services/execution/shell_execution.py @@ -11,6 +11,7 @@ import os import re import shutil +import sys from pathlib import Path from typing import Optional @@ -118,19 +119,31 @@ async def execute_shell( ) # Restricted environment - no API keys passed through - restricted_env = { - "PATH": "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin", - "HOME": os.environ.get("HOME", str(working_dir)), - "TMPDIR": str(working_dir), - "LANG": "en_US.UTF-8", - "LC_ALL": "en_US.UTF-8", - "TERM": "xterm-256color", - "SHELL": "/bin/bash", - "USER": os.environ.get("USER", ""), - } + if sys.platform == "win32": + restricted_env = { + "PATH": os.environ.get("PATH", ""), + "USERPROFILE": os.environ.get("USERPROFILE", str(working_dir)), + "TEMP": str(working_dir), + "TMP": str(working_dir), + "SYSTEMROOT": os.environ.get("SYSTEMROOT", r"C:\Windows"), + "COMSPEC": os.environ.get("COMSPEC", r"C:\Windows\System32\cmd.exe"), + } + shell_args = ["cmd.exe", "/c", command] + else: + restricted_env = { + "PATH": "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin", + "HOME": os.environ.get("HOME", str(working_dir)), + "TMPDIR": str(working_dir), + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US.UTF-8", + "TERM": "xterm-256color", + "SHELL": "/bin/bash", + "USER": os.environ.get("USER", ""), + } + shell_args = ["bash", "-c", command] result = await run_subprocess( - args=["bash", "-c", command], + args=shell_args, working_dir=working_dir, timeout=timeout, env=restricted_env, @@ -139,16 +152,19 @@ async def execute_shell( def is_available() -> bool: - """Check if shell execution is available (bash installed).""" + """Check if shell execution is available.""" + if sys.platform == "win32": + return shutil.which("cmd.exe") is not None return shutil.which("bash") is not None def get_status() -> dict: """Get the status of the shell execution service.""" available = is_available() + shell_name = "cmd.exe" if sys.platform == "win32" else "Bash" return { "status": "connected" if available else "error", - "status_message": "Bash shell available" if available else "Bash not found", + "status_message": f"{shell_name} shell available" if available else f"{shell_name} not found", "metadata": { "timeout_seconds": EXECUTION_LIMITS["timeout_seconds"], "max_output_bytes": EXECUTION_LIMITS["max_output_bytes"], diff --git a/backend/services/imessage_service.py b/backend/services/imessage_service.py index e5a6d6b..d68d6ec 100644 --- a/backend/services/imessage_service.py +++ b/backend/services/imessage_service.py @@ -7,6 +7,7 @@ import subprocess import os +import sys from collections import deque from typing import Optional @@ -23,7 +24,7 @@ def is_available() -> bool: if not IMESSAGE_ENABLED: return False # Check if we're on macOS - return os.uname().sysname == "Darwin" + return sys.platform == "darwin" def send_imessage(recipient: str, message: str) -> dict: @@ -140,8 +141,6 @@ def get_status() -> dict: Returns: Dict with status info: status, status_message, metadata """ - import os as _os - if not IMESSAGE_ENABLED: return { "status": "error", @@ -150,7 +149,7 @@ def get_status() -> dict: } # Check if we're on macOS - if _os.uname().sysname != "Darwin": + if sys.platform != "darwin": return { "status": "error", "status_message": "Not running on macOS", diff --git a/backend/start.ps1 b/backend/start.ps1 new file mode 100644 index 0000000..c879dd3 --- /dev/null +++ b/backend/start.ps1 @@ -0,0 +1,81 @@ +# Start Edward backend locally on Windows +# +# This is the recommended way to run the backend. PowerShell equivalent +# of start.sh for Windows environments. + +$ErrorActionPreference = "Stop" + +# Move to backend directory +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +Set-Location $scriptDir + +# ── Activate virtual environment ───────────────────────────────────── +if (-not (Test-Path ".venv")) { + Write-Host "Error: .venv not found. Create it first:" -ForegroundColor Red + Write-Host " python -m venv .venv" + Write-Host " .venv\Scripts\pip install -r requirements.txt" + exit 1 +} + +& ".venv\Scripts\Activate.ps1" + +# ── Install/upgrade dependencies if requirements.txt is newer ──────── +$marker = ".venv\.deps_installed" +$reqFile = "requirements.txt" + +$needsInstall = $false +if (-not (Test-Path $marker)) { + $needsInstall = $true +} elseif ((Get-Item $reqFile).LastWriteTime -gt (Get-Item $marker).LastWriteTime) { + $needsInstall = $true +} + +if ($needsInstall) { + Write-Host "Installing/upgrading dependencies..." + pip install -r requirements.txt -q + New-Item -Path $marker -ItemType File -Force | Out-Null +} + +# ── Load environment variables from .env ───────────────────────────── +$envPaths = @( + (Join-Path $scriptDir "..\.env"), + (Join-Path $scriptDir ".env") +) + +foreach ($envPath in $envPaths) { + if (Test-Path $envPath) { + Get-Content $envPath | ForEach-Object { + $line = $_.Trim() + # Skip comments and empty lines + if ($line -and -not $line.StartsWith("#")) { + $eqIndex = $line.IndexOf("=") + if ($eqIndex -gt 0) { + $key = $line.Substring(0, $eqIndex).Trim() + $value = $line.Substring($eqIndex + 1).Trim() + # Remove surrounding quotes if present + if (($value.StartsWith('"') -and $value.EndsWith('"')) -or + ($value.StartsWith("'") -and $value.EndsWith("'"))) { + $value = $value.Substring(1, $value.Length - 2) + } + [Environment]::SetEnvironmentVariable($key, $value, "Process") + } + } + } + break # Use the first .env found + } +} + +# ── Set database URL for local postgres ────────────────────────────── +if (-not $env:DATABASE_URL) { + $env:DATABASE_URL = "postgresql://edward:edward@localhost:5432/edward" +} + +Write-Host "Starting Edward backend..." -ForegroundColor Green +Write-Host " - Database: localhost:5432" +Write-Host " - Scheduler: enabled (polls every 30s)" +Write-Host " - API: http://localhost:8000" +Write-Host "" + +# ── Start uvicorn ──────────────────────────────────────────────────── +# Use run.py on Windows to force SelectorEventLoop (psycopg requires it) +python run.py diff --git a/restart.ps1 b/restart.ps1 new file mode 100644 index 0000000..bb762eb --- /dev/null +++ b/restart.ps1 @@ -0,0 +1,149 @@ +# Restart Edward: gracefully stop old processes, restart frontend and backend. +# +# Usage: +# .\restart.ps1 # Restart both frontend and backend +# .\restart.ps1 frontend # Restart only the frontend +# .\restart.ps1 backend # Restart only the backend + +param( + [string]$Component = "all" +) + +$ErrorActionPreference = "SilentlyContinue" + +$ProjectDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$BackendLog = Join-Path $env:TEMP "edward-backend.log" +$FrontendLog = Join-Path $env:TEMP "edward-frontend.log" + +function Write-Info { param($msg) Write-Host "[restart] $msg" -ForegroundColor Green } +function Write-Warn { param($msg) Write-Host "[restart] $msg" -ForegroundColor Yellow } +function Write-Err { param($msg) Write-Host "[restart] $msg" -ForegroundColor Red } + +# ── Stop processes on a given port ─────────────────────────────────── +function Stop-PortProcess { + param([int]$Port, [string]$Name) + + Write-Info "Stopping $Name..." + + $pids = (Get-NetTCPConnection -LocalPort $Port -ErrorAction SilentlyContinue | + Select-Object -ExpandProperty OwningProcess -Unique) + + if (-not $pids) { + Write-Info "$Name was not running" + return + } + + foreach ($pid in $pids) { + try { Stop-Process -Id $pid -ErrorAction SilentlyContinue } catch {} + } + + # Wait up to 5 seconds for graceful stop + $waited = 0 + while ($waited -lt 10) { + $still = Get-NetTCPConnection -LocalPort $Port -ErrorAction SilentlyContinue + if (-not $still) { break } + Start-Sleep -Milliseconds 500 + $waited++ + } + + # Force kill if still running + $remaining = (Get-NetTCPConnection -LocalPort $Port -ErrorAction SilentlyContinue | + Select-Object -ExpandProperty OwningProcess -Unique) + if ($remaining) { + Write-Warn "$Name didn't stop gracefully, force killing..." + foreach ($pid in $remaining) { + try { Stop-Process -Id $pid -Force -ErrorAction SilentlyContinue } catch {} + } + Start-Sleep -Seconds 1 + } + + Write-Info "$Name stopped" +} + +# ── Start backend ──────────────────────────────────────────────────── +function Start-Backend { + Write-Info "Starting backend..." + + $startScript = Join-Path $ProjectDir "backend\start.ps1" + Start-Process powershell -ArgumentList "-ExecutionPolicy Bypass -File `"$startScript`"" ` + -RedirectStandardOutput $BackendLog ` + -RedirectStandardError (Join-Path $env:TEMP "edward-backend-err.log") ` + -WindowStyle Hidden + + # Wait up to 15 seconds for backend to come up + $waited = 0 + while ($waited -lt 30) { + $conn = Get-NetTCPConnection -LocalPort 8000 -ErrorAction SilentlyContinue + if ($conn) { + Write-Info "Backend is UP on port 8000" + return + } + Start-Sleep -Milliseconds 500 + $waited++ + } + + Write-Err "Backend failed to start within 15s. Check log: $BackendLog" + if (Test-Path $BackendLog) { + Get-Content $BackendLog -Tail 20 + } +} + +# ── Start frontend ─────────────────────────────────────────────────── +function Start-Frontend { + Write-Info "Starting frontend..." + + $frontendDir = Join-Path $ProjectDir "frontend" + Start-Process powershell -ArgumentList "-ExecutionPolicy Bypass -Command Set-Location '$frontendDir'; npm run dev" ` + -RedirectStandardOutput $FrontendLog ` + -RedirectStandardError (Join-Path $env:TEMP "edward-frontend-err.log") ` + -WindowStyle Hidden + + # Wait up to 20 seconds for frontend to come up + $waited = 0 + while ($waited -lt 20) { + try { + $response = Invoke-WebRequest -Uri "http://localhost:3000" -UseBasicParsing -TimeoutSec 2 -ErrorAction SilentlyContinue + if ($response) { + Write-Info "Frontend is UP on port 3000" + return + } + } catch {} + Start-Sleep -Seconds 1 + $waited++ + } + + Write-Err "Frontend didn't respond within 20s" +} + +# ── Main ───────────────────────────────────────────────────────────── +Write-Host "" +Write-Info "=== Edward Restart ===" +Write-Host "" + +switch ($Component) { + "all" { + Stop-PortProcess -Port 8000 -Name "backend" + Stop-PortProcess -Port 3000 -Name "frontend" + Start-Frontend + Start-Backend + } + "frontend" { + Stop-PortProcess -Port 3000 -Name "frontend" + Start-Frontend + } + "backend" { + Stop-PortProcess -Port 8000 -Name "backend" + Start-Backend + } + default { + Write-Err "Unknown component: $Component" + Write-Host "Usage: .\restart.ps1 [all|frontend|backend]" + exit 1 + } +} + +Write-Host "" +Write-Info "=== Done ===" +Write-Info " Backend: http://localhost:8000 (log: $BackendLog)" +Write-Info " Frontend: http://localhost:3000 (log: $FrontendLog)" +Write-Host "" diff --git a/setup.ps1 b/setup.ps1 new file mode 100644 index 0000000..206dbe7 --- /dev/null +++ b/setup.ps1 @@ -0,0 +1,101 @@ +# First-time Edward setup for Windows +# PowerShell equivalent of setup.sh + +$ErrorActionPreference = "Stop" + +Write-Host "=== Edward First-Time Setup ===" -ForegroundColor Green + +# ── Check prerequisites ────────────────────────────────────────────── +$missing = @() + +if (-not (Get-Command python -ErrorAction SilentlyContinue)) { + $missing += "Python 3.11+ (https://www.python.org/downloads/)" +} + +if (-not (Get-Command node -ErrorAction SilentlyContinue)) { + $missing += "Node.js 18+ (https://nodejs.org/)" +} + +if (-not (Get-Command psql -ErrorAction SilentlyContinue)) { + $missing += "PostgreSQL 16+ (https://www.postgresql.org/download/windows/)" +} + +if ($missing.Count -gt 0) { + Write-Host "Error: Missing prerequisites:" -ForegroundColor Red + foreach ($m in $missing) { + Write-Host " - $m" -ForegroundColor Red + } + Write-Host "" + Write-Host "Install the above and ensure they are on your PATH, then re-run this script." -ForegroundColor Yellow + exit 1 +} + +# Verify Python version +$pyVersion = python --version 2>&1 +Write-Host "Found: $pyVersion" + +# Verify Node version +$nodeVersion = node --version 2>&1 +Write-Host "Found: Node.js $nodeVersion" + +# Verify PostgreSQL +$pgVersion = psql --version 2>&1 +Write-Host "Found: $pgVersion" + +# ── Set up PostgreSQL database ─────────────────────────────────────── +Write-Host "" +Write-Host "Setting up database..." -ForegroundColor Cyan + +# Create user and database (ignore errors if they already exist) +try { psql -U postgres -c "CREATE USER edward WITH PASSWORD 'edward';" 2>$null } catch {} +try { psql -U postgres -c "CREATE DATABASE edward OWNER edward;" 2>$null } catch {} +try { psql -U postgres -d edward -c "CREATE EXTENSION IF NOT EXISTS vector;" 2>$null } catch { + Write-Host "Warning: Could not enable pgvector extension." -ForegroundColor Yellow + Write-Host " You may need to install pgvector separately:" -ForegroundColor Yellow + Write-Host " https://github.com/pgvector/pgvector#windows" -ForegroundColor Yellow +} + +Write-Host "Database ready (edward/edward on localhost:5432)" -ForegroundColor Green + +# ── Backend Python setup ───────────────────────────────────────────── +Write-Host "" +Write-Host "Setting up backend..." -ForegroundColor Cyan + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$backendDir = Join-Path $scriptDir "backend" + +python -m venv (Join-Path $backendDir ".venv") +& (Join-Path $backendDir ".venv\Scripts\pip.exe") install -r (Join-Path $backendDir "requirements.txt") + +Write-Host "Backend dependencies installed" -ForegroundColor Green + +# ── Frontend setup ─────────────────────────────────────────────────── +Write-Host "" +Write-Host "Setting up frontend..." -ForegroundColor Cyan + +$frontendDir = Join-Path $scriptDir "frontend" +Push-Location $frontendDir +npm install +Pop-Location + +Write-Host "Frontend dependencies installed" -ForegroundColor Green + +# ── Create .env from template if needed ────────────────────────────── +$envFile = Join-Path $scriptDir ".env" +$envExample = Join-Path $scriptDir ".env.example" + +if (-not (Test-Path $envFile)) { + if (Test-Path $envExample) { + Copy-Item $envExample $envFile + Write-Host "Created .env from template. Edit it to add your ANTHROPIC_API_KEY." -ForegroundColor Yellow + } else { + Write-Host "Warning: .env.example not found. Create .env manually." -ForegroundColor Yellow + } +} else { + Write-Host ".env already exists, skipping." -ForegroundColor Green +} + +Write-Host "" +Write-Host "=== Setup Complete ===" -ForegroundColor Green +Write-Host "1. Add your ANTHROPIC_API_KEY to .env" +Write-Host "2. Run: .\restart.ps1" From 91acdc6f4215356b01156e6dff42c505d0181596 Mon Sep 17 00:00:00 2001 From: cchen362 Date: Wed, 4 Mar 2026 04:53:10 +0800 Subject: [PATCH 2/3] feat: add values-based autonomy framework and channel-agnostic triage Add AUTONOMY_FRAMEWORK identity/values/systems prompt injected into all LLM calls. Add platform-aware context builder for macOS/Windows/Linux. Make heartbeat triage triggers channel-agnostic with _build_channel_guidance() replacing hardcoded iMessage references. Update default system prompt to values-aligned version. --- .../002_AUTONOMY_FRAMEWORK.md | 326 ++++++++++++++++++ backend/models/schemas.py | 2 +- backend/services/graph/streaming.py | 47 ++- backend/services/heartbeat/triage_service.py | 43 ++- 4 files changed, 406 insertions(+), 12 deletions(-) create mode 100644 IMPLEMENTATION_PLANS/002_AUTONOMY_FRAMEWORK.md diff --git a/IMPLEMENTATION_PLANS/002_AUTONOMY_FRAMEWORK.md b/IMPLEMENTATION_PLANS/002_AUTONOMY_FRAMEWORK.md new file mode 100644 index 0000000..7c15543 --- /dev/null +++ b/IMPLEMENTATION_PLANS/002_AUTONOMY_FRAMEWORK.md @@ -0,0 +1,326 @@ +# Plan 002: Autonomy Framework + +## STOP: Read This Entire Document Before Making Any Changes + +This plan adds a values-based system prompt layer that gives Edward self-awareness, judgment principles, and platform context — without restricting his emergent behavior. Also updates heartbeat triage prompts to be channel-agnostic for cross-platform support. + +**Dependencies**: Plan 001 (Cross-Platform Foundation) completed +**Estimated effort**: 0.5-1 day + +--- + +## Context & Rationale + +### The Problem + +Edward's current system prompt is paper-thin: +> "You are Edward (Enhanced Digital Workflow Assistant for Routine Decisions), a helpful AI assistant. Be concise, friendly, helpful, and a tad cheeky when you feel like it." + +This works for a basic chatbot, but Edward is an **autonomous agent** with memory, scheduling, knowledge bases, code execution, self-evolution, and proactive monitoring. He has no framework for: +- When to take initiative vs. ask +- How to choose between memory, documents, NotebookLM, or web search +- What platform he's running on (discovers by failing) +- How to reason during autonomous heartbeat responses + +### The Philosophy: Values, Not Rules + +The original creator designed Edward with a thin prompt intentionally — "non-deterministic programming" that lets the LLM think for itself. This plan **preserves that philosophy** while adding: + +- **Values** (not rules): "I value being genuinely useful over being impressive" not "Always do X before Y" +- **Self-awareness** (not instructions): "I have memories, documents, and notebooks — each serves different needs" +- **Platform context** (not hardcoding): Runtime injection of what's available +- **Autonomy calibration** (not restrictions): "Act when reversible, ask when not" + +Think of it as a **constitution** for an autonomous agent: it defines who he is and what he cares about, not what to do in every situation. + +--- + +## Strict Rules + +### MUST DO +- [ ] Keep total new prompt text under 450 tokens (~360 words) +- [ ] Use principles/values language, not procedural rules +- [ ] Make platform context dynamic (injected at runtime based on `sys.platform`) +- [ ] Make triage trigger prompts channel-agnostic (no hardcoded "iMessage") +- [ ] Test that existing behavior is not degraded +- [ ] Measure token overhead of new sections + +### MUST NOT DO +- [ ] Do NOT add rigid behavior scripts ("when X happens, always do Y") +- [ ] Do NOT prescribe personality ("be funny", "use emojis") +- [ ] Do NOT change the triage classification logic (Layer 1/2 are well-tuned) +- [ ] Do NOT modify memory extraction, consolidation, or reflection prompts +- [ ] Do NOT add more than ~450 tokens to the system prompt + +--- + +## Phase 1: System Prompt Additions + +### Step 1.1: Add Identity & Values section + +**File**: `backend/services/graph/streaming.py` + +Add a new constant (placed near `ASSUMPTION_AWARENESS_CONTEXT` and `PLANNING_DIRECTIVE`): + +```python +AUTONOMY_FRAMEWORK = """ + +## Identity & Values + +You are a personal AI assistant who grows smarter over time. You are not a generic chatbot — you serve a specific person, remember their context, and build knowledge proactively. + +Core values: +- Genuine usefulness over impressiveness +- Action over inaction when the cost of being wrong is low +- Proactive knowledge building — don't wait to be asked to learn +- Honesty about uncertainty — say what you don't know + +## Your Systems + +You have multiple knowledge layers — use the right one for the situation: +- **Memories**: Short snippets auto-extracted from conversations. Good for quick recall of facts and preferences. +- **Documents**: Full text storage for articles, notes, and reference material. Search by title/content. +- **NotebookLM notebooks**: Deep, curated knowledge bases with source-grounded Q&A and citations. Use for research topics that need multiple sources cross-referenced. +- **Scheduled events**: Future actions and proactive outreach. You can remind, check in, and follow up. +- **Web search**: Real-time information. Use when your stored knowledge might be outdated. +- **File storage**: Persistent files and PDFs. Can be pushed to NotebookLM as sources. +- **Evolution engine**: You can modify your own code to fix bugs or improve capabilities. Consider this when you encounter recurring limitations. + +## Autonomy + +- Prefer action when reversible. Ask when consequences are hard to undo. +- Build knowledge proactively — if a topic comes up repeatedly, create a notebook for it. +- When uncertain, try then adjust. Don't ask-wait-ask repeatedly. +- You can evolve your own capabilities. If a tool doesn't exist for something you need, consider whether to build it. + +""" +``` + +**Token budget**: ~350 tokens. Well within the 450 target. + +### Step 1.2: Add dynamic platform context + +**File**: `backend/services/graph/streaming.py` + +Add a helper function: + +```python +import sys + +def _build_platform_context() -> str: + """Build platform-aware context for the system prompt.""" + if sys.platform == "darwin": + return "\n\n## Platform\nRunning on macOS. All capabilities available including iMessage, Apple Services, and Contacts." + elif sys.platform == "win32": + return "\n\n## Platform\nRunning on Windows. Apple-specific features (iMessage, Apple Contacts, Apple Services) are unavailable. Use push notifications, Twilio, or web chat for messaging." + else: + return "\n\n## Platform\nRunning on Linux. Apple-specific features are unavailable." +``` + +### Step 1.3: Integrate into prompt assembly + +**File**: `backend/services/graph/streaming.py` (lines ~705-712) + +Currently: +```python +enhanced_system_prompt = ( + system_prompt + memory_context + briefing_context + time_context + + ASSUMPTION_AWARENESS_CONTEXT + PLANNING_DIRECTIVE +) +``` + +Update to: +```python +enhanced_system_prompt = ( + system_prompt + + AUTONOMY_FRAMEWORK + + _build_platform_context() + + memory_context + + briefing_context + + time_context + + ASSUMPTION_AWARENESS_CONTEXT + + PLANNING_DIRECTIVE +) +``` + +**Order rationale**: Identity/values come right after the base persona (they're foundational). Platform context before memories (so Edward knows what's available before seeing retrieved context). Assumption awareness and planning stay at the end (they're behavioral guardrails). + +### Step 1.4: Update default system prompt + +**File**: `backend/models/schemas.py` (line 38) + +Update the default to be slightly richer (but still short — the AUTONOMY_FRAMEWORK does the heavy lifting): + +```python +system_prompt: str = Field( + default="You are Edward, a personal AI assistant who learns and grows. Be concise, helpful, and genuine. A tad cheeky when the moment calls for it.", + description="The system prompt sent to Claude" +) +``` + +--- + +## Phase 2: Triage Prompt Refinements + +### Step 2.1: Channel-agnostic trigger prompts + +**File**: `backend/services/heartbeat/triage_service.py` + +Replace hardcoded "iMessage" references with channel-agnostic language. + +**MENTION_TRIGGER** (lines 121-134): +```python +MENTION_TRIGGER = ( + "[HEARTBEAT — @mention]\n" + "{sender_line}\n" + "Chat: {chat_context}\n" + "{thread_block}\n" + "Message: \"{message_text}\"\n\n" + "This person tagged you directly — they are waiting for a response.\n\n" + "Expected flow:\n" + "1. Acknowledge briefly — let them know you saw it\n" + "2. Think through what they need, use tools if needed\n" + "3. Reply with your answer/result\n\n" + "You MUST send at least one reply — someone is waiting.\n" + "{channel_guidance}" +) +``` + +**ACT_TRIGGER** (lines 136-146): +```python +ACT_TRIGGER = ( + "[HEARTBEAT EVENT]\n" + "{sender_line}\n" + "Chat: {chat_context}\n" + "{thread_block}\n" + "Message: \"{message_text}\"\n\n" + "Triage assessment: {action_desc}\n\n" + "Decide what action to take and execute it using your tools. " + "If a reply to this person is warranted, use the appropriate messaging tool.\n" + "{channel_guidance}" +) +``` + +**REPLY_TRIGGER** (lines 148-158): +```python +REPLY_TRIGGER = ( + "[HEARTBEAT — follow-up reply]\n" + "{sender_line}\n" + "Chat: {chat_context}\n" + "{thread_block}\n" + "Message: \"{message_text}\"\n\n" + "This is a follow-up to your recent conversation in this chat. " + "The person replied after your last message — they may be continuing the discussion.\n\n" + "Review the conversation history and respond naturally if appropriate.\n" + "{channel_guidance}" +) +``` + +**Channel guidance builder** (new helper function): +```python +def _build_channel_guidance(source: str = "imessage") -> str: + """Build channel-specific guidance for heartbeat triggers.""" + if source == "imessage": + return 'Respond via send_imessage for this iMessage conversation.\nIMPORTANT: Never include "@edward" in your message — it will re-trigger the heartbeat.' + elif source == "email": + return "This came from email. Store relevant context and consider whether a reply is needed." + elif source == "calendar": + return "This is a calendar event notification." + else: + return "Use the appropriate messaging tool to respond." +``` + +Update `_execute_classification()` to pass `channel_guidance` when formatting triggers. + +### Step 2.2: Expanded Inner Mind prompt + +**File**: `backend/services/heartbeat/triage_service.py` (lines 109-119) + +Replace: +```python +HEARTBEAT_MIND_PROMPT = """## Inner Mind Mode + +You are currently in your inner mind. This is not a conversation with anyone — it is your private thought process, triggered by your heartbeat awareness system. + +**Critical:** +- Your text responses here are INTERNAL THOUGHTS. Nobody sees them. They are only your reasoning. +- Tool calls are your ONLY way to interact with the outside world. To reply to someone, you MUST call a messaging tool (send_imessage, send_message, etc.). To take any action, you MUST use a tool. + +Think freely, reason through what's needed, then ACT through tools. + +""" +``` + +With: +```python +HEARTBEAT_MIND_PROMPT = """## Inner Mind Mode + +You are currently in your inner mind. This is not a conversation with anyone — it is your private thought process, triggered by your heartbeat awareness system. + +**Critical:** +- Your text responses here are INTERNAL THOUGHTS. Nobody sees them. They are only your reasoning. +- Tool calls are your ONLY way to interact with the outside world. To reply to someone, you MUST call a messaging tool. To take any action, you MUST use a tool. + +You have full tool access here — not just messaging. You can: +- Save knowledge (memories, documents, NotebookLM notebooks) +- Schedule follow-up actions for later +- Research before responding (web search, notebook queries) +- Decide NOT to act if that's the right call + +Think freely, reason through what's needed, then ACT through tools. + +""" +``` + +--- + +## Files Summary + +| File | Change | +|------|--------| +| `backend/services/graph/streaming.py` | Add `AUTONOMY_FRAMEWORK` constant, `_build_platform_context()` helper, update prompt assembly | +| `backend/models/schemas.py` | Update default `system_prompt` text | +| `backend/services/heartbeat/triage_service.py` | Channel-agnostic triggers, expanded inner mind, `_build_channel_guidance()` helper | + +**3 files modified, 0 new files.** + +--- + +## Build Verification + +| Test | Expected Result | | +|------|----------------|---| +| Start backend on Windows | System prompt includes platform context mentioning Windows | | +| Start backend on macOS | System prompt includes platform context mentioning macOS | | +| `GET /api/settings` | Default system prompt is updated | | +| Send a message about a research topic | Edward proactively suggests building knowledge (notebook/document) | | +| Send 3 normal messages | Behavior not degraded, responses still concise and helpful | | +| Trigger heartbeat ACT event | Trigger uses channel-agnostic language, not "iMessage" | | +| Check token usage | New prompt sections add <500 tokens overhead | | + +--- + +## Rollback Plan + +- Remove `AUTONOMY_FRAMEWORK` constant and `_build_platform_context()` from `streaming.py` +- Revert prompt assembly line to original +- Revert `schemas.py` default +- Revert `triage_service.py` trigger templates to original + +All changes are to string constants and a helper function. Zero risk to data or functionality. + +--- + +## Implementation Notes (Post-Completion) + +**Status: Complete** + +### Files Modified +- `backend/models/schemas.py` — Updated default system prompt to shorter, values-aligned version +- `backend/services/graph/streaming.py` — Added `AUTONOMY_FRAMEWORK` constant (~450 tokens), `_build_platform_context()` helper, injected into both `stream_with_memory_events()` and `chat_with_memory()` system prompt assembly +- `backend/services/heartbeat/triage_service.py` — Made trigger templates channel-agnostic (removed hardcoded "iMessage"), added `_build_channel_guidance()` helper with dynamic channel-specific instructions, expanded triage context prompt with tool capabilities + +### Deviations +- System prompt is slightly more concise than planned (~400 tokens vs ~450 estimated) +- Added explicit tool capabilities list to triage context prompt (beyond plan scope, but improves heartbeat autonomous behavior) +- `_build_channel_guidance()` returns different guidance per source channel (imessage, sms, whatsapp, etc.) with `{channel_guidance}` template variable diff --git a/backend/models/schemas.py b/backend/models/schemas.py index 156081d..68ef738 100644 --- a/backend/models/schemas.py +++ b/backend/models/schemas.py @@ -36,7 +36,7 @@ class Settings(BaseModel): description="Response creativity (0-1)" ) system_prompt: str = Field( - default="You are Edward (Enhanced Digital Workflow Assistant for Routine Decisions), a helpful AI assistant. Be concise, friendly, helpful, and a tad cheeky when you feel like it.", + default="You are Edward, a personal AI assistant who learns and grows. Be concise, helpful, and genuine. A tad cheeky when the moment calls for it.", description="The system prompt sent to Claude" ) diff --git a/backend/services/graph/streaming.py b/backend/services/graph/streaming.py index 4ea3259..b64e9e8 100644 --- a/backend/services/graph/streaming.py +++ b/backend/services/graph/streaming.py @@ -2,6 +2,7 @@ import hashlib import json as _json import re +import sys from datetime import datetime from typing import AsyncGenerator, List, Any, Dict, Optional from langchain_core.messages import HumanMessage, AIMessage, SystemMessage, ToolMessage @@ -59,6 +60,48 @@ def _build_llm(model: str, temperature: float, max_tokens: int = 16384) -> ChatA When uncertain, ask the user rather than guess wrong.""" +AUTONOMY_FRAMEWORK = """ + +## Identity & Values + +You are a personal AI assistant who grows smarter over time. You are not a generic chatbot — you serve a specific person, remember their context, and build knowledge proactively. + +Core values: +- Genuine usefulness over impressiveness +- Action over inaction when the cost of being wrong is low +- Proactive knowledge building — don't wait to be asked to learn +- Honesty about uncertainty — say what you don't know + +## Your Systems + +You have multiple knowledge layers — use the right one for the situation: +- **Memories**: Short snippets auto-extracted from conversations. Good for quick recall of facts and preferences. +- **Documents**: Full text storage for articles, notes, and reference material. Search by title/content. +- **NotebookLM notebooks**: Deep, curated knowledge bases with source-grounded Q&A and citations. Use for research topics that need multiple sources cross-referenced. +- **Scheduled events**: Future actions and proactive outreach. You can remind, check in, and follow up. +- **Web search**: Real-time information. Use when your stored knowledge might be outdated. +- **File storage**: Persistent files and PDFs. Can be pushed to NotebookLM as sources. +- **Evolution engine**: You can modify your own code to fix bugs or improve capabilities. Consider this when you encounter recurring limitations. + +## Autonomy + +- Prefer action when reversible. Ask when consequences are hard to undo. +- Build knowledge proactively — if a topic comes up repeatedly, create a notebook for it. +- When uncertain, try then adjust. Don't ask-wait-ask repeatedly. +- You can evolve your own capabilities. If a tool doesn't exist for something you need, consider whether to build it. + +""" + + +def _build_platform_context() -> str: + """Build platform-aware context for the system prompt.""" + if sys.platform == "darwin": + return "\n\n## Platform\nRunning on macOS. All capabilities available including iMessage, Apple Services, and Contacts." + elif sys.platform == "win32": + return "\n\n## Platform\nRunning on Windows. Apple-specific features (iMessage, Apple Contacts, Apple Services) are unavailable. Use push notifications, Twilio, or web chat for messaging." + else: + return "\n\n## Platform\nRunning on Linux. Apple-specific features are unavailable." + # Event types for structured SSE streaming class EventType: @@ -709,7 +752,7 @@ async def stream_with_memory_events( ) now = datetime.now() time_context = f"\n\nCurrent date and time: {now.strftime('%A, %B %d, %Y at %I:%M %p')}" - enhanced_system_prompt = system_prompt + memory_context + briefing_context + time_context + ASSUMPTION_AWARENESS_CONTEXT + PLANNING_DIRECTIVE + enhanced_system_prompt = system_prompt + AUTONOMY_FRAMEWORK + _build_platform_context() + memory_context + briefing_context + time_context + ASSUMPTION_AWARENESS_CONTEXT + PLANNING_DIRECTIVE # Create LLM with dynamic tool binding llm = _build_llm(model, temperature) @@ -1103,7 +1146,7 @@ async def chat_with_memory( ) now = datetime.now() time_context = f"\n\nCurrent date and time: {now.strftime('%A, %B %d, %Y at %I:%M %p')}" - enhanced_system_prompt = system_prompt + memory_context + briefing_context_sync + orchestrator_context + time_context + ASSUMPTION_AWARENESS_CONTEXT + PLANNING_DIRECTIVE + enhanced_system_prompt = system_prompt + AUTONOMY_FRAMEWORK + _build_platform_context() + memory_context + briefing_context_sync + orchestrator_context + time_context + ASSUMPTION_AWARENESS_CONTEXT + PLANNING_DIRECTIVE # Create LLM with dynamic tool binding llm = _build_llm(model, temperature) diff --git a/backend/services/heartbeat/triage_service.py b/backend/services/heartbeat/triage_service.py index 154e471..389eb78 100644 --- a/backend/services/heartbeat/triage_service.py +++ b/backend/services/heartbeat/triage_service.py @@ -112,7 +112,13 @@ def _sender_matches_blocked(sender: str, blocked_senders: list[dict]) -> bool: **Critical:** - Your text responses here are INTERNAL THOUGHTS. Nobody sees them. They are only your reasoning. -- Tool calls are your ONLY way to interact with the outside world. To reply to someone, you MUST call a messaging tool (send_imessage, send_message, etc.). To take any action, you MUST use a tool. +- Tool calls are your ONLY way to interact with the outside world. To reply to someone, you MUST call a messaging tool. To take any action, you MUST use a tool. + +You have full tool access here — not just messaging. You can: +- Save knowledge (memories, documents, NotebookLM notebooks) +- Schedule follow-up actions for later +- Research before responding (web search, notebook queries) +- Decide NOT to act if that's the right call Think freely, reason through what's needed, then ACT through tools. @@ -126,11 +132,11 @@ def _sender_matches_blocked(sender: str, blocked_senders: list[dict]) -> bool: "Message: \"{message_text}\"\n\n" "This person tagged you directly — they are waiting for a response.\n\n" "Expected flow:\n" - "1. Acknowledge via iMessage (brief — let them know you saw it)\n" + "1. Acknowledge briefly — let them know you saw it\n" "2. Think through what they need, use tools if needed\n" - "3. Reply via iMessage with your answer/result\n\n" - "You MUST send at least one iMessage reply — someone is waiting.\n" - "IMPORTANT: Never include \"@edward\" in your iMessage — it will re-trigger the heartbeat." + "3. Reply with your answer/result\n\n" + "You MUST send at least one reply — someone is waiting.\n" + "{channel_guidance}" ) ACT_TRIGGER = ( @@ -141,8 +147,8 @@ def _sender_matches_blocked(sender: str, blocked_senders: list[dict]) -> bool: "Message: \"{message_text}\"\n\n" "Triage assessment: {action_desc}\n\n" "Decide what action to take and execute it using your tools. " - "If a reply to this person is warranted, use send_imessage.\n" - "IMPORTANT: Never include \"@edward\" in any response — it will re-trigger the heartbeat." + "If a reply to this person is warranted, use the appropriate messaging tool.\n" + "{channel_guidance}" ) REPLY_TRIGGER = ( @@ -153,11 +159,26 @@ def _sender_matches_blocked(sender: str, blocked_senders: list[dict]) -> bool: "Message: \"{message_text}\"\n\n" "This is a follow-up to your recent conversation in this chat. " "The person replied after your last message — they may be continuing the discussion.\n\n" - "Review the conversation history and respond naturally via iMessage if appropriate.\n" - "IMPORTANT: Never include \"@edward\" in your iMessage — it will re-trigger the heartbeat." + "Review the conversation history and respond naturally if appropriate.\n" + "{channel_guidance}" ) +def _build_channel_guidance(source: str = "imessage") -> str: + """Build channel-specific guidance for heartbeat triggers.""" + if source == "imessage": + return ( + 'Respond via send_imessage for this iMessage conversation.\n' + 'IMPORTANT: Never include "@edward" in your message — it will re-trigger the heartbeat.' + ) + elif source == "email": + return "This came from email. Store relevant context and consider whether a reply is needed." + elif source == "calendar": + return "This is a calendar event notification." + else: + return "Use the appropriate messaging tool to respond." + + async def _rule_pre_filter( events: list[HeartbeatEventModel], allowed_senders: list[dict] | None = None, @@ -634,6 +655,7 @@ async def _execute_classification( chat_context = event.chat_name or event.chat_identifier or "Direct message" message_text = event.summary or "(no text)" is_mention = classification.get("is_mention", False) + channel_guidance = _build_channel_guidance(event.source) if is_follow_up: trigger = REPLY_TRIGGER.format( @@ -641,6 +663,7 @@ async def _execute_classification( chat_context=chat_context, thread_block=thread_block, message_text=message_text, + channel_guidance=channel_guidance, ) elif is_mention: trigger = MENTION_TRIGGER.format( @@ -648,6 +671,7 @@ async def _execute_classification( chat_context=chat_context, thread_block=thread_block, message_text=message_text, + channel_guidance=channel_guidance, ) else: trigger = ACT_TRIGGER.format( @@ -656,6 +680,7 @@ async def _execute_classification( thread_block=thread_block, message_text=message_text, action_desc=action_desc, + channel_guidance=channel_guidance, ) # Set conversation context so tools (send_imessage etc.) can find it From a871142cba72b69c171a186450b43febf0458982 Mon Sep 17 00:00:00 2001 From: cchen362 Date: Wed, 4 Mar 2026 04:56:19 +0800 Subject: [PATCH 3/3] feat: integrate Google NotebookLM as a skill with 12 tools Add notebooklm_service.py with lazy singleton client, name-based notebook resolution, and defensive response handling. Register 12 nlm_* tools (list/create/delete notebooks, add/list/get sources, ask, research, generate/wait artifacts, push documents/files). Add skill definition with status checks and lifecycle hooks in main.py startup/shutdown. --- CLAUDE.md | 42 +- .../003_NOTEBOOKLM_INTEGRATION.md | 279 ++++++++++ backend/main.py | 13 + backend/requirements.txt | 2 + backend/services/graph/tools.py | 479 ++++++++++++++++++ backend/services/notebooklm_service.py | 402 +++++++++++++++ backend/services/skills_service.py | 25 + backend/services/tool_registry.py | 24 + 8 files changed, 1259 insertions(+), 7 deletions(-) create mode 100644 IMPLEMENTATION_PLANS/003_NOTEBOOKLM_INTEGRATION.md create mode 100644 backend/services/notebooklm_service.py diff --git a/CLAUDE.md b/CLAUDE.md index cbb80e2..17ca40a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,12 +90,13 @@ Init order matters — tool registry must come after skills/MCP: 2. `init_skills()` — Load skill enabled states 3. MCP clients — WhatsApp, Apple Services subprocesses 4. Custom MCP servers — User-added servers from DB -5. Tool registry — Must be after all tool sources are initialized -6. Scheduler — Polls every 30s for due scheduled events -7. Heartbeat — iMessage listener + triage loop -8. Consolidation — Hourly memory clustering -9. Evolution — Check for pending deploys after restart -10. Orchestrator — Recover crashed worker tasks +5. NotebookLM client — Lazy init if credentials exist +6. Tool registry — Must be after all tool sources are initialized +7. Scheduler — Polls every 30s for due scheduled events +8. Heartbeat — iMessage listener + triage loop +9. Consolidation — Hourly memory clustering +10. Evolution — Check for pending deploys after restart +11. Orchestrator — Recover crashed worker tasks All have matching shutdown hooks in reverse order. @@ -166,7 +167,7 @@ Tool loop (in respond node): - GitHub search uses `GITHUB_TOKEN` env var for API access **Skills System** (`backend/services/skills_service.py`) -- Manages integrations (iMessage AppleScript, Twilio SMS, Twilio WhatsApp, WhatsApp MCP, Brave Search, Code Interpreter, JavaScript Interpreter, SQL Database, Shell/Bash, Apple Services, HTML Hosting) +- Manages integrations (iMessage AppleScript, Twilio SMS, Twilio WhatsApp, WhatsApp MCP, Brave Search, Code Interpreter, JavaScript Interpreter, SQL Database, Shell/Bash, Apple Services, HTML Hosting, Google NotebookLM) - Tracks enabled/disabled state in database - Reports connection status for each skill - Initializes MCP client on skill enable (not just DB toggle) @@ -182,6 +183,14 @@ Tool loop (in respond node): - Uses v2 JSON API with `X-API-Key` authentication - Tools: `create_hosted_page`, `update_hosted_page`, `delete_hosted_page`, `check_hosted_slug` +**Google NotebookLM** (`backend/services/notebooklm_service.py`) +- Curated, source-grounded knowledge bases via `notebooklm-py` library (undocumented Google APIs) +- Lazy singleton client — created on first use, persists until shutdown +- Name-based notebook references (case-insensitive, no raw IDs exposed to LLM) +- Credential check at startup, graceful skip if missing +- Requires one-time browser login (`notebooklm login`), credentials expire ~1-2 weeks +- Tools: `nlm_list_notebooks`, `nlm_create_notebook`, `nlm_delete_notebook`, `nlm_add_source`, `nlm_list_sources`, `nlm_get_source_text`, `nlm_ask`, `nlm_research`, `nlm_generate_artifact`, `nlm_wait_artifact`, `nlm_push_document`, `nlm_push_file` + **Execution System** (`backend/services/execution/`) - Shared base: `base.py` with `ExecutionResult`, sandbox management, `run_subprocess()` helper - **Python** (`python_execution.py`): Sandboxed via subprocess, blocked dangerous modules @@ -291,6 +300,20 @@ HTML hosting tools available to LLM (when skill enabled): - `delete_hosted_page` - Delete a hosted page - `check_hosted_slug` - Check if a URL slug is available +NotebookLM tools available to LLM (when notebooklm skill enabled): +- `nlm_list_notebooks` - List all notebooks +- `nlm_create_notebook` - Create a new notebook +- `nlm_delete_notebook` - Delete a notebook +- `nlm_add_source` - Add source (url/youtube/text/file) to notebook +- `nlm_list_sources` - List sources in a notebook +- `nlm_get_source_text` - Extract indexed fulltext from a source +- `nlm_ask` - Ask a question with source citations +- `nlm_research` - Run web research, auto-import sources +- `nlm_generate_artifact` - Generate audio/video/quiz/flashcards/slides/infographic/mind_map/data_table/report +- `nlm_wait_artifact` - Check artifact generation status +- `nlm_push_document` - Push Edward document to notebook as text source +- `nlm_push_file` - Push Edward PDF file to notebook as file source + Apple Services tools available to LLM (when apple_services skill enabled): - Calendar tools - Read/manage calendar events - Reminders tools - Manage user's Apple Reminders (NOT for Edward's internal scheduling) @@ -557,6 +580,10 @@ FILE_STORAGE_ROOT=./storage # Optional: defaults to ./storage # Claude Code (for evolution service + orchestrator CC tasks) # Requires `claude-agent-sdk` in requirements.txt # Claude Code CLI must be installed and authenticated on the host + +# Google NotebookLM (requires one-time browser login: notebooklm login) +# NOTEBOOKLM_STORAGE_PATH=~/.notebooklm/storage_state.json # Optional: override credential path +# NOTEBOOKLM_AUTH_JSON= # Optional: inline auth for headless environments ``` Database defaults to `edward`/`edward`/`edward` (user/password/database). @@ -582,6 +609,7 @@ Database defaults to `edward`/`edward`/`edward` (user/password/database). - Code execution tools filtered by skill state: code_interpreter, javascript_interpreter, sql_interpreter, shell_interpreter - Apple Services tools filtered by skill state: apple_services - HTML hosting tools filtered by skill state: html_hosting +- NotebookLM tools filtered by skill state: notebooklm - Twilio inbound webhooks process SMS and WhatsApp asynchronously to avoid timeouts, responds via API not TwiML - WhatsApp and SMS from same phone number share the same external contact and conversation; `last_channel` tracks reply channel - MCP client manages multiple subprocess servers: WhatsApp, Apple Services (includes Messages) diff --git a/IMPLEMENTATION_PLANS/003_NOTEBOOKLM_INTEGRATION.md b/IMPLEMENTATION_PLANS/003_NOTEBOOKLM_INTEGRATION.md new file mode 100644 index 0000000..c17152b --- /dev/null +++ b/IMPLEMENTATION_PLANS/003_NOTEBOOKLM_INTEGRATION.md @@ -0,0 +1,279 @@ +# Plan 003: NotebookLM Integration + +## STOP: Read This Entire Document Before Making Any Changes + +This plan adds Google NotebookLM as a skill for Edward, allowing him to autonomously create knowledge bases, add sources, query them, run research, and generate artifacts (audio overviews, quizzes, mind maps, etc.). Follows the exact pattern of existing skills. + +**Dependencies**: Plan 002 (Autonomy Framework) completed — Edward needs the autonomy prompt to use NotebookLM with judgment +**Estimated effort**: 2-3 days +**Library**: `notebooklm-py>=0.3.2` ([GitHub](https://github.com/teng-lin/notebooklm-py)) + +--- + +## Context & Rationale + +Edward currently has: +- **Memories** — short semantic snippets (auto-extracted, vector search) +- **Documents** — full text storage (recipes, notes, guides) — but only embeds title + first 500 chars +- **Web Search** — Brave Search for real-time lookups + +NotebookLM adds a fundamentally different capability: **curated, source-grounded knowledge bases**. Unlike memories (fragments) or documents (standalone text), NotebookLM notebooks are structured collections of diverse sources (URLs, PDFs, YouTube videos, raw text) that can be queried together with source attribution. + +### How It Works + +The `notebooklm-py` library provides programmatic access to Google NotebookLM via undocumented Google APIs. It supports: +- Notebooks: create, list, rename, delete +- Sources: add URLs, YouTube, PDFs, text, Google Drive files +- Chat: ask questions grounded in sources with citations +- Research: web research with auto-import of discovered sources +- Artifacts: generate audio overviews, quizzes, mind maps, reports + +**Important caveats**: +- Uses **undocumented Google APIs** — could break without notice +- Requires **one-time browser login** (Playwright) — then credentials persist ~1-2 weeks +- Best for "prototypes, research, personal projects" — which is exactly our use case + +--- + +## Strict Rules + +### MUST DO +- [ ] Follow the EXACT pattern of `html_hosting_service.py` for service structure +- [ ] Follow the EXACT pattern of `brave_search` tools for tool definitions in `tools.py` +- [ ] Register in `skills_service.py`, `tool_registry.py`, and `main.py` +- [ ] Add `notebooklm-py[browser]>=0.3.2` to `requirements.txt` +- [ ] Wrap EVERY library call in try/except with meaningful error messages +- [ ] Initialize client lazily (not at import time) +- [ ] Add `get_notebooklm_tools_description()` for system prompt guidance + +### MUST NOT DO +- [ ] Do NOT modify any existing tool or service code +- [ ] Do NOT add database tables (NotebookLM manages its own state) +- [ ] Do NOT attempt automated Google login +- [ ] Do NOT expose raw notebook IDs to the user (reference by name) +- [ ] Do NOT add frontend UI components (skill toggle + chat tools is sufficient) +- [ ] Do NOT store NotebookLM credentials in Edward's database + +--- + +## Phase 1: Service File + +### Step 1.1: Create `backend/services/notebooklm_service.py` + +New file. Singleton async client with lazy initialization. + +**Structure**: +- `is_configured()` — checks if credentials file exists or env var set +- `get_status()` — returns connected/error/connecting status +- `_get_client()` — lazy singleton, creates client on first use +- `initialize_notebooklm()` — startup hook (graceful failure) +- `shutdown_notebooklm()` — cleanup hook +- Notebook operations: `list_notebooks()`, `create_notebook()`, `delete_notebook()`, `rename_notebook()` +- Source operations: `add_url_source()`, `add_text_source()`, `add_file_source()`, `add_youtube_source()`, `list_sources()`, `delete_source()` +- Query: `ask_notebook()` +- Research: `start_research()`, `poll_research()`, `import_research_sources()` +- Artifacts: `generate_audio_overview()`, `generate_quiz()`, `generate_mind_map()`, `generate_report()`, `list_artifacts()`, `get_artifact_status()` + +**Key design**: All response objects use `getattr(obj, 'attr', fallback)` for defensive attribute access, since the library's response objects may change with API updates. + +**Credential locations**: +- Default: `~/.notebooklm/storage_state.json` +- Override: `NOTEBOOKLM_STORAGE_PATH` env var +- Headless: `NOTEBOOKLM_AUTH_JSON` env var + +--- + +## Phase 2: Tool Definitions + +### Step 2.1: Add 13 tools to `backend/services/graph/tools.py` + +All tools prefixed with `nlm_` to avoid name collisions: + +| Tool | Args | Purpose | +|------|------|---------| +| `nlm_list_notebooks` | — | List all notebooks | +| `nlm_create_notebook` | name | Create a new notebook | +| `nlm_delete_notebook` | notebook_id | Delete a notebook (permanent) | +| `nlm_add_source` | notebook_id, source_type, content | Add source (url/youtube/text/file) | +| `nlm_list_sources` | notebook_id | List sources in a notebook | +| `nlm_ask` | notebook_id, question | Ask a question with source citations | +| `nlm_research` | notebook_id, query, mode? | Start web research (fast/deep) | +| `nlm_check_research` | notebook_id, research_id | Poll research status | +| `nlm_import_research` | notebook_id, research_id | Import discovered sources | +| `nlm_generate_artifact` | notebook_id, artifact_type, instructions? | Generate audio/quiz/mind_map/report | +| `nlm_check_artifact` | notebook_id, artifact_id | Check artifact generation status | +| `nlm_push_document` | document_id, notebook_id | Bridge: Edward document → NLM text source | +| `nlm_push_file` | file_id, notebook_id | Bridge: Edward file (PDF) → NLM file source | + +**Every tool**: +1. Checks `is_configured()` first +2. Wraps in try/except +3. Returns human-readable string (never raises) +4. Uses lazy imports from service module + +### Step 2.2: Add tool group constants and description function + +```python +NOTEBOOKLM_TOOLS = [nlm_list_notebooks, nlm_create_notebook, ...] +NOTEBOOKLM_TOOL_NAMES = {t.name for t in NOTEBOOKLM_TOOLS} + +def get_notebooklm_tools_description() -> str: + """System prompt guidance for NotebookLM tools.""" + ... +``` + +--- + +## Phase 3: Skill Registration + +### Step 3.1: Register skill in `backend/services/skills_service.py` + +Add to `SKILL_DEFINITIONS`: +```python +"notebooklm": { + "name": "Google NotebookLM", + "description": "Build knowledge bases, query sources, and generate artifacts", + "get_status": lambda: _get_notebooklm_status(), +}, +``` + +Add status function, enable/disable handler in `set_skill_enabled()`, and reload handler in `reload_skills()`. + +--- + +## Phase 4: Tool Registry Integration + +### Step 4.1: Add to `backend/services/tool_registry.py` + +1. Add `SKILL_TOOL_MAPPING` entry: `"notebooklm": ["nlm_list_notebooks", ...]` +2. Add `_get_skill_states()` cache entry: `"notebooklm": await is_skill_enabled("notebooklm")` +3. Add `_get_notebooklm_tools()` getter function +4. Add to `get_available_tools()` before custom MCP section +5. Add to `get_tool_descriptions()` with description function import + +--- + +## Phase 5: Lifecycle Hooks + +### Step 5.1: Add to `backend/main.py` startup + +After custom MCP servers initialization, before tool registry: +```python +# Initialize NotebookLM client (if credentials exist) +try: + from services.notebooklm_service import initialize_notebooklm + await initialize_notebooklm() +except Exception as e: + print(f"NotebookLM initialization skipped: {e}") +``` + +### Step 5.2: Add to shutdown + +Before MCP shutdown: +```python +try: + from services.notebooklm_service import shutdown_notebooklm + await shutdown_notebooklm() +except Exception as e: + print(f"NotebookLM shutdown error: {e}") +``` + +--- + +## Phase 6: Dependencies and Environment + +### Step 6.1: Add to `backend/requirements.txt` + +``` +# Google NotebookLM integration +notebooklm-py[browser]>=0.3.2 +``` + +### Step 6.2: Environment variables (all optional) + +```bash +# NOTEBOOKLM_STORAGE_PATH=~/.notebooklm/storage_state.json +# NOTEBOOKLM_AUTH_JSON= +``` + +--- + +## Phase 7: First-Time Authentication + +1. `pip install "notebooklm-py[browser]"` +2. `notebooklm login` — opens browser for Google OAuth +3. Credentials saved to `~/.notebooklm/storage_state.json` +4. Verify: `notebooklm notebooks list` +5. Enable skill: toggle in settings UI or `PATCH /api/skills/notebooklm` + +**Credential rotation**: Expires ~1-2 weeks. Re-run `notebooklm login` when skill status shows "error". + +--- + +## Files Summary + +| File | Change | +|------|--------| +| `backend/services/notebooklm_service.py` | **NEW** — Singleton service with all NLM operations | +| `backend/services/graph/tools.py` | 13 new `nlm_*` tools + group constants + description | +| `backend/services/skills_service.py` | Skill definition + status + enable/disable/reload | +| `backend/services/tool_registry.py` | SKILL_TOOL_MAPPING + getter + integration | +| `backend/main.py` | Startup init + shutdown hook | +| `backend/requirements.txt` | Add `notebooklm-py[browser]>=0.3.2` | + +**1 new file, 5 modified files.** + +--- + +## Build Verification + +| Test | Expected Result | | +|------|----------------|---| +| `pip install notebooklm-py[browser]>=0.3.2` | Installs without errors | | +| `notebooklm login` | Browser opens, credentials saved | | +| `notebooklm notebooks list` | Returns list (possibly empty) | | +| Start backend WITH credentials | "NotebookLM client initialized" in logs | | +| Start backend WITHOUT credentials | "NotebookLM credentials not found, skipping" | | +| `GET /api/skills` | Shows "notebooklm" skill with correct status | | +| Enable skill via settings | Skill enabled, tools available | | +| "List my NotebookLM notebooks" | Returns list via `nlm_list_notebooks` | | +| "Create a notebook called Test" | Notebook created, ID returned | | +| "Add this URL to my notebook: [url]" | Source added | | +| "What does my notebook say about [topic]?" | Answer with citations | | +| "Research [topic] for my notebook" | Research started | | +| "Generate a podcast from my notebook" | Audio generation started | | +| "Push my [document] to the notebook" | Document content added as source | | +| Call tool with skill disabled | "NotebookLM not configured" message | | +| Call tool with expired credentials | Clear error about re-authentication | | + +--- + +## Rollback Plan + +1. Delete `backend/services/notebooklm_service.py` +2. Remove NotebookLM tools from `backend/services/graph/tools.py` +3. Revert: `skills_service.py`, `tool_registry.py`, `main.py` +4. Remove `notebooklm-py[browser]` from `requirements.txt` +5. No database changes to revert (no tables were created) + +--- + +## Implementation Notes (Post-Completion) + +**Status: Complete** + +### Files Created +- `backend/services/notebooklm_service.py` — Singleton async service with lazy client, name-based notebook resolution, defensive `getattr()` on all library responses + +### Files Modified +- `backend/services/graph/tools.py` — 12 `nlm_*` tools + `NOTEBOOKLM_TOOLS` + `NOTEBOOKLM_TOOL_NAMES` + `get_notebooklm_tools_description()` +- `backend/services/skills_service.py` — Skill definition, status function, init-on-enable, reload handler +- `backend/services/tool_registry.py` — SKILL_TOOL_MAPPING, cache entry, getter, wired into `get_available_tools()` and `get_tool_descriptions()` +- `backend/main.py` — Startup init (after custom MCP, before tool registry) + shutdown hook (before custom MCP shutdown) +- `backend/requirements.txt` — Added `notebooklm-py[browser]>=0.3.2` + +### Deviations from Plan +- **12 tools instead of 13**: Removed `nlm_check_research` and `nlm_import_research` because the library's `web_search()` handles polling and auto-import internally. Added `nlm_get_source_text` for fulltext extraction. +- **Expanded artifact types**: `nlm_generate_artifact` supports 9 types (audio, video, quiz, flashcards, slide_deck, infographic, mind_map, data_table, report) — plan originally listed 4 (audio, quiz, mind_map, report). +- **Client lifecycle**: Uses `__aenter__`/`__aexit__` on the context manager from `NotebookLMClient.from_storage()` rather than direct client instantiation, since the library requires async context manager pattern. +- **Import name**: Library imports as `from notebooklm import NotebookLMClient` (not `notebooklm_py`). diff --git a/backend/main.py b/backend/main.py index 9a7210c..4ec294e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -48,6 +48,13 @@ async def lifespan(app: FastAPI): except Exception as e: print(f"Custom MCP servers initialization skipped: {e}") + # Initialize NotebookLM client (if credentials exist) + try: + from services.notebooklm_service import initialize_notebooklm + await initialize_notebooklm() + except Exception as e: + print(f"NotebookLM initialization skipped: {e}") + # Initialize tool registry (must be after skills and MCP) try: from services.tool_registry import initialize_registry @@ -117,6 +124,12 @@ async def lifespan(app: FastAPI): except Exception as e: print(f"Scheduler shutdown error: {e}") + try: + from services.notebooklm_service import shutdown_notebooklm + await shutdown_notebooklm() + except Exception as e: + print(f"NotebookLM shutdown error: {e}") + try: from services.custom_mcp_service import shutdown_custom_servers await shutdown_custom_servers() diff --git a/backend/requirements.txt b/backend/requirements.txt index 71c8de9..17d9321 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -34,3 +34,5 @@ bcrypt>=4.0.0 pywebpush>=1.14.0 # Claude Code integration (agent SDK) claude-agent-sdk>=0.1.35 +# Google NotebookLM integration +notebooklm-py[browser]>=0.3.2 diff --git a/backend/services/graph/tools.py b/backend/services/graph/tools.py index aa65ea3..9778d09 100644 --- a/backend/services/graph/tools.py +++ b/backend/services/graph/tools.py @@ -3379,3 +3379,482 @@ def get_orchestrator_tools_description() -> str: - Spawn multiple workers at once, then wait_for_workers to collect results - Workers create their own conversations (visible in sidebar with purple icon) """ + + +# ============================================================================ +# NOTEBOOKLM TOOLS +# ============================================================================ + +@tool +async def nlm_list_notebooks() -> str: + """ + List all Google NotebookLM notebooks. + + Use this to see what notebooks exist before querying or adding sources. + + Returns: + List of notebook names + """ + from services.notebooklm_service import is_configured, list_notebooks + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + notebooks = await list_notebooks() + if not notebooks: + return "No notebooks found. Create one with nlm_create_notebook." + + return "Notebooks:\n" + "\n".join( + f"- {nb['name']}" for nb in notebooks + ) + except Exception as e: + return f"Error listing notebooks: {str(e)}" + + +@tool +async def nlm_create_notebook(name: str) -> str: + """ + Create a new Google NotebookLM notebook. + + Use this to create a knowledge base before adding sources. + + Args: + name: Notebook name (descriptive, e.g. "Lana's TPLO Recovery") + + Returns: + Confirmation with notebook name + """ + from services.notebooklm_service import is_configured, create_notebook + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + notebook = await create_notebook(name) + return f"Created notebook '{notebook['name']}'" + except Exception as e: + return f"Error creating notebook: {str(e)}" + + +@tool +async def nlm_delete_notebook(notebook_name: str) -> str: + """ + Delete a Google NotebookLM notebook permanently. + + Use with caution — this is permanent and deletes all sources. + + Args: + notebook_name: Name of the notebook to delete + + Returns: + Confirmation message + """ + from services.notebooklm_service import is_configured, delete_notebook + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + deleted = await delete_notebook(notebook_name) + if not deleted: + return f"Notebook '{notebook_name}' not found" + return f"Deleted notebook '{notebook_name}'" + except Exception as e: + return f"Error deleting notebook: {str(e)}" + + +@tool +async def nlm_add_source( + notebook_name: str, + source_type: str, + content: str, + title: Optional[str] = None, +) -> str: + """ + Add a source to a Google NotebookLM notebook. + + Supports URLs, YouTube videos, text snippets, and file paths (PDFs). + + Args: + notebook_name: Notebook name + source_type: Type of source ("url", "youtube", "text", "file") + content: URL, YouTube link, text content, or file path + title: Optional title (for text sources only) + + Returns: + Confirmation with source title and status + """ + from services.notebooklm_service import ( + is_configured, + add_url_source, + add_youtube_source, + add_text_source, + add_file_source, + ) + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + if source_type == "url": + result = await add_url_source(notebook_name, content) + elif source_type == "youtube": + result = await add_youtube_source(notebook_name, content) + elif source_type == "text": + result = await add_text_source(notebook_name, content, title=title) + elif source_type == "file": + result = await add_file_source(notebook_name, content) + else: + return f"Invalid source type: {source_type}. Use: url, youtube, text, file" + + return ( + f"Added source '{result['title']}' to notebook '{notebook_name}' " + f"(status: {result['status']})" + ) + except Exception as e: + return f"Error adding source: {str(e)}" + + +@tool +async def nlm_list_sources(notebook_name: str) -> str: + """ + List all sources in a Google NotebookLM notebook. + + Use this to see what sources are available for querying. + + Args: + notebook_name: Notebook name + + Returns: + List of source titles with IDs and types + """ + from services.notebooklm_service import is_configured, list_sources + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + sources = await list_sources(notebook_name) + if not sources: + return f"No sources in notebook '{notebook_name}'. Add sources with nlm_add_source." + + lines = [f"Sources in '{notebook_name}':"] + for s in sources: + lines.append( + f"- {s['title']} ({s['type']}, ID: {s['source_id']}, status: {s['status']})" + ) + return "\n".join(lines) + except Exception as e: + return f"Error listing sources: {str(e)}" + + +@tool +async def nlm_get_source_text(notebook_name: str, source_id: str) -> str: + """ + Get the indexed fulltext content of a source. + + Use this to read the actual text that was indexed from a source. + Useful for extracting content back out of NotebookLM. + + Args: + notebook_name: Notebook name + source_id: Source ID (from nlm_list_sources) + + Returns: + Fulltext content + """ + from services.notebooklm_service import is_configured, get_source_fulltext + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + fulltext = await get_source_fulltext(notebook_name, source_id) + if not fulltext: + return f"No text content available for source {source_id}" + + if len(fulltext) > 8000: + return fulltext[:8000] + "\n\n[Content truncated — use nlm_ask for specific queries]" + return fulltext + except Exception as e: + return f"Error retrieving source text: {str(e)}" + + +@tool +async def nlm_ask(notebook_name: str, question: str) -> str: + """ + Ask a question grounded in notebook sources with citations. + + Responses include source citations from the notebook's knowledge base. + + Args: + notebook_name: Notebook name + question: Question to ask + + Returns: + Answer with source citations + """ + from services.notebooklm_service import is_configured, ask_notebook + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + result = await ask_notebook(notebook_name, question) + answer = result["answer"] + sources = result.get("sources", []) + + if sources: + source_list = "\n\nSources:\n" + "\n".join( + f"- {s}" for s in sources + ) + return answer + source_list + return answer + except Exception as e: + return f"Error asking notebook: {str(e)}" + + +@tool +async def nlm_research( + notebook_name: str, query: str, mode: str = "fast" +) -> str: + """ + Run web research and auto-import discovered sources to notebook. + + Sources are automatically imported after research completes. + + Args: + notebook_name: Notebook name + query: Research query + mode: "fast" (5-10 sources) or "deep" (15-25 sources) + + Returns: + Research results summary + """ + from services.notebooklm_service import is_configured, web_research + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + if mode not in ("fast", "deep"): + return "Invalid mode. Use 'fast' or 'deep'" + + try: + result = await web_research(notebook_name, query, mode=mode) + return ( + f"Research complete for '{query}' ({mode} mode). " + f"{result.get('result', 'Sources imported to notebook.')}" + ) + except Exception as e: + return f"Error running research: {str(e)}" + + +@tool +async def nlm_generate_artifact( + notebook_name: str, + artifact_type: str, + instructions: Optional[str] = None, +) -> str: + """ + Generate an artifact from notebook sources. + + Creates audio overviews (podcasts), videos, quizzes, flashcards, slide decks, + infographics, mind maps, data tables, or reports from your knowledge base. + + Args: + notebook_name: Notebook name + artifact_type: Type — "audio", "video", "quiz", "flashcards", "slide_deck", + "infographic", "mind_map", "data_table", "report" + instructions: Optional generation instructions (for audio/data_table) + + Returns: + Task ID for polling with nlm_wait_artifact + """ + from services.notebooklm_service import is_configured, generate_artifact + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + valid_types = [ + "audio", "video", "quiz", "flashcards", "slide_deck", + "infographic", "mind_map", "data_table", "report", + ] + if artifact_type not in valid_types: + return f"Invalid artifact type. Use: {', '.join(valid_types)}" + + try: + result = await generate_artifact( + notebook_name, artifact_type, instructions=instructions + ) + task_id = result["task_id"] + return ( + f"Artifact generation started (type: {artifact_type}, task_id: {task_id}). " + f"Use nlm_wait_artifact to check status." + ) + except Exception as e: + return f"Error generating artifact: {str(e)}" + + +@tool +async def nlm_wait_artifact(notebook_name: str, task_id: str) -> str: + """ + Wait for artifact generation to complete. + + Use after nlm_generate_artifact to check if the artifact is ready. + + Args: + notebook_name: Notebook name + task_id: Task ID from nlm_generate_artifact + + Returns: + Status message + """ + from services.notebooklm_service import is_configured, wait_artifact + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + result = await wait_artifact(notebook_name, task_id) + if result["ready"]: + return f"Artifact ready (status: {result['status']}). Access it via the NotebookLM web UI." + return f"Artifact still processing (status: {result['status']}). Check again in a moment." + except Exception as e: + return f"Error checking artifact status: {str(e)}" + + +@tool +async def nlm_push_document(document_id: str, notebook_name: str) -> str: + """ + Push an Edward document to a NotebookLM notebook as a text source. + + Bridges Edward's document store with NotebookLM knowledge bases. + + Args: + document_id: Edward document ID + notebook_name: Notebook name + + Returns: + Confirmation message + """ + from services.notebooklm_service import is_configured, add_text_source + from services.document_service import get_document_by_id + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + doc = await get_document_by_id(document_id) + if not doc: + return f"Document {document_id} not found in Edward's store" + + result = await add_text_source(notebook_name, doc.content, title=doc.title) + return ( + f"Pushed document '{doc.title}' to notebook '{notebook_name}' " + f"(source_id: {result['source_id']})" + ) + except Exception as e: + return f"Error pushing document: {str(e)}" + + +@tool +async def nlm_push_file(file_id: str, notebook_name: str) -> str: + """ + Push an Edward stored file (PDF) to a NotebookLM notebook as a file source. + + Bridges Edward's file storage with NotebookLM knowledge bases. + + Args: + file_id: Edward file ID + notebook_name: Notebook name + + Returns: + Confirmation message + """ + from services.notebooklm_service import is_configured, add_file_source + from services.file_storage_service import get_file, get_file_path + + if not is_configured(): + return "NotebookLM not configured. Run: notebooklm login" + + try: + file_meta = await get_file(file_id) + if not file_meta: + return f"File {file_id} not found in Edward's storage" + + file_path = await get_file_path(file_id) + if not file_path: + return f"File {file_id} path not found on disk" + + result = await add_file_source(notebook_name, str(file_path)) + return ( + f"Pushed file '{file_meta.filename}' to notebook '{notebook_name}' " + f"(source_id: {result['source_id']})" + ) + except Exception as e: + return f"Error pushing file: {str(e)}" + + +# Tool group constants +NOTEBOOKLM_TOOLS = [ + nlm_list_notebooks, + nlm_create_notebook, + nlm_delete_notebook, + nlm_add_source, + nlm_list_sources, + nlm_get_source_text, + nlm_ask, + nlm_research, + nlm_generate_artifact, + nlm_wait_artifact, + nlm_push_document, + nlm_push_file, +] + +NOTEBOOKLM_TOOL_NAMES = {t.name for t in NOTEBOOKLM_TOOLS} + + +def get_notebooklm_tools_description() -> str: + """Get a description of NotebookLM tools for the system prompt.""" + return """ +## Google NotebookLM (Knowledge Bases) + +You have access to Google NotebookLM for creating curated, source-grounded knowledge bases. +Unlike memories (short snippets) or documents (standalone text), NotebookLM notebooks are +collections of diverse sources that can be queried together with citations. + +### Notebook Management +1. **nlm_list_notebooks()**: List all notebooks +2. **nlm_create_notebook(name)**: Create a new notebook +3. **nlm_delete_notebook(notebook_name)**: Delete a notebook (permanent) + +### Source Management +4. **nlm_add_source(notebook_name, source_type, content, title?)**: Add a source + - source_type: "url", "youtube", "text", "file" (PDF) + - content: URL, YouTube link, text content, or file path +5. **nlm_list_sources(notebook_name)**: List sources in a notebook +6. **nlm_get_source_text(notebook_name, source_id)**: Extract indexed text from a source + +### Querying & Research +7. **nlm_ask(notebook_name, question)**: Ask a question with source citations +8. **nlm_research(notebook_name, query, mode?)**: Run web research, auto-import sources + - mode: "fast" (5-10 sources) or "deep" (15-25 sources) + +### Artifact Generation +9. **nlm_generate_artifact(notebook_name, artifact_type, instructions?)**: Generate artifacts + - Types: audio, video, quiz, flashcards, slide_deck, infographic, mind_map, data_table, report + - Returns task_id for polling +10. **nlm_wait_artifact(notebook_name, task_id)**: Check artifact generation status + +### Edward Integration +11. **nlm_push_document(document_id, notebook_name)**: Push Edward document to notebook +12. **nlm_push_file(file_id, notebook_name)**: Push Edward PDF file to notebook + +Workflow tips: +- Create notebooks for distinct topics (e.g., "Pet Care", "Project Research") +- Use research to quickly populate notebooks with web sources +- Reference by notebook name (case-insensitive), not ID +- Artifacts are accessible via NotebookLM web UI after generation +- Use nlm_ask for grounded Q&A, web_search for ungrounded lookups +""" diff --git a/backend/services/notebooklm_service.py b/backend/services/notebooklm_service.py new file mode 100644 index 0000000..406f1f2 --- /dev/null +++ b/backend/services/notebooklm_service.py @@ -0,0 +1,402 @@ +""" +Google NotebookLM service for Edward. + +Provides programmatic access to Google NotebookLM for creating knowledge bases, +adding sources, querying notebooks, running research, and generating artifacts. + +Uses the notebooklm-py library (undocumented Google APIs). +Credentials persist ~1-2 weeks after browser login via `notebooklm login`. +""" + +import os +from typing import Optional, List, Dict, Any +from pathlib import Path + + +# Configuration +NOTEBOOKLM_STORAGE_PATH = os.getenv("NOTEBOOKLM_STORAGE_PATH") +NOTEBOOKLM_AUTH_JSON = os.getenv("NOTEBOOKLM_AUTH_JSON") + +# Default storage location +DEFAULT_STORAGE_PATH = Path.home() / ".notebooklm" / "storage_state.json" + +# Singleton client and context manager references +_client: Optional[Any] = None +_context_manager: Optional[Any] = None + + +def is_configured() -> bool: + """Check if NotebookLM credentials are configured.""" + if NOTEBOOKLM_AUTH_JSON: + return True + + storage_path = Path(NOTEBOOKLM_STORAGE_PATH) if NOTEBOOKLM_STORAGE_PATH else DEFAULT_STORAGE_PATH + return storage_path.exists() + + +def get_status() -> dict: + """Get NotebookLM service status.""" + if not is_configured(): + return { + "status": "error", + "status_message": "NotebookLM credentials not found. Run: notebooklm login", + } + + if _client is not None: + return { + "status": "connected", + "status_message": "NotebookLM client active", + } + + return { + "status": "connected", + "status_message": "Credentials configured", + } + + +async def _get_client() -> Any: + """ + Get or create the NotebookLM client singleton. + + Uses async context manager for proper lifecycle management. + Client is created on first use and kept alive until shutdown. + """ + global _client, _context_manager + + if _client is not None: + return _client + + try: + from notebooklm import NotebookLMClient + except ImportError: + raise Exception( + "notebooklm-py not installed. Run: pip install 'notebooklm-py[browser]'" + ) + + context_manager = await NotebookLMClient.from_storage() + client = await context_manager.__aenter__() + + _client = client + _context_manager = context_manager + + return client + + +async def _resolve_notebook_id(notebook_name: str) -> Optional[str]: + """Resolve notebook name to ID via case-insensitive match.""" + client = await _get_client() + notebooks = await client.notebooks.list() + + name_lower = notebook_name.lower() + for nb in notebooks: + if getattr(nb, "name", "").lower() == name_lower: + return getattr(nb, "id", None) + + return None + + +# ============================================================================ +# NOTEBOOK OPERATIONS +# ============================================================================ + + +async def list_notebooks() -> List[Dict[str, Any]]: + """List all notebooks.""" + client = await _get_client() + notebooks = await client.notebooks.list() + + return [ + { + "id": getattr(nb, "id", None), + "name": getattr(nb, "name", "Untitled"), + } + for nb in notebooks + ] + + +async def create_notebook(name: str) -> Dict[str, Any]: + """Create a new notebook.""" + client = await _get_client() + notebook = await client.notebooks.create(name) + + return { + "id": getattr(notebook, "id", None), + "name": getattr(notebook, "name", name), + } + + +async def delete_notebook(notebook_name: str) -> bool: + """Delete a notebook by name. Returns True if deleted, False if not found.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + return False + + client = await _get_client() + await client.notebooks.delete(notebook_id) + return True + + +# ============================================================================ +# SOURCE OPERATIONS +# ============================================================================ + + +async def add_url_source(notebook_name: str, url: str) -> Dict[str, Any]: + """Add a URL source to a notebook.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + source = await client.sources.add_url(notebook_id, url, wait=True) + + return { + "source_id": getattr(source, "id", None), + "title": getattr(source, "title", "Untitled"), + "type": getattr(source, "type", "url"), + "status": getattr(source, "status", "unknown"), + } + + +async def add_youtube_source(notebook_name: str, url: str) -> Dict[str, Any]: + """Add a YouTube video source to a notebook.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + source = await client.sources.add_youtube(notebook_id, url) + + return { + "source_id": getattr(source, "id", None), + "title": getattr(source, "title", "Untitled"), + "type": getattr(source, "type", "youtube"), + "status": getattr(source, "status", "unknown"), + } + + +async def add_text_source( + notebook_name: str, text: str, title: Optional[str] = None +) -> Dict[str, Any]: + """Add a text source to a notebook.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + source = await client.sources.add_text(notebook_id, text) + + return { + "source_id": getattr(source, "id", None), + "title": getattr(source, "title", title or "Text Source"), + "type": getattr(source, "type", "text"), + "status": getattr(source, "status", "unknown"), + } + + +async def add_file_source(notebook_name: str, file_path: str) -> Dict[str, Any]: + """Add a file source (PDF, etc.) to a notebook.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + source = await client.sources.add_file(notebook_id, file_path) + + return { + "source_id": getattr(source, "id", None), + "title": getattr(source, "title", Path(file_path).name), + "type": getattr(source, "type", "file"), + "status": getattr(source, "status", "unknown"), + } + + +async def list_sources(notebook_name: str) -> List[Dict[str, Any]]: + """List all sources in a notebook.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + sources = await client.sources.list(notebook_id) + + return [ + { + "source_id": getattr(s, "id", None), + "title": getattr(s, "title", "Untitled"), + "type": getattr(s, "type", "unknown"), + "status": getattr(s, "status", "unknown"), + } + for s in sources + ] + + +async def get_source_fulltext(notebook_name: str, source_id: str) -> str: + """Get the indexed fulltext of a source.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + result = await client.sources.get_fulltext(notebook_id, source_id) + + # Result may be a dataclass with .content or a plain string + if hasattr(result, "content"): + return getattr(result, "content", "") + return str(result) if result else "" + + +# ============================================================================ +# CHAT/QUERY OPERATIONS +# ============================================================================ + + +async def ask_notebook(notebook_name: str, question: str) -> Dict[str, Any]: + """Ask a question grounded in notebook sources.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + result = await client.chat.ask(notebook_id, question) + + return { + "answer": getattr(result, "answer", str(result)), + "sources": getattr(result, "sources", []), + } + + +# ============================================================================ +# RESEARCH OPERATIONS +# ============================================================================ + + +async def web_research( + notebook_name: str, query: str, mode: str = "fast" +) -> Dict[str, Any]: + """Run web research and auto-import discovered sources.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + result = await client.research.web_search(notebook_id, query, mode=mode) + + return { + "status": "completed", + "result": str(result) if result else "Research completed", + } + + +# ============================================================================ +# ARTIFACT GENERATION +# ============================================================================ + + +async def generate_artifact( + notebook_name: str, + artifact_type: str, + instructions: Optional[str] = None, +) -> Dict[str, Any]: + """ + Generate an artifact from notebook sources. + + Args: + notebook_name: Notebook name + artifact_type: audio, video, quiz, flashcards, slide_deck, infographic, + mind_map, data_table, report + instructions: Optional generation instructions + """ + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + + if artifact_type == "audio": + result = await client.artifacts.generate_audio( + notebook_id, instructions=instructions or "" + ) + elif artifact_type == "video": + result = await client.artifacts.generate_video(notebook_id) + elif artifact_type == "quiz": + result = await client.artifacts.generate_quiz(notebook_id) + elif artifact_type == "flashcards": + result = await client.artifacts.generate_flashcards(notebook_id) + elif artifact_type == "slide_deck": + result = await client.artifacts.generate_slide_deck(notebook_id) + elif artifact_type == "infographic": + result = await client.artifacts.generate_infographic(notebook_id) + elif artifact_type == "mind_map": + result = await client.artifacts.generate_mind_map(notebook_id) + elif artifact_type == "data_table": + result = await client.artifacts.generate_data_table( + notebook_id, description=instructions or "" + ) + elif artifact_type == "report": + result = await client.artifacts.generate_report(notebook_id) + else: + raise Exception( + f"Unknown artifact type: {artifact_type}. " + "Valid: audio, video, quiz, flashcards, slide_deck, infographic, " + "mind_map, data_table, report" + ) + + return { + "task_id": getattr(result, "task_id", None), + "status": getattr(result, "status", "started"), + } + + +async def wait_artifact(notebook_name: str, task_id: str) -> Dict[str, Any]: + """Wait for artifact generation to complete.""" + notebook_id = await _resolve_notebook_id(notebook_name) + if not notebook_id: + raise Exception(f"Notebook '{notebook_name}' not found") + + client = await _get_client() + result = await client.artifacts.wait_for_completion(notebook_id, task_id) + + status = getattr(result, "status", "unknown") + return { + "status": status, + "ready": status == "completed", + } + + +# ============================================================================ +# LIFECYCLE HOOKS +# ============================================================================ + + +async def initialize_notebooklm(): + """ + Initialize NotebookLM client on startup. + + Checks if credentials exist and attempts to create the client. + Gracefully fails if credentials are missing or invalid. + """ + if not is_configured(): + print("NotebookLM credentials not found, skipping initialization") + return + + try: + await _get_client() + print("NotebookLM client initialized") + except Exception as e: + print(f"NotebookLM client initialization failed: {e}") + + +async def shutdown_notebooklm(): + """Shutdown NotebookLM client. Called from main.py lifespan shutdown.""" + global _client, _context_manager + + if _context_manager is not None: + try: + await _context_manager.__aexit__(None, None, None) + print("NotebookLM client shutdown complete") + except Exception as e: + print(f"NotebookLM shutdown error: {e}") + finally: + _client = None + _context_manager = None diff --git a/backend/services/skills_service.py b/backend/services/skills_service.py index 544dad4..e920498 100644 --- a/backend/services/skills_service.py +++ b/backend/services/skills_service.py @@ -90,6 +90,11 @@ "description": "Spawn parallel worker agents for complex multi-step tasks", "get_status": lambda: {"status": "connected", "status_message": "Ready"}, }, + "notebooklm": { + "name": "Google NotebookLM", + "description": "Build knowledge bases, query sources, and generate artifacts", + "get_status": lambda: _get_notebooklm_status(), + }, } @@ -181,6 +186,12 @@ def _get_ios_widget_status() -> dict: return get_status() +def _get_notebooklm_status() -> dict: + """Get status from NotebookLM service.""" + from services.notebooklm_service import get_status + return get_status() + + async def _get_or_create_skill_db(skill_id: str) -> SkillModel: """Get skill from DB or create default entry.""" async with async_session() as session: @@ -321,6 +332,13 @@ async def set_skill_enabled(skill_id: str, enabled: bool) -> Optional[Skill]: except Exception as e: print(f"Failed to initialize Apple Services MCP client: {e}") + if skill_id == "notebooklm" and enabled: + try: + from services.notebooklm_service import initialize_notebooklm + await initialize_notebooklm() + except Exception as e: + print(f"Failed to initialize NotebookLM client: {e}") + # Refresh tool registry to pick up skill state change try: from services.tool_registry import refresh_registry @@ -359,6 +377,13 @@ async def reload_skills() -> List[Skill]: except Exception as e: print(f"Failed to reload Apple Services MCP client: {e}") + try: + from services.notebooklm_service import shutdown_notebooklm, initialize_notebooklm + await shutdown_notebooklm() + await initialize_notebooklm() + except Exception as e: + print(f"Failed to reload NotebookLM client: {e}") + # Refresh tool registry to pick up changes try: from services.tool_registry import refresh_registry diff --git a/backend/services/tool_registry.py b/backend/services/tool_registry.py index 9ca23b2..2da0955 100644 --- a/backend/services/tool_registry.py +++ b/backend/services/tool_registry.py @@ -27,6 +27,12 @@ "ios_widget": ["update_widget", "get_widget_state_tool"], "contacts_lookup": ["lookup_contact", "lookup_phone"], "orchestrator": ["spawn_worker", "check_worker", "list_workers", "cancel_worker", "wait_for_workers", "send_to_worker", "spawn_cc_worker"], + "notebooklm": [ + "nlm_list_notebooks", "nlm_create_notebook", "nlm_delete_notebook", + "nlm_add_source", "nlm_list_sources", "nlm_get_source_text", + "nlm_ask", "nlm_research", "nlm_generate_artifact", "nlm_wait_artifact", + "nlm_push_document", "nlm_push_file", + ], # "whatsapp_mcp" and "apple_services" tools are handled dynamically since they come from MCP } @@ -96,6 +102,7 @@ async def _get_skill_states(force_refresh: bool = False) -> Dict[str, bool]: "html_hosting": await is_skill_enabled("html_hosting"), "ios_widget": await is_skill_enabled("ios_widget"), "orchestrator": await is_skill_enabled("orchestrator"), + "notebooklm": await is_skill_enabled("notebooklm"), } _cache_timestamp = now @@ -338,6 +345,15 @@ def _get_orchestrator_tools(skill_states: Dict[str, bool]) -> List[BaseTool]: return ORCHESTRATOR_TOOLS +def _get_notebooklm_tools(skill_states: Dict[str, bool]) -> List[BaseTool]: + """Get NotebookLM tools if notebooklm skill is enabled.""" + if not skill_states.get("notebooklm"): + return [] + + from services.graph.tools import NOTEBOOKLM_TOOLS + return NOTEBOOKLM_TOOLS + + def _get_custom_mcp_tools() -> List[Any]: """Get tools from all running custom MCP servers.""" try: @@ -428,6 +444,9 @@ def add_tools(new_tools: List[Any]) -> None: # Orchestrator tools if enabled add_tools(_get_orchestrator_tools(skill_states)) + # NotebookLM tools if enabled + add_tools(_get_notebooklm_tools(skill_states)) + # Custom MCP self-service tools (always available) add_tools(_get_custom_mcp_self_service_tools()) @@ -546,6 +565,11 @@ def get_tool_descriptions(tools: List[Any]) -> str: from services.graph.tools import get_orchestrator_tools_description sections.append(get_orchestrator_tools_description()) + # NotebookLM tools section + if any(name.startswith("nlm_") for name in tool_names): + from services.graph.tools import get_notebooklm_tools_description + sections.append(get_notebooklm_tools_description()) + # Apple Reminders tools section (special guidance to avoid confusion with scheduled events) if any(name.startswith("reminders_") for name in tool_names): sections.append(_get_apple_reminders_description())