From 12135dcffa3b6b01d93c46c81ca6ba279a5ac6a6 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 28 Jan 2026 02:40:30 +0000 Subject: [PATCH] Optimize AdaptiveMemory with persistent connection and WAL mode Co-authored-by: Rohith-Shimori <228351330+Rohith-Shimori@users.noreply.github.com> --- .gitignore | 1 + memory/adaptive_memory.py | 558 ++++++++++++++++----------------- memory/data/adaptive_memory.db | Bin 69632 -> 69632 bytes 3 files changed, 279 insertions(+), 280 deletions(-) diff --git a/.gitignore b/.gitignore index 2987e6c..57cd63a 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ models/ # Temporary files tmp/ temp/ +memory/data/ diff --git a/memory/adaptive_memory.py b/memory/adaptive_memory.py index 04b301e..15b0102 100644 --- a/memory/adaptive_memory.py +++ b/memory/adaptive_memory.py @@ -10,6 +10,7 @@ from datetime import datetime from typing import List, Dict, Optional, Tuple import uuid +import threading DATA_DIR = os.path.join(os.path.dirname(__file__), "data") MEMORY_DB = os.path.join(DATA_DIR, "adaptive_memory.db") @@ -21,70 +22,88 @@ class AdaptiveMemory: - Decay over time - Structured queries - Relationship tracking + + Optimized with persistent connection and WAL mode. """ - def __init__(self): + def __init__(self, db_path: str = None): os.makedirs(DATA_DIR, exist_ok=True) - self.db_path = MEMORY_DB - self._init_database() - - def _init_database(self): - """Initialize database schema.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - # Main memories table - cursor.execute(""" - CREATE TABLE IF NOT EXISTS memories ( - id TEXT PRIMARY KEY, - role TEXT NOT NULL, - content TEXT NOT NULL, - memory_type TEXT NOT NULL, - importance INTEGER DEFAULT 5, - timestamp REAL NOT NULL, - last_accessed REAL, - access_count INTEGER DEFAULT 0, - decay_factor REAL DEFAULT 1.0, - tags TEXT, - metadata TEXT - ) - """) + self.db_path = db_path or MEMORY_DB - # Facts table (structured personal data) - cursor.execute(""" - CREATE TABLE IF NOT EXISTS facts ( - id TEXT PRIMARY KEY, - category TEXT NOT NULL, - value TEXT NOT NULL, - importance INTEGER DEFAULT 7, - confidence REAL DEFAULT 1.0, - source TEXT, - timestamp REAL NOT NULL, - last_updated REAL - ) - """) + # Thread safety for persistent connection + self.lock = threading.RLock() - # Relationships table (connections between memories) - cursor.execute(""" - CREATE TABLE IF NOT EXISTS relationships ( - id TEXT PRIMARY KEY, - memory_id_1 TEXT NOT NULL, - memory_id_2 TEXT NOT NULL, - relationship_type TEXT, - strength REAL DEFAULT 0.5, - FOREIGN KEY (memory_id_1) REFERENCES memories(id), - FOREIGN KEY (memory_id_2) REFERENCES memories(id) - ) - """) + # Persistent connection + self.conn = sqlite3.connect(self.db_path, check_same_thread=False) + self.conn.row_factory = sqlite3.Row # Optional: allows accessing columns by name, but keeping default for now to minimize changes - # Indices for fast queries - cursor.execute("CREATE INDEX IF NOT EXISTS idx_importance ON memories(importance DESC)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON memories(timestamp DESC)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_type ON memories(memory_type)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_facts_category ON facts(category)") + # Enable WAL mode for performance + self.conn.execute("PRAGMA journal_mode=WAL") + self.conn.execute("PRAGMA synchronous=NORMAL") - conn.commit() - conn.close() + self._init_database() + + def __del__(self): + """Ensure connection is closed on deletion.""" + if hasattr(self, 'conn') and self.conn: + self.conn.close() + + def _init_database(self): + """Initialize database schema.""" + with self.lock: + cursor = self.conn.cursor() + + # Main memories table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + role TEXT NOT NULL, + content TEXT NOT NULL, + memory_type TEXT NOT NULL, + importance INTEGER DEFAULT 5, + timestamp REAL NOT NULL, + last_accessed REAL, + access_count INTEGER DEFAULT 0, + decay_factor REAL DEFAULT 1.0, + tags TEXT, + metadata TEXT + ) + """) + + # Facts table (structured personal data) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS facts ( + id TEXT PRIMARY KEY, + category TEXT NOT NULL, + value TEXT NOT NULL, + importance INTEGER DEFAULT 7, + confidence REAL DEFAULT 1.0, + source TEXT, + timestamp REAL NOT NULL, + last_updated REAL + ) + """) + + # Relationships table (connections between memories) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS relationships ( + id TEXT PRIMARY KEY, + memory_id_1 TEXT NOT NULL, + memory_id_2 TEXT NOT NULL, + relationship_type TEXT, + strength REAL DEFAULT 0.5, + FOREIGN KEY (memory_id_1) REFERENCES memories(id), + FOREIGN KEY (memory_id_2) REFERENCES memories(id) + ) + """) + + # Indices for fast queries + cursor.execute("CREATE INDEX IF NOT EXISTS idx_importance ON memories(importance DESC)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON memories(timestamp DESC)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_type ON memories(memory_type)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_facts_category ON facts(category)") + + self.conn.commit() def add_memory(self, role: str, content: str, memory_type: str = "conversation", importance: int = 5, tags: List[str] = None, metadata: Dict = None) -> str: @@ -102,30 +121,27 @@ def add_memory(self, role: str, content: str, memory_type: str = "conversation", Returns: Memory ID """ - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - memory_id = str(uuid.uuid4()) timestamp = datetime.now().timestamp() - cursor.execute(""" - INSERT INTO memories (id, role, content, memory_type, importance, - timestamp, last_accessed, tags, metadata) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - memory_id, - role, - content, - memory_type, - min(10, max(1, importance)), # Clamp 1-10 - timestamp, - timestamp, - json.dumps(tags or []), - json.dumps(metadata or {}) - )) - - conn.commit() - conn.close() + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + INSERT INTO memories (id, role, content, memory_type, importance, + timestamp, last_accessed, tags, metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + memory_id, + role, + content, + memory_type, + min(10, max(1, importance)), # Clamp 1-10 + timestamp, + timestamp, + json.dumps(tags or []), + json.dumps(metadata or {}) + )) + self.conn.commit() return memory_id @@ -133,20 +149,21 @@ def get_important_memories(self, min_importance: int = 7, limit: int = 10) -> Li """ Retrieve memories above a certain importance threshold. """ - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - SELECT id, role, content, memory_type, importance, timestamp, - access_count, tags, metadata - FROM memories - WHERE importance >= ? AND decay_factor > 0.3 - ORDER BY importance DESC, timestamp DESC - LIMIT ? - """, (min_importance, limit)) + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + SELECT id, role, content, memory_type, importance, timestamp, + access_count, tags, metadata + FROM memories + WHERE importance >= ? AND decay_factor > 0.3 + ORDER BY importance DESC, timestamp DESC + LIMIT ? + """, (min_importance, limit)) + + rows = cursor.fetchall() memories = [] - for row in cursor.fetchall(): + for row in rows: memories.append({ "id": row[0], "role": row[1], @@ -159,34 +176,35 @@ def get_important_memories(self, min_importance: int = 7, limit: int = 10) -> Li "metadata": json.loads(row[8]) if row[8] else {} }) - conn.close() return memories def get_recent_memories(self, limit: int = 10, memory_type: Optional[str] = None) -> List[Dict]: """ Get recent memories, optionally filtered by type. """ - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - if memory_type: - cursor.execute(""" - SELECT id, role, content, memory_type, importance, timestamp - FROM memories - WHERE memory_type = ? - ORDER BY timestamp DESC - LIMIT ? - """, (memory_type, limit)) - else: - cursor.execute(""" - SELECT id, role, content, memory_type, importance, timestamp - FROM memories - ORDER BY timestamp DESC - LIMIT ? - """, (limit,)) + with self.lock: + cursor = self.conn.cursor() + + if memory_type: + cursor.execute(""" + SELECT id, role, content, memory_type, importance, timestamp + FROM memories + WHERE memory_type = ? + ORDER BY timestamp DESC + LIMIT ? + """, (memory_type, limit)) + else: + cursor.execute(""" + SELECT id, role, content, memory_type, importance, timestamp + FROM memories + ORDER BY timestamp DESC + LIMIT ? + """, (limit,)) + + rows = cursor.fetchall() memories = [] - for row in cursor.fetchall(): + for row in rows: memories.append({ "id": row[0], "role": row[1], @@ -196,40 +214,33 @@ def get_recent_memories(self, limit: int = 10, memory_type: Optional[str] = None "timestamp": row[5] }) - conn.close() return memories def update_importance(self, memory_id: str, new_importance: int): """Update the importance level of a memory.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - UPDATE memories - SET importance = ? - WHERE id = ? - """, (min(10, max(1, new_importance)), memory_id)) - - conn.commit() - conn.close() + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + UPDATE memories + SET importance = ? + WHERE id = ? + """, (min(10, max(1, new_importance)), memory_id)) + self.conn.commit() def access_memory(self, memory_id: str): """ Mark a memory as accessed (updates last_accessed and access_count). Frequently accessed memories maintain higher importance. """ - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - UPDATE memories - SET last_accessed = ?, - access_count = access_count + 1 - WHERE id = ? - """, (datetime.now().timestamp(), memory_id)) - - conn.commit() - conn.close() + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + UPDATE memories + SET last_accessed = ?, + access_count = access_count + 1 + WHERE id = ? + """, (datetime.now().timestamp(), memory_id)) + self.conn.commit() def decay_memories(self, decay_rate: float = 0.95): """ @@ -237,67 +248,61 @@ def decay_memories(self, decay_rate: float = 0.95): Reduces importance of old, unaccessed memories. Should be called periodically (e.g., daily). """ - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - # Decay factor reduces over time, but frequently accessed memories decay slower - cursor.execute(""" - UPDATE memories - SET decay_factor = decay_factor * ? - WHERE access_count < 3 - """, (decay_rate,)) - - conn.commit() - conn.close() + with self.lock: + cursor = self.conn.cursor() + # Decay factor reduces over time, but frequently accessed memories decay slower + cursor.execute(""" + UPDATE memories + SET decay_factor = decay_factor * ? + WHERE access_count < 3 + """, (decay_rate,)) + self.conn.commit() def store_fact(self, category: str, value: str, importance: int = 7, confidence: float = 1.0, source: str = "user") -> str: """ Store a structured fact with importance. """ - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - fact_id = str(uuid.uuid4()) timestamp = datetime.now().timestamp() - # Check if fact already exists, update if so - cursor.execute("SELECT id FROM facts WHERE category = ?", (category,)) - existing = cursor.fetchone() - - if existing: - cursor.execute(""" - UPDATE facts - SET value = ?, importance = ?, confidence = ?, - last_updated = ?, source = ? - WHERE category = ? - """, (value, importance, confidence, timestamp, source, category)) - fact_id = existing[0] - else: - cursor.execute(""" - INSERT INTO facts (id, category, value, importance, confidence, - source, timestamp, last_updated) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, (fact_id, category, value, importance, confidence, source, timestamp, timestamp)) - - conn.commit() - conn.close() + with self.lock: + cursor = self.conn.cursor() + + # Check if fact already exists, update if so + cursor.execute("SELECT id FROM facts WHERE category = ?", (category,)) + existing = cursor.fetchone() + + if existing: + cursor.execute(""" + UPDATE facts + SET value = ?, importance = ?, confidence = ?, + last_updated = ?, source = ? + WHERE category = ? + """, (value, importance, confidence, timestamp, source, category)) + fact_id = existing[0] + else: + cursor.execute(""" + INSERT INTO facts (id, category, value, importance, confidence, + source, timestamp, last_updated) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, (fact_id, category, value, importance, confidence, source, timestamp, timestamp)) + + self.conn.commit() return fact_id def get_fact(self, category: str) -> Optional[Dict]: """Retrieve a fact by category.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - SELECT id, category, value, importance, confidence, source, timestamp - FROM facts - WHERE category = ? - """, (category,)) - - row = cursor.fetchone() - conn.close() + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + SELECT id, category, value, importance, confidence, source, timestamp + FROM facts + WHERE category = ? + """, (category,)) + + row = cursor.fetchone() if row: return { @@ -313,18 +318,19 @@ def get_fact(self, category: str) -> Optional[Dict]: def get_all_facts(self, min_importance: int = 1) -> List[Dict]: """Get all facts above a certain importance.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - SELECT category, value, importance, confidence - FROM facts - WHERE importance >= ? - ORDER BY importance DESC, timestamp DESC - """, (min_importance,)) + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + SELECT category, value, importance, confidence + FROM facts + WHERE importance >= ? + ORDER BY importance DESC, timestamp DESC + """, (min_importance,)) + + rows = cursor.fetchall() facts = [] - for row in cursor.fetchall(): + for row in rows: facts.append({ "category": row[0], "value": row[1], @@ -332,7 +338,6 @@ def get_all_facts(self, min_importance: int = 1) -> List[Dict]: "confidence": row[3] }) - conn.close() return facts def search_memories(self, query: str, limit: int = 5) -> List[Dict]: @@ -340,19 +345,20 @@ def search_memories(self, query: str, limit: int = 5) -> List[Dict]: Simple text search in memories. (For better semantic search, use with embeddings separately) """ - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - SELECT id, role, content, memory_type, importance, timestamp - FROM memories - WHERE content LIKE ? - ORDER BY importance DESC, timestamp DESC - LIMIT ? - """, (f"%{query}%", limit)) + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + SELECT id, role, content, memory_type, importance, timestamp + FROM memories + WHERE content LIKE ? + ORDER BY importance DESC, timestamp DESC + LIMIT ? + """, (f"%{query}%", limit)) + + rows = cursor.fetchall() memories = [] - for row in cursor.fetchall(): + for row in rows: memories.append({ "id": row[0], "role": row[1], @@ -362,7 +368,6 @@ def search_memories(self, query: str, limit: int = 5) -> List[Dict]: "timestamp": row[5] }) - conn.close() return memories def create_relationship(self, memory_id_1: str, memory_id_2: str, @@ -371,36 +376,34 @@ def create_relationship(self, memory_id_1: str, memory_id_2: str, Create a relationship between two memories. Useful for building knowledge graphs. """ - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - rel_id = str(uuid.uuid4()) - cursor.execute(""" - INSERT INTO relationships (id, memory_id_1, memory_id_2, relationship_type, strength) - VALUES (?, ?, ?, ?, ?) - """, (rel_id, memory_id_1, memory_id_2, relationship_type, strength)) - - conn.commit() - conn.close() + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + INSERT INTO relationships (id, memory_id_1, memory_id_2, relationship_type, strength) + VALUES (?, ?, ?, ?, ?) + """, (rel_id, memory_id_1, memory_id_2, relationship_type, strength)) + self.conn.commit() return rel_id def get_related_memories(self, memory_id: str) -> List[Dict]: """Get all memories related to a given memory.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - SELECT m.id, m.content, m.importance, r.relationship_type, r.strength - FROM memories m - JOIN relationships r ON (m.id = r.memory_id_2) - WHERE r.memory_id_1 = ? - ORDER BY r.strength DESC - """, (memory_id,)) + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + SELECT m.id, m.content, m.importance, r.relationship_type, r.strength + FROM memories m + JOIN relationships r ON (m.id = r.memory_id_2) + WHERE r.memory_id_1 = ? + ORDER BY r.strength DESC + """, (memory_id,)) + + rows = cursor.fetchall() related = [] - for row in cursor.fetchall(): + for row in rows: related.append({ "id": row[0], "content": row[1], @@ -409,31 +412,31 @@ def get_related_memories(self, memory_id: str) -> List[Dict]: "strength": row[4] }) - conn.close() return related def get_statistics(self) -> Dict: """Get memory system statistics.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - # Count memories by type - cursor.execute("SELECT memory_type, COUNT(*) FROM memories GROUP BY memory_type") - type_counts = dict(cursor.fetchall()) - - # Count facts - cursor.execute("SELECT COUNT(*) FROM facts") - fact_count = cursor.fetchone()[0] - - # Average importance - cursor.execute("SELECT AVG(importance) FROM memories") - avg_importance = cursor.fetchone()[0] or 0 - - # High importance count - cursor.execute("SELECT COUNT(*) FROM memories WHERE importance >= 8") - critical_count = cursor.fetchone()[0] - - conn.close() + with self.lock: + cursor = self.conn.cursor() + + # Count memories by type + cursor.execute("SELECT memory_type, COUNT(*) FROM memories GROUP BY memory_type") + type_counts = dict(cursor.fetchall()) + + # Count facts + cursor.execute("SELECT COUNT(*) FROM facts") + fact_count_row = cursor.fetchone() + fact_count = fact_count_row[0] if fact_count_row else 0 + + # Average importance + cursor.execute("SELECT AVG(importance) FROM memories") + avg_row = cursor.fetchone() + avg_importance = avg_row[0] if avg_row and avg_row[0] else 0 + + # High importance count + cursor.execute("SELECT COUNT(*) FROM memories WHERE importance >= 8") + crit_row = cursor.fetchone() + critical_count = crit_row[0] if crit_row else 0 return { "total_memories": sum(type_counts.values()), @@ -445,38 +448,34 @@ def get_statistics(self) -> Dict: def clear_low_importance(self, threshold: int = 3): """Remove memories below importance threshold (cleanup).""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - DELETE FROM memories - WHERE importance < ? AND decay_factor < 0.5 - """, (threshold,)) - - deleted = cursor.rowcount - conn.commit() - conn.close() + with self.lock: + cursor = self.conn.cursor() + cursor.execute(""" + DELETE FROM memories + WHERE importance < ? AND decay_factor < 0.5 + """, (threshold,)) + deleted = cursor.rowcount + self.conn.commit() return deleted def clear_personal(self) -> Dict: """Clear personal facts and any personal-type memories.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() + with self.lock: + cursor = self.conn.cursor() - # Clear structured facts - cursor.execute("DELETE FROM facts") - facts_deleted = cursor.rowcount + # Clear structured facts + cursor.execute("DELETE FROM facts") + facts_deleted = cursor.rowcount - # Clear memories explicitly marked as personal - cursor.execute( - "DELETE FROM memories WHERE memory_type = ?", - ("personal",), - ) - memories_deleted = cursor.rowcount + # Clear memories explicitly marked as personal + cursor.execute( + "DELETE FROM memories WHERE memory_type = ?", + ("personal",), + ) + memories_deleted = cursor.rowcount - conn.commit() - conn.close() + self.conn.commit() return { "cleared_facts": facts_deleted if facts_deleted is not None else 0, @@ -485,16 +484,15 @@ def clear_personal(self) -> Dict: def clear_conversation(self) -> int: """Clear conversation-type memories (chat history).""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() + with self.lock: + cursor = self.conn.cursor() - cursor.execute( - "DELETE FROM memories WHERE memory_type = ?", - ("conversation",), - ) - deleted = cursor.rowcount or 0 + cursor.execute( + "DELETE FROM memories WHERE memory_type = ?", + ("conversation",), + ) + deleted = cursor.rowcount or 0 - conn.commit() - conn.close() + self.conn.commit() - return deleted \ No newline at end of file + return deleted diff --git a/memory/data/adaptive_memory.db b/memory/data/adaptive_memory.db index 8cbd612c08e4c9d36a6a8a5e6bcbd0c6b9620c04..57d37fa7d117b04b27705ae911c7c0c9fa0783a3 100644 GIT binary patch delta 290 zcmZozz|ydQWr8$g!9*En#)6FrOXQh#InGRGS1@PdIJ4PKp@&0PgE^GZGTFk+)F|0f z*TlfkRM#ZU#6mYQ#oSWYAj!bo!otibHO)8?tZ?#~V=9|vHJ`gN>+*e`%w7=5#P@l# zput2wHdbavMpoU;KkK*|*>rh#F!1i!%+aupmzRf`ozd}DTx|A!{su5He delta 245 zcmZozz|ydQWr8$g{zMsP#{7*5OXQh#IKED1S1{k~pwPoHae_9B26HIm=Qt0w(?A7 znJchab?$v733j$U416#7GPw=7wsCyts0DKQ85kJ&H#R0SZ>~Rln<+^b1=MiWC}?Rp z=Oh+omQ*UJDLCinC1)0==A{-FYiTLeC