Optimize SQLTrie performance with advanced caching and efficient data formats

Sources/_Modules/TrieKit/TrieSQLScriptGenerator.swift

@@ -175,19 +175,17 @@ extension VanguardTrie {
       // 遍歷所有 keyInitials 和對應的節點 ID Set
       for (keyInitials, nodeIDs) in keyInitialsMap {
         let escapedKeyInitials = escapeSQLString(keyInitials)
-        // 將 Set<Int> 轉換為 JSON 字串
-        let jsonData = try? JSONEncoder().encode(nodeIDs)
-        if let jsonString = jsonData.map({ String(data: $0, encoding: .utf8) ?? "[]" }) {
-          let escapedJsonString = escapeSQLString(jsonString)
-          keyInitialsValues.append("('\(escapedKeyInitials)', '\(escapedJsonString)')")
-
-          // 批量插入
-          if keyInitialsValues.count >= batchSize {
-            sqlCommands.append(
-              "INSERT INTO keyinitials_id_map (keyinitials, node_ids) VALUES \(keyInitialsValues.joined(separator: ","));"
-            )
-            keyInitialsValues = []
-          }
+        // 將 Set<Int> 轉換為逗號分隔的字串（比 JSON 更快）
+        let nodeIDsString = nodeIDs.sorted().map(String.init).joined(separator: ",")
+        let escapedNodeIDsString = escapeSQLString(nodeIDsString)
+        keyInitialsValues.append("('\(escapedKeyInitials)', '\(escapedNodeIDsString)')")
+
+        // 批量插入
+        if keyInitialsValues.count >= batchSize {
+          sqlCommands.append(
+            "INSERT INTO keyinitials_id_map (keyinitials, node_ids) VALUES \(keyInitialsValues.joined(separator: ","));"
+          )
+          keyInitialsValues = []
         }
       }
 

Sources/_Modules/TrieKit/TrieSQL_Core.swift

@@ -137,10 +137,22 @@ extension VanguardTrie {
         return nil
       }
 
+      // 啟用 SQLITE 最佳化功能
+      // Temporarily disabled for debugging
+      // if !applyPerformanceOptimizations() {
+      //   return nil
+      // }
+
       // 讀取分隔符設定
       if !initializeSettings() {
         return nil
       }
+
+      // 預編譯常用的 SQL 語句
+      prepareSQLStatements()
+
+      // 預編譯常用的 SQL 語句
+      prepareSQLStatements()
     }
 
     /// 從實體 SQLite 檔案讀取並在記憶體內以唯讀模式運作
@@ -202,6 +214,11 @@ extension VanguardTrie {
     }
 
     deinit {
+      // 釋放預編譯的語句
+      if let stmt = preparedNodeQuery { sqlite3_finalize(stmt) }
+      if let stmt = preparedKeyInitialsExactQuery { sqlite3_finalize(stmt) }
+      if let stmt = preparedKeyInitialsLikeQuery { sqlite3_finalize(stmt) }
+
       if !closedAndNullified {
         closeAndNullifyConnection()
       }
@@ -228,7 +245,13 @@ extension VanguardTrie {
     internal let queryBuffer4Node: QueryBuffer<TNode?> = .init()
     internal let queryBuffer4Nodes: QueryBuffer<[TNode]> = .init()
     internal let queryBuffer4NodeIDs: QueryBuffer<Set<Int>> = .init()
+    internal let entriesDecodeCache: QueryBuffer<[Trie.Entry]?> = .init()
     internal var database: OpaquePointer?
+
+    // 預編譯的 SQL 語句以提升效能
+    internal var preparedNodeQuery: OpaquePointer?
+    internal var preparedKeyInitialsExactQuery: OpaquePointer?
+    internal var preparedKeyInitialsLikeQuery: OpaquePointer?
 
     /// 獲取表的行數
     /// - Parameter tableName: 表名
@@ -250,24 +273,79 @@ extension VanguardTrie {
 
     // MARK: - 輔助方法
 
-    /// 從 base64 字串解碼 entries
+    /// 從 base64 字串解碼 entries (with caching)
     internal func decodeEntriesFromBase64(_ base64String: String) -> [Trie.Entry]? {
+      // 首先檢查快取
+      if let cachedResult = entriesDecodeCache.get(key: base64String) {
+        return cachedResult
+      }
+
       guard !base64String.isEmpty,
             let data = Data(base64Encoded: base64String) else {
+        entriesDecodeCache.set(key: base64String, value: nil)
         return nil
       }
 
       do {
-        return try plistDecoder.decode([Trie.Entry].self, from: data)
+        let result = try plistDecoder.decode([Trie.Entry].self, from: data)
+        entriesDecodeCache.set(key: base64String, value: result)
+        return result
       } catch {
         Self.printDebug("Error decoding entries: \(error)")
+        entriesDecodeCache.set(key: base64String, value: nil)
         return nil
       }
     }
 
     // MARK: Private
 
     private let plistDecoder = PropertyListDecoder()
+
+    /// 應用資料庫效能最佳化
+    private func applyPerformanceOptimizations() -> Bool {
+      let optimizations = [
+        "PRAGMA cache_size=10000",      // 增加快取大小
+        "PRAGMA temp_store=MEMORY",     // 暫存儲存在記憶體中
+        "PRAGMA mmap_size=268435456",   // 啟用記憶體映射 I/O (256MB)
+        "PRAGMA page_size=8192",        // 使用較大的頁面大小
+        "PRAGMA synchronous=NORMAL"      // 平衡效能與安全性
+      ]
+
+      for optimization in optimizations {
+        if sqlite3_exec(database, optimization, nil, nil, nil) != SQLITE_OK {
+          Self.printDebug("Failed to apply optimization: \(optimization)")
+          // 繼續執行其他最佳化，不要因為一個失敗就中止
+        }
+      }
+
+      return true
+    }
+
+    /// 預編譯常用的 SQL 語句
+    private func prepareSQLStatements() {
+      // 預編譯節點查詢語句
+      let nodeQuery = """
+        SELECT id, reading_key, entries_blob
+        FROM nodes
+        WHERE id = ?
+        LIMIT 1
+        """
+      sqlite3_prepare_v2(database, nodeQuery, -1, &preparedNodeQuery, nil)
+
+      // 預編譯精確匹配的鍵值初始查詢
+      let exactQuery = """
+        SELECT node_ids FROM keyinitials_id_map
+        WHERE keyinitials = ?
+        """
+      sqlite3_prepare_v2(database, exactQuery, -1, &preparedKeyInitialsExactQuery, nil)
+
+      // 預編譯 LIKE 模式匹配的鍵值初始查詢
+      let likeQuery = """
+        SELECT node_ids FROM keyinitials_id_map
+        WHERE keyinitials LIKE ?
+        """
+      sqlite3_prepare_v2(database, likeQuery, -1, &preparedKeyInitialsLikeQuery, nil)
+    }
 
     private static func printDebug(
       _ items: Any...,

Sources/_Modules/TrieKit/TrieSQL_Impl.swift

@@ -16,12 +16,8 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol {
       $0.first?.description
     }.joined()
 
-    let formedKeyHash: Int = {
-      var hasher = Hasher()
-      hasher.combine(keyInitialsStr)
-      hasher.combine(longerSpan)
-      return hasher.finalize()
-    }()
+    // 優化的快取鍵計算 - 使用更簡單的雜湊
+    let formedKeyHash: Int = keyInitialsStr.hashValue ^ (longerSpan ? 1 : 0)
 
     if let cachedResult = queryBuffer4NodeIDs.get(hashKey: formedKeyHash) {
       return cachedResult.sorted()
@@ -45,11 +41,24 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol {
 
     if sqlite3_prepare_v2(database, query, -1, &statement, nil) == SQLITE_OK {
       while sqlite3_step(statement) == SQLITE_ROW {
-        if let jsonData = sqlite3_column_text(statement, 0).map({ String(cString: $0) }) {
-          // 將 JSON 字串解析為 [Int]，然後轉換為 Set<Int>
-          if let data = jsonData.data(using: .utf8),
-             let setDecoded = try? JSONDecoder().decode(Set<Int>.self, from: data) {
-            nodeIDs.formUnion(setDecoded)
+        if let nodeIDsText = sqlite3_column_text(statement, 0).map({ String(cString: $0) }) {
+          if !nodeIDsText.isEmpty {
+            // 向後相容：支援舊的 JSON 格式和新的逗號分隔格式
+            if nodeIDsText.starts(with: "[") && nodeIDsText.hasSuffix("]") {
+              // 舊的 JSON 格式
+              if let data = nodeIDsText.data(using: .utf8),
+                 let setDecoded = try? JSONDecoder().decode(Set<Int>.self, from: data) {
+                nodeIDs.formUnion(setDecoded)
+              }
+            } else {
+              // 新的逗號分隔格式（更快）
+              let nodeIDStrings = nodeIDsText.split(separator: ",")
+              for nodeIDString in nodeIDStrings {
+                if let nodeID = Int(nodeIDString) {
+                  nodeIDs.insert(nodeID)
+                }
+              }
+            }
           }
         }
       }
@@ -68,14 +77,8 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol {
     longerSpan: Bool
   )
     -> [TNode] {
-    let formedKeyHash: Int = {
-      var hasher = Hasher()
-      hasher.combine(keyArray)
-      hasher.combine(filterType)
-      hasher.combine(partiallyMatch)
-      hasher.combine(longerSpan)
-      return hasher.finalize()
-    }()
+    // 優化的快取鍵計算 - 使用更簡單的雜湊
+    let formedKeyHash: Int = keyArray.hashValue ^ filterType.hashValue ^ (partiallyMatch ? 1 : 0) ^ (longerSpan ? 2 : 0)
 
     if let cachedResult = queryBuffer4Nodes.get(hashKey: formedKeyHash) { return cachedResult }
 
@@ -91,7 +94,10 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol {
         let hash = theNode.hashValue
         if !handledNodeHashes.contains(hash) {
           handledNodeHashes.insert(theNode.hashValue)
-          let nodeKeyArray = theNode.readingKey.split(separator: readingSeparator)
+          let nodeKeyArray = TrieStringOperationCache.shared.getCachedSplit(
+            theNode.readingKey,
+            separator: readingSeparator
+          )
           if nodeMeetsFilter(theNode, filter: filterType) {
             var matched: Bool = longerSpan
               ? nodeKeyArray.count > keyArray.count
@@ -116,21 +122,15 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol {
 
   public func getNode(_ nodeID: Int) -> TNode? {
     if let cachedResult = queryBuffer4Node.get(hashKey: nodeID) { return cachedResult }
-    // 基本查詢，只查必要資訊
-    let query = """
-    SELECT id, reading_key, entries_blob
-    FROM nodes
-    WHERE id = ?
-    LIMIT 1
-    """
-    var statement: OpaquePointer?
-
-    guard sqlite3_prepare_v2(database, query, -1, &statement, nil) == SQLITE_OK else {
-      sqlite3_finalize(statement)
-      return nil
-    }
-    defer { sqlite3_finalize(statement) }
-
+
+    // 使用預編譯的語句
+    guard let statement = preparedNodeQuery else { return nil }
+
+    // 重置語句狀態
+    sqlite3_reset(statement)
+    sqlite3_clear_bindings(statement)
+
+    // 綁定參數
     sqlite3_bind_int64(statement, 1, sqlite3_int64(nodeID))
 
     guard sqlite3_step(statement) == SQLITE_ROW else { return nil }