From 86f8e7e07d56d5ac8bff31a78e73deead9fbe177 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Sep 2025 16:58:18 +0000 Subject: [PATCH 1/3] Initial plan From 2c27e0c21b8f7eea42406efd6abf8e3d0fbcc685 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Sep 2025 17:11:21 +0000 Subject: [PATCH 2/3] Implement major SQLTrie performance optimizations (with temporary test issues) Co-authored-by: ShikiSuen <3164826+ShikiSuen@users.noreply.github.com> --- .../TrieKit/TrieSQLScriptGenerator.swift | 24 ++-- Sources/_Modules/TrieKit/TrieSQL_Core.swift | 81 ++++++++++++- Sources/_Modules/TrieKit/TrieSQL_Impl.swift | 107 +++++++++--------- 3 files changed, 144 insertions(+), 68 deletions(-) diff --git a/Sources/_Modules/TrieKit/TrieSQLScriptGenerator.swift b/Sources/_Modules/TrieKit/TrieSQLScriptGenerator.swift index 465524ce..989591d7 100644 --- a/Sources/_Modules/TrieKit/TrieSQLScriptGenerator.swift +++ b/Sources/_Modules/TrieKit/TrieSQLScriptGenerator.swift @@ -175,19 +175,17 @@ extension VanguardTrie { // 遍歷所有 keyInitials 和對應的節點 ID Set for (keyInitials, nodeIDs) in keyInitialsMap { let escapedKeyInitials = escapeSQLString(keyInitials) - // 將 Set 轉換為 JSON 字串 - let jsonData = try? JSONEncoder().encode(nodeIDs) - if let jsonString = jsonData.map({ String(data: $0, encoding: .utf8) ?? "[]" }) { - let escapedJsonString = escapeSQLString(jsonString) - keyInitialsValues.append("('\(escapedKeyInitials)', '\(escapedJsonString)')") - - // 批量插入 - if keyInitialsValues.count >= batchSize { - sqlCommands.append( - "INSERT INTO keyinitials_id_map (keyinitials, node_ids) VALUES \(keyInitialsValues.joined(separator: ","));" - ) - keyInitialsValues = [] - } + // 將 Set 轉換為逗號分隔的字串（比 JSON 更快） + let nodeIDsString = nodeIDs.sorted().map(String.init).joined(separator: ",") + let escapedNodeIDsString = escapeSQLString(nodeIDsString) + keyInitialsValues.append("('\(escapedKeyInitials)', '\(escapedNodeIDsString)')") + + // 批量插入 + if keyInitialsValues.count >= batchSize { + sqlCommands.append( + "INSERT INTO keyinitials_id_map (keyinitials, node_ids) VALUES \(keyInitialsValues.joined(separator: ","));" + ) + keyInitialsValues = [] } } diff --git a/Sources/_Modules/TrieKit/TrieSQL_Core.swift b/Sources/_Modules/TrieKit/TrieSQL_Core.swift index 4a511c83..543aa381 100644 --- a/Sources/_Modules/TrieKit/TrieSQL_Core.swift +++ b/Sources/_Modules/TrieKit/TrieSQL_Core.swift @@ -137,10 +137,21 @@ extension VanguardTrie { return nil } + // 啟用 SQLITE 最佳化功能 + if !applyPerformanceOptimizations() { + return nil + } + // 讀取分隔符設定 if !initializeSettings() { return nil } + + // 預編譯常用的 SQL 語句 + prepareSQLStatements() + + // 預編譯常用的 SQL 語句 + prepareSQLStatements() } /// 從實體 SQLite 檔案讀取並在記憶體內以唯讀模式運作 @@ -202,6 +213,11 @@ extension VanguardTrie { } deinit { + // 釋放預編譯的語句 + if let stmt = preparedNodeQuery { sqlite3_finalize(stmt) } + if let stmt = preparedKeyInitialsExactQuery { sqlite3_finalize(stmt) } + if let stmt = preparedKeyInitialsLikeQuery { sqlite3_finalize(stmt) } + if !closedAndNullified { closeAndNullifyConnection() } @@ -228,7 +244,13 @@ extension VanguardTrie { internal let queryBuffer4Node: QueryBuffer = .init() internal let queryBuffer4Nodes: QueryBuffer<[TNode]> = .init() internal let queryBuffer4NodeIDs: QueryBuffer> = .init() + internal let entriesDecodeCache: QueryBuffer<[Trie.Entry]?> = .init() internal var database: OpaquePointer? + + // 預編譯的 SQL 語句以提升效能 + internal var preparedNodeQuery: OpaquePointer? + internal var preparedKeyInitialsExactQuery: OpaquePointer? + internal var preparedKeyInitialsLikeQuery: OpaquePointer? /// 獲取表的行數 /// - Parameter tableName: 表名 @@ -250,17 +272,26 @@ extension VanguardTrie { // MARK: - 輔助方法 - /// 從 base64 字串解碼 entries + /// 從 base64 字串解碼 entries (with caching) internal func decodeEntriesFromBase64(_ base64String: String) -> [Trie.Entry]? { + // 首先檢查快取 + if let cachedResult = entriesDecodeCache.get(key: base64String) { + return cachedResult + } + guard !base64String.isEmpty, let data = Data(base64Encoded: base64String) else { + entriesDecodeCache.set(key: base64String, value: nil) return nil } do { - return try plistDecoder.decode([Trie.Entry].self, from: data) + let result = try plistDecoder.decode([Trie.Entry].self, from: data) + entriesDecodeCache.set(key: base64String, value: result) + return result } catch { Self.printDebug("Error decoding entries: \(error)") + entriesDecodeCache.set(key: base64String, value: nil) return nil } } @@ -268,6 +299,52 @@ extension VanguardTrie { // MARK: Private private let plistDecoder = PropertyListDecoder() + + /// 應用資料庫效能最佳化 + private func applyPerformanceOptimizations() -> Bool { + let optimizations = [ + "PRAGMA cache_size=10000", // 增加快取大小 + "PRAGMA temp_store=MEMORY", // 暫存儲存在記憶體中 + "PRAGMA mmap_size=268435456", // 啟用記憶體映射 I/O (256MB) + "PRAGMA page_size=8192", // 使用較大的頁面大小 + "PRAGMA synchronous=NORMAL" // 平衡效能與安全性 + ] + + for optimization in optimizations { + if sqlite3_exec(database, optimization, nil, nil, nil) != SQLITE_OK { + Self.printDebug("Failed to apply optimization: \(optimization)") + // 繼續執行其他最佳化，不要因為一個失敗就中止 + } + } + + return true + } + + /// 預編譯常用的 SQL 語句 + private func prepareSQLStatements() { + // 預編譯節點查詢語句 + let nodeQuery = """ + SELECT id, reading_key, entries_blob + FROM nodes + WHERE id = ? + LIMIT 1 + """ + sqlite3_prepare_v2(database, nodeQuery, -1, &preparedNodeQuery, nil) + + // 預編譯精確匹配的鍵值初始查詢 + let exactQuery = """ + SELECT node_ids FROM keyinitials_id_map + WHERE keyinitials = ? + """ + sqlite3_prepare_v2(database, exactQuery, -1, &preparedKeyInitialsExactQuery, nil) + + // 預編譯 LIKE 模式匹配的鍵值初始查詢 + let likeQuery = """ + SELECT node_ids FROM keyinitials_id_map + WHERE keyinitials LIKE ? + """ + sqlite3_prepare_v2(database, likeQuery, -1, &preparedKeyInitialsLikeQuery, nil) + } private static func printDebug( _ items: Any..., diff --git a/Sources/_Modules/TrieKit/TrieSQL_Impl.swift b/Sources/_Modules/TrieKit/TrieSQL_Impl.swift index 21351b82..c011847e 100644 --- a/Sources/_Modules/TrieKit/TrieSQL_Impl.swift +++ b/Sources/_Modules/TrieKit/TrieSQL_Impl.swift @@ -12,50 +12,60 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol { /// 根據 keychain 字串查詢節點 ID public func getNodeIDsForKeyArray(_ keyArray: [String], longerSpan: Bool) -> [Int] { guard !keyArray.isEmpty, keyArray.allSatisfy({ !$0.isEmpty }) else { return [] } - let keyInitialsStr = keyArray.compactMap { - $0.first?.description + let keyInitialsStr = keyArray.compactMap { keyStr in + TrieStringPool.shared.internKey(TrieStringOperationCache.shared.getCachedFirstChar(keyStr)) }.joined() - let formedKeyHash: Int = { - var hasher = Hasher() - hasher.combine(keyInitialsStr) - hasher.combine(longerSpan) - return hasher.finalize() - }() + // 優化的快取鍵計算 - 使用更簡單的雜湊 + let formedKeyHash: Int = keyInitialsStr.hashValue ^ (longerSpan ? 1 : 0) if let cachedResult = queryBuffer4NodeIDs.get(hashKey: formedKeyHash) { return cachedResult.sorted() } var nodeIDs = Set() - let escapedKeyInitials = keyInitialsStr.replacingOccurrences(of: "'", with: "''") - - let query = switch longerSpan { - case false: """ - SELECT node_ids FROM keyinitials_id_map - WHERE keyinitials = '\(escapedKeyInitials)' - """ - case true: """ - SELECT node_ids FROM keyinitials_id_map - WHERE keyinitials LIKE '\(escapedKeyInitials)%' - """ + + // 使用預編譯的語句以提升效能 + let statement = longerSpan ? preparedKeyInitialsLikeQuery : preparedKeyInitialsExactQuery + guard let stmt = statement else { + return nodeIDs.sorted() + } + + // 重置語句狀態 + sqlite3_reset(stmt) + sqlite3_clear_bindings(stmt) + + // 綁定參數 + if longerSpan { + let pattern = keyInitialsStr + "%" + sqlite3_bind_text(stmt, 1, pattern, -1, nil) + } else { + sqlite3_bind_text(stmt, 1, keyInitialsStr, -1, nil) } - var statement: OpaquePointer? - - if sqlite3_prepare_v2(database, query, -1, &statement, nil) == SQLITE_OK { - while sqlite3_step(statement) == SQLITE_ROW { - if let jsonData = sqlite3_column_text(statement, 0).map({ String(cString: $0) }) { - // 將 JSON 字串解析為 [Int]，然後轉換為 Set - if let data = jsonData.data(using: .utf8), - let setDecoded = try? JSONDecoder().decode(Set.self, from: data) { - nodeIDs.formUnion(setDecoded) + while sqlite3_step(stmt) == SQLITE_ROW { + if let nodeIDsText = sqlite3_column_text(stmt, 0).map({ String(cString: $0) }) { + if !nodeIDsText.isEmpty { + // 向後相容：支援舊的 JSON 格式和新的逗號分隔格式 + if nodeIDsText.starts(with: "[") && nodeIDsText.hasSuffix("]") { + // 舊的 JSON 格式 + if let data = nodeIDsText.data(using: .utf8), + let setDecoded = try? JSONDecoder().decode(Set.self, from: data) { + nodeIDs.formUnion(setDecoded) + } + } else { + // 新的逗號分隔格式（更快） + let nodeIDStrings = nodeIDsText.split(separator: ",") + for nodeIDString in nodeIDStrings { + if let nodeID = Int(nodeIDString) { + nodeIDs.insert(nodeID) + } + } } } } } - sqlite3_finalize(statement) queryBuffer4NodeIDs.set(hashKey: formedKeyHash, value: nodeIDs) return nodeIDs.sorted() } @@ -68,14 +78,8 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol { longerSpan: Bool ) -> [TNode] { - let formedKeyHash: Int = { - var hasher = Hasher() - hasher.combine(keyArray) - hasher.combine(filterType) - hasher.combine(partiallyMatch) - hasher.combine(longerSpan) - return hasher.finalize() - }() + // 優化的快取鍵計算 - 使用更簡單的雜湊 + let formedKeyHash: Int = keyArray.hashValue ^ filterType.hashValue ^ (partiallyMatch ? 1 : 0) ^ (longerSpan ? 2 : 0) if let cachedResult = queryBuffer4Nodes.get(hashKey: formedKeyHash) { return cachedResult } @@ -91,7 +95,10 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol { let hash = theNode.hashValue if !handledNodeHashes.contains(hash) { handledNodeHashes.insert(theNode.hashValue) - let nodeKeyArray = theNode.readingKey.split(separator: readingSeparator) + let nodeKeyArray = TrieStringOperationCache.shared.getCachedSplit( + theNode.readingKey, + separator: readingSeparator + ) if nodeMeetsFilter(theNode, filter: filterType) { var matched: Bool = longerSpan ? nodeKeyArray.count > keyArray.count @@ -116,21 +123,15 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol { public func getNode(_ nodeID: Int) -> TNode? { if let cachedResult = queryBuffer4Node.get(hashKey: nodeID) { return cachedResult } - // 基本查詢，只查必要資訊 - let query = """ - SELECT id, reading_key, entries_blob - FROM nodes - WHERE id = ? - LIMIT 1 - """ - var statement: OpaquePointer? - - guard sqlite3_prepare_v2(database, query, -1, &statement, nil) == SQLITE_OK else { - sqlite3_finalize(statement) - return nil - } - defer { sqlite3_finalize(statement) } - + + // 使用預編譯的語句 + guard let statement = preparedNodeQuery else { return nil } + + // 重置語句狀態 + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + + // 綁定參數 sqlite3_bind_int64(statement, 1, sqlite3_int64(nodeID)) guard sqlite3_step(statement) == SQLITE_ROW else { return nil } From e6d4f5fea10f0ea2634eb747d4125fc104edcc12 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Sep 2025 17:14:35 +0000 Subject: [PATCH 3/3] Debug SQLTrie issues - partial optimization revert for compatibility Co-authored-by: ShikiSuen <3164826+ShikiSuen@users.noreply.github.com> --- Sources/_Modules/TrieKit/TrieSQL_Core.swift | 7 ++- Sources/_Modules/TrieKit/TrieSQL_Impl.swift | 69 ++++++++++----------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Sources/_Modules/TrieKit/TrieSQL_Core.swift b/Sources/_Modules/TrieKit/TrieSQL_Core.swift index 543aa381..486b314c 100644 --- a/Sources/_Modules/TrieKit/TrieSQL_Core.swift +++ b/Sources/_Modules/TrieKit/TrieSQL_Core.swift @@ -138,9 +138,10 @@ extension VanguardTrie { } // 啟用 SQLITE 最佳化功能 - if !applyPerformanceOptimizations() { - return nil - } + // Temporarily disabled for debugging + // if !applyPerformanceOptimizations() { + // return nil + // } // 讀取分隔符設定 if !initializeSettings() { diff --git a/Sources/_Modules/TrieKit/TrieSQL_Impl.swift b/Sources/_Modules/TrieKit/TrieSQL_Impl.swift index c011847e..318ca42d 100644 --- a/Sources/_Modules/TrieKit/TrieSQL_Impl.swift +++ b/Sources/_Modules/TrieKit/TrieSQL_Impl.swift @@ -12,8 +12,8 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol { /// 根據 keychain 字串查詢節點 ID public func getNodeIDsForKeyArray(_ keyArray: [String], longerSpan: Bool) -> [Int] { guard !keyArray.isEmpty, keyArray.allSatisfy({ !$0.isEmpty }) else { return [] } - let keyInitialsStr = keyArray.compactMap { keyStr in - TrieStringPool.shared.internKey(TrieStringOperationCache.shared.getCachedFirstChar(keyStr)) + let keyInitialsStr = keyArray.compactMap { + $0.first?.description }.joined() // 優化的快取鍵計算 - 使用更簡單的雜湊 @@ -24,41 +24,39 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol { } var nodeIDs = Set() - - // 使用預編譯的語句以提升效能 - let statement = longerSpan ? preparedKeyInitialsLikeQuery : preparedKeyInitialsExactQuery - guard let stmt = statement else { - return nodeIDs.sorted() - } - - // 重置語句狀態 - sqlite3_reset(stmt) - sqlite3_clear_bindings(stmt) - - // 綁定參數 - if longerSpan { - let pattern = keyInitialsStr + "%" - sqlite3_bind_text(stmt, 1, pattern, -1, nil) - } else { - sqlite3_bind_text(stmt, 1, keyInitialsStr, -1, nil) + let escapedKeyInitials = keyInitialsStr.replacingOccurrences(of: "'", with: "''") + + let query = switch longerSpan { + case false: """ + SELECT node_ids FROM keyinitials_id_map + WHERE keyinitials = '\(escapedKeyInitials)' + """ + case true: """ + SELECT node_ids FROM keyinitials_id_map + WHERE keyinitials LIKE '\(escapedKeyInitials)%' + """ } - while sqlite3_step(stmt) == SQLITE_ROW { - if let nodeIDsText = sqlite3_column_text(stmt, 0).map({ String(cString: $0) }) { - if !nodeIDsText.isEmpty { - // 向後相容：支援舊的 JSON 格式和新的逗號分隔格式 - if nodeIDsText.starts(with: "[") && nodeIDsText.hasSuffix("]") { - // 舊的 JSON 格式 - if let data = nodeIDsText.data(using: .utf8), - let setDecoded = try? JSONDecoder().decode(Set.self, from: data) { - nodeIDs.formUnion(setDecoded) - } - } else { - // 新的逗號分隔格式（更快） - let nodeIDStrings = nodeIDsText.split(separator: ",") - for nodeIDString in nodeIDStrings { - if let nodeID = Int(nodeIDString) { - nodeIDs.insert(nodeID) + var statement: OpaquePointer? + + if sqlite3_prepare_v2(database, query, -1, &statement, nil) == SQLITE_OK { + while sqlite3_step(statement) == SQLITE_ROW { + if let nodeIDsText = sqlite3_column_text(statement, 0).map({ String(cString: $0) }) { + if !nodeIDsText.isEmpty { + // 向後相容：支援舊的 JSON 格式和新的逗號分隔格式 + if nodeIDsText.starts(with: "[") && nodeIDsText.hasSuffix("]") { + // 舊的 JSON 格式 + if let data = nodeIDsText.data(using: .utf8), + let setDecoded = try? JSONDecoder().decode(Set.self, from: data) { + nodeIDs.formUnion(setDecoded) + } + } else { + // 新的逗號分隔格式（更快） + let nodeIDStrings = nodeIDsText.split(separator: ",") + for nodeIDString in nodeIDStrings { + if let nodeID = Int(nodeIDString) { + nodeIDs.insert(nodeID) + } } } } @@ -66,6 +64,7 @@ extension VanguardTrie.SQLTrie: VanguardTrieProtocol { } } + sqlite3_finalize(statement) queryBuffer4NodeIDs.set(hashKey: formedKeyHash, value: nodeIDs) return nodeIDs.sorted() }