Skip to content

Commit 542ca6e

Browse files
SurviveMSurviveM
committed
fix: scan full session files for keyword hits
1 parent 76f50df commit 542ca6e

3 files changed

Lines changed: 179 additions & 48 deletions

File tree

‎cli.js‎

Lines changed: 123 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -4770,25 +4770,7 @@ function extractMessageFromRecord(record, source) {
47704770
return { role, text };
47714771
}
47724772

4773-
function scanSessionContentForQuery(session, tokens, options = {}) {
4774-
if (!session || !Array.isArray(tokens) || tokens.length === 0) {
4775-
return { hit: false, count: 0, snippets: [] };
4776-
}
4777-
4778-
const filePath = resolveSessionFilePath(session.source, session.filePath, session.sessionId);
4779-
if (!filePath) {
4780-
return { hit: false, count: 0, snippets: [] };
4781-
}
4782-
4783-
const maxBytes = Number.isFinite(Number(options.maxBytes))
4784-
? Math.max(1024, Number(options.maxBytes))
4785-
: SESSION_CONTENT_READ_BYTES;
4786-
const headText = getFileHeadText(filePath, maxBytes);
4787-
if (!headText) {
4788-
return { hit: false, count: 0, snippets: [] };
4789-
}
4790-
4791-
const records = parseJsonlContent(headText);
4773+
function createSessionQueryScanState(tokens, options = {}) {
47924774
const mode = normalizeQueryMode(options.mode);
47934775
const roleFilter = normalizeRoleFilter(options.roleFilter);
47944776
const maxMatches = Number.isFinite(Number(options.maxMatches))
@@ -4798,43 +4780,137 @@ function scanSessionContentForQuery(session, tokens, options = {}) {
47984780
? Math.max(0, Number(options.snippetLimit))
47994781
: 0;
48004782

4801-
const messages = [];
4783+
return {
4784+
tokens,
4785+
mode,
4786+
roleFilter,
4787+
maxMatches,
4788+
snippetLimit,
4789+
count: 0,
4790+
snippets: [],
4791+
leadingSystem: roleFilter !== 'system'
4792+
};
4793+
}
4794+
4795+
function consumeSessionQueryMessage(state, message) {
4796+
if (!state || typeof state !== 'object' || !message) {
4797+
return false;
4798+
}
4799+
4800+
const role = normalizeRole(message.role);
4801+
const text = typeof message.text === 'string' ? message.text : '';
4802+
if (!role || !text) {
4803+
return false;
4804+
}
4805+
4806+
if (state.leadingSystem && (role === 'system' || isBootstrapLikeText(text))) {
4807+
return false;
4808+
}
4809+
state.leadingSystem = false;
4810+
4811+
if (state.roleFilter !== 'all' && role !== state.roleFilter) {
4812+
return false;
4813+
}
4814+
if (!matchTokensInText(text, state.tokens, state.mode)) {
4815+
return false;
4816+
}
4817+
4818+
state.count += 1;
4819+
if (state.snippetLimit > 0 && state.snippets.length < state.snippetLimit) {
4820+
state.snippets.push(truncateText(text));
4821+
}
4822+
return state.count >= state.maxMatches;
4823+
}
4824+
4825+
function buildSessionQueryScanResult(state) {
4826+
return {
4827+
hit: !!(state && state.count > 0),
4828+
count: state && Number.isFinite(state.count) ? state.count : 0,
4829+
snippets: state && Array.isArray(state.snippets) ? state.snippets : []
4830+
};
4831+
}
4832+
4833+
function scanSessionContentForQueryInRecords(records, source, state) {
4834+
if (!Array.isArray(records) || !state) {
4835+
return buildSessionQueryScanResult(state);
4836+
}
4837+
48024838
for (const record of records) {
4803-
const message = extractMessageFromRecord(record, session.source);
4804-
if (!message || !message.text) {
4839+
const message = extractMessageFromRecord(record, source);
4840+
if (!message) {
48054841
continue;
48064842
}
4807-
messages.push(message);
4843+
if (consumeSessionQueryMessage(state, message)) {
4844+
break;
4845+
}
48084846
}
48094847

4810-
const filteredMessages = roleFilter === 'system'
4811-
? messages
4812-
: removeLeadingSystemMessage(messages);
4848+
return buildSessionQueryScanResult(state);
4849+
}
48134850

4814-
let count = 0;
4815-
const snippets = [];
4851+
async function scanSessionContentForQuery(session, tokens, options = {}) {
4852+
if (!session || !Array.isArray(tokens) || tokens.length === 0) {
4853+
return { hit: false, count: 0, snippets: [] };
4854+
}
48164855

4817-
for (const message of filteredMessages) {
4818-
if (roleFilter !== 'all' && message.role !== roleFilter) {
4819-
continue;
4820-
}
4821-
if (!matchTokensInText(message.text, tokens, mode)) {
4822-
continue;
4856+
const filePath = resolveSessionFilePath(session.source, session.filePath, session.sessionId);
4857+
if (!filePath) {
4858+
return { hit: false, count: 0, snippets: [] };
4859+
}
4860+
4861+
const rawMaxBytes = Number(options.maxBytes);
4862+
const maxBytes = Number.isFinite(rawMaxBytes) && rawMaxBytes > 0
4863+
? Math.max(1024, rawMaxBytes)
4864+
: 0;
4865+
const state = createSessionQueryScanState(tokens, options);
4866+
let stream;
4867+
let rl;
4868+
try {
4869+
stream = fs.createReadStream(filePath, { encoding: 'utf-8' });
4870+
rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
4871+
4872+
let bytesRead = 0;
4873+
for await (const line of rl) {
4874+
if (maxBytes > 0 && bytesRead >= maxBytes) {
4875+
break;
4876+
}
4877+
4878+
bytesRead += Buffer.byteLength(line, 'utf-8') + 1;
4879+
const trimmed = line.trim();
4880+
if (!trimmed) {
4881+
continue;
4882+
}
4883+
4884+
let record;
4885+
try {
4886+
record = JSON.parse(trimmed);
4887+
} catch (e) {
4888+
continue;
4889+
}
4890+
4891+
const message = extractMessageFromRecord(record, session.source);
4892+
if (!message) {
4893+
continue;
4894+
}
4895+
if (consumeSessionQueryMessage(state, message)) {
4896+
break;
4897+
}
48234898
}
48244899

4825-
count += 1;
4826-
if (snippetLimit > 0 && snippets.length < snippetLimit) {
4827-
snippets.push(truncateText(message.text));
4900+
return buildSessionQueryScanResult(state);
4901+
} catch (e) {
4902+
return scanSessionContentForQueryInRecords(readJsonlRecords(filePath), session.source, state);
4903+
} finally {
4904+
if (rl) {
4905+
try { rl.close(); } catch (e) {}
48284906
}
4829-
if (count >= maxMatches) {
4830-
break;
4907+
if (stream && !stream.destroyed && stream.destroy) {
4908+
try { stream.destroy(); } catch (e) {}
48314909
}
48324910
}
4833-
4834-
return { hit: count > 0, count, snippets };
48354911
}
48364912

4837-
function applySessionQueryFilter(sessions, options = {}) {
4913+
async function applySessionQueryFilter(sessions, options = {}) {
48384914
const tokens = Array.isArray(options.tokens) ? options.tokens : [];
48394915
if (tokens.length === 0) {
48404916
return sessions;
@@ -4848,7 +4924,7 @@ function applySessionQueryFilter(sessions, options = {}) {
48484924
: DEFAULT_CONTENT_SCAN_LIMIT;
48494925
const contentScanBytes = Number.isFinite(Number(options.contentScanBytes))
48504926
? Math.max(1024, Number(options.contentScanBytes))
4851-
: SESSION_CONTENT_READ_BYTES;
4927+
: 0;
48524928

48534929
let scanned = 0;
48544930
const results = [];
@@ -4866,7 +4942,7 @@ function applySessionQueryFilter(sessions, options = {}) {
48664942
const shouldScanContent = scope === 'content' || scope === 'all' || !summaryHit;
48674943
if (shouldScanContent && scanned < contentScanLimit) {
48684944
scanned += 1;
4869-
contentInfo = scanSessionContentForQuery(session, tokens, {
4945+
contentInfo = await scanSessionContentForQuery(session, tokens, {
48704946
mode,
48714947
roleFilter,
48724948
maxBytes: contentScanBytes,
@@ -5341,7 +5417,7 @@ function listClaudeSessions(limit, options = {}) {
53415417
return mergeAndLimitSessions(sessions, limit);
53425418
}
53435419

5344-
function listAllSessions(params = {}) {
5420+
async function listAllSessions(params = {}) {
53455421
const source = params.source === 'codex' || params.source === 'claude'
53465422
? params.source
53475423
: 'all';
@@ -5383,7 +5459,7 @@ function listAllSessions(params = {}) {
53835459

53845460
let result = sessions;
53855461
if (hasQuery) {
5386-
result = applySessionQueryFilter(result, {
5462+
result = await applySessionQueryFilter(result, {
53875463
tokens: queryTokens,
53885464
queryMode: params.queryMode,
53895465
queryScope: params.queryScope,
@@ -5419,7 +5495,7 @@ async function listAllSessionsData(params = {}) {
54195495
}
54205496
}
54215497

5422-
const sessions = listAllSessions(params);
5498+
const sessions = await listAllSessions(params);
54235499
const hydratedSessions = await hydrateSessionItemsExactMessageCount(sessions);
54245500
const result = hydratedSessions.map((item) => {
54255501
if (!item || typeof item !== 'object' || Array.isArray(item)) {

‎tests/e2e/test-session-search.js‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ async function fetchHtml(port) {
2222
}
2323

2424
module.exports = async function testSessionSearch(ctx) {
25-
const { api, sessionId, claudeSessionId, daudeSessionId } = ctx;
25+
const { api, sessionId, claudeSessionId, daudeSessionId, lateKeywordSessionId, lateKeywordMessage } = ctx;
2626

2727
// ========== Basic Query Tests ==========
2828
const claudeSearch = await api('list-sessions', { source: 'claude', query: 'claudecode', limit: 20, forceRefresh: true });
@@ -102,6 +102,30 @@ module.exports = async function testSessionSearch(ctx) {
102102
snippet => typeof snippet === 'string' && snippet.includes('222')
103103
), '222 snippets missing numeric token');
104104

105+
// ========== Late Content Query Tests ==========
106+
const lateKeywordDetail = await api('session-detail', {
107+
source: 'codex',
108+
sessionId: lateKeywordSessionId
109+
});
110+
assert(Array.isArray(lateKeywordDetail.messages), 'late keyword session detail missing messages');
111+
assert(lateKeywordDetail.messages.some(
112+
message => message && typeof message.text === 'string' && message.text.includes(lateKeywordMessage)
113+
), 'late keyword session detail should expose the tail message');
114+
115+
const lateKeywordSearch = await api('list-sessions', {
116+
source: 'codex',
117+
query: '提示 通过',
118+
queryScope: 'content',
119+
limit: 20,
120+
forceRefresh: true
121+
});
122+
const lateKeywordHit = lateKeywordSearch.sessions.find(item => item.sessionId === lateKeywordSessionId);
123+
assert(lateKeywordHit, 'late keyword query should find the tail-only session');
124+
assert(lateKeywordHit.match && lateKeywordHit.match.hit === true, 'late keyword query missing match metadata');
125+
assert(Array.isArray(lateKeywordHit.match.snippets) && lateKeywordHit.match.snippets.some(
126+
snippet => typeof snippet === 'string' && snippet.includes('提示') && snippet.includes('通过')
127+
), 'late keyword query snippets should include the tail message');
128+
105129
// ========== Pagination Tests ==========
106130
const paged = await api('list-sessions', {
107131
source: 'claude',

‎tests/e2e/test-setup.js‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,34 @@ module.exports = async function testSetup(ctx) {
112112
];
113113
fs.writeFileSync(daudeSessionPath, daudeRecords.map(record => JSON.stringify(record)).join('\n') + '\n', 'utf-8');
114114

115+
const lateKeywordSessionId = 'late-keyword-e2e-session';
116+
const lateKeywordSessionPath = path.join(sessionsDir, `${lateKeywordSessionId}.jsonl`);
117+
const lateKeywordMessage = '这是后段命中的提示，测试已经通过。';
118+
const lateKeywordRecords = [
119+
{
120+
type: 'session_meta',
121+
payload: { id: lateKeywordSessionId, cwd: '/tmp/late-keyword' },
122+
timestamp: '2025-02-10T00:00:00.000Z'
123+
}
124+
];
125+
for (let i = 0; i < 32; i++) {
126+
lateKeywordRecords.push({
127+
type: 'response_item',
128+
payload: {
129+
type: 'message',
130+
role: i % 2 === 0 ? 'user' : 'assistant',
131+
content: `padding-${String(i).padStart(2, '0')}-${'x'.repeat(12 * 1024)}`
132+
},
133+
timestamp: `2025-02-10T00:00:${String(i + 1).padStart(2, '0')}.000Z`
134+
});
135+
}
136+
lateKeywordRecords.push({
137+
type: 'response_item',
138+
payload: { type: 'message', role: 'assistant', content: lateKeywordMessage },
139+
timestamp: '2025-02-10T00:01:00.000Z'
140+
});
141+
fs.writeFileSync(lateKeywordSessionPath, lateKeywordRecords.map(record => JSON.stringify(record)).join('\n') + '\n', 'utf-8');
142+
115143
const claudeProjectsDir = path.join(tmpHome, '.claude', 'projects');
116144
const claudeProjectDir = path.join(claudeProjectsDir, 'e2e-project');
117145
fs.mkdirSync(claudeProjectDir, { recursive: true });
@@ -155,6 +183,9 @@ module.exports = async function testSetup(ctx) {
155183
sessionPath,
156184
daudeSessionId,
157185
daudeSessionPath,
186+
lateKeywordSessionId,
187+
lateKeywordSessionPath,
188+
lateKeywordMessage,
158189
claudeSessionId,
159190
claudeSessionPath,
160191
noModelsUrl,

0 commit comments

Comments
 (0)