Skip to content

Commit b4313c8

Browse files
cardernecarderne
authored
feat: logs search v2 (#4615)
1 parent 158f695 commit b4313c8

36 files changed

Lines changed: 2368 additions & 328 deletions

‎.env.example‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ NODE_ENV=development
2323
CLICKHOUSE_URL=http://default:password@localhost:8123
2424
RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@localhost:8123
2525
RUN_REPLICATION_ENABLED=1
26+
# LOGS_SEARCH_PROJECTOR_ENABLED=1
27+
# LOGS_SEARCH_PROJECTOR_PREVIEW_ENABLED=1
2628
# Store task run spans/traces in ClickHouse so the dashboard trace view is
2729
# populated in local dev. The local stack is ClickHouse-backed (see above), so
2830
# leaving this unset falls back to the "postgres" store and dev run traces show

‎.server-changes/improve-global-log-search.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
area: webapp
3+
type: improvement
4+
---
5+
6+
Global log search now supports faster bounded substring matching and clearer time-range expansion.

‎apps/webapp/app/components/navigation/SideMenu.tsx‎

Lines changed: 54 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -825,7 +825,7 @@ export function SideMenu({
825825
});
826826
}
827827

828-
if (isAdmin || featureFlags.hasQueryAccess) {
828+
if (isAdmin || featureFlags.hasQueryAccess || featureFlags.hasLogsPageAccess) {
829829
staticSections.push({
830830
id: "metrics",
831831
title: "Observability",
@@ -843,55 +843,59 @@ export function SideMenu({
843843
} satisfies SideMenuItemConfig,
844844
]
845845
: []),
846-
{
847-
id: "errors",
848-
name: "Errors",
849-
icon: BugIcon,
850-
activeIconColor: "text-errors",
851-
to: v3ErrorsPath(organization, project, environment),
852-
dataAction: "errors",
853-
},
854-
{
855-
id: "query",
856-
name: "Query",
857-
icon: CodeSquareIcon,
858-
activeIconColor: "text-query",
859-
to: queryPath(organization, project, environment),
860-
dataAction: "query",
861-
},
862-
{
863-
id: "queues",
864-
name: "Queues",
865-
icon: QueuesIcon,
866-
activeIconColor: "text-queues",
867-
to: v3QueuesPath(organization, project, environment),
868-
dataAction: "queues",
869-
},
870-
{
871-
id: "dashboards",
872-
name: "Dashboards",
873-
icon: ChartBarIcon,
874-
activeIconColor: "text-metrics",
875-
to: v3DashboardsLandingPath(organization, project, environment),
876-
dataAction: "dashboards-landing",
877-
action: (
878-
<CreateDashboardButton
879-
organization={organization}
880-
project={project}
881-
environment={environment}
882-
isCollapsed={isCollapsed}
883-
/>
884-
),
885-
after: (
886-
<DashboardList
887-
organization={organization}
888-
project={project}
889-
environment={environment}
890-
isCollapsed={isCollapsed}
891-
user={user}
892-
/>
893-
),
894-
},
846+
...(isAdmin || featureFlags.hasQueryAccess
847+
? [
848+
{
849+
id: "errors",
850+
name: "Errors",
851+
icon: BugIcon,
852+
activeIconColor: "text-errors",
853+
to: v3ErrorsPath(organization, project, environment),
854+
dataAction: "errors",
855+
},
856+
{
857+
id: "query",
858+
name: "Query",
859+
icon: CodeSquareIcon,
860+
activeIconColor: "text-query",
861+
to: queryPath(organization, project, environment),
862+
dataAction: "query",
863+
},
864+
{
865+
id: "queues",
866+
name: "Queues",
867+
icon: QueuesIcon,
868+
activeIconColor: "text-queues",
869+
to: v3QueuesPath(organization, project, environment),
870+
dataAction: "queues",
871+
},
872+
{
873+
id: "dashboards",
874+
name: "Dashboards",
875+
icon: ChartBarIcon,
876+
activeIconColor: "text-metrics",
877+
to: v3DashboardsLandingPath(organization, project, environment),
878+
dataAction: "dashboards-landing",
879+
action: (
880+
<CreateDashboardButton
881+
organization={organization}
882+
project={project}
883+
environment={environment}
884+
isCollapsed={isCollapsed}
885+
/>
886+
),
887+
after: (
888+
<DashboardList
889+
organization={organization}
890+
project={project}
891+
environment={environment}
892+
isCollapsed={isCollapsed}
893+
user={user}
894+
/>
895+
),
896+
},
897+
]
898+
: []),
895899
],
896900
});
897901
}

‎apps/webapp/app/components/primitives/SearchInput.tsx‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ export type SearchInputProps = {
1414
/** Additional URL params to reset when searching or clearing (e.g. pagination). Defaults to ["cursor", "direction"]. */
1515
resetParams?: string[];
1616
autoFocus?: boolean;
17+
minLength?: number;
18+
/** Normalize the submitted value before applying minLength validation. */
19+
normalizeForValidation?: (value: string) => string;
1720
/**
1821
* Controlled value. When provided alongside `onValueChange`, the input
1922
* skips URL params entirely and acts as a controlled component — useful
@@ -34,6 +37,8 @@ export function SearchInput({
3437
paramName = "search",
3538
resetParams = ["cursor", "direction"],
3639
autoFocus,
40+
minLength,
41+
normalizeForValidation,
3742
value: controlledValue,
3843
onValueChange,
3944
}: SearchInputProps) {
@@ -70,20 +75,33 @@ export function SearchInput({
7075
}, [isControlled, controlledValue, value, isFocused, paramName]);
7176

7277
const updateText = (next: string) => {
78+
inputRef.current?.setCustomValidity("");
7379
setText(next);
7480
if (isControlled) {
7581
onValueChange?.(next);
7682
}
7783
};
7884

7985
const handleSubmit = () => {
86+
const trimmedText = text.trim();
87+
const validationText = normalizeForValidation?.(trimmedText) ?? trimmedText;
88+
if (
89+
minLength !== undefined &&
90+
trimmedText.length > 0 &&
91+
[...validationText].length < minLength
92+
) {
93+
inputRef.current?.setCustomValidity(`Enter at least ${minLength} characters`);
94+
inputRef.current?.reportValidity();
95+
return;
96+
}
97+
inputRef.current?.setCustomValidity("");
8098
if (isControlled) {
8199
// Live updates already fired through onValueChange; submit is a no-op.
82100
return;
83101
}
84102
const resetValues = Object.fromEntries(resetParams.map((p) => [p, undefined]));
85-
if (text.trim()) {
86-
replace({ [paramName]: text.trim(), ...resetValues });
103+
if (trimmedText) {
104+
replace({ [paramName]: trimmedText, ...resetValues });
87105
} else {
88106
del([paramName, ...resetParams]);
89107
}
@@ -116,7 +134,10 @@ export function SearchInput({
116134
variant="secondary-small"
117135
placeholder={placeholder}
118136
value={text}
119-
onChange={(e) => updateText(e.target.value)}
137+
onChange={(e) => {
138+
e.currentTarget.setCustomValidity("");
139+
updateText(e.target.value);
140+
}}
120141
fullWidth
121142
autoFocus={autoFocus}
122143
className={cn("", isFocused && "placeholder:text-text-dimmed/70")}

‎apps/webapp/app/entry.server.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { PassThrough } from "stream";
1010
import { initMollifierDrainerWorker } from "~/v3/mollifierDrainerWorker.server";
1111
import { initMollifierStaleSweepWorker } from "~/v3/mollifierStaleSweepWorker.server";
1212
import { initBillingLimitWorker } from "~/v3/billingLimitWorker.server";
13+
import { initLogsSearchProjectorWorker } from "~/v3/logsSearchProjectorWorker.server";
1314
import { initQueueMetricsConsumer, initQueueMetricsEmitter } from "~/v3/queueMetrics.server";
1415
import { bootstrap } from "./bootstrap";
1516
import { LocaleContextProvider } from "./components/primitives/LocaleProvider";
@@ -277,6 +278,7 @@ export const handleError = wrapHandleErrorWithSentry((error, { request }) => {
277278
initMollifierDrainerWorker();
278279
initMollifierStaleSweepWorker();
279280
initBillingLimitWorker();
281+
initLogsSearchProjectorWorker();
280282
initQueueMetricsEmitter();
281283
initQueueMetricsConsumer();
282284

‎apps/webapp/app/env.server.ts‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2094,20 +2094,28 @@ const EnvironmentSchema = z
20942094
.nonnegative()
20952095
.optional(),
20962096

2097-
// Logs list pagination tuning (page sizing + recent-first probe windows).
2097+
// Scheduled logs-search projection. Disabled by default. LOGS_CLICKHOUSE_URL, or the
2098+
// CLICKHOUSE_URL fallback, must reach both source and destination tables and allow writes.
2099+
LOGS_SEARCH_PROJECTOR_ENABLED: BoolEnv.default(false),
2100+
LOGS_SEARCH_PROJECTOR_PREVIEW_ENABLED: BoolEnv.default(false),
2101+
LOGS_SEARCH_PROJECTOR_MAX_WINDOWS_PER_TICK: z.coerce.number().int().min(1).max(20).default(5),
2102+
LOGS_SEARCH_PROJECTOR_MAX_EXECUTION_TIME_SECONDS: z.coerce
2103+
.number()
2104+
.int()
2105+
.min(1)
2106+
.max(300)
2107+
.default(120),
2108+
LOGS_SEARCH_PROJECTOR_MAX_ROWS_TO_READ: z.coerce.number().int().positive().default(10_000_000),
2109+
LOGS_SEARCH_PROJECTOR_MAX_MEMORY_USAGE: z.coerce
2110+
.number()
2111+
.int()
2112+
.positive()
2113+
.default(1_500_000_000),
2114+
LOGS_SEARCH_PROJECTOR_MAX_THREADS: z.coerce.number().int().min(1).max(8).default(2),
2115+
2116+
// Logs list pagination tuning.
20982117
LOGS_LIST_DEFAULT_PAGE_SIZE: z.coerce.number().int().positive().default(50),
20992118
LOGS_LIST_MAX_PAGE_SIZE: z.coerce.number().int().positive().default(100),
2100-
// Days back from the page ceiling to probe before widening to the full requested window,
2101-
// comma-separated. Empty disables narrowing (a single full-window query).
2102-
LOGS_LIST_RECENT_FIRST_PROBE_DAYS: z
2103-
.string()
2104-
.default("1,7")
2105-
.transform((s) =>
2106-
s
2107-
.split(",")
2108-
.map((v) => Number(v.trim()))
2109-
.filter((n) => Number.isFinite(n) && n > 0)
2110-
),
21112119

21122120
// Query feature flag
21132121
QUERY_FEATURE_ENABLED: z.string().default("1"),
@@ -2116,10 +2124,7 @@ const EnvironmentSchema = z
21162124
AI_FEATURES_ENABLED: z.string().default("0"),
21172125

21182126
// Logs page ClickHouse URL (for logs queries)
2119-
LOGS_CLICKHOUSE_URL: z
2120-
.string()
2121-
.optional()
2122-
.transform((v) => v ?? process.env.CLICKHOUSE_READER_URL ?? process.env.CLICKHOUSE_URL),
2127+
LOGS_CLICKHOUSE_URL: z.string().optional(),
21232128

21242129
// Query page ClickHouse limits (for TSQL queries)
21252130
QUERY_CLICKHOUSE_URL: z

0 commit comments

Comments
 (0)