From 2cbcec69125a9369f5e956d8daee7272e94de8c8 Mon Sep 17 00:00:00 2001 From: Youhao Wei Date: Sat, 11 Jul 2026 22:06:34 -0700 Subject: [PATCH 01/11] feat: onboard remote data connectors for v0.3 --- .../src/functions/app-artifacts.test.ts | 30 ++++ apps/server/src/functions/app-artifacts.ts | 16 ++- packages/app-data/src/index.ts | 6 + packages/app-data/src/postgres.ts | 43 ++++++ .../src/app/_components/OnboardingView.tsx | 6 +- .../data-sources/AddConnectionPanel.tsx | 7 +- .../data-sources/DataPickerContent.tsx | 136 ++++++++++++++++-- .../data-sources/DataSourceDisplay.tsx | 76 +--------- .../renderers/ConnectorCardWithForm.tsx | 7 +- packages/app/src/lib/connectors/registry.ts | 6 +- .../lib/remote-connector-onboarding.test.ts | 62 ++++++++ .../src/lib/remote-connector-onboarding.ts | 64 +++++++++ .../src/lib/remote-table-materialization.ts | 66 +++++++++ packages/types/src/data-sources.ts | 2 + 14 files changed, 422 insertions(+), 105 deletions(-) create mode 100644 packages/app-data/src/postgres.ts create mode 100644 packages/app/src/lib/remote-connector-onboarding.test.ts create mode 100644 packages/app/src/lib/remote-connector-onboarding.ts create mode 100644 packages/app/src/lib/remote-table-materialization.ts diff --git a/apps/server/src/functions/app-artifacts.test.ts b/apps/server/src/functions/app-artifacts.test.ts index d04a5d2f..c0667617 100644 --- a/apps/server/src/functions/app-artifacts.test.ts +++ b/apps/server/src/functions/app-artifacts.test.ts @@ -721,4 +721,34 @@ describe("addDataSource / updateDataSource — same-operation minted-ref rollbac expect(await vault.has(config.apiKey as SecretRef)).toBe(true); expect(rows[0]?.id).toBe(result.id); }); + + it("persists connector-specific config beside a vault-backed credential", async () => { + await call("addDataSource", { + type: "postgres", + name: "Warehouse", + connectionString: "postgres://user:secret@host/db", + config: { defaultSchema: "analytics" }, + }); + + const rows = await db.select().from(dataSources); + const config = rows[0]?.config as { + connectionString?: unknown; + defaultSchema?: unknown; + }; + expect(config.defaultSchema).toBe("analytics"); + expect(isSecretRef(config.connectionString)).toBe(true); + expect(JSON.stringify(config)).not.toContain("postgres://user:secret"); + }); + + it("rejects credentials smuggled through connector-specific config", async () => { + await expect( + call("addDataSource", { + type: "postgres", + name: "Unsafe", + config: { connectionString: "postgres://plaintext" }, + }), + ).rejects.toThrow(/typed credential fields/); + + expect(await db.select().from(dataSources)).toHaveLength(0); + }); }); diff --git a/apps/server/src/functions/app-artifacts.ts b/apps/server/src/functions/app-artifacts.ts index cf44ae73..be02eaa7 100644 --- a/apps/server/src/functions/app-artifacts.ts +++ b/apps/server/src/functions/app-artifacts.ts @@ -478,15 +478,27 @@ const addDataSource = mutation({ name: text, apiKey: text.optional(), connectionString: text.optional(), + config: jsonb.optional(), }, handler: async ( ctx, - { type, name, apiKey, connectionString }, + { type, name, apiKey, connectionString, config: rawConfig }, ): Promise<{ id: string }> => { const vault = vaultFromCtx(ctx); // The id is generated by the DB default; use a pre-generated UUID as hint. const rowId = crypto.randomUUID(); - const config: DataSourceConfig = {}; + if (rawConfig !== undefined && !isRecord(rawConfig)) { + throw new Error("Data source config must be an object"); + } + if ( + rawConfig && + ("apiKey" in rawConfig || "connectionString" in rawConfig) + ) { + throw new Error( + "Data source credentials must use typed credential fields, not config", + ); + } + const config: DataSourceConfig = { ...(rawConfig ?? {}) }; // Refs minted in THIS call only — a fresh insert has no pre-existing config, // so every ref that lands in `config` here was just minted by this operation. const minted: SecretRef[] = []; diff --git a/packages/app-data/src/index.ts b/packages/app-data/src/index.ts index 9137b7fb..3b7dbbbb 100644 --- a/packages/app-data/src/index.ts +++ b/packages/app-data/src/index.ts @@ -111,6 +111,12 @@ export { type NotionQueryResult, } from "./notion"; +export { + usePostgresMutations, + type PostgresQueryResult, + type PostgresTableRef, +} from "./postgres"; + export { DatabaseProvider, useDatabase } from "./compat"; // Preview batch — SPLIT-TIER: returns metadata only, no row data over the wire. diff --git a/packages/app-data/src/postgres.ts b/packages/app-data/src/postgres.ts new file mode 100644 index 00000000..0f383e6f --- /dev/null +++ b/packages/app-data/src/postgres.ts @@ -0,0 +1,43 @@ +import type { Field, UUID } from "@dashframe/types"; +import { useMutation } from "@wystack/client"; +import { useMemo } from "react"; + +import { api } from "./api"; +import { loose } from "./wystack-args"; + +export interface PostgresTableRef { + id: string; + title: string; +} + +export interface PostgresQueryResult { + arrowBuffer: string; + fieldIds: string[]; + fields: Field[]; + rowCount: number; +} + +/** Server-backed Postgres operations. Credentials remain in SecretVault. */ +export function usePostgresMutations() { + const listMutation = useMutation(api.listPostgresTables); + const queryMutation = useMutation(api.queryPostgresTable); + + return useMemo( + () => ({ + listTables: async (dataSourceId: UUID): Promise => + (await listMutation.mutateAsync({ + dataSourceId, + })) as PostgresTableRef[], + queryTable: async ( + dataSourceId: UUID, + databaseId: string, + tableId: UUID, + limit?: number, + ): Promise => + (await queryMutation.mutateAsync( + loose({ dataSourceId, databaseId, tableId, limit }), + )) as PostgresQueryResult, + }), + [listMutation, queryMutation], + ); +} diff --git a/packages/app/src/app/_components/OnboardingView.tsx b/packages/app/src/app/_components/OnboardingView.tsx index dfe49aed..1db48640 100644 --- a/packages/app/src/app/_components/OnboardingView.tsx +++ b/packages/app/src/app/_components/OnboardingView.tsx @@ -28,10 +28,8 @@ export function OnboardingView() { onTableSelect={createInsightFromTable} onInsightSelect={(id, name) => createInsightFromInsight(id, name)} showInsights={true} - // Notion connector is hidden until the integration moves off - // web tRPC (see: https://www.notion.so/360d48ccaf5481749ae1f0eeed29361b). - // The `/api/trpc` endpoint isn't served by Vite. - showNotion={false} + showNotion={true} + showPostgres={true} /> diff --git a/packages/app/src/components/data-sources/AddConnectionPanel.tsx b/packages/app/src/components/data-sources/AddConnectionPanel.tsx index ac71c132..8b2eae2a 100644 --- a/packages/app/src/components/data-sources/AddConnectionPanel.tsx +++ b/packages/app/src/components/data-sources/AddConnectionPanel.tsx @@ -20,12 +20,11 @@ export interface AddConnectionPanelProps { onConnect: ( connector: RemoteApiConnector, credentials: Record, - ) => void; - /** Whether to show Notion connector (feature flag) */ + ) => Promise; + /** Whether to show Notion connector on this surface. */ showNotion?: boolean; /** - * Whether to show Postgres connector (feature flag, default false). - * Hidden until the renderer-side connect UI is complete. + * Whether to show Postgres connector on this surface. */ showPostgres?: boolean; } diff --git a/packages/app/src/components/data-sources/DataPickerContent.tsx b/packages/app/src/components/data-sources/DataPickerContent.tsx index 552dac87..83ab703c 100644 --- a/packages/app/src/components/data-sources/DataPickerContent.tsx +++ b/packages/app/src/components/data-sources/DataPickerContent.tsx @@ -1,15 +1,26 @@ import { getConnectorById } from "@/lib/connectors/registry"; import { handleFileConnectorResult } from "@/lib/local-csv-handler"; +import { + connectRemoteSource, + type RemoteResource, + type SupportedRemoteConnectorId, +} from "@/lib/remote-connector-onboarding"; +import { materializeRemoteTable } from "@/lib/remote-table-materialization"; import { useDataFrames, + useDataSourceMutations, useDataSources, + useDataTableMutations, useDataTables, useInsights, + useNotionMutations, + usePostgresMutations, } from "@dashframe/core"; import type { FileSourceConnector, RemoteApiConnector, } from "@dashframe/engine"; +import type { UUID } from "@dashframe/types"; import { Button, SectionList } from "@wystack/ui"; import { ArrowLeftIcon } from "@wystack/ui-icons"; import { useCallback, useMemo, useState } from "react"; @@ -42,12 +53,12 @@ export interface DataPickerContentProps { onCancel?: () => void; /** * Whether to show Notion connection option - * @default false + * @default true */ showNotion?: boolean; /** - * Whether to show Postgres connection option (connect UI not yet wired) - * @default false + * Whether to show Postgres connection option + * @default true */ showPostgres?: boolean; /** @@ -57,6 +68,12 @@ export interface DataPickerContentProps { showInsights?: boolean; } +interface RemoteResourceState { + connectorId: SupportedRemoteConnectorId; + sourceId: UUID; + resources: RemoteResource[]; +} + /** * Reusable data picker content for selecting insights or tables. * @@ -73,18 +90,27 @@ export function DataPickerContent({ excludeInsightIds = [], excludeTableIds = [], onCancel, - showNotion = false, - showPostgres = false, + showNotion = true, + showPostgres = true, showInsights = true, }: DataPickerContentProps) { const { data: dataSources = [] } = useDataSources(); const { data: allDataTables = [] } = useDataTables(); const { data: allInsights = [] } = useInsights(); const { data: dataFrames = [] } = useDataFrames(); + const dataSourceMutations = useDataSourceMutations(); + const tableMutations = useDataTableMutations(); + const notionMutations = useNotionMutations(); + const postgresMutations = usePostgresMutations(); // Local state const [selectedSourceId, setSelectedSourceId] = useState(null); const [error, setError] = useState(null); + const [remoteResourceState, setRemoteResourceState] = + useState(null); + const [materializingResourceId, setMaterializingResourceId] = useState< + string | null + >(null); // Transform sources for DataSourceList const dataSourcesInfo: DataSourceInfo[] = useMemo(() => { @@ -215,15 +241,75 @@ export function DataPickerContent({ [onTableSelect, allDataTables], ); - // Handle remote connector form submission (Notion, Airtable, etc.). - // The renderer never calls connector.connect(); the credential is resolved - // server-side. Full implementation: create the DataSource, then list databases - // via the listNotionDatabases mutation. Stubbed pending the add-remote flow. const handleConnect = useCallback( - (connector: RemoteApiConnector, _credentials: Record) => { - console.log(`Connector form submitted: ${connector.name}`); + async ( + connector: RemoteApiConnector, + credentials: Record, + ) => { + if (connector.id !== "notion" && connector.id !== "postgres") { + throw new Error(`${connector.name} onboarding is not supported yet`); + } + + setError(null); + setRemoteResourceState( + await connectRemoteSource({ + connectorId: connector.id, + connectorName: connector.name, + credentials, + addSource: dataSourceMutations.add, + removeSource: dataSourceMutations.remove, + listNotionDatabases: notionMutations.listDatabases, + listPostgresTables: postgresMutations.listTables, + }), + ); }, - [], + [dataSourceMutations, notionMutations, postgresMutations], + ); + + const handleRemoteResourceSelect = useCallback( + async (resource: { id: string; title: string }) => { + if (!remoteResourceState) return; + setMaterializingResourceId(resource.id); + setError(null); + let tableId: UUID | null = null; + try { + tableId = await tableMutations.add( + remoteResourceState.sourceId, + resource.title, + resource.id, + ); + const result = + remoteResourceState.connectorId === "notion" + ? await notionMutations.queryDatabase( + remoteResourceState.sourceId, + resource.id, + tableId, + ) + : await postgresMutations.queryTable( + remoteResourceState.sourceId, + resource.id, + tableId, + ); + await materializeRemoteTable({ id: tableId }, result, resource.title); + onTableSelect(tableId, resource.title); + } catch (cause) { + if (tableId) await tableMutations.remove(tableId).catch(() => {}); + setError( + cause instanceof Error + ? cause.message + : "Failed to import remote table", + ); + } finally { + setMaterializingResourceId(null); + } + }, + [ + notionMutations, + onTableSelect, + postgresMutations, + remoteResourceState, + tableMutations, + ], ); const hasInsights = @@ -273,7 +359,7 @@ export function DataPickerContent({ )} {/* Section: Add New Source */} - {!selectedSourceId && ( + {!selectedSourceId && !remoteResourceState && ( )} + + {remoteResourceState && !selectedSourceId && ( + +
+ {remoteResourceState.resources.length === 0 ? ( +

+ The connection succeeded, but no databases or tables were + found. +

+ ) : ( + remoteResourceState.resources.map((resource) => ( +
+ + )} {/* Footer */} diff --git a/packages/app/src/components/data-sources/DataSourceDisplay.tsx b/packages/app/src/components/data-sources/DataSourceDisplay.tsx index d0cc5ff3..713bc930 100644 --- a/packages/app/src/components/data-sources/DataSourceDisplay.tsx +++ b/packages/app/src/components/data-sources/DataSourceDisplay.tsx @@ -1,16 +1,12 @@ import { useDataFrameData } from "@/hooks/useDataFrameData"; import { getConnectorById } from "@/lib/connectors/registry"; +import { materializeRemoteTable } from "@/lib/remote-table-materialization"; import { - addDataFrameEntry, - getDataTable, - replaceDataFrame, - updateDataTable, useDataSources, useDataTables, useNotionMutations, } from "@dashframe/core"; -import { DataFrame } from "@dashframe/engine-browser"; -import type { DataTable, Field, UUID } from "@dashframe/types"; +import type { DataTable, Field } from "@dashframe/types"; import { VirtualTable, type VirtualTableColumn } from "@dashframe/ui"; import { Button, @@ -480,72 +476,6 @@ function NotionDataSourceView({ ); } -// Decode a base64 Arrow IPC buffer (as returned by the server) into bytes the -// browser engine can persist. Lives at module scope so it's testable in -// isolation and out of the hook's complexity budget. -function decodeBase64ToBytes(base64: string): Uint8Array { - const binary = atob(base64); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - return bytes; -} - -interface MaterializedNotion { - dataFrameId: UUID; - rowCount: number; - columnCount: number; -} - -// Persist the server's serializable Notion result as a durable browser -// DataFrame and link it to the DataTable. This is what makes an added/synced -// Notion source survive a reload: the Arrow bytes land in IndexedDB, the -// DataFrame entry is registered, and the table is updated with its fields + -// dataFrameId (replacing any prior frame). Mirrors the local-CSV ingest path. -async function materializeNotionTable( - table: { id: string }, - result: { - arrowBuffer: string; - fieldIds: string[]; - fields: Field[]; - rowCount: number; - }, - name: string, -): Promise { - const fields = result.fields; - const columnCount = result.fieldIds.length; - const bytes = decodeBase64ToBytes(result.arrowBuffer); - const dataFrame = await DataFrame.create(bytes, result.fieldIds as UUID[]); - - // Re-read the table to resolve its current frame (avoids a stale prop on - // re-sync) and replace-in-place rather than orphaning the old Arrow blob. - const existing = await getDataTable(table.id as UUID); - let dataFrameId: UUID; - if (existing?.dataFrameId) { - await replaceDataFrame(existing.dataFrameId, dataFrame, { - rowCount: result.rowCount, - columnCount, - }); - dataFrameId = existing.dataFrameId; - } else { - await addDataFrameEntry(dataFrame, { - name, - rowCount: result.rowCount, - columnCount, - }); - dataFrameId = dataFrame.id as UUID; - } - - // Persist fields AND the frame link, then stamp lastFetchedAt. After this the - // table has a real schema and a durable frame — both survive a reload. - await updateDataTable(table.id as UUID, { - fields, - dataFrameId, - lastFetchedAt: Date.now(), - }); - - return { dataFrameId, rowCount: result.rowCount, columnCount }; -} - // Hook that owns Notion query state and the server-side query call. // Extracted from DataSourceDisplay so the branch-heavy async body does not // contribute toward the component function's cognitive complexity budget. @@ -602,7 +532,7 @@ function useNotionSync( // Persist the frame + fields BEFORE reporting success, so a reload finds // the data and schema intact (not just a refreshed timestamp). - const materialized = await materializeNotionTable( + const materialized = await materializeRemoteTable( selectedDataTable, result, selectedDataTable.name, diff --git a/packages/app/src/components/data-sources/renderers/ConnectorCardWithForm.tsx b/packages/app/src/components/data-sources/renderers/ConnectorCardWithForm.tsx index ae7df0d7..b82e8959 100644 --- a/packages/app/src/components/data-sources/renderers/ConnectorCardWithForm.tsx +++ b/packages/app/src/components/data-sources/renderers/ConnectorCardWithForm.tsx @@ -26,7 +26,7 @@ interface ConnectorCardWithFormProps { onConnect: ( connector: RemoteApiConnector, credentials: Record, - ) => void; + ) => Promise; } /** @@ -83,10 +83,7 @@ export function ConnectorCardWithForm({ // resolver throws by design). execute() validates the form and returns the // credential values; the parent creates the DataSource (storing the key as a // vault SecretRef) and lists databases via the listNotionDatabases mutation. - const credentials = await execute(async (data) => data); - if (credentials) { - onConnect(connector, credentials); - } + await execute((data) => onConnect(connector, data)); }; return ( diff --git a/packages/app/src/lib/connectors/registry.ts b/packages/app/src/lib/connectors/registry.ts index e56c16fd..c268dcd0 100644 --- a/packages/app/src/lib/connectors/registry.ts +++ b/packages/app/src/lib/connectors/registry.ts @@ -119,14 +119,12 @@ export function clearConnectorRegistry(): void { * Options for filtering connectors */ export interface GetConnectorsOptions { - /** Show Notion connector (default: false for feature flag control) */ + /** Show Notion connector (default: false for explicit surface control) */ showNotion?: boolean; /** * Show Postgres connector (default: false). * - * Postgres is registered for static metadata and server-side query routing - * but the renderer-side connect() flow is not yet wired. Keep hidden until - * the connect UI is complete. + * Postgres is registered for static metadata and server-side query routing. */ showPostgres?: boolean; /** Filter by source type */ diff --git a/packages/app/src/lib/remote-connector-onboarding.test.ts b/packages/app/src/lib/remote-connector-onboarding.test.ts new file mode 100644 index 00000000..57b25754 --- /dev/null +++ b/packages/app/src/lib/remote-connector-onboarding.test.ts @@ -0,0 +1,62 @@ +import type { UUID } from "@dashframe/types"; +import { describe, expect, it, vi } from "vitest"; + +import { connectRemoteSource } from "./remote-connector-onboarding"; + +const SOURCE_ID = "11111111-1111-4111-8111-111111111111" as UUID; + +describe("connectRemoteSource", () => { + it("creates and probes a Postgres source with non-secret config separated", async () => { + const addSource = vi.fn(async () => SOURCE_ID); + const removeSource = vi.fn(async () => {}); + const listPostgresTables = vi.fn(async () => [ + { id: "analytics.orders", title: "orders" }, + ]); + + const result = await connectRemoteSource({ + connectorId: "postgres", + connectorName: "Postgres", + credentials: { + connectionString: "postgres://user:secret@host/db", + defaultSchema: " analytics ", + }, + addSource, + removeSource, + listNotionDatabases: vi.fn(), + listPostgresTables, + }); + + expect(addSource).toHaveBeenCalledWith({ + type: "postgres", + name: "Postgres", + apiKey: undefined, + connectionString: "postgres://user:secret@host/db", + config: { defaultSchema: "analytics" }, + }); + expect(listPostgresTables).toHaveBeenCalledWith(SOURCE_ID); + expect(removeSource).not.toHaveBeenCalled(); + expect(result.resources).toEqual([ + { id: "analytics.orders", title: "orders" }, + ]); + }); + + it("removes a newly-created source when the connection probe fails", async () => { + const removeSource = vi.fn(async () => {}); + + await expect( + connectRemoteSource({ + connectorId: "notion", + connectorName: "Notion", + credentials: { apiKey: "secret_test" }, + addSource: vi.fn(async () => SOURCE_ID), + removeSource, + listNotionDatabases: vi.fn(async () => { + throw new Error("invalid token"); + }), + listPostgresTables: vi.fn(), + }), + ).rejects.toThrow("invalid token"); + + expect(removeSource).toHaveBeenCalledWith(SOURCE_ID); + }); +}); diff --git a/packages/app/src/lib/remote-connector-onboarding.ts b/packages/app/src/lib/remote-connector-onboarding.ts new file mode 100644 index 00000000..3af87564 --- /dev/null +++ b/packages/app/src/lib/remote-connector-onboarding.ts @@ -0,0 +1,64 @@ +import type { CreateDataSourceInput, UUID } from "@dashframe/types"; + +export type SupportedRemoteConnectorId = "notion" | "postgres"; + +export interface RemoteResource { + id: string; + title: string; +} + +interface ConnectRemoteSourceOptions { + connectorId: SupportedRemoteConnectorId; + connectorName: string; + credentials: Record; + addSource: (input: CreateDataSourceInput) => Promise; + removeSource: (id: UUID) => Promise; + listNotionDatabases: (id: UUID) => Promise; + listPostgresTables: (id: UUID) => Promise; +} + +/** Create, probe, and compensate a remote source as one onboarding operation. */ +export async function connectRemoteSource({ + connectorId, + connectorName, + credentials, + addSource, + removeSource, + listNotionDatabases, + listPostgresTables, +}: ConnectRemoteSourceOptions): Promise<{ + connectorId: SupportedRemoteConnectorId; + sourceId: UUID; + resources: RemoteResource[]; +}> { + const apiKey = + typeof credentials.apiKey === "string" ? credentials.apiKey : undefined; + const connectionString = + typeof credentials.connectionString === "string" + ? credentials.connectionString + : undefined; + const defaultSchema = + typeof credentials.defaultSchema === "string" && + credentials.defaultSchema.trim() + ? credentials.defaultSchema.trim() + : undefined; + + let sourceId: UUID | null = null; + try { + sourceId = await addSource({ + type: connectorId, + name: connectorName, + apiKey, + connectionString, + config: defaultSchema ? { defaultSchema } : undefined, + }); + const resources = + connectorId === "notion" + ? await listNotionDatabases(sourceId) + : await listPostgresTables(sourceId); + return { connectorId, sourceId, resources }; + } catch (cause) { + if (sourceId) await removeSource(sourceId).catch(() => {}); + throw cause; + } +} diff --git a/packages/app/src/lib/remote-table-materialization.ts b/packages/app/src/lib/remote-table-materialization.ts new file mode 100644 index 00000000..8236a3c3 --- /dev/null +++ b/packages/app/src/lib/remote-table-materialization.ts @@ -0,0 +1,66 @@ +import { + addDataFrameEntry, + getDataTable, + replaceDataFrame, + updateDataTable, +} from "@dashframe/core"; +import { DataFrame } from "@dashframe/engine-browser"; +import type { Field, UUID } from "@dashframe/types"; + +export interface RemoteQueryResult { + arrowBuffer: string; + fieldIds: string[]; + fields: Field[]; + rowCount: number; +} + +export interface MaterializedRemoteTable { + dataFrameId: UUID; + rowCount: number; + columnCount: number; +} + +function decodeBase64ToBytes(base64: string): Uint8Array { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes; +} + +/** Persist a server-returned Arrow result and link it to its DataTable. */ +export async function materializeRemoteTable( + table: { id: string }, + result: RemoteQueryResult, + name: string, +): Promise { + const columnCount = result.fieldIds.length; + const dataFrame = await DataFrame.create( + decodeBase64ToBytes(result.arrowBuffer), + result.fieldIds as UUID[], + ); + const existing = await getDataTable(table.id as UUID); + let dataFrameId: UUID; + + if (existing?.dataFrameId) { + await replaceDataFrame(existing.dataFrameId, dataFrame, { + rowCount: result.rowCount, + columnCount, + }); + dataFrameId = existing.dataFrameId; + } else { + await addDataFrameEntry(dataFrame, { + name, + rowCount: result.rowCount, + columnCount, + }); + dataFrameId = dataFrame.id as UUID; + } + + await updateDataTable(table.id as UUID, { + fields: result.fields, + dataFrameId, + lastFetchedAt: Date.now(), + }); + + return { dataFrameId, rowCount: result.rowCount, columnCount }; +} diff --git a/packages/types/src/data-sources.ts b/packages/types/src/data-sources.ts index 81f53b68..c7a136f3 100644 --- a/packages/types/src/data-sources.ts +++ b/packages/types/src/data-sources.ts @@ -53,6 +53,8 @@ export interface CreateDataSourceInput { name: string; apiKey?: string; connectionString?: string; + /** Connector-specific, non-credential settings. */ + config?: Record; } // ============================================================================ From fc3a5efc2dd12d3d177cd6b8ab2e79fd05c0e23d Mon Sep 17 00:00:00 2001 From: Youhao Wei Date: Mon, 13 Jul 2026 21:34:19 -0700 Subject: [PATCH 02/11] fix: recover clawpatch orphaned review locks --- .../data-sources/DataPickerContent.tsx | 51 +++++++++------ scripts/clawpatch.sh | 10 +++ scripts/test-clawpatch-wrapper.sh | 65 +++++++++++++++++++ 3 files changed, 105 insertions(+), 21 deletions(-) create mode 100755 scripts/test-clawpatch-wrapper.sh diff --git a/packages/app/src/components/data-sources/DataPickerContent.tsx b/packages/app/src/components/data-sources/DataPickerContent.tsx index 83ab703c..1e75e67d 100644 --- a/packages/app/src/components/data-sources/DataPickerContent.tsx +++ b/packages/app/src/components/data-sources/DataPickerContent.tsx @@ -372,27 +372,36 @@ export function DataPickerContent({ )} {remoteResourceState && !selectedSourceId && ( - -
- {remoteResourceState.resources.length === 0 ? ( -

- The connection succeeded, but no databases or tables were - found. -

- ) : ( - remoteResourceState.resources.map((resource) => ( -
- + <> + + ); +} + export function InsightConfigPanel({ insight, dataTable, @@ -74,6 +127,7 @@ export function InsightConfigPanel({ name, onNameChange, }: InsightConfigPanelProps) { + const [activeSection, setActiveSection] = useState("model"); // Modal states const [isFieldEditorOpen, setIsFieldEditorOpen] = useState(false); const [isMetricEditorOpen, setIsMetricEditorOpen] = useState(false); @@ -169,6 +223,15 @@ export function InsightConfigPanel({ [insight.filters], ); + const sorts = insight.sorts ?? []; + + const handleSortsChange = useCallback( + (nextSorts: InsightSort[]) => { + updateInsight(insight.id, { sorts: nextSorts }); + }, + [insight.id, updateInsight], + ); + // --- Field handlers --- const handleFieldsReorder = useCallback( (newOrder: string[]) => { @@ -366,46 +429,111 @@ export function InsightConfigPanel({ return ( - +
+
+ +
+
+ } + label="Model" + onClick={() => setActiveSection("model")} + /> + } + label="Fields" + onClick={() => setActiveSection("fields")} + /> + } + label="Metrics" + onClick={() => setActiveSection("metrics")} + /> + } + label="Filters" + onClick={() => setActiveSection("filters")} + /> + } + label="Sort" + onClick={() => setActiveSection("sort")} + /> +
} > -
- {/* Fields Section */} - setIsFieldEditorOpen(true)} - /> - - {/* Metrics Section */} - setIsMetricEditorOpen(true)} - /> - - {/* Filters Section */} - setFilterToEdit("new")} - /> +
+ {activeSection === "model" && ( +
+ +
+ )} + {activeSection === "fields" && ( + setIsFieldEditorOpen(true)} + embedded + /> + )} + {activeSection === "metrics" && ( + setIsMetricEditorOpen(true)} + embedded + /> + )} + {activeSection === "filters" && ( + setFilterToEdit("new")} + embedded + /> + )} + {activeSection === "sort" && ( + + )}
void; onAddClick: () => void; defaultOpen?: boolean; + embedded?: boolean; } /** @@ -44,6 +45,7 @@ export function MetricsSection({ onEditClick, onAddClick, defaultOpen = true, + embedded = false, }: MetricsSectionProps) { const [isOpen, setIsOpen] = useState(defaultOpen); @@ -61,6 +63,49 @@ export function MetricsSection({ [onReorder], ); + const content = ( +
+ {embedded && ( +
+
+

Metrics

+

+ Define the values to aggregate. +

+
+
+ )} + {sortableItems.length > 0 ? ( + ( + onRemove(item.id)} + onEditClick={() => onEditClick(item.metric)} + /> + )} + /> + ) : ( +

+ No metrics configured. +

+ )} +
+ ); + + if (embedded) return content; + return (
@@ -95,29 +140,7 @@ export function MetricsSection({ onClick={onAddClick} />
- -
- {sortableItems.length > 0 ? ( - ( - onRemove(item.id)} - onEditClick={() => onEditClick(item.metric)} - /> - )} - /> - ) : ( -

- No metrics configured. -

- )} -
- + {content}
); diff --git a/packages/app/src/app/insights/[insightId]/_components/config-panel/SortSection.test.tsx b/packages/app/src/app/insights/[insightId]/_components/config-panel/SortSection.test.tsx new file mode 100644 index 00000000..2e44fd5a --- /dev/null +++ b/packages/app/src/app/insights/[insightId]/_components/config-panel/SortSection.test.tsx @@ -0,0 +1,49 @@ +import type { CombinedField } from "@/lib/insights/compute-combined-fields"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import { SortSection } from "./SortSection"; + +const field = { + id: "field-1", + name: "Created at", + columnName: "created_at", + displayName: "Created at", + type: "date", + sourceTableId: "table-1", +} as CombinedField; + +describe("SortSection", () => { + it("adds the first available result field as an ascending sort", () => { + const onChange = vi.fn(); + + render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Add" })); + + expect(onChange).toHaveBeenCalledWith([ + { field: "created_at", direction: "asc" }, + ]); + }); + + it("does not offer another sort when all available fields are used", () => { + render( + , + ); + + expect( + screen.getByRole("button", { name: "Add" }).hasAttribute("disabled"), + ).toBe(true); + }); +}); diff --git a/packages/app/src/app/insights/[insightId]/_components/config-panel/SortSection.tsx b/packages/app/src/app/insights/[insightId]/_components/config-panel/SortSection.tsx new file mode 100644 index 00000000..10d8f873 --- /dev/null +++ b/packages/app/src/app/insights/[insightId]/_components/config-panel/SortSection.tsx @@ -0,0 +1,136 @@ +import type { CombinedField } from "@/lib/insights/compute-combined-fields"; +import type { InsightMetric, InsightSort } from "@dashframe/types"; +import { + Button, + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@wystack/ui"; +import { CloseIcon, PlusIcon } from "@wystack/ui-icons"; + +interface SortSectionProps { + sorts: InsightSort[]; + fields: CombinedField[]; + metrics: InsightMetric[]; + onChange: (sorts: InsightSort[]) => void; +} + +export function SortSection({ + sorts, + fields, + metrics, + onChange, +}: SortSectionProps) { + const options = [ + ...fields.map((field) => ({ + value: field.columnName ?? field.name, + label: field.displayName, + })), + ...metrics.map((metric) => ({ value: metric.name, label: metric.name })), + ]; + + const addSort = () => { + const firstUnused = options.find( + (option) => !sorts.some((sort) => sort.field === option.value), + ); + if (firstUnused) + onChange([...sorts, { field: firstUnused.value, direction: "asc" }]); + }; + + return ( +
+
+
+

Sort

+

+ Set the default order of the result. +

+
+
+ + {sorts.length === 0 ? ( +

+ No default sort configured. +

+ ) : ( +
+ {sorts.map((sort, index) => ( +
+ + +
+ ))} +
+ )} +
+ ); +} diff --git a/packages/app/src/app/insights/[insightId]/_components/sections/DataModelSection.tsx b/packages/app/src/app/insights/[insightId]/_components/sections/DataModelSection.tsx index c88ba8b5..6d6eb673 100644 --- a/packages/app/src/app/insights/[insightId]/_components/sections/DataModelSection.tsx +++ b/packages/app/src/app/insights/[insightId]/_components/sections/DataModelSection.tsx @@ -22,13 +22,14 @@ import { ExternalLinkIcon, PlusIcon, } from "@wystack/ui-icons"; -import { memo, useCallback, useMemo, useState } from "react"; +import { memo, useCallback, useMemo, useState, type ReactNode } from "react"; interface DataModelSectionProps { insight: Insight; dataTable: DataTable; allDataTables: DataTable[]; combinedFieldCount: number; + compact?: boolean; } /** @@ -48,7 +49,8 @@ interface DataModelSectionProps { * any unregistered type. `size` controls the rendered dimensions. */ function getSourceTypeIcon(type: string, size: "sm" | "xs") { - const className = size === "sm" ? "h-4 w-4" : "h-3 w-3"; + const className = + size === "sm" ? "inline-block h-4 w-4" : "inline-block h-3 w-3"; const connector = getConnectorById(type); if (connector) { return ; @@ -93,11 +95,59 @@ function getDisplayFileName(table: DataTable, maxLength = 24): string { return fullName.slice(0, startLength) + "..." + fullName.slice(-endLength); } +function CompactDataModelItem({ item }: { item: ListItem }) { + const ItemIcon = typeof item.icon === "function" ? item.icon : null; + const iconNode: ReactNode = ItemIcon ? ( + + ) : ( + (item.icon as ReactNode) + ); + + return ( +
+
+
+ {iconNode} +
+
+
+

+ {item.title} +

+ {item.badge && ( + + {item.badge} + + )} +
+
{item.content}
+
+ {item.actions?.map((action) => { + const ActionIcon = action.icon; + return ( + + ); + })} +
+
+ ); +} + export const DataModelSection = memo(function DataModelSection({ insight, dataTable, allDataTables, combinedFieldCount, + compact = false, }: DataModelSectionProps) { const navigate = useNavigate(); const [isJoinFlowOpen, setIsJoinFlowOpen] = useState(false); @@ -281,9 +331,12 @@ export const DataModelSection = memo(function DataModelSection({ > : undefined + } emptyMessage="No data sources" emptyIcon={} /> From d877466713fb4b7d0e73e8cbb1bcf192aefb04d8 Mon Sep 17 00:00:00 2001 From: Youhao Wei Date: Tue, 14 Jul 2026 17:05:16 -0700 Subject: [PATCH 08/11] polish insight model navigation --- bun.lock | 1 + packages/app/package.json | 1 + .../config-panel/InsightConfigPanel.tsx | 26 +++++++++---------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/bun.lock b/bun.lock index ca2fc961..876cdf2c 100644 --- a/bun.lock +++ b/bun.lock @@ -419,6 +419,7 @@ "dompurify": "^3.2.4", "idb-keyval": "^6.2.2", "immer": "^11.0.1", + "lucide-react": "^0.563.0", "papaparse": "5.5.3", "react": "19.2.4", "react-dom": "19.2.4", diff --git a/packages/app/package.json b/packages/app/package.json index 34a8b102..9c54a3b9 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -53,6 +53,7 @@ "dompurify": "^3.2.4", "idb-keyval": "^6.2.2", "immer": "^11.0.1", + "lucide-react": "^0.563.0", "papaparse": "5.5.3", "react": "19.2.4", "react-dom": "19.2.4", diff --git a/packages/app/src/app/insights/[insightId]/_components/config-panel/InsightConfigPanel.tsx b/packages/app/src/app/insights/[insightId]/_components/config-panel/InsightConfigPanel.tsx index 0e13c4c6..1527b94e 100644 --- a/packages/app/src/app/insights/[insightId]/_components/config-panel/InsightConfigPanel.tsx +++ b/packages/app/src/app/insights/[insightId]/_components/config-panel/InsightConfigPanel.tsx @@ -19,12 +19,12 @@ import type { import { InputField } from "@dashframe/ui"; import { Badge, Panel, cn } from "@wystack/ui"; import { - CalculatorIcon, - DatabaseIcon, - ListIcon, - NumberTypeIcon, - SettingsIcon, -} from "@wystack/ui-icons"; + ArrowUpDown, + Columns3, + ListFilter, + Sigma, + Workflow, +} from "lucide-react"; import { useCallback, useMemo, useState, type ReactNode } from "react"; import { DataModelSection } from "../sections/DataModelSection"; import { @@ -99,7 +99,7 @@ function ConfigSectionButton({ type="button" onClick={onClick} className={cn( - "flex h-9 min-w-0 flex-[1_1_7rem] items-center gap-1.5 rounded-md px-2 text-xs font-medium transition-colors", + "flex h-9 min-w-0 w-full items-center gap-1.5 rounded-md px-2 text-xs font-medium transition-colors", "focus-visible:ring-2 focus-visible:ring-palette-primary focus-visible:outline-none", active ? "bg-neutral-bg-emphasis text-neutral-fg shadow-sm" @@ -440,41 +440,41 @@ export function InsightConfigPanel({ />
} + icon={} label="Model" onClick={() => setActiveSection("model")} /> } + icon={} label="Fields" onClick={() => setActiveSection("fields")} /> } + icon={} label="Metrics" onClick={() => setActiveSection("metrics")} /> } + icon={} label="Filters" onClick={() => setActiveSection("filters")} /> } + icon={} label="Sort" onClick={() => setActiveSection("sort")} /> From c264e18e68a04b10aedbb650c0ec61e54b041507 Mon Sep 17 00:00:00 2001 From: Youhao Wei Date: Tue, 14 Jul 2026 17:18:00 -0700 Subject: [PATCH 09/11] fix chart renderer blinking --- .../src/components/Chart.test.tsx | 43 +++++++++++++++++++ .../visualization/src/components/Chart.tsx | 10 ++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/packages/visualization/src/components/Chart.test.tsx b/packages/visualization/src/components/Chart.test.tsx index eceee2ba..5b845755 100644 --- a/packages/visualization/src/components/Chart.test.tsx +++ b/packages/visualization/src/components/Chart.test.tsx @@ -127,4 +127,47 @@ describe("Chart renderer resolution", () => { // No render happened — the wrong-engine flash is prevented. expect(calls).toHaveLength(0); }); + + it("keeps the renderer mounted when an equivalent encoding object is recreated", () => { + const cleanup = vi.fn(); + const renderer: ChartRenderer = { + supportedTypes: ["barY"] as readonly VisualizationType[], + render: vi.fn(() => cleanup), + }; + mockUseVisualization.mockReturnValue({ + renderer, + isReady: true, + error: null, + }); + + const { rerender } = render( + , + ); + + rerender( + , + ); + + expect(renderer.render).toHaveBeenCalledTimes(1); + expect(cleanup).not.toHaveBeenCalled(); + + rerender( + , + ); + + expect(renderer.render).toHaveBeenCalledTimes(2); + expect(cleanup).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/visualization/src/components/Chart.tsx b/packages/visualization/src/components/Chart.tsx index 1a704e1a..4d404218 100644 --- a/packages/visualization/src/components/Chart.tsx +++ b/packages/visualization/src/components/Chart.tsx @@ -188,6 +188,12 @@ export function Chart({ }: ChartProps) { const cleanupRef = useRef<(() => void) | null>(null); + // Suggestions and persisted visualizations can recreate an equivalent + // encoding object during unrelated store updates. Key renderer lifecycle to + // the serialized value instead of object identity so those renders do not + // tear down a healthy chart and briefly leave its container empty. + const serializedEncoding = JSON.stringify(encoding); + // Track chart colors to detect theme changes const chartColors = useChartColors(); @@ -276,7 +282,7 @@ export function Chart({ // Build config with resolved dimensions const config: ChartConfig = { tableName, - encoding, + encoding: JSON.parse(serializedEncoding) as ChartEncoding, width: resolvedWidth, height: resolvedHeight, preview, @@ -310,7 +316,7 @@ export function Chart({ }, [ tableName, visualizationType, - encoding, + serializedEncoding, resolvedWidth, resolvedHeight, preview, From 665482df89aecc5dc6c0fdf68adb1a09768a44c6 Mon Sep 17 00:00:00 2001 From: Youhao Wei Date: Tue, 14 Jul 2026 17:28:22 -0700 Subject: [PATCH 10/11] clarify insight view controls --- .../[insightId]/_components/InsightView.tsx | 89 +++++++++++-------- packages/ui/src/components/ControlTooltip.tsx | 42 +++++++++ packages/ui/src/index.ts | 5 ++ 3 files changed, 101 insertions(+), 35 deletions(-) create mode 100644 packages/ui/src/components/ControlTooltip.tsx diff --git a/packages/app/src/app/insights/[insightId]/_components/InsightView.tsx b/packages/app/src/app/insights/[insightId]/_components/InsightView.tsx index 6ab70c79..7638e348 100644 --- a/packages/app/src/app/insights/[insightId]/_components/InsightView.tsx +++ b/packages/app/src/app/insights/[insightId]/_components/InsightView.tsx @@ -47,6 +47,7 @@ import type { import { CHART_TYPE_METADATA } from "@dashframe/types"; import { CHART_ICONS, + ControlTooltip, VirtualTable, type VirtualTableColumnConfig, } from "@dashframe/ui"; @@ -54,7 +55,6 @@ import { Chart } from "@dashframe/visualization"; import { useNavigate } from "@tanstack/react-router"; import { Button, cn } from "@wystack/ui"; import { - CheckIcon, DashboardIcon, PlusIcon, SparklesIcon, @@ -679,34 +679,35 @@ function CanvasViewButton({ muted = false, icon, label, + description, onClick, }: { active: boolean; muted?: boolean; icon: ReactNode; label: string; + description: string; onClick: () => void; }) { return ( - + + + ); } @@ -1363,8 +1364,8 @@ export function InsightView({ autoPinAttemptRef.current = insightId; pinChartSuggestion(firstChartSuggestion).catch((error) => { - console.error("[InsightView] Auto-pin failed:", error); - toast.error("Couldn't pin the chart view"); + console.error("[InsightView] Auto-save failed:", error); + toast.error("Couldn't save the chart"); }); }, [ firstChartSuggestion, @@ -1378,10 +1379,10 @@ export function InsightView({ if (!activeChartSuggestion) return; try { await pinChartSuggestion(activeChartSuggestion); - toast.success("Chart view pinned"); + toast.success("Chart saved"); } catch (error) { - console.error("[InsightView] Pin failed:", error); - toast.error("Couldn't pin chart view"); + console.error("[InsightView] Save failed:", error); + toast.error("Couldn't save the chart"); } }, [activeChartSuggestion, pinChartSuggestion]); @@ -1488,13 +1489,13 @@ export function InsightView({ if (activeView.kind === "chart") { activeViewLabel = CHART_TYPE_METADATA[activeView.chartType].displayName; } else if (activeView.kind === "visualization") { - activeViewLabel = activeVisualization?.name ?? "Pinned view"; + activeViewLabel = activeVisualization?.name ?? "Saved chart"; } let activeViewDescription = "Rows produced by the current data model"; if (activeView.kind === "chart") { - activeViewDescription = "Unpinned chart view"; + activeViewDescription = "Chart preview — changes are not saved"; } else if (activeView.kind === "visualization") { - activeViewDescription = "Pinned visualization"; + activeViewDescription = "Saved chart — reusable in dashboards"; } const canPinActiveChart = activeView.kind === "chart" && activeChartSuggestion !== undefined; @@ -1532,12 +1533,14 @@ export function InsightView({ active={activeView.kind === "table"} icon={} label="Data" + description="View the rows produced by the current data model." onClick={() => handleSetActiveView(TABLE_CANVAS_VIEW)} /> } label="Visualize" + description="Explore chart types before saving a chart." onClick={handleSelectVisualMode} />
@@ -1552,13 +1555,18 @@ export function InsightView({
{canPinActiveChart && ( -