diff --git a/apps/api/.env.example b/apps/api/.env.example index ac0418d..59be6df 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -8,3 +8,19 @@ RESEND_API_KEY=re_xxxxx RESEND_FROM_EMAIL=Verio GOOGLE_CLIENT_ID=xxxxx GOOGLE_CLIENT_SECRET=xxxxx + +# Import pipeline +# Leave false to run every connector against built-in fixtures. The full pipeline — +# extraction, mapping, matching, loading — works end to end with no provider credentials. +INTEGRATION_LIVE_FETCH_ENABLED=false +# Encrypts stored OAuth and API tokens. Falls back to BETTER_AUTH_SECRET when unset. +INTEGRATION_TOKEN_SECRET= + +# Outlook / Microsoft Graph (contacts, sent mail, calendar through one grant) +MICROSOFT_CLIENT_ID=xxxxx +MICROSOFT_CLIENT_SECRET=xxxxx +MICROSOFT_TENANT_ID=common + +# Use https://eu.posthog.com for EU projects, or a self-hosted host +POSTHOG_API_HOST=https://us.posthog.com +CALENDLY_API_HOST=https://api.calendly.com diff --git a/apps/api/package.json b/apps/api/package.json index 5e61133..7c1da82 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -8,6 +8,7 @@ "build": "tsc", "start": "bun run src/server.ts", "worker:sequences": "bun run src/workers/sequence-worker.ts", + "worker:imports": "bun run src/workers/import-worker.ts", "check-types": "tsc --noEmit && tsc --noEmit -p test/tsconfig.json", "lint": "eslint .", "test": "TZ=UTC vitest run", diff --git a/apps/api/src/config/env.config.ts b/apps/api/src/config/env.config.ts index 8f694e8..1374993 100644 --- a/apps/api/src/config/env.config.ts +++ b/apps/api/src/config/env.config.ts @@ -13,6 +13,13 @@ const rawEnv = { GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET, SEQUENCE_LIVE_SEND_ENABLED: process.env.SEQUENCE_LIVE_SEND_ENABLED, GROQ_API_KEY: process.env.GROQ_API_KEY, + INTEGRATION_LIVE_FETCH_ENABLED: process.env.INTEGRATION_LIVE_FETCH_ENABLED, + INTEGRATION_TOKEN_SECRET: process.env.INTEGRATION_TOKEN_SECRET, + MICROSOFT_CLIENT_ID: process.env.MICROSOFT_CLIENT_ID, + MICROSOFT_CLIENT_SECRET: process.env.MICROSOFT_CLIENT_SECRET, + MICROSOFT_TENANT_ID: process.env.MICROSOFT_TENANT_ID, + POSTHOG_API_HOST: process.env.POSTHOG_API_HOST, + CALENDLY_API_HOST: process.env.CALENDLY_API_HOST, }; const parsedEnv = apiEnvSchema.safeParse(rawEnv); diff --git a/apps/api/src/controllers/imports.controller.ts b/apps/api/src/controllers/imports.controller.ts new file mode 100644 index 0000000..0e35b09 --- /dev/null +++ b/apps/api/src/controllers/imports.controller.ts @@ -0,0 +1,185 @@ +import type { Context } from "hono"; +import type { + CreateConnectionInput, + ListImportJobsQuery, + ListImportRecordsQuery, + StartImportJobInput, + UpdateConnectionInput, + UpdateImportJobInput, + WebhookIngestInput, +} from "@workspace/validators/schemas/import"; +import type { ImportEntityType } from "@workspace/validators/types/import"; +import { STATUS_CODES } from "@/constants/status-codes.js"; +import { sendSuccess } from "@/lib/api-response.js"; +import { AppError } from "@/lib/app-error.js"; +import { getSessionWorkspaceId } from "@/lib/workspace.js"; +import { + cancelImportJob, + commitImportJob, + createApiKey, + createConnection, + deleteConnection, + getImportJob, + ingestWebhookRecords, + listApiKeys, + listConnections, + listImportJobs, + listImportRecords, + previewSourceFields, + resolveApiKey, + revokeApiKey, + startImportJob, + updateConnection, + updateImportJob, +} from "@/services/import.service.js"; + +export async function listConnectionsController(c: Context) { + const workspaceId = getSessionWorkspaceId(c); + const connections = await listConnections(workspaceId); + + return sendSuccess(c, { connections }, STATUS_CODES.OK); +} + +export async function createConnectionController(c: Context, payload: CreateConnectionInput) { + const workspaceId = getSessionWorkspaceId(c); + const user = c.get("user"); + const connection = await createConnection(workspaceId, user.id, payload); + + return sendSuccess(c, { connection }, STATUS_CODES.CREATED); +} + +export async function updateConnectionController( + c: Context, + id: string, + payload: UpdateConnectionInput, +) { + const workspaceId = getSessionWorkspaceId(c); + const connection = await updateConnection(workspaceId, id, payload); + + return sendSuccess(c, { connection }, STATUS_CODES.OK); +} + +export async function deleteConnectionController(c: Context, id: string) { + const workspaceId = getSessionWorkspaceId(c); + const connection = await deleteConnection(workspaceId, id); + + return sendSuccess(c, { connection }, STATUS_CODES.OK); +} + +export async function previewConnectionFieldsController( + c: Context, + id: string, + entityType: ImportEntityType, +) { + const workspaceId = getSessionWorkspaceId(c); + const preview = await previewSourceFields(workspaceId, id, entityType); + + return sendSuccess(c, preview, STATUS_CODES.OK); +} + +export async function listApiKeysController(c: Context) { + const workspaceId = getSessionWorkspaceId(c); + const apiKeys = await listApiKeys(workspaceId); + + return sendSuccess(c, { apiKeys }, STATUS_CODES.OK); +} + +export async function createApiKeyController(c: Context, payload: { name: string }) { + const workspaceId = getSessionWorkspaceId(c); + const user = c.get("user"); + const result = await createApiKey(workspaceId, user.id, payload.name); + + return sendSuccess( + c, + result, + STATUS_CODES.CREATED, + "Store this key now. It cannot be shown again.", + ); +} + +export async function revokeApiKeyController(c: Context, id: string) { + const workspaceId = getSessionWorkspaceId(c); + const apiKey = await revokeApiKey(workspaceId, id); + + return sendSuccess(c, { apiKey }, STATUS_CODES.OK); +} + +export async function startImportJobController(c: Context, payload: StartImportJobInput) { + const workspaceId = getSessionWorkspaceId(c); + const user = c.get("user"); + const job = await startImportJob(workspaceId, user.id, payload); + + return sendSuccess(c, { job }, STATUS_CODES.CREATED); +} + +export async function listImportJobsController(c: Context, query: ListImportJobsQuery) { + const workspaceId = getSessionWorkspaceId(c); + const result = await listImportJobs(workspaceId, query); + + return sendSuccess(c, result, STATUS_CODES.OK); +} + +export async function getImportJobController(c: Context, id: string) { + const workspaceId = getSessionWorkspaceId(c); + const job = await getImportJob(workspaceId, id); + + return sendSuccess(c, { job }, STATUS_CODES.OK); +} + +export async function listImportRecordsController( + c: Context, + id: string, + query: ListImportRecordsQuery, +) { + const workspaceId = getSessionWorkspaceId(c); + const result = await listImportRecords(workspaceId, id, query); + + return sendSuccess(c, result, STATUS_CODES.OK); +} + +export async function updateImportJobController( + c: Context, + id: string, + payload: UpdateImportJobInput, +) { + const workspaceId = getSessionWorkspaceId(c); + const job = await updateImportJob(workspaceId, id, payload); + + return sendSuccess(c, { job }, STATUS_CODES.OK); +} + +export async function commitImportJobController(c: Context, id: string) { + const workspaceId = getSessionWorkspaceId(c); + const job = await commitImportJob(workspaceId, id); + + return sendSuccess(c, { job }, STATUS_CODES.ACCEPTED); +} + +export async function cancelImportJobController(c: Context, id: string) { + const workspaceId = getSessionWorkspaceId(c); + const job = await cancelImportJob(workspaceId, id); + + return sendSuccess(c, { job }, STATUS_CODES.OK); +} + +/** + * Unauthenticated by session — this is the endpoint Zapier, Make, n8n, and custom scripts + * push to, so it authenticates with a workspace API key instead. + */ +export async function ingestWebhookController(c: Context, payload: WebhookIngestInput) { + const header = c.req.header("authorization") ?? ""; + const token = header.toLowerCase().startsWith("bearer ") ? header.slice(7).trim() : ""; + + if (token === "") { + throw new AppError("Missing API key", STATUS_CODES.UNAUTHORIZED); + } + + const apiKey = await resolveApiKey(token); + if (!apiKey) { + throw new AppError("Invalid or revoked API key", STATUS_CODES.UNAUTHORIZED); + } + + const result = await ingestWebhookRecords(apiKey.workspaceId, payload); + + return sendSuccess(c, result, STATUS_CODES.ACCEPTED); +} diff --git a/apps/api/src/db/drizzle/0004_public_mesmero.sql b/apps/api/src/db/drizzle/0004_public_mesmero.sql new file mode 100644 index 0000000..8591718 --- /dev/null +++ b/apps/api/src/db/drizzle/0004_public_mesmero.sql @@ -0,0 +1,126 @@ +CREATE TYPE "public"."import_connection_status" AS ENUM('connected', 'reconnect_required', 'disconnected');--> statement-breakpoint +CREATE TYPE "public"."import_entity_type" AS ENUM('person', 'org');--> statement-breakpoint +CREATE TYPE "public"."import_job_status" AS ENUM('pending', 'extracting', 'ready_for_review', 'loading', 'completed', 'failed', 'canceled');--> statement-breakpoint +CREATE TYPE "public"."import_match_reason" AS ENUM('external_identity', 'email', 'domain', 'name', 'none');--> statement-breakpoint +CREATE TYPE "public"."import_provider" AS ENUM('csv', 'webhook', 'gmail', 'google_calendar', 'calendly', 'google_sheets', 'posthog', 'outlook');--> statement-breakpoint +CREATE TYPE "public"."import_record_status" AS ENUM('pending', 'valid', 'invalid', 'loaded', 'skipped', 'duplicate');--> statement-breakpoint +ALTER TYPE "public"."people_source" ADD VALUE 'import';--> statement-breakpoint +CREATE TABLE "external_identities" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "provider" "import_provider" NOT NULL, + "external_id" varchar(255) NOT NULL, + "entity_type" "import_entity_type" NOT NULL, + "person_id" uuid, + "org_id" uuid, + "profile" jsonb DEFAULT '{}'::jsonb NOT NULL, + "last_seen_at" timestamp DEFAULT now() NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "external_identities_workspace_provider_external_unique" UNIQUE("workspace_id","provider","external_id") +); +--> statement-breakpoint +CREATE TABLE "import_jobs" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "connection_id" uuid, + "created_by_id" uuid, + "provider" "import_provider" NOT NULL, + "entity_type" "import_entity_type" DEFAULT 'person' NOT NULL, + "status" "import_job_status" DEFAULT 'pending' NOT NULL, + "mapping" jsonb DEFAULT '{"fields":[]}'::jsonb NOT NULL, + "options" jsonb NOT NULL, + "source_config" jsonb DEFAULT '{}'::jsonb NOT NULL, + "stats" jsonb NOT NULL, + "cursor" jsonb, + "attempts" integer DEFAULT 0 NOT NULL, + "next_attempt_at" timestamp, + "started_at" timestamp, + "completed_at" timestamp, + "last_error" text, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "import_records" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "job_id" uuid NOT NULL, + "external_id" varchar(255), + "row_number" integer NOT NULL, + "raw" jsonb NOT NULL, + "normalized" jsonb, + "status" "import_record_status" DEFAULT 'pending' NOT NULL, + "match_reason" "import_match_reason" DEFAULT 'none' NOT NULL, + "match_person_id" uuid, + "match_org_id" uuid, + "errors" jsonb DEFAULT '[]'::jsonb NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "import_records_job_row_unique" UNIQUE("job_id","row_number") +); +--> statement-breakpoint +CREATE TABLE "integration_api_keys" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "created_by_id" uuid, + "name" varchar(255) NOT NULL, + "token_hash" varchar(128) NOT NULL, + "token_prefix" varchar(16) NOT NULL, + "last_used_at" timestamp, + "revoked_at" timestamp, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "integration_api_keys_token_hash_unique" UNIQUE("token_hash") +); +--> statement-breakpoint +CREATE TABLE "integration_connections" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "user_id" uuid NOT NULL, + "provider" "import_provider" NOT NULL, + "display_name" varchar(255) NOT NULL, + "external_account_id" varchar(255), + "status" "import_connection_status" DEFAULT 'connected' NOT NULL, + "granted_scopes" jsonb DEFAULT '[]'::jsonb NOT NULL, + "access_token_encrypted" text NOT NULL, + "refresh_token_encrypted" text, + "token_expires_at" timestamp, + "config" jsonb DEFAULT '{}'::jsonb NOT NULL, + "cursor" jsonb, + "last_sync_at" timestamp, + "last_error" text, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "integration_connections_workspace_provider_account_unique" UNIQUE("workspace_id","provider","external_account_id") +); +--> statement-breakpoint +ALTER TABLE "external_identities" ADD CONSTRAINT "external_identities_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "external_identities" ADD CONSTRAINT "external_identities_person_id_people_id_fk" FOREIGN KEY ("person_id") REFERENCES "public"."people"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "external_identities" ADD CONSTRAINT "external_identities_org_id_org_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."org"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_jobs" ADD CONSTRAINT "import_jobs_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_jobs" ADD CONSTRAINT "import_jobs_connection_id_integration_connections_id_fk" FOREIGN KEY ("connection_id") REFERENCES "public"."integration_connections"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_jobs" ADD CONSTRAINT "import_jobs_created_by_id_user_id_fk" FOREIGN KEY ("created_by_id") REFERENCES "public"."user"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_records" ADD CONSTRAINT "import_records_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_records" ADD CONSTRAINT "import_records_job_id_import_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."import_jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_records" ADD CONSTRAINT "import_records_match_person_id_people_id_fk" FOREIGN KEY ("match_person_id") REFERENCES "public"."people"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "import_records" ADD CONSTRAINT "import_records_match_org_id_org_id_fk" FOREIGN KEY ("match_org_id") REFERENCES "public"."org"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "integration_api_keys" ADD CONSTRAINT "integration_api_keys_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "integration_api_keys" ADD CONSTRAINT "integration_api_keys_created_by_id_user_id_fk" FOREIGN KEY ("created_by_id") REFERENCES "public"."user"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "integration_connections" ADD CONSTRAINT "integration_connections_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "integration_connections" ADD CONSTRAINT "integration_connections_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "external_identities_workspace_id_idx" ON "external_identities" USING btree ("workspace_id");--> statement-breakpoint +CREATE INDEX "external_identities_person_id_idx" ON "external_identities" USING btree ("person_id");--> statement-breakpoint +CREATE INDEX "external_identities_org_id_idx" ON "external_identities" USING btree ("org_id");--> statement-breakpoint +CREATE INDEX "import_jobs_workspace_id_idx" ON "import_jobs" USING btree ("workspace_id");--> statement-breakpoint +CREATE INDEX "import_jobs_connection_id_idx" ON "import_jobs" USING btree ("connection_id");--> statement-breakpoint +CREATE INDEX "import_jobs_status_next_attempt_idx" ON "import_jobs" USING btree ("status","next_attempt_at");--> statement-breakpoint +CREATE INDEX "import_jobs_provider_idx" ON "import_jobs" USING btree ("provider");--> statement-breakpoint +CREATE INDEX "import_records_workspace_id_idx" ON "import_records" USING btree ("workspace_id");--> statement-breakpoint +CREATE INDEX "import_records_job_status_idx" ON "import_records" USING btree ("job_id","status");--> statement-breakpoint +CREATE INDEX "import_records_external_id_idx" ON "import_records" USING btree ("external_id");--> statement-breakpoint +CREATE INDEX "integration_api_keys_workspace_id_idx" ON "integration_api_keys" USING btree ("workspace_id");--> statement-breakpoint +CREATE INDEX "integration_connections_workspace_id_idx" ON "integration_connections" USING btree ("workspace_id");--> statement-breakpoint +CREATE INDEX "integration_connections_user_id_idx" ON "integration_connections" USING btree ("user_id");--> statement-breakpoint +CREATE INDEX "integration_connections_provider_idx" ON "integration_connections" USING btree ("provider");--> statement-breakpoint +CREATE INDEX "integration_connections_status_idx" ON "integration_connections" USING btree ("status"); \ No newline at end of file diff --git a/apps/api/src/db/drizzle/meta/0004_snapshot.json b/apps/api/src/db/drizzle/meta/0004_snapshot.json new file mode 100644 index 0000000..74082d6 --- /dev/null +++ b/apps/api/src/db/drizzle/meta/0004_snapshot.json @@ -0,0 +1,4805 @@ +{ + "id": "a81f471b-d1a5-4887-8e2d-221e5a7bb7c7", + "prevId": "8c70f1a0-a2c7-493e-8fea-63858e646e63", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_provider_idx": { + "name": "account_provider_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crm_tour_completed_at": { + "name": "crm_tour_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "user_email_idx": { + "name": "user_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.crm_custom_field_definitions": { + "name": "crm_custom_field_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "crm_custom_field_entity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "field_type": { + "name": "field_type", + "type": "crm_custom_field_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crm_custom_field_definitions_workspace_entity_idx": { + "name": "crm_custom_field_definitions_workspace_entity_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crm_custom_field_definitions_workspace_id_idx": { + "name": "crm_custom_field_definitions_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crm_custom_field_definitions_workspace_id_workspaces_id_fk": { + "name": "crm_custom_field_definitions_workspace_id_workspaces_id_fk", + "tableFrom": "crm_custom_field_definitions", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "crm_custom_field_definitions_workspace_entity_label_unique": { + "name": "crm_custom_field_definitions_workspace_entity_label_unique", + "nullsNotDistinct": false, + "columns": [ + "workspace_id", + "entity_type", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deals": { + "name": "deals", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "person_id": { + "name": "person_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "currency": { + "name": "currency", + "type": "varchar(3)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "stage": { + "name": "stage", + "type": "deal_stage", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'new'" + }, + "close_date": { + "name": "close_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "deals_workspace_id_idx": { + "name": "deals_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deals_person_id_idx": { + "name": "deals_person_id_idx", + "columns": [ + { + "expression": "person_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deals_org_id_idx": { + "name": "deals_org_id_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deals_owner_id_idx": { + "name": "deals_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deals_stage_idx": { + "name": "deals_stage_idx", + "columns": [ + { + "expression": "stage", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deals_close_date_idx": { + "name": "deals_close_date_idx", + "columns": [ + { + "expression": "close_date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deals_workspace_id_workspaces_id_fk": { + "name": "deals_workspace_id_workspaces_id_fk", + "tableFrom": "deals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deals_person_id_people_id_fk": { + "name": "deals_person_id_people_id_fk", + "tableFrom": "deals", + "tableTo": "people", + "columnsFrom": [ + "person_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "deals_org_id_org_id_fk": { + "name": "deals_org_id_org_id_fk", + "tableFrom": "deals", + "tableTo": "org", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "deals_owner_id_user_id_fk": { + "name": "deals_owner_id_user_id_fk", + "tableFrom": "deals", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org": { + "name": "org", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "owner_id": { + "name": "owner_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "industry": { + "name": "industry", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "size": { + "name": "size", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "location": { + "name": "location", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "custom_fields": { + "name": "custom_fields", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_workspace_id_idx": { + "name": "org_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_owner_id_idx": { + "name": "org_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "org_workspace_id_workspaces_id_fk": { + "name": "org_workspace_id_workspaces_id_fk", + "tableFrom": "org", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "org_owner_id_user_id_fk": { + "name": "org_owner_id_user_id_fk", + "tableFrom": "org", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "org_workspace_name_unique": { + "name": "org_workspace_name_unique", + "nullsNotDistinct": false, + "columns": [ + "workspace_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.people": { + "name": "people", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "phone": { + "name": "phone", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "job_title": { + "name": "job_title", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "linkedin_url": { + "name": "linkedin_url", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "people_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'lead'" + }, + "source": { + "name": "source", + "type": "people_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "last_contacted_at": { + "name": "last_contacted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "custom_fields": { + "name": "custom_fields", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "people_workspace_id_idx": { + "name": "people_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "people_org_id_idx": { + "name": "people_org_id_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "people_owner_id_idx": { + "name": "people_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "people_email_idx": { + "name": "people_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "people_status_idx": { + "name": "people_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "people_workspace_id_workspaces_id_fk": { + "name": "people_workspace_id_workspaces_id_fk", + "tableFrom": "people", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "people_org_id_org_id_fk": { + "name": "people_org_id_org_id_fk", + "tableFrom": "people", + "tableTo": "org", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "people_owner_id_user_id_fk": { + "name": "people_owner_id_user_id_fk", + "tableFrom": "people", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "people_workspace_email_unique": { + "name": "people_workspace_email_unique", + "nullsNotDistinct": false, + "columns": [ + "workspace_id", + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.external_identities": { + "name": "external_identities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "import_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "import_entity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "person_id": { + "name": "person_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "profile": { + "name": "profile", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "external_identities_workspace_id_idx": { + "name": "external_identities_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "external_identities_person_id_idx": { + "name": "external_identities_person_id_idx", + "columns": [ + { + "expression": "person_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "external_identities_org_id_idx": { + "name": "external_identities_org_id_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "external_identities_workspace_id_workspaces_id_fk": { + "name": "external_identities_workspace_id_workspaces_id_fk", + "tableFrom": "external_identities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "external_identities_person_id_people_id_fk": { + "name": "external_identities_person_id_people_id_fk", + "tableFrom": "external_identities", + "tableTo": "people", + "columnsFrom": [ + "person_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "external_identities_org_id_org_id_fk": { + "name": "external_identities_org_id_org_id_fk", + "tableFrom": "external_identities", + "tableTo": "org", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "external_identities_workspace_provider_external_unique": { + "name": "external_identities_workspace_provider_external_unique", + "nullsNotDistinct": false, + "columns": [ + "workspace_id", + "provider", + "external_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.import_jobs": { + "name": "import_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_by_id": { + "name": "created_by_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "import_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "import_entity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'person'" + }, + "status": { + "name": "status", + "type": "import_job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "mapping": { + "name": "mapping", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{\"fields\":[]}'::jsonb" + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source_config": { + "name": "source_config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "stats": { + "name": "stats", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "cursor": { + "name": "cursor", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "import_jobs_workspace_id_idx": { + "name": "import_jobs_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "import_jobs_connection_id_idx": { + "name": "import_jobs_connection_id_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "import_jobs_status_next_attempt_idx": { + "name": "import_jobs_status_next_attempt_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "import_jobs_provider_idx": { + "name": "import_jobs_provider_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "import_jobs_workspace_id_workspaces_id_fk": { + "name": "import_jobs_workspace_id_workspaces_id_fk", + "tableFrom": "import_jobs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "import_jobs_connection_id_integration_connections_id_fk": { + "name": "import_jobs_connection_id_integration_connections_id_fk", + "tableFrom": "import_jobs", + "tableTo": "integration_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "import_jobs_created_by_id_user_id_fk": { + "name": "import_jobs_created_by_id_user_id_fk", + "tableFrom": "import_jobs", + "tableTo": "user", + "columnsFrom": [ + "created_by_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.import_records": { + "name": "import_records", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "row_number": { + "name": "row_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raw": { + "name": "raw", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "normalized": { + "name": "normalized", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "import_record_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "match_reason": { + "name": "match_reason", + "type": "import_match_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "match_person_id": { + "name": "match_person_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "match_org_id": { + "name": "match_org_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "import_records_workspace_id_idx": { + "name": "import_records_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "import_records_job_status_idx": { + "name": "import_records_job_status_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "import_records_external_id_idx": { + "name": "import_records_external_id_idx", + "columns": [ + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "import_records_workspace_id_workspaces_id_fk": { + "name": "import_records_workspace_id_workspaces_id_fk", + "tableFrom": "import_records", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "import_records_job_id_import_jobs_id_fk": { + "name": "import_records_job_id_import_jobs_id_fk", + "tableFrom": "import_records", + "tableTo": "import_jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "import_records_match_person_id_people_id_fk": { + "name": "import_records_match_person_id_people_id_fk", + "tableFrom": "import_records", + "tableTo": "people", + "columnsFrom": [ + "match_person_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "import_records_match_org_id_org_id_fk": { + "name": "import_records_match_org_id_org_id_fk", + "tableFrom": "import_records", + "tableTo": "org", + "columnsFrom": [ + "match_org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "import_records_job_row_unique": { + "name": "import_records_job_row_unique", + "nullsNotDistinct": false, + "columns": [ + "job_id", + "row_number" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_api_keys": { + "name": "integration_api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "created_by_id": { + "name": "created_by_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "token_prefix": { + "name": "token_prefix", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_api_keys_workspace_id_idx": { + "name": "integration_api_keys_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_api_keys_workspace_id_workspaces_id_fk": { + "name": "integration_api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "integration_api_keys", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "integration_api_keys_created_by_id_user_id_fk": { + "name": "integration_api_keys_created_by_id_user_id_fk", + "tableFrom": "integration_api_keys", + "tableTo": "user", + "columnsFrom": [ + "created_by_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integration_api_keys_token_hash_unique": { + "name": "integration_api_keys_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_connections": { + "name": "integration_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "import_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "external_account_id": { + "name": "external_account_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "import_connection_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "granted_scopes": { + "name": "granted_scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "access_token_encrypted": { + "name": "access_token_encrypted", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token_encrypted": { + "name": "refresh_token_encrypted", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_expires_at": { + "name": "token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "cursor": { + "name": "cursor", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_connections_workspace_id_idx": { + "name": "integration_connections_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_connections_user_id_idx": { + "name": "integration_connections_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_connections_provider_idx": { + "name": "integration_connections_provider_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_connections_status_idx": { + "name": "integration_connections_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_connections_workspace_id_workspaces_id_fk": { + "name": "integration_connections_workspace_id_workspaces_id_fk", + "tableFrom": "integration_connections", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "integration_connections_user_id_user_id_fk": { + "name": "integration_connections_user_id_user_id_fk", + "tableFrom": "integration_connections", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integration_connections_workspace_provider_account_unique": { + "name": "integration_connections_workspace_provider_account_unique", + "nullsNotDistinct": false, + "columns": [ + "workspace_id", + "provider", + "external_account_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.gmail_integrations": { + "name": "gmail_integrations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "gmail_connection_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "granted_scopes": { + "name": "granted_scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "access_token_encrypted": { + "name": "access_token_encrypted", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token_encrypted": { + "name": "refresh_token_encrypted", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_expires_at": { + "name": "token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "timezone": { + "name": "timezone", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "sending_window": { + "name": "sending_window", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "daily_limit": { + "name": "daily_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 50 + }, + "hourly_limit": { + "name": "hourly_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "gmail_integrations_workspace_id_idx": { + "name": "gmail_integrations_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "gmail_integrations_user_id_idx": { + "name": "gmail_integrations_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "gmail_integrations_status_idx": { + "name": "gmail_integrations_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "gmail_integrations_workspace_id_workspaces_id_fk": { + "name": "gmail_integrations_workspace_id_workspaces_id_fk", + "tableFrom": "gmail_integrations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "gmail_integrations_user_id_user_id_fk": { + "name": "gmail_integrations_user_id_user_id_fk", + "tableFrom": "gmail_integrations", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "gmail_integrations_workspace_user_email_unique": { + "name": "gmail_integrations_workspace_user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "workspace_id", + "user_id", + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequence_activity_events": { + "name": "sequence_activity_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "sequence_id": { + "name": "sequence_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "enrollment_id": { + "name": "enrollment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "person_id": { + "name": "person_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "sequence_activity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequence_activity_events_workspace_id_idx": { + "name": "sequence_activity_events_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_activity_events_sequence_id_idx": { + "name": "sequence_activity_events_sequence_id_idx", + "columns": [ + { + "expression": "sequence_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_activity_events_enrollment_id_idx": { + "name": "sequence_activity_events_enrollment_id_idx", + "columns": [ + { + "expression": "enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_activity_events_person_id_idx": { + "name": "sequence_activity_events_person_id_idx", + "columns": [ + { + "expression": "person_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_activity_events_type_idx": { + "name": "sequence_activity_events_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_activity_events_created_at_idx": { + "name": "sequence_activity_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequence_activity_events_workspace_id_workspaces_id_fk": { + "name": "sequence_activity_events_workspace_id_workspaces_id_fk", + "tableFrom": "sequence_activity_events", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_activity_events_sequence_id_sequences_id_fk": { + "name": "sequence_activity_events_sequence_id_sequences_id_fk", + "tableFrom": "sequence_activity_events", + "tableTo": "sequences", + "columnsFrom": [ + "sequence_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_activity_events_enrollment_id_sequence_enrollments_id_fk": { + "name": "sequence_activity_events_enrollment_id_sequence_enrollments_id_fk", + "tableFrom": "sequence_activity_events", + "tableTo": "sequence_enrollments", + "columnsFrom": [ + "enrollment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_activity_events_person_id_people_id_fk": { + "name": "sequence_activity_events_person_id_people_id_fk", + "tableFrom": "sequence_activity_events", + "tableTo": "people", + "columnsFrom": [ + "person_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequence_enrollment_steps": { + "name": "sequence_enrollment_steps", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "enrollment_id": { + "name": "enrollment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "step_id": { + "name": "step_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "sequence_step_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "step_snapshot": { + "name": "step_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "sequence_enrollment_step_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "due_at": { + "name": "due_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gmail_message_id": { + "name": "gmail_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gmail_thread_id": { + "name": "gmail_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_task_id": { + "name": "created_task_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequence_enrollment_steps_workspace_id_idx": { + "name": "sequence_enrollment_steps_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_enrollment_steps_enrollment_id_idx": { + "name": "sequence_enrollment_steps_enrollment_id_idx", + "columns": [ + { + "expression": "enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_enrollment_steps_status_due_idx": { + "name": "sequence_enrollment_steps_status_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_enrollment_steps_thread_idx": { + "name": "sequence_enrollment_steps_thread_idx", + "columns": [ + { + "expression": "gmail_thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequence_enrollment_steps_workspace_id_workspaces_id_fk": { + "name": "sequence_enrollment_steps_workspace_id_workspaces_id_fk", + "tableFrom": "sequence_enrollment_steps", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_enrollment_steps_enrollment_id_sequence_enrollments_id_fk": { + "name": "sequence_enrollment_steps_enrollment_id_sequence_enrollments_id_fk", + "tableFrom": "sequence_enrollment_steps", + "tableTo": "sequence_enrollments", + "columnsFrom": [ + "enrollment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sequence_enrollment_steps_enrollment_position_unique": { + "name": "sequence_enrollment_steps_enrollment_position_unique", + "nullsNotDistinct": false, + "columns": [ + "enrollment_id", + "position" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequence_enrollments": { + "name": "sequence_enrollments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "sequence_id": { + "name": "sequence_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "person_id": { + "name": "person_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "gmail_integration_id": { + "name": "gmail_integration_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "enrolled_by_id": { + "name": "enrolled_by_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "sequence_enrollment_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "current_position": { + "name": "current_position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_step_due_at": { + "name": "next_step_due_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "paused_at": { + "name": "paused_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequence_enrollments_workspace_id_idx": { + "name": "sequence_enrollments_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_enrollments_sequence_id_idx": { + "name": "sequence_enrollments_sequence_id_idx", + "columns": [ + { + "expression": "sequence_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_enrollments_version_id_idx": { + "name": "sequence_enrollments_version_id_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_enrollments_person_id_idx": { + "name": "sequence_enrollments_person_id_idx", + "columns": [ + { + "expression": "person_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_enrollments_gmail_integration_id_idx": { + "name": "sequence_enrollments_gmail_integration_id_idx", + "columns": [ + { + "expression": "gmail_integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_enrollments_status_due_idx": { + "name": "sequence_enrollments_status_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_step_due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequence_enrollments_workspace_id_workspaces_id_fk": { + "name": "sequence_enrollments_workspace_id_workspaces_id_fk", + "tableFrom": "sequence_enrollments", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_enrollments_sequence_id_sequences_id_fk": { + "name": "sequence_enrollments_sequence_id_sequences_id_fk", + "tableFrom": "sequence_enrollments", + "tableTo": "sequences", + "columnsFrom": [ + "sequence_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_enrollments_version_id_sequence_versions_id_fk": { + "name": "sequence_enrollments_version_id_sequence_versions_id_fk", + "tableFrom": "sequence_enrollments", + "tableTo": "sequence_versions", + "columnsFrom": [ + "version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "sequence_enrollments_person_id_people_id_fk": { + "name": "sequence_enrollments_person_id_people_id_fk", + "tableFrom": "sequence_enrollments", + "tableTo": "people", + "columnsFrom": [ + "person_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_enrollments_gmail_integration_id_gmail_integrations_id_fk": { + "name": "sequence_enrollments_gmail_integration_id_gmail_integrations_id_fk", + "tableFrom": "sequence_enrollments", + "tableTo": "gmail_integrations", + "columnsFrom": [ + "gmail_integration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "sequence_enrollments_enrolled_by_id_user_id_fk": { + "name": "sequence_enrollments_enrolled_by_id_user_id_fk", + "tableFrom": "sequence_enrollments", + "tableTo": "user", + "columnsFrom": [ + "enrolled_by_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequence_steps": { + "name": "sequence_steps", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "sequence_id": { + "name": "sequence_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "sequence_step_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequence_steps_workspace_id_idx": { + "name": "sequence_steps_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_steps_sequence_id_idx": { + "name": "sequence_steps_sequence_id_idx", + "columns": [ + { + "expression": "sequence_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequence_steps_workspace_id_workspaces_id_fk": { + "name": "sequence_steps_workspace_id_workspaces_id_fk", + "tableFrom": "sequence_steps", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_steps_sequence_id_sequences_id_fk": { + "name": "sequence_steps_sequence_id_sequences_id_fk", + "tableFrom": "sequence_steps", + "tableTo": "sequences", + "columnsFrom": [ + "sequence_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sequence_steps_sequence_position_unique": { + "name": "sequence_steps_sequence_position_unique", + "nullsNotDistinct": false, + "columns": [ + "sequence_id", + "position" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequence_suppressions": { + "name": "sequence_suppressions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "default": "'unsubscribe'" + }, + "source": { + "name": "source", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "default": "'public_unsubscribe'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequence_suppressions_workspace_id_idx": { + "name": "sequence_suppressions_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_suppressions_email_idx": { + "name": "sequence_suppressions_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequence_suppressions_workspace_id_workspaces_id_fk": { + "name": "sequence_suppressions_workspace_id_workspaces_id_fk", + "tableFrom": "sequence_suppressions", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sequence_suppressions_workspace_email_unique": { + "name": "sequence_suppressions_workspace_email_unique", + "nullsNotDistinct": false, + "columns": [ + "workspace_id", + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequence_tasks": { + "name": "sequence_tasks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "sequence_id": { + "name": "sequence_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "enrollment_id": { + "name": "enrollment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "enrollment_step_id": { + "name": "enrollment_step_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "person_id": { + "name": "person_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "assigned_to_id": { + "name": "assigned_to_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "sequence_task_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "sequence_task_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "title": { + "name": "title", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "due_at": { + "name": "due_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequence_tasks_workspace_id_idx": { + "name": "sequence_tasks_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_tasks_sequence_id_idx": { + "name": "sequence_tasks_sequence_id_idx", + "columns": [ + { + "expression": "sequence_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_tasks_enrollment_id_idx": { + "name": "sequence_tasks_enrollment_id_idx", + "columns": [ + { + "expression": "enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_tasks_enrollment_step_id_idx": { + "name": "sequence_tasks_enrollment_step_id_idx", + "columns": [ + { + "expression": "enrollment_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_tasks_person_id_idx": { + "name": "sequence_tasks_person_id_idx", + "columns": [ + { + "expression": "person_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_tasks_assigned_to_id_idx": { + "name": "sequence_tasks_assigned_to_id_idx", + "columns": [ + { + "expression": "assigned_to_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_tasks_status_due_idx": { + "name": "sequence_tasks_status_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequence_tasks_workspace_id_workspaces_id_fk": { + "name": "sequence_tasks_workspace_id_workspaces_id_fk", + "tableFrom": "sequence_tasks", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_tasks_sequence_id_sequences_id_fk": { + "name": "sequence_tasks_sequence_id_sequences_id_fk", + "tableFrom": "sequence_tasks", + "tableTo": "sequences", + "columnsFrom": [ + "sequence_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_tasks_enrollment_id_sequence_enrollments_id_fk": { + "name": "sequence_tasks_enrollment_id_sequence_enrollments_id_fk", + "tableFrom": "sequence_tasks", + "tableTo": "sequence_enrollments", + "columnsFrom": [ + "enrollment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_tasks_enrollment_step_id_sequence_enrollment_steps_id_fk": { + "name": "sequence_tasks_enrollment_step_id_sequence_enrollment_steps_id_fk", + "tableFrom": "sequence_tasks", + "tableTo": "sequence_enrollment_steps", + "columnsFrom": [ + "enrollment_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "sequence_tasks_person_id_people_id_fk": { + "name": "sequence_tasks_person_id_people_id_fk", + "tableFrom": "sequence_tasks", + "tableTo": "people", + "columnsFrom": [ + "person_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "sequence_tasks_assigned_to_id_user_id_fk": { + "name": "sequence_tasks_assigned_to_id_user_id_fk", + "tableFrom": "sequence_tasks", + "tableTo": "user", + "columnsFrom": [ + "assigned_to_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequence_unsubscribe_tokens": { + "name": "sequence_unsubscribe_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "sequence_id": { + "name": "sequence_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "enrollment_id": { + "name": "enrollment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "person_id": { + "name": "person_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequence_unsubscribe_tokens_workspace_id_idx": { + "name": "sequence_unsubscribe_tokens_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_unsubscribe_tokens_enrollment_id_idx": { + "name": "sequence_unsubscribe_tokens_enrollment_id_idx", + "columns": [ + { + "expression": "enrollment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_unsubscribe_tokens_email_idx": { + "name": "sequence_unsubscribe_tokens_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequence_unsubscribe_tokens_workspace_id_workspaces_id_fk": { + "name": "sequence_unsubscribe_tokens_workspace_id_workspaces_id_fk", + "tableFrom": "sequence_unsubscribe_tokens", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_unsubscribe_tokens_sequence_id_sequences_id_fk": { + "name": "sequence_unsubscribe_tokens_sequence_id_sequences_id_fk", + "tableFrom": "sequence_unsubscribe_tokens", + "tableTo": "sequences", + "columnsFrom": [ + "sequence_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_unsubscribe_tokens_enrollment_id_sequence_enrollments_id_fk": { + "name": "sequence_unsubscribe_tokens_enrollment_id_sequence_enrollments_id_fk", + "tableFrom": "sequence_unsubscribe_tokens", + "tableTo": "sequence_enrollments", + "columnsFrom": [ + "enrollment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_unsubscribe_tokens_person_id_people_id_fk": { + "name": "sequence_unsubscribe_tokens_person_id_people_id_fk", + "tableFrom": "sequence_unsubscribe_tokens", + "tableTo": "people", + "columnsFrom": [ + "person_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sequence_unsubscribe_tokens_hash_unique": { + "name": "sequence_unsubscribe_tokens_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequence_versions": { + "name": "sequence_versions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "sequence_id": { + "name": "sequence_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_number": { + "name": "version_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "steps_snapshot": { + "name": "steps_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "published_by_id": { + "name": "published_by_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequence_versions_workspace_id_idx": { + "name": "sequence_versions_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequence_versions_sequence_id_idx": { + "name": "sequence_versions_sequence_id_idx", + "columns": [ + { + "expression": "sequence_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequence_versions_workspace_id_workspaces_id_fk": { + "name": "sequence_versions_workspace_id_workspaces_id_fk", + "tableFrom": "sequence_versions", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_versions_sequence_id_sequences_id_fk": { + "name": "sequence_versions_sequence_id_sequences_id_fk", + "tableFrom": "sequence_versions", + "tableTo": "sequences", + "columnsFrom": [ + "sequence_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequence_versions_published_by_id_user_id_fk": { + "name": "sequence_versions_published_by_id_user_id_fk", + "tableFrom": "sequence_versions", + "tableTo": "user", + "columnsFrom": [ + "published_by_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sequence_versions_sequence_version_unique": { + "name": "sequence_versions_sequence_version_unique", + "nullsNotDistinct": false, + "columns": [ + "sequence_id", + "version_number" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sequences": { + "name": "sequences", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "created_by_id": { + "name": "created_by_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "latest_published_version_id": { + "name": "latest_published_version_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "sequence_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'draft'" + }, + "timezone": { + "name": "timezone", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "sending_window": { + "name": "sending_window", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sequences_workspace_id_idx": { + "name": "sequences_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequences_created_by_id_idx": { + "name": "sequences_created_by_id_idx", + "columns": [ + { + "expression": "created_by_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sequences_status_idx": { + "name": "sequences_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sequences_workspace_id_workspaces_id_fk": { + "name": "sequences_workspace_id_workspaces_id_fk", + "tableFrom": "sequences", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sequences_created_by_id_user_id_fk": { + "name": "sequences_created_by_id_user_id_fk", + "tableFrom": "sequences", + "tableTo": "user", + "columnsFrom": [ + "created_by_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_invites": { + "name": "workspace_invites", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "workspace_invite_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "token": { + "name": "token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "workspace_invite_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_invites_workspace_id_idx": { + "name": "workspace_invites_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_invites_email_idx": { + "name": "workspace_invites_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_invites_status_idx": { + "name": "workspace_invites_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_invites_expires_at_idx": { + "name": "workspace_invites_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_invites_created_by_idx": { + "name": "workspace_invites_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_invites_workspace_id_workspaces_id_fk": { + "name": "workspace_invites_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_invites", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_invites_created_by_user_id_fk": { + "name": "workspace_invites_created_by_user_id_fk", + "tableFrom": "workspace_invites", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_invites_token_unique": { + "name": "workspace_invites_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_members": { + "name": "workspace_members", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "workspace_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "joined_at": { + "name": "joined_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_members_workspace_id_idx": { + "name": "workspace_members_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_members_user_id_idx": { + "name": "workspace_members_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_members_workspace_id_workspaces_id_fk": { + "name": "workspace_members_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_members", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_members_user_id_user_id_fk": { + "name": "workspace_members_user_id_user_id_fk", + "tableFrom": "workspace_members", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_members_workspace_user_unique": { + "name": "workspace_members_workspace_user_unique", + "nullsNotDistinct": false, + "columns": [ + "workspace_id", + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner_id": { + "name": "owner_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_owner_id_idx": { + "name": "workspaces_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspaces_slug_idx": { + "name": "workspaces_slug_idx", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspaces_owner_id_user_id_fk": { + "name": "workspaces_owner_id_user_id_fk", + "tableFrom": "workspaces", + "tableTo": "user", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_slug_unique": { + "name": "workspaces_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.crm_custom_field_entity_type": { + "name": "crm_custom_field_entity_type", + "schema": "public", + "values": [ + "people", + "org" + ] + }, + "public.crm_custom_field_type": { + "name": "crm_custom_field_type", + "schema": "public", + "values": [ + "text", + "number", + "select", + "dateTime" + ] + }, + "public.deal_stage": { + "name": "deal_stage", + "schema": "public", + "values": [ + "new", + "contacted", + "demo", + "proposal", + "won", + "lost" + ] + }, + "public.gmail_connection_status": { + "name": "gmail_connection_status", + "schema": "public", + "values": [ + "connected", + "reconnect_required", + "disconnected" + ] + }, + "public.import_connection_status": { + "name": "import_connection_status", + "schema": "public", + "values": [ + "connected", + "reconnect_required", + "disconnected" + ] + }, + "public.import_entity_type": { + "name": "import_entity_type", + "schema": "public", + "values": [ + "person", + "org" + ] + }, + "public.import_job_status": { + "name": "import_job_status", + "schema": "public", + "values": [ + "pending", + "extracting", + "ready_for_review", + "loading", + "completed", + "failed", + "canceled" + ] + }, + "public.import_match_reason": { + "name": "import_match_reason", + "schema": "public", + "values": [ + "external_identity", + "email", + "domain", + "name", + "none" + ] + }, + "public.import_provider": { + "name": "import_provider", + "schema": "public", + "values": [ + "csv", + "webhook", + "gmail", + "google_calendar", + "calendly", + "google_sheets", + "posthog", + "outlook" + ] + }, + "public.import_record_status": { + "name": "import_record_status", + "schema": "public", + "values": [ + "pending", + "valid", + "invalid", + "loaded", + "skipped", + "duplicate" + ] + }, + "public.people_source": { + "name": "people_source", + "schema": "public", + "values": [ + "manual", + "csv", + "api", + "import" + ] + }, + "public.people_status": { + "name": "people_status", + "schema": "public", + "values": [ + "lead", + "prospect", + "qualified", + "customer", + "churned" + ] + }, + "public.sequence_activity_type": { + "name": "sequence_activity_type", + "schema": "public", + "values": [ + "sequence_created", + "sequence_updated", + "sequence_published", + "sequence_archived", + "enrollment_started", + "email_sent", + "email_would_send", + "email_failed", + "reply_detected", + "unsubscribed", + "task_created", + "task_completed", + "paused", + "resumed", + "completed", + "gmail_warning" + ] + }, + "public.sequence_enrollment_status": { + "name": "sequence_enrollment_status", + "schema": "public", + "values": [ + "active", + "paused", + "failed", + "replied", + "completed", + "unsubscribed" + ] + }, + "public.sequence_enrollment_step_status": { + "name": "sequence_enrollment_step_status", + "schema": "public", + "values": [ + "pending", + "processing", + "completed", + "failed", + "skipped" + ] + }, + "public.sequence_status": { + "name": "sequence_status", + "schema": "public", + "values": [ + "draft", + "published", + "paused", + "archived" + ] + }, + "public.sequence_step_type": { + "name": "sequence_step_type", + "schema": "public", + "values": [ + "email", + "wait", + "linkedin_task", + "general_task" + ] + }, + "public.sequence_task_status": { + "name": "sequence_task_status", + "schema": "public", + "values": [ + "open", + "completed", + "canceled" + ] + }, + "public.sequence_task_type": { + "name": "sequence_task_type", + "schema": "public", + "values": [ + "linkedin_task", + "general_task", + "failure", + "gmail_warning" + ] + }, + "public.workspace_invite_role": { + "name": "workspace_invite_role", + "schema": "public", + "values": [ + "admin", + "member" + ] + }, + "public.workspace_invite_status": { + "name": "workspace_invite_status", + "schema": "public", + "values": [ + "pending", + "accepted", + "rejected", + "canceled" + ] + }, + "public.workspace_role": { + "name": "workspace_role", + "schema": "public", + "values": [ + "owner", + "admin", + "member" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/drizzle/meta/_journal.json b/apps/api/src/db/drizzle/meta/_journal.json index 69850d2..bc784b9 100644 --- a/apps/api/src/db/drizzle/meta/_journal.json +++ b/apps/api/src/db/drizzle/meta/_journal.json @@ -29,6 +29,13 @@ "when": 1783389401615, "tag": "0003_dusty_thor", "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1784563850047, + "tag": "0004_public_mesmero", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/db/schema/enums.schema.ts b/apps/api/src/db/schema/enums.schema.ts index e69aab1..27aec14 100644 --- a/apps/api/src/db/schema/enums.schema.ts +++ b/apps/api/src/db/schema/enums.schema.ts @@ -19,7 +19,7 @@ export const peopleStatusEnum = pgEnum("people_status", [ "churned", ]); -export const peopleSourceEnum = pgEnum("people_source", ["manual", "csv", "api"]); +export const peopleSourceEnum = pgEnum("people_source", ["manual", "csv", "api", "import"]); export const dealStageEnum = pgEnum("deal_stage", [ "new", @@ -110,3 +110,49 @@ export const gmailConnectionStatusEnum = pgEnum("gmail_connection_status", [ "reconnect_required", "disconnected", ]); + +export const importProviderEnum = pgEnum("import_provider", [ + "csv", + "webhook", + "gmail", + "google_calendar", + "calendly", + "google_sheets", + "posthog", + "outlook", +]); + +export const importEntityTypeEnum = pgEnum("import_entity_type", ["person", "org"]); + +export const importJobStatusEnum = pgEnum("import_job_status", [ + "pending", + "extracting", + "ready_for_review", + "loading", + "completed", + "failed", + "canceled", +]); + +export const importRecordStatusEnum = pgEnum("import_record_status", [ + "pending", + "valid", + "invalid", + "loaded", + "skipped", + "duplicate", +]); + +export const importConnectionStatusEnum = pgEnum("import_connection_status", [ + "connected", + "reconnect_required", + "disconnected", +]); + +export const importMatchReasonEnum = pgEnum("import_match_reason", [ + "external_identity", + "email", + "domain", + "name", + "none", +]); diff --git a/apps/api/src/db/schema/import.schema.ts b/apps/api/src/db/schema/import.schema.ts new file mode 100644 index 0000000..49039e4 --- /dev/null +++ b/apps/api/src/db/schema/import.schema.ts @@ -0,0 +1,275 @@ +import { relations } from "drizzle-orm"; +import { + index, + integer, + jsonb, + pgTable, + text, + timestamp, + unique, + uuid, + varchar, +} from "drizzle-orm/pg-core"; +import type { ImportFieldMapping } from "@workspace/validators/schemas/import"; +import type { ImportConflictPolicy } from "@workspace/validators/types/import"; +import { id, timestamps } from "./common.schema.js"; +import { user } from "./auth.schema.js"; +import { workspaces } from "./workspace.schema.js"; +import { org, people } from "./crm.schema.js"; +import { + importConnectionStatusEnum, + importEntityTypeEnum, + importJobStatusEnum, + importMatchReasonEnum, + importProviderEnum, + importRecordStatusEnum, + peopleStatusEnum, +} from "./enums.schema.js"; + +export type ImportJobMapping = { + fields: ImportFieldMapping[]; +}; + +export type ImportJobOptionsSnapshot = { + conflictPolicy: ImportConflictPolicy; + defaultStatus?: (typeof peopleStatusEnum.enumValues)[number]; + defaultOwnerId: string | null; + normalizeSubaddressing: boolean; + createMissingOrgs: boolean; + skipRecordsWithoutEmail: boolean; +}; + +export type ImportJobStats = { + extracted: number; + valid: number; + invalid: number; + created: number; + updated: number; + skipped: number; + duplicate: number; +}; + +/** + * Opaque per-provider pagination state. Persisted after every page so a rate-limited or + * crashed run resumes instead of restarting. + */ +export type ImportCursor = Record; + +export type ImportRecordError = { + field: string; + message: string; +}; + +/** + * Generalizes the `gmail_integrations` pattern to every provider. Left as a separate table + * because `gmail_integrations` carries sequence-specific sending windows and rate limits + * that do not generalize, and is load-bearing for autosend. + */ +export const integrationConnections = pgTable( + "integration_connections", + { + ...id, + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + userId: uuid("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + provider: importProviderEnum("provider").notNull(), + displayName: varchar("display_name", { length: 255 }).notNull(), + externalAccountId: varchar("external_account_id", { length: 255 }), + status: importConnectionStatusEnum("status").notNull().default("connected"), + grantedScopes: jsonb("granted_scopes").$type().default([]).notNull(), + accessTokenEncrypted: text("access_token_encrypted").notNull(), + /** Null for API-key providers such as PostHog and Calendly personal tokens. */ + refreshTokenEncrypted: text("refresh_token_encrypted"), + tokenExpiresAt: timestamp("token_expires_at"), + config: jsonb("config").$type>().default({}).notNull(), + cursor: jsonb("cursor").$type(), + lastSyncAt: timestamp("last_sync_at"), + lastError: text("last_error"), + ...timestamps, + }, + (table) => [ + unique("integration_connections_workspace_provider_account_unique").on( + table.workspaceId, + table.provider, + table.externalAccountId, + ), + index("integration_connections_workspace_id_idx").on(table.workspaceId), + index("integration_connections_user_id_idx").on(table.userId), + index("integration_connections_provider_idx").on(table.provider), + index("integration_connections_status_idx").on(table.status), + ], +); + +/** + * Authenticates inbound webhook pushes. Only the hash is stored; the plaintext key is + * shown once at creation, matching the unsubscribe-token handling in sequences. + */ +export const integrationApiKeys = pgTable( + "integration_api_keys", + { + ...id, + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + createdById: uuid("created_by_id").references(() => user.id, { onDelete: "set null" }), + name: varchar("name", { length: 255 }).notNull(), + tokenHash: varchar("token_hash", { length: 128 }).notNull(), + tokenPrefix: varchar("token_prefix", { length: 16 }).notNull(), + lastUsedAt: timestamp("last_used_at"), + revokedAt: timestamp("revoked_at"), + ...timestamps, + }, + (table) => [ + unique("integration_api_keys_token_hash_unique").on(table.tokenHash), + index("integration_api_keys_workspace_id_idx").on(table.workspaceId), + ], +); + +export const importJobs = pgTable( + "import_jobs", + { + ...id, + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + connectionId: uuid("connection_id").references(() => integrationConnections.id, { + onDelete: "set null", + }), + createdById: uuid("created_by_id").references(() => user.id, { onDelete: "set null" }), + provider: importProviderEnum("provider").notNull(), + entityType: importEntityTypeEnum("entity_type").notNull().default("person"), + status: importJobStatusEnum("status").notNull().default("pending"), + mapping: jsonb("mapping").$type().default({ fields: [] }).notNull(), + options: jsonb("options").$type().notNull(), + sourceConfig: jsonb("source_config").$type>().default({}).notNull(), + stats: jsonb("stats").$type().notNull(), + cursor: jsonb("cursor").$type(), + attempts: integer("attempts").notNull().default(0), + nextAttemptAt: timestamp("next_attempt_at"), + startedAt: timestamp("started_at"), + completedAt: timestamp("completed_at"), + lastError: text("last_error"), + ...timestamps, + }, + (table) => [ + index("import_jobs_workspace_id_idx").on(table.workspaceId), + index("import_jobs_connection_id_idx").on(table.connectionId), + index("import_jobs_status_next_attempt_idx").on(table.status, table.nextAttemptAt), + index("import_jobs_provider_idx").on(table.provider), + ], +); + +/** + * Staging. One row per source record, holding the untouched payload alongside the mapped + * result. Keeping `raw` means a mapping correction re-runs the load without re-extracting. + */ +export const importRecords = pgTable( + "import_records", + { + ...id, + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + jobId: uuid("job_id") + .notNull() + .references(() => importJobs.id, { onDelete: "cascade" }), + externalId: varchar("external_id", { length: 255 }), + rowNumber: integer("row_number").notNull(), + raw: jsonb("raw").$type>().notNull(), + normalized: jsonb("normalized").$type>(), + status: importRecordStatusEnum("status").notNull().default("pending"), + matchReason: importMatchReasonEnum("match_reason").notNull().default("none"), + matchPersonId: uuid("match_person_id").references(() => people.id, { onDelete: "set null" }), + matchOrgId: uuid("match_org_id").references(() => org.id, { onDelete: "set null" }), + errors: jsonb("errors").$type().default([]).notNull(), + ...timestamps, + }, + (table) => [ + unique("import_records_job_row_unique").on(table.jobId, table.rowNumber), + index("import_records_workspace_id_idx").on(table.workspaceId), + index("import_records_job_status_idx").on(table.jobId, table.status), + index("import_records_external_id_idx").on(table.externalId), + ], +); + +/** + * The table that makes re-sync idempotent. Without it, any source yielding people without + * email addresses duplicates on every run, because `people_workspace_email_unique` is + * scoped to a nullable column and Postgres permits unlimited NULLs. + */ +export const externalIdentities = pgTable( + "external_identities", + { + ...id, + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + provider: importProviderEnum("provider").notNull(), + externalId: varchar("external_id", { length: 255 }).notNull(), + entityType: importEntityTypeEnum("entity_type").notNull(), + personId: uuid("person_id").references(() => people.id, { onDelete: "cascade" }), + orgId: uuid("org_id").references(() => org.id, { onDelete: "cascade" }), + profile: jsonb("profile").$type>().default({}).notNull(), + lastSeenAt: timestamp("last_seen_at").defaultNow().notNull(), + ...timestamps, + }, + (table) => [ + unique("external_identities_workspace_provider_external_unique").on( + table.workspaceId, + table.provider, + table.externalId, + ), + index("external_identities_workspace_id_idx").on(table.workspaceId), + index("external_identities_person_id_idx").on(table.personId), + index("external_identities_org_id_idx").on(table.orgId), + ], +); + +export const integrationConnectionsRelations = relations( + integrationConnections, + ({ one, many }) => ({ + workspace: one(workspaces, { + fields: [integrationConnections.workspaceId], + references: [workspaces.id], + }), + user: one(user, { fields: [integrationConnections.userId], references: [user.id] }), + jobs: many(importJobs), + }), +); + +export const importJobsRelations = relations(importJobs, ({ one, many }) => ({ + workspace: one(workspaces, { fields: [importJobs.workspaceId], references: [workspaces.id] }), + connection: one(integrationConnections, { + fields: [importJobs.connectionId], + references: [integrationConnections.id], + }), + createdBy: one(user, { fields: [importJobs.createdById], references: [user.id] }), + records: many(importRecords), +})); + +export const importRecordsRelations = relations(importRecords, ({ one }) => ({ + workspace: one(workspaces, { fields: [importRecords.workspaceId], references: [workspaces.id] }), + job: one(importJobs, { fields: [importRecords.jobId], references: [importJobs.id] }), + matchPerson: one(people, { fields: [importRecords.matchPersonId], references: [people.id] }), + matchOrg: one(org, { fields: [importRecords.matchOrgId], references: [org.id] }), +})); + +export const externalIdentitiesRelations = relations(externalIdentities, ({ one }) => ({ + workspace: one(workspaces, { + fields: [externalIdentities.workspaceId], + references: [workspaces.id], + }), + person: one(people, { fields: [externalIdentities.personId], references: [people.id] }), + org: one(org, { fields: [externalIdentities.orgId], references: [org.id] }), +})); + +export const integrationApiKeysRelations = relations(integrationApiKeys, ({ one }) => ({ + workspace: one(workspaces, { + fields: [integrationApiKeys.workspaceId], + references: [workspaces.id], + }), + createdBy: one(user, { fields: [integrationApiKeys.createdById], references: [user.id] }), +})); diff --git a/apps/api/src/db/schema/index.ts b/apps/api/src/db/schema/index.ts index f3b1ce5..14025a9 100644 --- a/apps/api/src/db/schema/index.ts +++ b/apps/api/src/db/schema/index.ts @@ -2,4 +2,5 @@ export * from "./auth.schema.js"; export * from "./workspace.schema.js"; export * from "./crm.schema.js"; export * from "./sequence.schema.js"; +export * from "./import.schema.js"; export * from "./enums.schema.js"; diff --git a/apps/api/src/routes/import-webhook.route.ts b/apps/api/src/routes/import-webhook.route.ts new file mode 100644 index 0000000..54dfe49 --- /dev/null +++ b/apps/api/src/routes/import-webhook.route.ts @@ -0,0 +1,18 @@ +import { Hono } from "hono"; +import { webhookIngestSchema } from "@workspace/validators/schemas/import"; +import { ingestWebhookController } from "@/controllers/imports.controller.js"; +import { VALIDATION_TARGET } from "@/constants/validation-targets.js"; +import { validateRequest } from "@/middlewares/validate-request.js"; + +/** + * Deliberately outside the session-auth middleware. This is the generic inbound endpoint + * that Zapier, Make, n8n, and custom scripts push to, so it authenticates with a workspace + * API key in the Authorization header instead of a browser session. + * + * One endpoint here substitutes for a long tail of native connectors. + */ +export const importWebhookRoutes = new Hono().post( + "/", + validateRequest(VALIDATION_TARGET.JSON, webhookIngestSchema), + (c) => ingestWebhookController(c, c.req.valid(VALIDATION_TARGET.JSON)), +); diff --git a/apps/api/src/routes/imports.route.ts b/apps/api/src/routes/imports.route.ts new file mode 100644 index 0000000..27d3dc6 --- /dev/null +++ b/apps/api/src/routes/imports.route.ts @@ -0,0 +1,128 @@ +import { Hono } from "hono"; +import { z } from "zod"; +import { + connectionParamsSchema, + createConnectionSchema, + importEntityTypeSchema, + importJobParamsSchema, + listImportJobsQuerySchema, + listImportRecordsQuerySchema, + startImportJobSchema, + updateConnectionSchema, + updateImportJobSchema, +} from "@workspace/validators/schemas/import"; +import { + cancelImportJobController, + commitImportJobController, + createApiKeyController, + createConnectionController, + deleteConnectionController, + getImportJobController, + listApiKeysController, + listConnectionsController, + listImportJobsController, + listImportRecordsController, + previewConnectionFieldsController, + revokeApiKeyController, + startImportJobController, + updateConnectionController, + updateImportJobController, +} from "@/controllers/imports.controller.js"; +import { VALIDATION_TARGET } from "@/constants/validation-targets.js"; +import { authMiddleware } from "@/middlewares/auth-middleware.js"; +import { validateRequest } from "@/middlewares/validate-request.js"; + +const createApiKeySchema = z.object({ name: z.string().trim().min(1).max(255) }); +const previewQuerySchema = z.object({ entityType: importEntityTypeSchema.default("person") }); + +export const importRoutes = new Hono() + .use("*", authMiddleware) + .get("/connections", (c) => listConnectionsController(c)) + .post( + "/connections", + validateRequest(VALIDATION_TARGET.JSON, createConnectionSchema), + (c) => createConnectionController(c, c.req.valid(VALIDATION_TARGET.JSON)), + ) + .get( + "/connections/:id/fields", + validateRequest(VALIDATION_TARGET.PARAM, connectionParamsSchema), + validateRequest(VALIDATION_TARGET.QUERY, previewQuerySchema), + (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + const { entityType } = c.req.valid(VALIDATION_TARGET.QUERY); + return previewConnectionFieldsController(c, id, entityType); + }, + ) + .patch( + "/connections/:id", + validateRequest(VALIDATION_TARGET.PARAM, connectionParamsSchema), + validateRequest(VALIDATION_TARGET.JSON, updateConnectionSchema), + (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + return updateConnectionController(c, id, c.req.valid(VALIDATION_TARGET.JSON)); + }, + ) + .delete( + "/connections/:id", + validateRequest(VALIDATION_TARGET.PARAM, connectionParamsSchema), + (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + return deleteConnectionController(c, id); + }, + ) + .get("/api-keys", (c) => listApiKeysController(c)) + .post("/api-keys", validateRequest(VALIDATION_TARGET.JSON, createApiKeySchema), (c) => + createApiKeyController(c, c.req.valid(VALIDATION_TARGET.JSON)), + ) + .delete( + "/api-keys/:id", + validateRequest(VALIDATION_TARGET.PARAM, connectionParamsSchema), + (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + return revokeApiKeyController(c, id); + }, + ) + .get("/jobs", validateRequest(VALIDATION_TARGET.QUERY, listImportJobsQuerySchema), (c) => + listImportJobsController(c, c.req.valid(VALIDATION_TARGET.QUERY)), + ) + .post("/jobs", validateRequest(VALIDATION_TARGET.JSON, startImportJobSchema), (c) => + startImportJobController(c, c.req.valid(VALIDATION_TARGET.JSON)), + ) + .get("/jobs/:id", validateRequest(VALIDATION_TARGET.PARAM, importJobParamsSchema), (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + return getImportJobController(c, id); + }) + .get( + "/jobs/:id/records", + validateRequest(VALIDATION_TARGET.PARAM, importJobParamsSchema), + validateRequest(VALIDATION_TARGET.QUERY, listImportRecordsQuerySchema), + (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + return listImportRecordsController(c, id, c.req.valid(VALIDATION_TARGET.QUERY)); + }, + ) + .patch( + "/jobs/:id", + validateRequest(VALIDATION_TARGET.PARAM, importJobParamsSchema), + validateRequest(VALIDATION_TARGET.JSON, updateImportJobSchema), + (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + return updateImportJobController(c, id, c.req.valid(VALIDATION_TARGET.JSON)); + }, + ) + .post( + "/jobs/:id/commit", + validateRequest(VALIDATION_TARGET.PARAM, importJobParamsSchema), + (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + return commitImportJobController(c, id); + }, + ) + .post( + "/jobs/:id/cancel", + validateRequest(VALIDATION_TARGET.PARAM, importJobParamsSchema), + (c) => { + const { id } = c.req.valid(VALIDATION_TARGET.PARAM); + return cancelImportJobController(c, id); + }, + ); diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index 7297437..440aae5 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -3,6 +3,8 @@ import { analyticsRoutes } from "./analytics.route.js"; import { authRoutes } from "./auth.route.js"; import { dealRoutes } from "./deals.route.js"; import { healthRoutes } from "./health.route.js"; +import { importRoutes } from "./imports.route.js"; +import { importWebhookRoutes } from "./import-webhook.route.js"; import { orgRoutes } from "./org.route.js"; import { peopleRoutes } from "./people.route.js"; import { onboardingRoutes } from "./onboarding.route.js"; @@ -19,4 +21,6 @@ export function registerRoutes(app: Hono) { app.route("/onboarding", onboardingRoutes); app.route("/sequences", sequenceRoutes); app.route("/sequence-unsubscribe", sequenceUnsubscribeRoutes); + app.route("/imports", importRoutes); + app.route("/import-webhook", importWebhookRoutes); } diff --git a/apps/api/src/services/import-connectors/calendly.ts b/apps/api/src/services/import-connectors/calendly.ts new file mode 100644 index 0000000..301d7b8 --- /dev/null +++ b/apps/api/src/services/import-connectors/calendly.ts @@ -0,0 +1,128 @@ +import { env } from "@/config/env.config.js"; +import { deriveNameFromEmail, isValidEmail, normalizeEmail } from "@/services/import-engine.js"; +import { buildUrl, requestJson } from "./http.js"; +import { + readConfigString, + readCursorString, + type Connector, + type ConnectorContext, + type ExtractPage, + type RawRecord, +} from "./types.js"; + +type CalendlyEventsResponse = { + collection?: { + uri: string; + name?: string; + start_time?: string; + status?: string; + }[]; + pagination?: { next_page_token?: string | null }; +}; + +type CalendlyInviteesResponse = { + collection?: { + uri: string; + email?: string; + name?: string; + status?: string; + timezone?: string; + created_at?: string; + questions_and_answers?: { question: string; answer: string }[]; + }[]; +}; + +function eventUuid(uri: string) { + return uri.split("/").pop() ?? uri; +} + +/** + * Someone who booked a meeting is qualified by definition, which makes Calendly one of the + * best value-to-effort sources available. + * + * Booking forms usually capture company and other custom questions, so answers are flattened + * onto the record as `question:` keys and become mappable to custom fields. + */ +export const calendlyConnector: Connector = { + provider: "calendly", + entityTypes: ["person"], + + async listFields(ctx: ConnectorContext) { + const base = ["name", "email", "lastContactedAt", "eventName", "timezone", "status"]; + const page = await this.extract({ ...ctx, pageSize: 5, cursor: null }); + const questionKeys = new Set(); + + for (const record of page.records) { + for (const key of Object.keys(record.data)) { + if (key.startsWith("question:")) questionKeys.add(key); + } + } + + return [...base, ...questionKeys]; + }, + + async extract(ctx: ConnectorContext): Promise { + const organization = readConfigString(ctx.config, "organization"); + if (!organization) { + throw new Error("Calendly import requires an organization URI in the source config"); + } + + const pageToken = readCursorString(ctx.cursor, "pageToken"); + const minStartTime = + readConfigString(ctx.config, "minStartTime") ?? + new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toISOString(); + + const events = await requestJson({ + url: buildUrl(`${env.CALENDLY_API_HOST}/scheduled_events`, { + organization, + count: Math.min(ctx.pageSize, 100), + min_start_time: minStartTime, + sort: "start_time:asc", + page_token: pageToken ?? undefined, + }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + const records: RawRecord[] = []; + + for (const event of events.collection ?? []) { + const invitees = await requestJson({ + url: buildUrl(`${env.CALENDLY_API_HOST}/scheduled_events/${eventUuid(event.uri)}/invitees`, { + count: 100, + }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + for (const invitee of invitees.collection ?? []) { + if (!invitee.email) continue; + + const email = normalizeEmail(invitee.email); + if (!isValidEmail(email)) continue; + + const data: Record = { + name: invitee.name ?? deriveNameFromEmail(email), + email, + lastContactedAt: event.start_time ?? invitee.created_at ?? null, + eventName: event.name ?? null, + timezone: invitee.timezone ?? null, + status: invitee.status ?? event.status ?? null, + }; + + for (const answer of invitee.questions_and_answers ?? []) { + if (answer.answer.trim() === "") continue; + data[`question:${answer.question}`] = answer.answer; + } + + records.push({ externalId: invitee.uri, data }); + } + } + + const nextPageToken = events.pagination?.next_page_token; + return { + records, + nextCursor: nextPageToken ? { pageToken: nextPageToken } : null, + }; + }, +}; diff --git a/apps/api/src/services/import-connectors/fixtures.ts b/apps/api/src/services/import-connectors/fixtures.ts new file mode 100644 index 0000000..23fc0f4 --- /dev/null +++ b/apps/api/src/services/import-connectors/fixtures.ts @@ -0,0 +1,186 @@ +import type { ImportProvider } from "@workspace/validators/types/import"; +import type { ExtractPage } from "./types.js"; + +/** + * Deterministic sample pages served when INTEGRATION_LIVE_FETCH_ENABLED is false, mirroring + * the dry-run behaviour of SEQUENCE_LIVE_SEND_ENABLED in sequence-adapters. + * + * This keeps the whole pipeline — extraction, mapping, matching, loading — exercisable + * before any provider credentials exist, and gives tests a network-free path. + * + * Fixtures are intentionally messy: mixed casing, a free-mail address, a missing name, and + * a duplicate across pages, so the normalization and dedupe paths are actually hit. + */ +const FIXTURES: Record = { + gmail: { + records: [ + { + externalId: "dana.reeves@northwind.example", + data: { + name: "Dana Reeves", + email: "dana.reeves@northwind.example", + lastContactedAt: "2026-06-02T10:15:00.000Z", + messageCount: 4, + source: "gmail_sent", + }, + }, + { + externalId: "s.patel@brightline.example", + data: { + name: null, + email: "S.Patel@Brightline.example", + lastContactedAt: "2026-05-28T08:00:00.000Z", + messageCount: 1, + source: "gmail_sent", + }, + }, + ], + nextCursor: null, + }, + + google_calendar: { + records: [ + { + externalId: "dana.reeves@northwind.example", + data: { + name: "Dana Reeves", + email: "dana.reeves@northwind.example", + lastContactedAt: "2026-06-10T14:00:00.000Z", + meetingCount: 2, + lastMeetingTitle: "Northwind / Stallion intro", + source: "google_calendar", + }, + }, + { + externalId: "marcus@lumen-labs.example", + data: { + name: "Marcus Oyelaran", + email: "marcus@lumen-labs.example", + lastContactedAt: "2026-06-11T09:30:00.000Z", + meetingCount: 1, + lastMeetingTitle: "Pricing walkthrough", + source: "google_calendar", + }, + }, + ], + nextCursor: null, + }, + + calendly: { + records: [ + { + externalId: "https://api.calendly.com/scheduled_events/abc/invitees/001", + data: { + name: "Priya Raman", + email: "priya@vantage.example", + lastContactedAt: "2026-06-14T16:00:00.000Z", + eventName: "30 Minute Demo", + timezone: "Asia/Kolkata", + status: "active", + "question:What are you hoping to solve?": "Replacing spreadsheets", + "question:Company": "Vantage Systems", + }, + }, + ], + nextCursor: null, + }, + + google_sheets: { + records: [ + { + externalId: "fixture-sheet:2", + data: { + Name: "Alexei Petrov", + Email: "alexei@harborworks.example", + Company: "Harborworks", + "Job Title": "Head of Ops", + }, + }, + { + externalId: "fixture-sheet:3", + data: { + Name: "Fen Zhao", + Email: "fen.zhao@gmail.com", + Company: "", + "Job Title": "Consultant", + }, + }, + ], + nextCursor: null, + }, + + posthog: { + records: [ + { + externalId: "01917c3e-0000-7000-8000-000000000001", + data: { + distinctId: "user_8812", + email: "marcus@lumen-labs.example", + name: "Marcus Oyelaran", + createdAt: "2026-04-02T11:00:00.000Z", + company: "Lumen Labs", + jobTitle: null, + phone: null, + "property:plan": "trial", + "property:seats": "12", + }, + }, + { + externalId: "01917c3e-0000-7000-8000-000000000002", + data: { + distinctId: "user_9134", + email: null, + name: null, + createdAt: "2026-05-19T07:45:00.000Z", + company: null, + jobTitle: null, + phone: null, + "property:plan": "free", + }, + }, + ], + nextCursor: null, + }, + + outlook: { + records: [ + { + externalId: "AAMkAGI2THVSAAA=", + data: { + name: "Rosa Iglesias", + email: "rosa.iglesias@meridian.example", + jobTitle: "VP Revenue", + phone: "+1 (415) 555-0142", + orgName: "Meridian Group", + source: "outlook_contacts", + }, + }, + ], + nextCursor: null, + }, + + csv: { records: [], nextCursor: null }, + webhook: { records: [], nextCursor: null }, +}; + +export function getFixturePage(provider: ImportProvider): ExtractPage { + const page = FIXTURES[provider]; + + // Structured-clone so a caller mutating record data cannot corrupt later runs. + return { + records: page.records.map((record) => ({ + externalId: record.externalId, + data: { ...record.data }, + })), + nextCursor: page.nextCursor, + }; +} + +export function getFixtureFields(provider: ImportProvider): string[] { + const fields = new Set(); + for (const record of FIXTURES[provider].records) { + for (const key of Object.keys(record.data)) fields.add(key); + } + + return [...fields]; +} diff --git a/apps/api/src/services/import-connectors/google.ts b/apps/api/src/services/import-connectors/google.ts new file mode 100644 index 0000000..de94188 --- /dev/null +++ b/apps/api/src/services/import-connectors/google.ts @@ -0,0 +1,288 @@ +import { + deriveNameFromEmail, + isAutomatedEmail, + isValidEmail, + normalizeEmail, + parseAddressList, +} from "@/services/import-engine.js"; +import { buildUrl, requestJson } from "./http.js"; +import { + readConfigString, + readCursorNumber, + readCursorString, + type Connector, + type ConnectorContext, + type ExtractPage, + type RawRecord, +} from "./types.js"; + +const GMAIL_BASE = "https://gmail.googleapis.com/gmail/v1/users/me"; +const CALENDAR_BASE = "https://www.googleapis.com/calendar/v3"; +const SHEETS_BASE = "https://sheets.googleapis.com/v4/spreadsheets"; + +type GmailListResponse = { + messages?: { id: string }[]; + nextPageToken?: string; +}; + +type GmailMessageResponse = { + id: string; + internalDate?: string; + payload?: { headers?: { name: string; value: string }[] }; +}; + +function headerValue(message: GmailMessageResponse, name: string) { + const headers = message.payload?.headers ?? []; + const match = headers.find((header) => header.name.toLowerCase() === name.toLowerCase()); + return match?.value ?? ""; +} + +/** + * Mines sent mail for recipients. A person the user has emailed is far stronger evidence + * of a real relationship than membership in any directory, which is why this seeds + * `qualified` rather than `lead`. + * + * Gmail has no bulk metadata endpoint, so each message in a page needs its own request. + * Page size is kept small by default to bound that fan-out. + */ +export const gmailConnector: Connector = { + provider: "gmail", + entityTypes: ["person"], + + async listFields() { + return ["name", "email", "lastContactedAt", "messageCount", "source"]; + }, + + async extract(ctx: ConnectorContext): Promise { + const pageToken = readCursorString(ctx.cursor, "pageToken"); + const query = readConfigString(ctx.config, "query") ?? "in:sent"; + + const list = await requestJson({ + url: buildUrl(`${GMAIL_BASE}/messages`, { + q: query, + maxResults: Math.min(ctx.pageSize, 50), + pageToken: pageToken ?? undefined, + }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + const messages = list.messages ?? []; + const byEmail = new Map(); + + for (const summary of messages) { + const message = await requestJson({ + url: buildUrl(`${GMAIL_BASE}/messages/${summary.id}`, { + format: "metadata", + metadataHeaders: "To", + }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + const sentAt = message.internalDate + ? new Date(Number(message.internalDate)).toISOString() + : null; + + const recipients = [ + ...parseAddressList(headerValue(message, "To")), + ...parseAddressList(headerValue(message, "Cc")), + ]; + + for (const recipient of recipients) { + if (isAutomatedEmail(recipient.email)) continue; + + const existing = byEmail.get(recipient.email); + if (existing) { + const count = Number(existing.data.messageCount ?? 1) + 1; + existing.data.messageCount = count; + // Keep the most recent contact date across the page. + if (sentAt && (!existing.data.lastContactedAt || sentAt > String(existing.data.lastContactedAt))) { + existing.data.lastContactedAt = sentAt; + } + continue; + } + + byEmail.set(recipient.email, { + externalId: recipient.email, + data: { + name: recipient.name ?? deriveNameFromEmail(recipient.email), + email: recipient.email, + lastContactedAt: sentAt, + messageCount: 1, + source: "gmail_sent", + }, + }); + } + } + + return { + records: [...byEmail.values()], + nextCursor: list.nextPageToken ? { pageToken: list.nextPageToken } : null, + }; + }, +}; + +type CalendarEventsResponse = { + items?: { + id: string; + start?: { dateTime?: string; date?: string }; + summary?: string; + attendees?: { + email?: string; + displayName?: string; + self?: boolean; + resource?: boolean; + organizer?: boolean; + responseStatus?: string; + }[]; + }[]; + nextPageToken?: string; +}; + +/** + * Mines calendar attendees. Someone who took a meeting is a stronger signal than someone + * who was merely emailed, so both Google sources seed `qualified`. + * + * Rooms and equipment are excluded via `resource`, and the connected user via `self`. + */ +export const googleCalendarConnector: Connector = { + provider: "google_calendar", + entityTypes: ["person"], + + async listFields() { + return ["name", "email", "lastContactedAt", "meetingCount", "lastMeetingTitle", "source"]; + }, + + async extract(ctx: ConnectorContext): Promise { + const pageToken = readCursorString(ctx.cursor, "pageToken"); + const calendarId = readConfigString(ctx.config, "calendarId") ?? "primary"; + const timeMin = + readConfigString(ctx.config, "timeMin") ?? + new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toISOString(); + + const response = await requestJson({ + url: buildUrl(`${CALENDAR_BASE}/calendars/${encodeURIComponent(calendarId)}/events`, { + timeMin, + maxResults: Math.min(ctx.pageSize, 250), + singleEvents: "true", + orderBy: "startTime", + pageToken: pageToken ?? undefined, + }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + const byEmail = new Map(); + + for (const event of response.items ?? []) { + const startedAt = event.start?.dateTime ?? event.start?.date ?? null; + + for (const attendee of event.attendees ?? []) { + if (!attendee.email || attendee.self || attendee.resource) continue; + + const email = normalizeEmail(attendee.email); + if (!isValidEmail(email) || isAutomatedEmail(email)) continue; + + const existing = byEmail.get(email); + if (existing) { + existing.data.meetingCount = Number(existing.data.meetingCount ?? 1) + 1; + if (startedAt && (!existing.data.lastContactedAt || startedAt > String(existing.data.lastContactedAt))) { + existing.data.lastContactedAt = startedAt; + existing.data.lastMeetingTitle = event.summary ?? null; + } + continue; + } + + byEmail.set(email, { + externalId: email, + data: { + name: attendee.displayName ?? deriveNameFromEmail(email), + email, + lastContactedAt: startedAt, + meetingCount: 1, + lastMeetingTitle: event.summary ?? null, + source: "google_calendar", + }, + }); + } + } + + return { + records: [...byEmail.values()], + nextCursor: response.nextPageToken ? { pageToken: response.nextPageToken } : null, + }; + }, +}; + +type SheetsValuesResponse = { + values?: string[][]; +}; + +/** + * Reads a sheet as a header row plus data rows, so it maps through exactly the same path + * as a CSV upload. Paginates by row offset because the Sheets values endpoint has no + * cursor of its own. + */ +export const googleSheetsConnector: Connector = { + provider: "google_sheets", + entityTypes: ["person", "org"], + + async listFields(ctx: ConnectorContext) { + const headerRow = await fetchSheetRange(ctx, 1, 1); + return (headerRow[0] ?? []).map((header) => header.trim()).filter((header) => header !== ""); + }, + + async extract(ctx: ConnectorContext): Promise { + const startRow = readCursorNumber(ctx.cursor, "startRow") ?? 2; + const headerRow = await fetchSheetRange(ctx, 1, 1); + const headers = (headerRow[0] ?? []).map((header) => header.trim()); + + const endRow = startRow + ctx.pageSize - 1; + const rows = await fetchSheetRange(ctx, startRow, endRow); + + const records: RawRecord[] = rows.map((cells, index) => { + const data: Record = {}; + headers.forEach((header, columnIndex) => { + if (header === "") return; + data[header] = (cells[columnIndex] ?? "").trim(); + }); + + const rowNumber = startRow + index; + return { + // The sheet row is the only stable identifier available; re-running the same sheet + // therefore updates in place rather than duplicating. + externalId: `${readConfigString(ctx.config, "spreadsheetId") ?? "sheet"}:${rowNumber}`, + data, + }; + }); + + const hasMore = rows.length === ctx.pageSize; + return { + records: records.filter((record) => Object.values(record.data).some((value) => value !== "")), + nextCursor: hasMore ? { startRow: endRow + 1 } : null, + }; + }, +}; + +async function fetchSheetRange(ctx: ConnectorContext, startRow: number, endRow: number) { + const spreadsheetId = readConfigString(ctx.config, "spreadsheetId"); + if (!spreadsheetId) { + throw new Error("Google Sheets import requires a spreadsheetId in the source config"); + } + + const sheetName = readConfigString(ctx.config, "sheetName"); + const range = sheetName + ? `${sheetName}!A${startRow}:ZZ${endRow}` + : `A${startRow}:ZZ${endRow}`; + + const response = await requestJson({ + url: buildUrl(`${SHEETS_BASE}/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}`, { + majorDimension: "ROWS", + }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + return response.values ?? []; +} diff --git a/apps/api/src/services/import-connectors/http.ts b/apps/api/src/services/import-connectors/http.ts new file mode 100644 index 0000000..36e8b4b --- /dev/null +++ b/apps/api/src/services/import-connectors/http.ts @@ -0,0 +1,109 @@ +import { AppError } from "@/lib/app-error.js"; +import { STATUS_CODES } from "@/constants/status-codes.js"; +import type { FetchLike } from "./types.js"; + +/** + * Signals that extraction should pause and resume from the persisted cursor rather than + * fail the job. The worker reschedules using `retryAfterMs`. + */ +export class RateLimitError extends Error { + readonly retryAfterMs: number; + + constructor(message: string, retryAfterMs: number) { + super(message); + this.name = "RateLimitError"; + this.retryAfterMs = retryAfterMs; + } +} + +/** + * Signals expired or revoked credentials. The service flags the connection + * `reconnect_required` instead of retrying, because retrying cannot succeed. + */ +export class ConnectionAuthError extends Error { + constructor(message: string) { + super(message); + this.name = "ConnectionAuthError"; + } +} + +const DEFAULT_RETRY_AFTER_MS = 60_000; +const MAX_RETRY_AFTER_MS = 3_600_000; + +/** + * `Retry-After` is either delta-seconds or an HTTP date. Both appear in the wild; + * Google uses seconds, Microsoft Graph sometimes returns a date. + */ +export function parseRetryAfter(header: string | null, now: Date): number { + if (!header) return DEFAULT_RETRY_AFTER_MS; + + const trimmed = header.trim(); + const seconds = Number(trimmed); + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS); + } + + const retryDate = new Date(trimmed); + if (!Number.isNaN(retryDate.getTime())) { + const delta = retryDate.getTime() - now.getTime(); + return Math.min(Math.max(delta, 0), MAX_RETRY_AFTER_MS); + } + + return DEFAULT_RETRY_AFTER_MS; +} + +export type JsonRequestInput = { + url: string; + accessToken: string; + fetchImpl: FetchLike; + method?: string; + body?: unknown; + headers?: Record; + now?: Date; +}; + +export async function requestJson(input: JsonRequestInput): Promise { + const now = input.now ?? new Date(); + const response = await input.fetchImpl(input.url, { + method: input.method ?? "GET", + headers: { + authorization: `Bearer ${input.accessToken}`, + accept: "application/json", + ...(input.body === undefined ? {} : { "content-type": "application/json" }), + ...input.headers, + }, + ...(input.body === undefined ? {} : { body: JSON.stringify(input.body) }), + }); + + if (response.status === 429 || response.status === 503) { + throw new RateLimitError( + `Rate limited by ${new URL(input.url).host}`, + parseRetryAfter(response.headers.get("retry-after"), now), + ); + } + + if (response.status === 401 || response.status === 403) { + throw new ConnectionAuthError( + `Credentials rejected by ${new URL(input.url).host} with status ${response.status}`, + ); + } + + if (!response.ok) { + throw new AppError( + `Request to ${new URL(input.url).host} failed with status ${response.status}`, + STATUS_CODES.BAD_GATEWAY, + ); + } + + return (await response.json()) as T; +} + +export function buildUrl(base: string, params: Record) { + const url = new URL(base); + for (const [key, value] of Object.entries(params)) { + if (value === undefined || value === "") continue; + url.searchParams.set(key, String(value)); + } + + return url.toString(); +} diff --git a/apps/api/src/services/import-connectors/index.ts b/apps/api/src/services/import-connectors/index.ts new file mode 100644 index 0000000..b09d56d --- /dev/null +++ b/apps/api/src/services/import-connectors/index.ts @@ -0,0 +1,81 @@ +import type { ImportProvider } from "@workspace/validators/types/import"; +import { IMPORT_PUSH_PROVIDERS } from "@workspace/validators/types/import"; +import { env } from "@/config/env.config.js"; +import { AppError } from "@/lib/app-error.js"; +import { STATUS_CODES } from "@/constants/status-codes.js"; +import { calendlyConnector } from "./calendly.js"; +import { getFixtureFields, getFixturePage } from "./fixtures.js"; +import { gmailConnector, googleCalendarConnector, googleSheetsConnector } from "./google.js"; +import { outlookConnector } from "./outlook.js"; +import { posthogConnector } from "./posthog.js"; +import type { Connector, ConnectorContext, ExtractPage } from "./types.js"; + +/** + * `csv` and `webhook` are push sources: records arrive by upload or HTTP push and are staged + * at that moment, so there is nothing to pull. They are registered as empty connectors so + * the rest of the pipeline can treat every provider uniformly. + */ +const pushConnector = (provider: ImportProvider): Connector => ({ + provider, + entityTypes: ["person", "org"], + async listFields() { + return []; + }, + async extract() { + return { records: [], nextCursor: null }; + }, +}); + +const CONNECTORS: Record = { + gmail: gmailConnector, + google_calendar: googleCalendarConnector, + google_sheets: googleSheetsConnector, + calendly: calendlyConnector, + posthog: posthogConnector, + outlook: outlookConnector, + csv: pushConnector("csv"), + webhook: pushConnector("webhook"), +}; + +export function getConnector(provider: ImportProvider): Connector { + const connector = CONNECTORS[provider]; + if (!connector) { + throw new AppError(`Unsupported import provider: ${provider}`, STATUS_CODES.BAD_REQUEST); + } + + return connector; +} + +export function isPushProvider(provider: ImportProvider) { + return IMPORT_PUSH_PROVIDERS.includes(provider); +} + +/** + * Single place the live/fixture switch is applied, so no connector has to know about it. + * With INTEGRATION_LIVE_FETCH_ENABLED off, the pipeline runs end to end against fixtures. + */ +export async function extractPage( + provider: ImportProvider, + ctx: ConnectorContext, +): Promise { + if (!env.INTEGRATION_LIVE_FETCH_ENABLED) { + // Fixtures are a single page; a stored cursor means that page was already consumed. + return ctx.cursor ? { records: [], nextCursor: null } : getFixturePage(provider); + } + + return getConnector(provider).extract(ctx); +} + +export async function listSourceFields( + provider: ImportProvider, + ctx: ConnectorContext, +): Promise { + if (!env.INTEGRATION_LIVE_FETCH_ENABLED) { + return getFixtureFields(provider); + } + + return getConnector(provider).listFields(ctx); +} + +export { ConnectionAuthError, RateLimitError } from "./http.js"; +export type { Connector, ConnectorContext, ExtractPage, RawRecord } from "./types.js"; diff --git a/apps/api/src/services/import-connectors/outlook.ts b/apps/api/src/services/import-connectors/outlook.ts new file mode 100644 index 0000000..b4b7343 --- /dev/null +++ b/apps/api/src/services/import-connectors/outlook.ts @@ -0,0 +1,247 @@ +import { + deriveNameFromEmail, + isAutomatedEmail, + isValidEmail, + normalizeEmail, +} from "@/services/import-engine.js"; +import { buildUrl, requestJson } from "./http.js"; +import { + readConfigString, + readCursorString, + type Connector, + type ConnectorContext, + type ExtractPage, + type RawRecord, +} from "./types.js"; + +const GRAPH_BASE = "https://graph.microsoft.com/v1.0/me"; + +export const OUTLOOK_MODES = ["contacts", "sent_mail", "calendar"] as const; +export type OutlookMode = (typeof OUTLOOK_MODES)[number]; + +type GraphRecipient = { + emailAddress?: { address?: string; name?: string }; +}; + +type GraphContactsResponse = { + value?: { + id: string; + displayName?: string; + jobTitle?: string; + companyName?: string; + mobilePhone?: string; + businessPhones?: string[]; + emailAddresses?: { address?: string; name?: string }[]; + }[]; + "@odata.nextLink"?: string; +}; + +type GraphMessagesResponse = { + value?: { + id: string; + sentDateTime?: string; + toRecipients?: GraphRecipient[]; + ccRecipients?: GraphRecipient[]; + }[]; + "@odata.nextLink"?: string; +}; + +type GraphEventsResponse = { + value?: { + id: string; + subject?: string; + start?: { dateTime?: string }; + attendees?: { + emailAddress?: { address?: string; name?: string }; + type?: string; + }[]; + }[]; + "@odata.nextLink"?: string; +}; + +function readMode(ctx: ConnectorContext): OutlookMode { + const configured = readConfigString(ctx.config, "mode"); + return (OUTLOOK_MODES as readonly string[]).includes(configured ?? "") + ? (configured as OutlookMode) + : "contacts"; +} + +/** + * Microsoft Graph is unusually efficient to integrate: contacts, sent mail, and calendar + * arrive through one OAuth grant and one API surface, so a single connector covers what + * takes two on the Google side. + * + * Which of the three a job pulls is chosen by `mode` in the source config. + */ +export const outlookConnector: Connector = { + provider: "outlook", + entityTypes: ["person"], + + async listFields(ctx: ConnectorContext) { + const mode = readMode(ctx); + if (mode === "contacts") { + return ["name", "email", "jobTitle", "phone", "orgName", "source"]; + } + if (mode === "calendar") { + return ["name", "email", "lastContactedAt", "meetingCount", "lastMeetingTitle", "source"]; + } + + return ["name", "email", "lastContactedAt", "messageCount", "source"]; + }, + + async extract(ctx: ConnectorContext): Promise { + const mode = readMode(ctx); + if (mode === "contacts") return extractContacts(ctx); + if (mode === "calendar") return extractCalendar(ctx); + + return extractSentMail(ctx); + }, +}; + +async function extractContacts(ctx: ConnectorContext): Promise { + const nextLink = readCursorString(ctx.cursor, "nextLink"); + const response = await requestJson({ + url: nextLink ?? buildUrl(`${GRAPH_BASE}/contacts`, { $top: Math.min(ctx.pageSize, 100) }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + const records: RawRecord[] = []; + + for (const contact of response.value ?? []) { + const rawEmail = contact.emailAddresses?.[0]?.address; + const email = rawEmail && isValidEmail(rawEmail) ? normalizeEmail(rawEmail) : null; + if (!email && !contact.displayName) continue; + + records.push({ + externalId: contact.id, + data: { + name: contact.displayName ?? (email ? deriveNameFromEmail(email) : null), + email, + jobTitle: contact.jobTitle ?? null, + phone: contact.mobilePhone ?? contact.businessPhones?.[0] ?? null, + orgName: contact.companyName ?? null, + source: "outlook_contacts", + }, + }); + } + + return { + records, + nextCursor: response["@odata.nextLink"] ? { nextLink: response["@odata.nextLink"] } : null, + }; +} + +async function extractSentMail(ctx: ConnectorContext): Promise { + const nextLink = readCursorString(ctx.cursor, "nextLink"); + const response = await requestJson({ + url: + nextLink ?? + buildUrl(`${GRAPH_BASE}/mailFolders/sentitems/messages`, { + $top: Math.min(ctx.pageSize, 100), + $select: "toRecipients,ccRecipients,sentDateTime", + $orderby: "sentDateTime desc", + }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + const byEmail = new Map(); + + for (const message of response.value ?? []) { + const recipients = [...(message.toRecipients ?? []), ...(message.ccRecipients ?? [])]; + + for (const recipient of recipients) { + const address = recipient.emailAddress?.address; + if (!address) continue; + + const email = normalizeEmail(address); + if (!isValidEmail(email) || isAutomatedEmail(email)) continue; + + const sentAt = message.sentDateTime ?? null; + const existing = byEmail.get(email); + if (existing) { + existing.data.messageCount = Number(existing.data.messageCount ?? 1) + 1; + if (sentAt && (!existing.data.lastContactedAt || sentAt > String(existing.data.lastContactedAt))) { + existing.data.lastContactedAt = sentAt; + } + continue; + } + + byEmail.set(email, { + externalId: email, + data: { + name: recipient.emailAddress?.name ?? deriveNameFromEmail(email), + email, + lastContactedAt: sentAt, + messageCount: 1, + source: "outlook_sent", + }, + }); + } + } + + return { + records: [...byEmail.values()], + nextCursor: response["@odata.nextLink"] ? { nextLink: response["@odata.nextLink"] } : null, + }; +} + +async function extractCalendar(ctx: ConnectorContext): Promise { + const nextLink = readCursorString(ctx.cursor, "nextLink"); + const response = await requestJson({ + url: + nextLink ?? + buildUrl(`${GRAPH_BASE}/events`, { + $top: Math.min(ctx.pageSize, 100), + $select: "subject,start,attendees", + $orderby: "start/dateTime desc", + }), + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + const byEmail = new Map(); + + for (const event of response.value ?? []) { + const startedAt = event.start?.dateTime ?? null; + + for (const attendee of event.attendees ?? []) { + // Rooms and equipment come through as `resource` attendees and are not people. + if (attendee.type === "resource") continue; + + const address = attendee.emailAddress?.address; + if (!address) continue; + + const email = normalizeEmail(address); + if (!isValidEmail(email) || isAutomatedEmail(email)) continue; + + const existing = byEmail.get(email); + if (existing) { + existing.data.meetingCount = Number(existing.data.meetingCount ?? 1) + 1; + if (startedAt && (!existing.data.lastContactedAt || startedAt > String(existing.data.lastContactedAt))) { + existing.data.lastContactedAt = startedAt; + existing.data.lastMeetingTitle = event.subject ?? null; + } + continue; + } + + byEmail.set(email, { + externalId: email, + data: { + name: attendee.emailAddress?.name ?? deriveNameFromEmail(email), + email, + lastContactedAt: startedAt, + meetingCount: 1, + lastMeetingTitle: event.subject ?? null, + source: "outlook_calendar", + }, + }); + } + } + + return { + records: [...byEmail.values()], + nextCursor: response["@odata.nextLink"] ? { nextLink: response["@odata.nextLink"] } : null, + }; +} diff --git a/apps/api/src/services/import-connectors/posthog.ts b/apps/api/src/services/import-connectors/posthog.ts new file mode 100644 index 0000000..f33b66d --- /dev/null +++ b/apps/api/src/services/import-connectors/posthog.ts @@ -0,0 +1,142 @@ +import { env } from "@/config/env.config.js"; +import { deriveNameFromEmail, isValidEmail, normalizeEmail } from "@/services/import-engine.js"; +import { buildUrl, requestJson } from "./http.js"; +import { + readConfigString, + readCursorString, + type Connector, + type ConnectorContext, + type ExtractPage, + type RawRecord, +} from "./types.js"; + +type PostHogPersonsResponse = { + results?: { + id: string | number; + distinct_ids?: string[]; + name?: string; + created_at?: string; + properties?: Record; + }[]; + next?: string | null; +}; + +/** + * PostHog property keys that carry CRM meaning. Everything else is surfaced as a mappable + * field so a workspace's own property naming can be routed to custom fields. + */ +const KNOWN_PROPERTY_FIELDS = [ + "email", + "name", + "$name", + "first_name", + "last_name", + "company", + "company_name", + "organization", + "$initial_referring_domain", + "job_title", + "title", + "phone", +]; + +function readProperty(properties: Record, key: string): string | null { + const value = properties[key]; + if (typeof value === "string" && value.trim() !== "") return value.trim(); + if (typeof value === "number" || typeof value === "boolean") return String(value); + return null; +} + +function apiHost(ctx: ConnectorContext) { + return readConfigString(ctx.config, "apiHost") ?? env.POSTHOG_API_HOST; +} + +/** + * Answers "who is using the product but has not been contacted", which is what makes the + * CRM product-led rather than a rolodex. + * + * Authenticates with a personal API key rather than OAuth, so the connection carries no + * refresh token. + */ +export const posthogConnector: Connector = { + provider: "posthog", + entityTypes: ["person"], + + async listFields(ctx: ConnectorContext) { + const page = await this.extract({ ...ctx, pageSize: 20, cursor: null }); + const fields = new Set(["distinctId", "email", "name", "createdAt"]); + + for (const record of page.records) { + for (const key of Object.keys(record.data)) fields.add(key); + } + + return [...fields]; + }, + + async extract(ctx: ConnectorContext): Promise { + const projectId = readConfigString(ctx.config, "projectId"); + if (!projectId) { + throw new Error("PostHog import requires a projectId in the source config"); + } + + // PostHog returns a fully-qualified `next` URL, so a stored cursor is used verbatim. + const nextUrl = readCursorString(ctx.cursor, "next"); + const url = + nextUrl ?? + buildUrl(`${apiHost(ctx)}/api/projects/${encodeURIComponent(projectId)}/persons/`, { + limit: Math.min(ctx.pageSize, 100), + }); + + const response = await requestJson({ + url, + accessToken: ctx.auth.accessToken, + fetchImpl: ctx.fetchImpl, + }); + + const records: RawRecord[] = []; + + for (const person of response.results ?? []) { + const properties = person.properties ?? {}; + const rawEmail = readProperty(properties, "email"); + const email = rawEmail && isValidEmail(rawEmail) ? normalizeEmail(rawEmail) : null; + + const firstName = readProperty(properties, "first_name"); + const lastName = readProperty(properties, "last_name"); + const composedName = + readProperty(properties, "name") ?? + readProperty(properties, "$name") ?? + person.name ?? + (firstName || lastName ? [firstName, lastName].filter(Boolean).join(" ") : null); + + const data: Record = { + distinctId: person.distinct_ids?.[0] ?? null, + email, + name: composedName ?? (email ? deriveNameFromEmail(email) : null), + createdAt: person.created_at ?? null, + company: + readProperty(properties, "company") ?? + readProperty(properties, "company_name") ?? + readProperty(properties, "organization"), + jobTitle: readProperty(properties, "job_title") ?? readProperty(properties, "title"), + phone: readProperty(properties, "phone"), + }; + + // Surface remaining custom properties so a workspace can map its own naming. + for (const [key, value] of Object.entries(properties)) { + if (KNOWN_PROPERTY_FIELDS.includes(key)) continue; + if (key.startsWith("$")) continue; + + const readable = readProperty(properties, key); + if (readable !== null) data[`property:${key}`] = readable; + else if (value === null) data[`property:${key}`] = null; + } + + records.push({ externalId: String(person.id), data }); + } + + return { + records, + nextCursor: response.next ? { next: response.next } : null, + }; + }, +}; diff --git a/apps/api/src/services/import-connectors/types.ts b/apps/api/src/services/import-connectors/types.ts new file mode 100644 index 0000000..463aa84 --- /dev/null +++ b/apps/api/src/services/import-connectors/types.ts @@ -0,0 +1,64 @@ +import type { + ImportEntityType, + ImportProvider, +} from "@workspace/validators/types/import"; +import type { ImportCursor } from "@/db/schema/import.schema.js"; + +export type FetchLike = typeof fetch; + +export type ConnectorAuth = { + accessToken: string; + refreshToken: string | null; +}; + +export type ConnectorContext = { + auth: ConnectorAuth; + /** Provider-scoped extraction settings: spreadsheet id, PostHog project, Outlook mode. */ + config: Record; + cursor: ImportCursor | null; + pageSize: number; + fetchImpl: FetchLike; +}; + +export type RawRecord = { + /** + * Stable identifier at the source. Written to `external_identities`, which is what makes + * a re-sync update rather than duplicate. Null when a source offers nothing stable. + */ + externalId: string | null; + data: Record; +}; + +export type ExtractPage = { + records: RawRecord[]; + /** Null signals extraction is complete. */ + nextCursor: ImportCursor | null; +}; + +export type Connector = { + provider: ImportProvider; + entityTypes: ImportEntityType[]; + /** Drives the mapping UI, so mapping is discovered rather than hardcoded per provider. */ + listFields(ctx: ConnectorContext): Promise; + extract(ctx: ConnectorContext): Promise; +}; + +export function readConfigString( + config: Record, + key: string, +): string | null { + const value = config[key]; + return typeof value === "string" && value.trim() !== "" ? value.trim() : null; +} + +export function readCursorString(cursor: ImportCursor | null, key: string): string | null { + if (!cursor) return null; + const value = cursor[key]; + return typeof value === "string" && value !== "" ? value : null; +} + +export function readCursorNumber(cursor: ImportCursor | null, key: string): number | null { + if (!cursor) return null; + const value = cursor[key]; + return typeof value === "number" && Number.isFinite(value) ? value : null; +} diff --git a/apps/api/src/services/import-csv.ts b/apps/api/src/services/import-csv.ts new file mode 100644 index 0000000..6011cb8 --- /dev/null +++ b/apps/api/src/services/import-csv.ts @@ -0,0 +1,125 @@ +import { AppError } from "@/lib/app-error.js"; +import { STATUS_CODES } from "@/constants/status-codes.js"; + +export type ParsedCsv = { + headers: string[]; + rows: Record[]; +}; + +/** + * RFC 4180 parser. Written rather than pulled in as a dependency because the surface we + * need is small and the pipeline requires it to be pure and fully testable. + * + * Handles quoted fields, escaped quotes (`""`), embedded newlines and commas, CRLF and LF + * line endings, and a UTF-8 BOM. Does not handle alternate delimiters. + */ +export function parseCsv(content: string): ParsedCsv { + const withoutBom = content.startsWith("") ? content.slice(1) : content; + const rows = parseRows(withoutBom); + + if (rows.length === 0) { + throw new AppError("The uploaded file is empty", STATUS_CODES.UNPROCESSABLE_ENTITY); + } + + const headers = dedupeHeaders(rows[0]!.map((header) => header.trim())); + if (headers.every((header) => header === "")) { + throw new AppError("The uploaded file has no header row", STATUS_CODES.UNPROCESSABLE_ENTITY); + } + + const dataRows = rows.slice(1).filter((cells) => !isBlankRow(cells)); + const parsedRows = dataRows.map((cells) => { + const row: Record = {}; + headers.forEach((header, index) => { + if (header === "") return; + row[header] = (cells[index] ?? "").trim(); + }); + return row; + }); + + return { headers: headers.filter((header) => header !== ""), rows: parsedRows }; +} + +function parseRows(content: string): string[][] { + const rows: string[][] = []; + let cells: string[] = []; + let value = ""; + let inQuotes = false; + let index = 0; + + const endCell = () => { + cells.push(value); + value = ""; + }; + + const endRow = () => { + endCell(); + rows.push(cells); + cells = []; + }; + + while (index < content.length) { + const char = content[index]!; + + if (inQuotes) { + if (char === '"') { + if (content[index + 1] === '"') { + value += '"'; + index += 2; + continue; + } + inQuotes = false; + index += 1; + continue; + } + value += char; + index += 1; + continue; + } + + if (char === '"') { + inQuotes = true; + index += 1; + continue; + } + + if (char === ",") { + endCell(); + index += 1; + continue; + } + + if (char === "\r" || char === "\n") { + endRow(); + index += char === "\r" && content[index + 1] === "\n" ? 2 : 1; + continue; + } + + value += char; + index += 1; + } + + // A trailing newline produces no final row; anything else is a real last row. + if (value !== "" || cells.length > 0) endRow(); + + return rows; +} + +function isBlankRow(cells: string[]) { + return cells.every((cell) => cell.trim() === ""); +} + +/** + * Spreadsheet exports frequently repeat a header. Suffixing keeps every column addressable + * in the mapping UI instead of silently dropping all but the last. + */ +function dedupeHeaders(headers: string[]) { + const seen = new Map(); + + return headers.map((header) => { + if (header === "") return header; + + const count = seen.get(header) ?? 0; + seen.set(header, count + 1); + return count === 0 ? header : `${header} (${count + 1})`; + }); +} diff --git a/apps/api/src/services/import-engine.ts b/apps/api/src/services/import-engine.ts new file mode 100644 index 0000000..5038591 --- /dev/null +++ b/apps/api/src/services/import-engine.ts @@ -0,0 +1,402 @@ +import { createHash } from "node:crypto"; +import type { ImportMatchReason } from "@workspace/validators/types/import"; + +/** + * Local parts that identify automated senders rather than people. Gmail and Outlook mining + * surface these constantly, and importing them produces CRM records nobody can reply to. + */ +const AUTOMATED_LOCAL_PARTS = new Set([ + "noreply", + "no-reply", + "donotreply", + "do-not-reply", + "notifications", + "notification", + "mailer-daemon", + "postmaster", + "bounce", + "bounces", + "automated", + "alerts", + "alert", +]); + +const ORG_SUFFIXES = [ + "incorporated", + "inc", + "llc", + "l.l.c", + "ltd", + "limited", + "corp", + "corporation", + "co", + "company", + "gmbh", + "bv", + "nv", + "plc", + "pty", + "ag", + "sa", + "srl", + "oy", + "ab", +]; + +/** + * Domains where the mailbox belongs to a person rather than an organization, so the domain + * must never be used to infer or match an org. + */ +const FREE_EMAIL_DOMAINS = new Set([ + "gmail.com", + "googlemail.com", + "yahoo.com", + "yahoo.co.uk", + "hotmail.com", + "hotmail.co.uk", + "outlook.com", + "live.com", + "msn.com", + "aol.com", + "icloud.com", + "me.com", + "mac.com", + "proton.me", + "protonmail.com", + "gmx.com", + "gmx.de", + "mail.com", + "zoho.com", + "yandex.com", + "fastmail.com", + "hey.com", +]); + +const SUBADDRESSING_DOMAINS = new Set(["gmail.com", "googlemail.com"]); + +const EMAIL_PATTERN = /^[^\s@]+@[^\s@.]+\.[^\s@]+$/; + +export function normalizeEmail(value: string) { + return value.trim().toLowerCase(); +} + +export function isValidEmail(value: string) { + return EMAIL_PATTERN.test(normalizeEmail(value)); +} + +export function getEmailDomain(value: string) { + const normalized = normalizeEmail(value); + const domain = normalized.split("@")[1]; + return domain ?? null; +} + +export function getEmailLocalPart(value: string) { + const normalized = normalizeEmail(value); + const localPart = normalized.split("@")[0]; + return localPart ?? null; +} + +export function isAutomatedEmail(value: string) { + const localPart = getEmailLocalPart(value); + if (!localPart) return false; + + const withoutTag = localPart.split("+")[0] ?? localPart; + return AUTOMATED_LOCAL_PARTS.has(withoutTag); +} + +export function isFreeEmailDomain(domain: string | null) { + if (!domain) return false; + return FREE_EMAIL_DOMAINS.has(domain.toLowerCase()); +} + +/** + * Collapses dots and plus-addressing so `a.b+tag@gmail.com` matches `ab@gmail.com`. + * Only applied to providers that actually treat those as equivalent, and only when the + * job opts in — it is wrong for domains that route them to distinct mailboxes. + */ +export function normalizeEmailForMatching(value: string, collapseSubaddressing: boolean) { + const normalized = normalizeEmail(value); + if (!collapseSubaddressing) return normalized; + + const [localPart, domain] = normalized.split("@"); + if (!localPart || !domain || !SUBADDRESSING_DOMAINS.has(domain)) return normalized; + + const withoutTag = localPart.split("+")[0] ?? localPart; + return `${withoutTag.replaceAll(".", "")}@${domain}`; +} + +/** + * Reduces a URL, bare hostname, or email domain to a comparable apex-ish form. + * Returns null for free mailbox providers so a personal address never resolves to an org. + */ +export function normalizeDomain(value: string): string | null { + const trimmed = value.trim().toLowerCase(); + if (trimmed === "") return null; + + const withoutScheme = trimmed.replace(/^[a-z][a-z0-9+.-]*:\/\//, ""); + const withoutAuth = withoutScheme.includes("@") + ? (withoutScheme.split("@").pop() ?? withoutScheme) + : withoutScheme; + const hostname = withoutAuth.split("/")[0]?.split("?")[0]?.split("#")[0] ?? ""; + const withoutPort = hostname.split(":")[0] ?? ""; + const withoutWww = withoutPort.replace(/^www\./, ""); + + if (withoutWww === "" || !withoutWww.includes(".") || withoutWww.startsWith(".")) return null; + if (FREE_EMAIL_DOMAINS.has(withoutWww)) return null; + + return withoutWww; +} + +export function normalizePersonName(value: string) { + return value.replace(/\s+/g, " ").trim(); +} + +/** + * Case, punctuation, and legal-suffix insensitive org key. This is what makes "Acme", + * "acme", and "Acme, Inc." resolve to one organization instead of three. + */ +export function normalizeOrgName(value: string): string | null { + const collapsed = value.toLowerCase().replace(/[.,]/g, " ").replace(/\s+/g, " ").trim(); + if (collapsed === "") return null; + + const words = collapsed.split(" "); + while (words.length > 1) { + const last = words[words.length - 1]!; + if (!ORG_SUFFIXES.includes(last)) break; + words.pop(); + } + + const result = words.join(" ").replace(/[^a-z0-9 &-]/g, "").replace(/\s+/g, " ").trim(); + return result === "" ? null : result; +} + +export function normalizePhone(value: string): string | null { + const trimmed = value.trim(); + if (trimmed === "") return null; + + const hasPlus = trimmed.startsWith("+"); + const digits = trimmed.replace(/\D/g, ""); + if (digits.length < 7) return null; + + return hasPlus ? `+${digits}` : digits; +} + +export function normalizeLinkedinUrl(value: string): string | null { + const trimmed = value.trim(); + if (trimmed === "") return null; + + const match = trimmed.match(/linkedin\.com\/(in|company)\/([^/?#\s]+)/i); + if (!match) return null; + + return `https://www.linkedin.com/${match[1]!.toLowerCase()}/${match[2]}`; +} + +/** + * Parses an RFC 5322 address list of the form `Name , c@d.com`. + * Quoted display names containing commas are handled; groups and comments are not, + * because neither appears in Gmail or Graph metadata headers in practice. + */ +export function parseAddressList(header: string): { name: string | null; email: string }[] { + const results: { name: string | null; email: string }[] = []; + let current = ""; + let inQuotes = false; + let inAngle = false; + + const flush = () => { + const entry = parseSingleAddress(current); + if (entry) results.push(entry); + current = ""; + }; + + for (const char of header) { + if (char === '"') inQuotes = !inQuotes; + else if (char === "<" && !inQuotes) inAngle = true; + else if (char === ">" && !inQuotes) inAngle = false; + + if (char === "," && !inQuotes && !inAngle) { + flush(); + continue; + } + current += char; + } + flush(); + + return results; +} + +function parseSingleAddress(raw: string): { name: string | null; email: string } | null { + const trimmed = raw.trim(); + if (trimmed === "") return null; + + const angleMatch = trimmed.match(/^(.*)<([^>]+)>$/); + const emailPart = angleMatch ? angleMatch[2]!.trim() : trimmed; + const email = normalizeEmail(emailPart); + if (!isValidEmail(email)) return null; + + const rawName = angleMatch ? angleMatch[1]!.trim().replace(/^"|"$/g, "").trim() : ""; + const name = rawName === "" ? null : normalizePersonName(rawName); + + return { name, email }; +} + +/** + * Falls back to a human-ish name when a source gives an address but no display name, + * so imported records are not a wall of raw email addresses. + */ +export function deriveNameFromEmail(email: string): string | null { + // Guarded so a malformed value can never be laundered into a person's name. + if (!isValidEmail(email)) return null; + + const localPart = getEmailLocalPart(email); + if (!localPart) return null; + + const withoutTag = localPart.split("+")[0] ?? localPart; + const words = withoutTag + .split(/[._-]+/) + .filter((word) => word !== "" && !/^\d+$/.test(word)) + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)); + + return words.length === 0 ? null : words.join(" "); +} + +/** + * Synthetic identifier for sources that supply nothing stable — a CSV upload, or a webhook + * push without an `externalId`. + * + * Without this, a record with no email address has nothing to match on: the email rule + * cannot fire, and `people_workspace_email_unique` is scoped to a nullable column, so + * Postgres happily accepts unlimited NULL-email rows. Re-uploading the same file would + * create the same people again on every run. + * + * Hashing the identifying fields makes an identical row resolve to an identical id, so a + * re-upload updates instead of duplicating. It is intentionally content-derived: change a + * person's details and it becomes a different record, which is the correct trade for a + * source that offers no real identity. + */ +export function deriveRecordFingerprint(parts: { + name: string | null; + email: string | null; + phone: string | null; + orgName: string | null; +}) { + const canonical = [ + parts.name?.toLowerCase() ?? "", + parts.email ?? "", + parts.phone ?? "", + parts.orgName?.toLowerCase() ?? "", + ].join("|"); + + return `fp_${createHash("sha256").update(canonical).digest("hex").slice(0, 32)}`; +} + +export type PersonMatchCandidate = { + externalId: string | null; + email: string | null; + normalizedEmail: string | null; +}; + +export type PersonMatchLookups = { + byExternalId: Map; + byEmail: Map; +}; + +export type PersonMatchResult = { + personId: string | null; + reason: ImportMatchReason; +}; + +/** + * Match order, first hit wins. External identity comes first because it is the only rule + * that works for records with no email, and the only one that survives a person changing + * their address at the source. + * + * Name-based matching is deliberately absent: it belongs in the review step as a + * suggestion, never as an automatic write. + */ +export function resolvePersonMatch( + candidate: PersonMatchCandidate, + lookups: PersonMatchLookups, +): PersonMatchResult { + if (candidate.externalId) { + const byIdentity = lookups.byExternalId.get(candidate.externalId); + if (byIdentity) return { personId: byIdentity, reason: "external_identity" }; + } + + if (candidate.normalizedEmail) { + const byEmail = lookups.byEmail.get(candidate.normalizedEmail); + if (byEmail) return { personId: byEmail, reason: "email" }; + } + + return { personId: null, reason: "none" }; +} + +export type OrgMatchCandidate = { + domain: string | null; + normalizedName: string | null; +}; + +export type OrgMatchLookups = { + byDomain: Map; + byNormalizedName: Map; +}; + +export type OrgMatchResult = { + orgId: string | null; + reason: ImportMatchReason; +}; + +/** + * Domain before name, because domain is stable and names drift. Both are advisory — + * `org_workspace_name_unique` still governs the actual write. + */ +export function resolveOrgMatch( + candidate: OrgMatchCandidate, + lookups: OrgMatchLookups, +): OrgMatchResult { + if (candidate.domain) { + const byDomain = lookups.byDomain.get(candidate.domain); + if (byDomain) return { orgId: byDomain, reason: "domain" }; + } + + if (candidate.normalizedName) { + const byName = lookups.byNormalizedName.get(candidate.normalizedName); + if (byName) return { orgId: byName, reason: "name" }; + } + + return { orgId: null, reason: "none" }; +} + +/** + * Applies the job's conflict policy to a single field. `fill_empty` is the default because + * most import sources are lower-trust than data a user typed into the CRM by hand. + * + * `crm_wins` is deliberately absent here: it means "leave the matched record untouched" + * and is enforced at record level before any field is considered. Treating it as a + * per-field rule would make it identical to `fill_empty`. + */ +export function resolveFieldValue( + existing: T | null | undefined, + incoming: T | null | undefined, + policy: "source_wins" | "fill_empty", +): T | null | undefined { + const incomingIsEmpty = incoming === null || incoming === undefined || incoming === ""; + if (incomingIsEmpty) return existing; + + if (policy === "source_wins") return incoming; + + const existingIsEmpty = existing === null || existing === undefined || existing === ""; + return existingIsEmpty ? incoming : existing; +} + +/** + * Under `crm_wins` a matched record is left exactly as the user left it. New records are + * still created — the policy governs conflicts, not whether the import runs. + */ +export function shouldSkipExistingRecord(policy: "source_wins" | "crm_wins" | "fill_empty") { + return policy === "crm_wins"; +} + +export function fieldPolicy(policy: "source_wins" | "crm_wins" | "fill_empty") { + return policy === "source_wins" ? ("source_wins" as const) : ("fill_empty" as const); +} diff --git a/apps/api/src/services/import-mapping.ts b/apps/api/src/services/import-mapping.ts new file mode 100644 index 0000000..a69f751 --- /dev/null +++ b/apps/api/src/services/import-mapping.ts @@ -0,0 +1,298 @@ +import type { ImportFieldMapping, ImportJobOptions } from "@workspace/validators/schemas/import"; +import type { + ImportEntityType, + ImportOrgTargetField, + ImportPersonTargetField, +} from "@workspace/validators/types/import"; +import { + IMPORT_ORG_TARGET_FIELD_VALUES, + IMPORT_PERSON_TARGET_FIELD_VALUES, +} from "@workspace/validators/types/import"; +import type { ImportRecordError } from "@/db/schema/import.schema.js"; +import { + deriveNameFromEmail, + getEmailDomain, + isAutomatedEmail, + isValidEmail, + normalizeDomain, + normalizeEmail, + normalizeEmailForMatching, + normalizeLinkedinUrl, + normalizeOrgName, + normalizePersonName, + normalizePhone, +} from "@/services/import-engine.js"; + +/** + * Header aliases for mapping auto-detection. Keys are the normalized source header, + * values the CRM target. Detection is a convenience — the user always confirms in review. + */ +const PERSON_FIELD_ALIASES: Record = { + name: "name", + fullname: "name", + full_name: "name", + contactname: "name", + person: "name", + email: "email", + emailaddress: "email", + email_address: "email", + workemail: "email", + primaryemail: "email", + mail: "email", + phone: "phone", + phonenumber: "phone", + phone_number: "phone", + mobile: "phone", + telephone: "phone", + tel: "phone", + jobtitle: "jobTitle", + job_title: "jobTitle", + title: "jobTitle", + role: "jobTitle", + position: "jobTitle", + linkedin: "linkedinUrl", + linkedinurl: "linkedinUrl", + linkedin_url: "linkedinUrl", + linkedinprofile: "linkedinUrl", + company: "orgName", + companyname: "orgName", + company_name: "orgName", + organization: "orgName", + organisation: "orgName", + account: "orgName", + employer: "orgName", + domain: "orgDomain", + website: "orgDomain", + companydomain: "orgDomain", + companywebsite: "orgDomain", + status: "status", + stage: "status", + lastcontacted: "lastContactedAt", + last_contacted: "lastContactedAt", + lastcontactedat: "lastContactedAt", +}; + +const ORG_FIELD_ALIASES: Record = { + name: "name", + companyname: "name", + company_name: "name", + company: "name", + organization: "name", + account: "name", + domain: "domain", + website: "domain", + url: "domain", + industry: "industry", + sector: "industry", + size: "size", + employees: "size", + headcount: "size", + companysize: "size", + location: "location", + city: "location", + country: "location", + address: "location", +}; + +function normalizeHeader(header: string) { + return header.toLowerCase().replace(/[\s\-.]/g, ""); +} + +/** + * Best-effort header matching so a well-formed CSV needs no manual mapping. Unmatched + * headers are returned with a null target rather than dropped, so they stay visible in the + * review step and can be routed to a custom field. + */ +export function autoDetectMapping( + sourceFields: string[], + entityType: ImportEntityType, +): ImportFieldMapping[] { + const aliases = entityType === "org" ? ORG_FIELD_ALIASES : PERSON_FIELD_ALIASES; + const claimed = new Set(); + + return sourceFields.map((sourceField) => { + const normalized = normalizeHeader(sourceField); + const underscored = normalized.replace(/_/g, ""); + const target = aliases[normalized] ?? aliases[underscored]; + + // First header to claim a target wins; later duplicates stay unmapped. + if (!target || claimed.has(target)) { + return { sourceField, targetField: null, customFieldId: null }; + } + + claimed.add(target); + return { sourceField, targetField: target, customFieldId: null }; + }); +} + +export type MappedPerson = { + name: string | null; + email: string | null; + normalizedEmail: string | null; + phone: string | null; + jobTitle: string | null; + linkedinUrl: string | null; + status: string | null; + orgName: string | null; + orgNormalizedName: string | null; + orgDomain: string | null; + lastContactedAt: Date | null; + customFields: Record; +}; + +export type MappedOrg = { + name: string | null; + normalizedName: string | null; + domain: string | null; + industry: string | null; + size: string | null; + location: string | null; + customFields: Record; +}; + +function readString(raw: Record, key: string): string | null { + const value = raw[key]; + if (value === null || value === undefined) return null; + if (typeof value === "string") return value.trim() === "" ? null : value.trim(); + if (typeof value === "number" || typeof value === "boolean") return String(value); + return null; +} + +function toDateOrNull(value: string | null): Date | null { + if (!value) return null; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +function collectTargets(raw: Record, mapping: ImportFieldMapping[]) { + const targets = new Map(); + const customFields: Record = {}; + + for (const field of mapping) { + const value = readString(raw, field.sourceField); + if (value === null) continue; + + if (field.customFieldId) { + customFields[field.customFieldId] = value; + continue; + } + if (field.targetField && !targets.has(field.targetField)) { + targets.set(field.targetField, value); + } + } + + return { targets, customFields }; +} + +export function applyPersonMapping( + raw: Record, + mapping: ImportFieldMapping[], + options: Pick, +): MappedPerson { + const { targets, customFields } = collectTargets(raw, mapping); + + const rawEmail = targets.get("email"); + const email = rawEmail && isValidEmail(rawEmail) ? normalizeEmail(rawEmail) : null; + + const rawName = targets.get("name"); + const name = rawName + ? normalizePersonName(rawName) + : email + ? deriveNameFromEmail(email) + : null; + + const orgNameValue = targets.get("orgName") ?? null; + const orgDomainValue = targets.get("orgDomain") ?? null; + + // Falling back to the email domain only works for corporate mailboxes; normalizeDomain + // returns null for free providers so personal addresses never invent an org. + const orgDomain = orgDomainValue + ? normalizeDomain(orgDomainValue) + : email + ? normalizeDomain(getEmailDomain(email) ?? "") + : null; + + return { + name, + email, + normalizedEmail: email + ? normalizeEmailForMatching(email, options.normalizeSubaddressing) + : null, + phone: targets.has("phone") ? normalizePhone(targets.get("phone")!) : null, + jobTitle: targets.get("jobTitle") ?? null, + linkedinUrl: targets.has("linkedinUrl") + ? normalizeLinkedinUrl(targets.get("linkedinUrl")!) + : null, + status: targets.get("status") ?? null, + orgName: orgNameValue ? normalizePersonName(orgNameValue) : null, + orgNormalizedName: orgNameValue ? normalizeOrgName(orgNameValue) : null, + orgDomain, + lastContactedAt: toDateOrNull(targets.get("lastContactedAt") ?? null), + customFields, + }; +} + +export function applyOrgMapping( + raw: Record, + mapping: ImportFieldMapping[], +): MappedOrg { + const { targets, customFields } = collectTargets(raw, mapping); + const nameValue = targets.get("name") ?? null; + const domainValue = targets.get("domain") ?? null; + + return { + name: nameValue ? normalizePersonName(nameValue) : null, + normalizedName: nameValue ? normalizeOrgName(nameValue) : null, + domain: domainValue ? normalizeDomain(domainValue) : null, + industry: targets.get("industry") ?? null, + size: targets.get("size") ?? null, + location: targets.get("location") ?? null, + customFields, + }; +} + +/** + * A record is invalid only when it cannot become a usable CRM row. Missing optional fields + * are not errors — they are the normal case for mined sources. + */ +export function validateMappedPerson( + person: MappedPerson, + options: Pick, + rawEmail: string | null, +): ImportRecordError[] { + const errors: ImportRecordError[] = []; + + if (rawEmail && !person.email) { + errors.push({ field: "email", message: `"${rawEmail}" is not a valid email address` }); + } + + if (!person.name && !person.email) { + errors.push({ field: "name", message: "Record has neither a name nor an email address" }); + } + + if (options.skipRecordsWithoutEmail && !person.email) { + errors.push({ field: "email", message: "Record has no email address" }); + } + + if (person.email && isAutomatedEmail(person.email)) { + errors.push({ field: "email", message: "Address belongs to an automated sender" }); + } + + return errors; +} + +export function validateMappedOrg(orgValue: MappedOrg): ImportRecordError[] { + if (!orgValue.name) { + return [{ field: "name", message: "Organization has no name" }]; + } + + return []; +} + +export function isKnownPersonTarget(value: string): value is ImportPersonTargetField { + return (IMPORT_PERSON_TARGET_FIELD_VALUES as readonly string[]).includes(value); +} + +export function isKnownOrgTarget(value: string): value is ImportOrgTargetField { + return (IMPORT_ORG_TARGET_FIELD_VALUES as readonly string[]).includes(value); +} diff --git a/apps/api/src/services/import.service.ts b/apps/api/src/services/import.service.ts new file mode 100644 index 0000000..b5fe5f3 --- /dev/null +++ b/apps/api/src/services/import.service.ts @@ -0,0 +1,1427 @@ +import { and, asc, count, eq, inArray, isNotNull, lte, or, sql } from "drizzle-orm"; +import type { + CreateConnectionInput, + ImportFieldMapping, + ListImportJobsQuery, + ListImportRecordsQuery, + StartImportJobInput, + UpdateConnectionInput, + UpdateImportJobInput, + WebhookIngestInput, +} from "@workspace/validators/schemas/import"; +import type { ImportEntityType, ImportProvider } from "@workspace/validators/types/import"; +import { + IMPORT_EXTRACT_PAGE_SIZE, + IMPORT_LOAD_BATCH_SIZE, + IMPORT_MAX_RECORDS_PER_JOB, + IMPORT_PROVIDER_DEFAULT_STATUS, +} from "@workspace/validators/types/import"; +import { env } from "@/config/env.config.js"; +import { db } from "@/db/client.js"; +import { + externalIdentities, + importJobs, + importRecords, + integrationApiKeys, + integrationConnections, + org, + people, + type ImportCursor, + type ImportJobOptionsSnapshot, + type ImportJobStats, + type ImportRecordError, +} from "@/db/schema/index.js"; +import { STATUS_CODES } from "@/constants/status-codes.js"; +import { AppError } from "@/lib/app-error.js"; +import { + createPublicToken, + decryptSecretToken, + encryptSecretToken, + hashPublicToken, +} from "@/lib/token-crypto.js"; +import { parseCsv } from "@/services/import-csv.js"; +import { + deriveRecordFingerprint, + fieldPolicy, + normalizeDomain, + resolveFieldValue, + resolveOrgMatch, + resolvePersonMatch, + shouldSkipExistingRecord, +} from "@/services/import-engine.js"; +import { + applyOrgMapping, + applyPersonMapping, + autoDetectMapping, + validateMappedOrg, + validateMappedPerson, + type MappedOrg, + type MappedPerson, +} from "@/services/import-mapping.js"; +import { + ConnectionAuthError, + extractPage, + isPushProvider, + listSourceFields, + RateLimitError, +} from "@/services/import-connectors/index.js"; + +const EMPTY_STATS: ImportJobStats = { + extracted: 0, + valid: 0, + invalid: 0, + created: 0, + updated: 0, + skipped: 0, + duplicate: 0, +}; + +const MAX_JOB_ATTEMPTS = 5; + +function tokenSecret() { + return env.INTEGRATION_TOKEN_SECRET ?? env.BETTER_AUTH_SECRET; +} + +function assertFound(value: T | undefined, message: string): T { + if (!value) throw new AppError(message, STATUS_CODES.NOT_FOUND); + return value; +} + +/** + * The identifier a record matches on across runs. A real source id (Gmail address, PostHog + * id, webhook-supplied externalId) is authoritative and preserved. When the source offers + * nothing — a CSV row, an anonymous push — a content fingerprint stands in, so re-uploading + * the same file updates rather than duplicates. + * + * Synthetic ids carry an `fp_` prefix so they can be told apart from real ones and + * recomputed from scratch on every pass; that keeps them correct after a remap changes the + * identifying fields, instead of pinning to a stale value persisted on the first pass. + */ +function effectiveExternalId( + rawExternalId: string | null, + mapped: Pick | null, +): string | null { + const sourceId = rawExternalId && !rawExternalId.startsWith("fp_") ? rawExternalId : null; + if (sourceId) return sourceId; + if (!mapped) return rawExternalId; + + return deriveRecordFingerprint({ + name: mapped.name, + email: mapped.email, + phone: mapped.phone, + orgName: mapped.orgName, + }); +} + +// --------------------------------------------------------------------------- +// Connections +// --------------------------------------------------------------------------- + +export async function listConnections(workspaceId: string) { + const rows = await db + .select({ + id: integrationConnections.id, + provider: integrationConnections.provider, + displayName: integrationConnections.displayName, + externalAccountId: integrationConnections.externalAccountId, + status: integrationConnections.status, + grantedScopes: integrationConnections.grantedScopes, + config: integrationConnections.config, + lastSyncAt: integrationConnections.lastSyncAt, + lastError: integrationConnections.lastError, + tokenExpiresAt: integrationConnections.tokenExpiresAt, + createdAt: integrationConnections.createdAt, + }) + .from(integrationConnections) + .where(eq(integrationConnections.workspaceId, workspaceId)) + .orderBy(asc(integrationConnections.createdAt)); + + return rows; +} + +export async function createConnection( + workspaceId: string, + userId: string, + input: CreateConnectionInput, +) { + const [connection] = await db + .insert(integrationConnections) + .values({ + workspaceId, + userId, + provider: input.provider, + displayName: input.displayName, + externalAccountId: input.externalAccountId, + status: "connected", + grantedScopes: input.grantedScopes, + accessTokenEncrypted: encryptSecretToken(input.accessToken, tokenSecret()), + refreshTokenEncrypted: input.refreshToken + ? encryptSecretToken(input.refreshToken, tokenSecret()) + : null, + tokenExpiresAt: input.tokenExpiresAt, + config: input.config, + }) + .onConflictDoUpdate({ + target: [ + integrationConnections.workspaceId, + integrationConnections.provider, + integrationConnections.externalAccountId, + ], + set: { + displayName: input.displayName, + status: "connected", + grantedScopes: input.grantedScopes, + accessTokenEncrypted: encryptSecretToken(input.accessToken, tokenSecret()), + refreshTokenEncrypted: input.refreshToken + ? encryptSecretToken(input.refreshToken, tokenSecret()) + : null, + tokenExpiresAt: input.tokenExpiresAt, + config: input.config, + lastError: null, + updatedAt: new Date(), + }, + }) + .returning(); + + return connection; +} + +export async function updateConnection( + workspaceId: string, + connectionId: string, + input: UpdateConnectionInput, +) { + const [connection] = await db + .update(integrationConnections) + .set({ ...input, updatedAt: new Date() }) + .where( + and( + eq(integrationConnections.id, connectionId), + eq(integrationConnections.workspaceId, workspaceId), + ), + ) + .returning(); + + return assertFound(connection, "Connection not found"); +} + +export async function deleteConnection(workspaceId: string, connectionId: string) { + const [connection] = await db + .delete(integrationConnections) + .where( + and( + eq(integrationConnections.id, connectionId), + eq(integrationConnections.workspaceId, workspaceId), + ), + ) + .returning({ id: integrationConnections.id }); + + return assertFound(connection, "Connection not found"); +} + +// --------------------------------------------------------------------------- +// API keys for inbound webhook pushes +// --------------------------------------------------------------------------- + +export async function listApiKeys(workspaceId: string) { + return db + .select({ + id: integrationApiKeys.id, + name: integrationApiKeys.name, + tokenPrefix: integrationApiKeys.tokenPrefix, + lastUsedAt: integrationApiKeys.lastUsedAt, + revokedAt: integrationApiKeys.revokedAt, + createdAt: integrationApiKeys.createdAt, + }) + .from(integrationApiKeys) + .where(eq(integrationApiKeys.workspaceId, workspaceId)) + .orderBy(asc(integrationApiKeys.createdAt)); +} + +/** + * Returns the plaintext key exactly once. Only the hash is persisted, matching how + * unsubscribe tokens are handled in sequences. + */ +export async function createApiKey(workspaceId: string, userId: string, name: string) { + const token = createPublicToken(); + const [apiKey] = await db + .insert(integrationApiKeys) + .values({ + workspaceId, + createdById: userId, + name, + tokenHash: hashPublicToken(token), + tokenPrefix: token.slice(0, 8), + }) + .returning({ id: integrationApiKeys.id, name: integrationApiKeys.name }); + + return { apiKey, token }; +} + +export async function revokeApiKey(workspaceId: string, apiKeyId: string) { + const [apiKey] = await db + .update(integrationApiKeys) + .set({ revokedAt: new Date(), updatedAt: new Date() }) + .where( + and(eq(integrationApiKeys.id, apiKeyId), eq(integrationApiKeys.workspaceId, workspaceId)), + ) + .returning({ id: integrationApiKeys.id }); + + return assertFound(apiKey, "API key not found"); +} + +export async function resolveApiKey(token: string) { + const apiKey = await db.query.integrationApiKeys.findFirst({ + where: eq(integrationApiKeys.tokenHash, hashPublicToken(token)), + }); + + if (!apiKey || apiKey.revokedAt) return null; + + await db + .update(integrationApiKeys) + .set({ lastUsedAt: new Date() }) + .where(eq(integrationApiKeys.id, apiKey.id)); + + return apiKey; +} + +// --------------------------------------------------------------------------- +// Job lifecycle +// --------------------------------------------------------------------------- + +function toOptionsSnapshot( + input: StartImportJobInput, + provider: ImportProvider, +): ImportJobOptionsSnapshot { + return { + conflictPolicy: input.options.conflictPolicy, + defaultStatus: input.options.defaultStatus ?? IMPORT_PROVIDER_DEFAULT_STATUS[provider], + defaultOwnerId: input.options.defaultOwnerId, + normalizeSubaddressing: input.options.normalizeSubaddressing, + createMissingOrgs: input.options.createMissingOrgs, + skipRecordsWithoutEmail: input.options.skipRecordsWithoutEmail, + }; +} + +export async function startImportJob( + workspaceId: string, + userId: string, + input: StartImportJobInput, +) { + if (input.provider === "csv" && !input.csvContent) { + throw new AppError("A CSV import requires file content", STATUS_CODES.UNPROCESSABLE_ENTITY); + } + + if (!isPushProvider(input.provider) && !input.connectionId) { + throw new AppError( + `A ${input.provider} import requires a connected account`, + STATUS_CODES.BAD_REQUEST, + ); + } + + if (input.connectionId) { + const connection = await db.query.integrationConnections.findFirst({ + where: and( + eq(integrationConnections.id, input.connectionId), + eq(integrationConnections.workspaceId, workspaceId), + ), + }); + assertFound(connection, "Connection not found"); + } + + const [job] = await db + .insert(importJobs) + .values({ + workspaceId, + connectionId: input.connectionId, + createdById: userId, + provider: input.provider, + entityType: input.entityType, + status: "pending", + mapping: { fields: input.mapping.fields }, + options: toOptionsSnapshot(input, input.provider), + sourceConfig: input.sourceConfig, + stats: { ...EMPTY_STATS }, + nextAttemptAt: new Date(), + }) + .returning(); + + const created = assertFound(job, "Import job could not be created"); + + // CSV arrives complete, so it is staged inline and lands in review immediately rather + // than waiting a worker tick. + if (input.provider === "csv" && input.csvContent) { + const parsed = parseCsv(input.csvContent); + if (parsed.rows.length > IMPORT_MAX_RECORDS_PER_JOB) { + throw new AppError( + `Imports are limited to ${IMPORT_MAX_RECORDS_PER_JOB} rows per run`, + STATUS_CODES.UNPROCESSABLE_ENTITY, + ); + } + + await stageRecords( + created.id, + workspaceId, + parsed.rows.map((row, index) => ({ externalId: null, data: row, rowNumber: index + 1 })), + ); + + const mapping = + input.mapping.fields.length > 0 + ? input.mapping.fields + : autoDetectMapping(parsed.headers, input.entityType); + + await db + .update(importJobs) + .set({ mapping: { fields: mapping }, updatedAt: new Date() }) + .where(eq(importJobs.id, created.id)); + + return evaluateJob(created.id); + } + + return created; +} + +type StagedRecord = { + externalId: string | null; + data: Record; + rowNumber: number; +}; + +async function stageRecords(jobId: string, workspaceId: string, records: StagedRecord[]) { + if (records.length === 0) return 0; + + for (let index = 0; index < records.length; index += IMPORT_LOAD_BATCH_SIZE) { + const batch = records.slice(index, index + IMPORT_LOAD_BATCH_SIZE); + await db + .insert(importRecords) + .values( + batch.map((record) => ({ + workspaceId, + jobId, + externalId: record.externalId, + rowNumber: record.rowNumber, + raw: record.data, + status: "pending" as const, + })), + ) + .onConflictDoNothing(); + } + + return records.length; +} + +/** + * Applies the job mapping to every staged record and records the match decision, without + * touching the CRM. This is what lets a user fix a mapping and re-preview without paying + * for another extraction. + */ +export async function evaluateJob(jobId: string) { + const job = assertFound( + await db.query.importJobs.findFirst({ where: eq(importJobs.id, jobId) }), + "Import job not found", + ); + + const staged = await db + .select() + .from(importRecords) + .where(eq(importRecords.jobId, jobId)) + .orderBy(asc(importRecords.rowNumber)); + + // Pass one is pure: map every row and settle its effective external id before any lookup + // is built, so fingerprints assigned here are visible to the identity query below. + const prepared = staged.map((record) => { + const mapped = + job.entityType === "org" + ? null + : applyPersonMapping(record.raw, job.mapping.fields, job.options); + + return { record, mapped, externalId: effectiveExternalId(record.externalId, mapped) }; + }); + + const lookups = await buildLookups( + job.workspaceId, + job.provider, + prepared.map((entry) => ({ externalId: entry.externalId })), + ); + + const seenEmails = new Set(); + const seenExternalIds = new Set(); + + let valid = 0; + let invalid = 0; + let duplicate = 0; + + for (const { record, externalId } of prepared) { + if (record.status === "loaded") continue; + + const evaluated = + job.entityType === "org" + ? evaluateOrgRecord(record.raw, job.mapping.fields, lookups) + : evaluatePersonRecord(record.raw, externalId, job.mapping.fields, job.options, lookups); + + // Within-run duplicates are marked rather than loaded twice. The first occurrence wins, + // which keeps the earliest row number authoritative. + const dedupeKey = evaluated.dedupeKey; + const isRunDuplicate = + (dedupeKey !== null && seenEmails.has(dedupeKey)) || + (externalId !== null && seenExternalIds.has(externalId)); + + if (dedupeKey !== null) seenEmails.add(dedupeKey); + if (externalId !== null) seenExternalIds.add(externalId); + + let status: "valid" | "invalid" | "duplicate" = "valid"; + if (evaluated.errors.length > 0) status = "invalid"; + else if (isRunDuplicate) status = "duplicate"; + + if (status === "valid") valid += 1; + else if (status === "invalid") invalid += 1; + else duplicate += 1; + + await db + .update(importRecords) + .set({ + externalId, + normalized: evaluated.normalized, + errors: evaluated.errors, + status, + matchReason: evaluated.matchReason, + matchPersonId: evaluated.matchPersonId, + matchOrgId: evaluated.matchOrgId, + updatedAt: new Date(), + }) + .where(eq(importRecords.id, record.id)); + } + + const [updated] = await db + .update(importJobs) + .set({ + status: "ready_for_review", + stats: { ...job.stats, extracted: staged.length, valid, invalid, duplicate }, + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)) + .returning(); + + return assertFound(updated, "Import job not found"); +} + +type EvaluatedRecord = { + normalized: Record; + errors: ImportRecordError[]; + matchReason: "external_identity" | "email" | "domain" | "name" | "none"; + matchPersonId: string | null; + matchOrgId: string | null; + dedupeKey: string | null; +}; + +function evaluatePersonRecord( + raw: Record, + externalId: string | null, + mapping: ImportFieldMapping[], + options: ImportJobOptionsSnapshot, + lookups: Lookups, +): EvaluatedRecord { + const mapped = applyPersonMapping(raw, mapping, options); + const rawEmailField = mapping.find((field) => field.targetField === "email")?.sourceField; + const rawEmail = rawEmailField ? ((raw[rawEmailField] as string | undefined) ?? null) : null; + + const errors = validateMappedPerson(mapped, options, rawEmail); + const personMatch = resolvePersonMatch( + { externalId, email: mapped.email, normalizedEmail: mapped.normalizedEmail }, + { byExternalId: lookups.identityByExternalId, byEmail: lookups.personByEmail }, + ); + + const orgMatch = resolveOrgMatch( + { domain: mapped.orgDomain, normalizedName: mapped.orgNormalizedName }, + { byDomain: lookups.orgByDomain, byNormalizedName: lookups.orgByName }, + ); + + return { + normalized: toNormalizedPayload(mapped), + errors, + matchReason: personMatch.reason, + matchPersonId: personMatch.personId, + matchOrgId: orgMatch.orgId, + dedupeKey: mapped.normalizedEmail, + }; +} + +function evaluateOrgRecord( + raw: Record, + mapping: ImportFieldMapping[], + lookups: Lookups, +): EvaluatedRecord { + const mapped = applyOrgMapping(raw, mapping); + const errors = validateMappedOrg(mapped); + const orgMatch = resolveOrgMatch( + { domain: mapped.domain, normalizedName: mapped.normalizedName }, + { byDomain: lookups.orgByDomain, byNormalizedName: lookups.orgByName }, + ); + + return { + normalized: { ...mapped }, + errors, + matchReason: orgMatch.reason, + matchPersonId: null, + matchOrgId: orgMatch.orgId, + dedupeKey: mapped.normalizedName, + }; +} + +function toNormalizedPayload(mapped: MappedPerson): Record { + return { + ...mapped, + lastContactedAt: mapped.lastContactedAt ? mapped.lastContactedAt.toISOString() : null, + }; +} + +type Lookups = { + identityByExternalId: Map; + personByEmail: Map; + orgByDomain: Map; + orgByName: Map; +}; + +/** + * Pre-loads every lookup a job needs in a bounded number of queries, so evaluation and + * loading do not issue one query per record. + */ +async function buildLookups( + workspaceId: string, + provider: ImportProvider, + staged: { externalId: string | null }[], +): Promise { + const externalIds = staged + .map((record) => record.externalId) + .filter((value): value is string => value !== null); + + const [identities, existingPeople, existingOrgs] = await Promise.all([ + externalIds.length === 0 + ? Promise.resolve([]) + : db + .select({ + externalId: externalIdentities.externalId, + personId: externalIdentities.personId, + }) + .from(externalIdentities) + .where( + and( + eq(externalIdentities.workspaceId, workspaceId), + eq(externalIdentities.provider, provider), + inArray(externalIdentities.externalId, externalIds), + isNotNull(externalIdentities.personId), + ), + ), + db + .select({ id: people.id, email: people.email }) + .from(people) + .where(and(eq(people.workspaceId, workspaceId), isNotNull(people.email))), + db + .select({ id: org.id, name: org.name, domain: org.domain }) + .from(org) + .where(eq(org.workspaceId, workspaceId)), + ]); + + const identityByExternalId = new Map(); + for (const identity of identities) { + if (identity.personId) identityByExternalId.set(identity.externalId, identity.personId); + } + + const personByEmail = new Map(); + for (const person of existingPeople) { + if (person.email) personByEmail.set(person.email.toLowerCase(), person.id); + } + + const orgByDomain = new Map(); + const orgByName = new Map(); + for (const organization of existingOrgs) { + const domain = organization.domain ? normalizeDomain(organization.domain) : null; + if (domain && !orgByDomain.has(domain)) orgByDomain.set(domain, organization.id); + + const normalizedName = normalizeOrgKey(organization.name); + if (normalizedName && !orgByName.has(normalizedName)) { + orgByName.set(normalizedName, organization.id); + } + } + + return { identityByExternalId, personByEmail, orgByDomain, orgByName }; +} + +function normalizeOrgKey(name: string) { + return name.toLowerCase().replace(/[.,]/g, " ").replace(/\s+/g, " ").trim() || null; +} + +export async function updateImportJob( + workspaceId: string, + jobId: string, + input: UpdateImportJobInput, +) { + const job = assertFound( + await db.query.importJobs.findFirst({ + where: and(eq(importJobs.id, jobId), eq(importJobs.workspaceId, workspaceId)), + }), + "Import job not found", + ); + + if (job.status === "loading" || job.status === "completed") { + throw new AppError( + "A job that has started loading can no longer be remapped", + STATUS_CODES.CONFLICT, + ); + } + + await db + .update(importJobs) + .set({ + mapping: input.mapping ? { fields: input.mapping.fields } : job.mapping, + options: input.options ? { ...job.options, ...input.options } : job.options, + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)); + + // Re-evaluating replays the mapping over staged rows; the source is never re-fetched. + return evaluateJob(jobId); +} + +export async function commitImportJob(workspaceId: string, jobId: string) { + const job = assertFound( + await db.query.importJobs.findFirst({ + where: and(eq(importJobs.id, jobId), eq(importJobs.workspaceId, workspaceId)), + }), + "Import job not found", + ); + + if (job.status !== "ready_for_review") { + throw new AppError( + `Only a job awaiting review can be committed, this one is ${job.status}`, + STATUS_CODES.CONFLICT, + ); + } + + const [updated] = await db + .update(importJobs) + .set({ + status: "loading", + startedAt: new Date(), + nextAttemptAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)) + .returning(); + + return assertFound(updated, "Import job not found"); +} + +export async function cancelImportJob(workspaceId: string, jobId: string) { + const [job] = await db + .update(importJobs) + .set({ status: "canceled", completedAt: new Date(), updatedAt: new Date() }) + .where( + and( + eq(importJobs.id, jobId), + eq(importJobs.workspaceId, workspaceId), + inArray(importJobs.status, ["pending", "extracting", "ready_for_review", "loading"]), + ), + ) + .returning(); + + return assertFound(job, "No cancelable import job was found"); +} + +export async function listImportJobs(workspaceId: string, query: ListImportJobsQuery) { + const conditions = [eq(importJobs.workspaceId, workspaceId)]; + if (query.provider) conditions.push(eq(importJobs.provider, query.provider)); + if (query.status) conditions.push(eq(importJobs.status, query.status)); + + const whereClause = and(...conditions); + const offset = (query.page - 1) * query.pageSize; + + const [rows, totalResult] = await Promise.all([ + db + .select() + .from(importJobs) + .where(whereClause) + .orderBy(sql`${importJobs.createdAt} desc`) + .limit(query.pageSize) + .offset(offset), + db.select({ totalCount: count() }).from(importJobs).where(whereClause), + ]); + + const totalCount = Number(totalResult[0]?.totalCount ?? 0); + + return { + jobs: rows, + meta: { + page: query.page, + pageSize: query.pageSize, + totalCount, + totalPages: totalCount === 0 ? 0 : Math.ceil(totalCount / query.pageSize), + }, + }; +} + +export async function getImportJob(workspaceId: string, jobId: string) { + const job = await db.query.importJobs.findFirst({ + where: and(eq(importJobs.id, jobId), eq(importJobs.workspaceId, workspaceId)), + }); + + return assertFound(job, "Import job not found"); +} + +export async function listImportRecords( + workspaceId: string, + jobId: string, + query: ListImportRecordsQuery, +) { + await getImportJob(workspaceId, jobId); + + const conditions = [eq(importRecords.jobId, jobId)]; + if (query.status) conditions.push(eq(importRecords.status, query.status)); + + const whereClause = and(...conditions); + const offset = (query.page - 1) * query.pageSize; + + const [rows, totalResult] = await Promise.all([ + db + .select() + .from(importRecords) + .where(whereClause) + .orderBy(asc(importRecords.rowNumber)) + .limit(query.pageSize) + .offset(offset), + db.select({ totalCount: count() }).from(importRecords).where(whereClause), + ]); + + const totalCount = Number(totalResult[0]?.totalCount ?? 0); + + return { + records: rows, + meta: { + page: query.page, + pageSize: query.pageSize, + totalCount, + totalPages: totalCount === 0 ? 0 : Math.ceil(totalCount / query.pageSize), + }, + }; +} + +export async function previewSourceFields( + workspaceId: string, + connectionId: string, + entityType: ImportEntityType, +) { + const connection = assertFound( + await db.query.integrationConnections.findFirst({ + where: and( + eq(integrationConnections.id, connectionId), + eq(integrationConnections.workspaceId, workspaceId), + ), + }), + "Connection not found", + ); + + const fields = await listSourceFields(connection.provider, { + auth: connectionAuth(connection), + config: connection.config, + cursor: null, + pageSize: IMPORT_EXTRACT_PAGE_SIZE, + fetchImpl: fetch, + }); + + return { fields, suggestedMapping: autoDetectMapping(fields, entityType) }; +} + +function connectionAuth(connection: { + accessTokenEncrypted: string; + refreshTokenEncrypted: string | null; +}) { + return { + accessToken: decryptSecretToken(connection.accessTokenEncrypted, tokenSecret()), + refreshToken: connection.refreshTokenEncrypted + ? decryptSecretToken(connection.refreshTokenEncrypted, tokenSecret()) + : null, + }; +} + +// --------------------------------------------------------------------------- +// Inbound webhook +// --------------------------------------------------------------------------- + +/** + * Stages a pushed batch as its own completed-extraction job. Every push runs through the + * same staging, mapping, and matching path as a CSV or a connector, so a Zapier scenario + * gets identical dedupe behaviour to a native integration. + */ +export async function ingestWebhookRecords(workspaceId: string, input: WebhookIngestInput) { + const [job] = await db + .insert(importJobs) + .values({ + workspaceId, + provider: "webhook", + entityType: input.entityType, + status: "pending", + mapping: { fields: [] }, + options: { + conflictPolicy: "fill_empty", + defaultStatus: IMPORT_PROVIDER_DEFAULT_STATUS.webhook, + defaultOwnerId: null, + normalizeSubaddressing: false, + createMissingOrgs: true, + skipRecordsWithoutEmail: false, + }, + sourceConfig: {}, + stats: { ...EMPTY_STATS }, + }) + .returning(); + + const created = assertFound(job, "Import job could not be created"); + + const fieldNames = new Set(); + for (const record of input.records) { + for (const key of Object.keys(record)) fieldNames.add(key); + } + + await stageRecords( + created.id, + workspaceId, + input.records.map((record, index) => { + const { externalId, ...rest } = record; + return { + externalId: externalId ?? null, + data: rest as Record, + rowNumber: index + 1, + }; + }), + ); + + await db + .update(importJobs) + .set({ + mapping: { fields: autoDetectMapping([...fieldNames], input.entityType) }, + updatedAt: new Date(), + }) + .where(eq(importJobs.id, created.id)); + + const evaluated = await evaluateJob(created.id); + + // Pushes are unattended, so they commit straight through instead of waiting for review. + await db + .update(importJobs) + .set({ status: "loading", startedAt: new Date(), nextAttemptAt: new Date() }) + .where(eq(importJobs.id, created.id)); + + return { jobId: evaluated.id, staged: input.records.length }; +} + +// --------------------------------------------------------------------------- +// Worker +// --------------------------------------------------------------------------- + +/** + * Worker entry point, mirroring `executeDueSequenceWork`. Claims due jobs and advances each + * by one unit of work — a single extraction page or a single load batch — so no tick holds + * a long transaction or blocks other jobs. + */ +export async function executeDueImportWork(now = new Date()) { + const due = await db + .select({ id: importJobs.id, status: importJobs.status }) + .from(importJobs) + .where( + and( + inArray(importJobs.status, ["pending", "extracting", "loading"]), + or(lte(importJobs.nextAttemptAt, now), sql`${importJobs.nextAttemptAt} is null`), + ), + ) + .orderBy(asc(importJobs.nextAttemptAt)) + .limit(10); + + const results: { jobId: string; outcome: string }[] = []; + + for (const entry of due) { + try { + const outcome = + entry.status === "loading" + ? await runLoadBatch(entry.id, now) + : await runExtractionPage(entry.id, now); + results.push({ jobId: entry.id, outcome }); + } catch (error) { + results.push({ jobId: entry.id, outcome: "failed" }); + await handleJobFailure(entry.id, error, now); + } + } + + return results; +} + +async function handleJobFailure(jobId: string, error: unknown, now: Date) { + const job = await db.query.importJobs.findFirst({ where: eq(importJobs.id, jobId) }); + if (!job) return; + + const message = error instanceof Error ? error.message : "Unknown import failure"; + + // A rate limit is not a failure — it is a scheduling instruction. The cursor is already + // persisted, so the run resumes from where it stopped. + if (error instanceof RateLimitError) { + await db + .update(importJobs) + .set({ + nextAttemptAt: new Date(now.getTime() + error.retryAfterMs), + lastError: message, + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)); + return; + } + + // Credentials cannot fix themselves by retrying, so the job stops and the connection is + // flagged for the user to reconnect. + if (error instanceof ConnectionAuthError) { + await db + .update(importJobs) + .set({ status: "failed", lastError: message, completedAt: now, updatedAt: new Date() }) + .where(eq(importJobs.id, jobId)); + + if (job.connectionId) { + await db + .update(integrationConnections) + .set({ status: "reconnect_required", lastError: message, updatedAt: new Date() }) + .where(eq(integrationConnections.id, job.connectionId)); + } + return; + } + + const attempts = job.attempts + 1; + const exhausted = attempts >= MAX_JOB_ATTEMPTS; + + await db + .update(importJobs) + .set({ + attempts, + status: exhausted ? "failed" : job.status, + lastError: message, + completedAt: exhausted ? now : null, + // Exponential backoff, matching the retry shape used for sequence steps. + nextAttemptAt: exhausted ? null : new Date(now.getTime() + 2 ** attempts * 30_000), + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)); +} + +async function runExtractionPage(jobId: string, now: Date) { + const job = assertFound( + await db.query.importJobs.findFirst({ where: eq(importJobs.id, jobId) }), + "Import job not found", + ); + + // Push providers have nothing to pull; their records were staged at ingest time. + if (isPushProvider(job.provider)) { + await evaluateJob(jobId); + return "evaluated"; + } + + const connection = job.connectionId + ? await db.query.integrationConnections.findFirst({ + where: eq(integrationConnections.id, job.connectionId), + }) + : null; + + if (!connection) { + throw new AppError("Import job has no usable connection", STATUS_CODES.BAD_REQUEST); + } + + await db + .update(importJobs) + .set({ status: "extracting", startedAt: job.startedAt ?? now, updatedAt: new Date() }) + .where(eq(importJobs.id, jobId)); + + const page = await extractPage(job.provider, { + auth: connectionAuth(connection), + config: { ...connection.config, ...job.sourceConfig }, + cursor: job.cursor ?? null, + pageSize: IMPORT_EXTRACT_PAGE_SIZE, + fetchImpl: fetch, + }); + + const [{ existing } = { existing: 0 }] = await db + .select({ existing: count() }) + .from(importRecords) + .where(eq(importRecords.jobId, jobId)); + + const offset = Number(existing); + await stageRecords( + jobId, + job.workspaceId, + page.records.map((record, index) => ({ + externalId: record.externalId, + data: record.data, + rowNumber: offset + index + 1, + })), + ); + + const extracted = offset + page.records.length; + const reachedLimit = extracted >= IMPORT_MAX_RECORDS_PER_JOB; + + if (page.nextCursor && !reachedLimit) { + await db + .update(importJobs) + .set({ + cursor: page.nextCursor as ImportCursor, + stats: { ...job.stats, extracted }, + nextAttemptAt: now, + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)); + + return "extracted_page"; + } + + await db + .update(importJobs) + .set({ + cursor: null, + stats: { ...job.stats, extracted }, + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)); + + // Mapping is auto-detected on first completion only when the user supplied none. + if (job.mapping.fields.length === 0) { + const fieldNames = new Set(); + for (const record of page.records) { + for (const key of Object.keys(record.data)) fieldNames.add(key); + } + + await db + .update(importJobs) + .set({ + mapping: { fields: autoDetectMapping([...fieldNames], job.entityType) }, + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)); + } + + await evaluateJob(jobId); + await db + .update(integrationConnections) + .set({ lastSyncAt: now, lastError: null, updatedAt: new Date() }) + .where(eq(integrationConnections.id, connection.id)); + + return "extraction_complete"; +} + +/** + * Loads one batch. Deliberately not one transaction for the whole job: a 5,000-row import + * must not hold a single long transaction, and a failure at row 4,900 must not roll back + * the 4,899 rows that already succeeded. + */ +async function runLoadBatch(jobId: string, now: Date) { + const job = assertFound( + await db.query.importJobs.findFirst({ where: eq(importJobs.id, jobId) }), + "Import job not found", + ); + + const batch = await db + .select() + .from(importRecords) + .where(and(eq(importRecords.jobId, jobId), eq(importRecords.status, "valid"))) + .orderBy(asc(importRecords.rowNumber)) + .limit(IMPORT_LOAD_BATCH_SIZE); + + if (batch.length === 0) { + const [completed] = await db + .update(importJobs) + .set({ status: "completed", completedAt: now, nextAttemptAt: null, updatedAt: new Date() }) + .where(eq(importJobs.id, jobId)) + .returning(); + + return completed ? "completed" : "noop"; + } + + const lookups = await buildLookups(job.workspaceId, job.provider, batch); + let created = 0; + let updated = 0; + let skipped = 0; + + for (const record of batch) { + try { + const result = await db.transaction(async (tx) => + job.entityType === "org" + ? loadOrgRecord(tx, job, record, lookups) + : loadPersonRecord(tx, job, record, lookups), + ); + + if (result === "created") created += 1; + else if (result === "updated") updated += 1; + else skipped += 1; + + await db + .update(importRecords) + .set({ status: "loaded", updatedAt: new Date() }) + .where(eq(importRecords.id, record.id)); + } catch (error) { + // A single bad row must not fail the run. It is marked and the batch continues. + skipped += 1; + await db + .update(importRecords) + .set({ + status: "invalid", + errors: [ + { + field: "record", + message: error instanceof Error ? error.message : "Failed to load record", + }, + ], + updatedAt: new Date(), + }) + .where(eq(importRecords.id, record.id)); + } + } + + await db + .update(importJobs) + .set({ + stats: { + ...job.stats, + created: job.stats.created + created, + updated: job.stats.updated + updated, + skipped: job.stats.skipped + skipped, + }, + nextAttemptAt: now, + updatedAt: new Date(), + }) + .where(eq(importJobs.id, jobId)); + + return "loaded_batch"; +} + +type Tx = Parameters[0]>[0]; + +async function loadPersonRecord( + tx: Tx, + job: { workspaceId: string; provider: ImportProvider; options: ImportJobOptionsSnapshot }, + record: { id: string; externalId: string | null; normalized: Record | null }, + lookups: Lookups, +) { + const mapped = record.normalized as unknown as MappedPerson | null; + if (!mapped) return "skipped" as const; + + const orgId = await resolveOrgId(tx, job, mapped, lookups); + + const match = resolvePersonMatch( + { + externalId: record.externalId, + email: mapped.email, + normalizedEmail: mapped.normalizedEmail, + }, + { byExternalId: lookups.identityByExternalId, byEmail: lookups.personByEmail }, + ); + + const lastContactedAt = mapped.lastContactedAt ? new Date(mapped.lastContactedAt) : null; + + if (match.personId) { + const existing = await tx.query.people.findFirst({ where: eq(people.id, match.personId) }); + if (existing && shouldSkipExistingRecord(job.options.conflictPolicy)) { + // Still record the identity so the next run matches without re-deriving it. + await upsertIdentity(tx, job, record.externalId, match.personId, null); + return "skipped" as const; + } + + if (existing) { + const policy = fieldPolicy(job.options.conflictPolicy); + await tx + .update(people) + .set({ + name: resolveFieldValue(existing.name, mapped.name, policy) ?? existing.name, + email: resolveFieldValue(existing.email, mapped.email, policy), + phone: resolveFieldValue(existing.phone, mapped.phone, policy), + jobTitle: resolveFieldValue(existing.jobTitle, mapped.jobTitle, policy), + linkedinUrl: resolveFieldValue(existing.linkedinUrl, mapped.linkedinUrl, policy), + orgId: existing.orgId ?? orgId, + // Status is CRM-owned once the record exists. An import never downgrades a + // customer to a lead because a low-trust list happened to contain their address. + lastContactedAt: pickLatestDate(existing.lastContactedAt, lastContactedAt), + customFields: { ...(existing.customFields ?? {}), ...mapped.customFields }, + updatedAt: new Date(), + }) + .where(eq(people.id, match.personId)); + + await upsertIdentity(tx, job, record.externalId, match.personId, null); + return "updated" as const; + } + } + + const [inserted] = await tx + .insert(people) + .values({ + workspaceId: job.workspaceId, + orgId, + ownerId: job.options.defaultOwnerId, + name: mapped.name ?? mapped.email ?? "Unknown", + email: mapped.email, + phone: mapped.phone, + jobTitle: mapped.jobTitle, + linkedinUrl: mapped.linkedinUrl, + source: "import", + status: job.options.defaultStatus ?? IMPORT_PROVIDER_DEFAULT_STATUS[job.provider], + lastContactedAt, + customFields: mapped.customFields, + }) + .onConflictDoUpdate({ + target: [people.workspaceId, people.email], + set: { updatedAt: new Date() }, + }) + .returning({ id: people.id }); + + if (!inserted) return "skipped" as const; + + // Keeping the in-memory maps current is what stops two rows with the same address inside + // one batch from creating two people before the next lookup refresh. + if (mapped.normalizedEmail) lookups.personByEmail.set(mapped.normalizedEmail, inserted.id); + await upsertIdentity(tx, job, record.externalId, inserted.id, null); + + return "created" as const; +} + +async function loadOrgRecord( + tx: Tx, + job: { workspaceId: string; provider: ImportProvider; options: ImportJobOptionsSnapshot }, + record: { externalId: string | null; normalized: Record | null }, + lookups: Lookups, +) { + const mapped = record.normalized as unknown as MappedOrg | null; + if (!mapped?.name) return "skipped" as const; + + const match = resolveOrgMatch( + { domain: mapped.domain, normalizedName: mapped.normalizedName }, + { byDomain: lookups.orgByDomain, byNormalizedName: lookups.orgByName }, + ); + + if (match.orgId) { + const existing = await tx.query.org.findFirst({ where: eq(org.id, match.orgId) }); + if (existing && shouldSkipExistingRecord(job.options.conflictPolicy)) { + await upsertIdentity(tx, job, record.externalId, null, match.orgId); + return "skipped" as const; + } + + if (existing) { + const policy = fieldPolicy(job.options.conflictPolicy); + await tx + .update(org) + .set({ + domain: resolveFieldValue(existing.domain, mapped.domain, policy), + industry: resolveFieldValue(existing.industry, mapped.industry, policy), + size: resolveFieldValue(existing.size, mapped.size, policy), + location: resolveFieldValue(existing.location, mapped.location, policy), + customFields: { ...(existing.customFields ?? {}), ...mapped.customFields }, + updatedAt: new Date(), + }) + .where(eq(org.id, match.orgId)); + + await upsertIdentity(tx, job, record.externalId, null, match.orgId); + return "updated" as const; + } + } + + const [inserted] = await tx + .insert(org) + .values({ + workspaceId: job.workspaceId, + ownerId: job.options.defaultOwnerId, + name: mapped.name, + domain: mapped.domain, + industry: mapped.industry, + size: mapped.size, + location: mapped.location, + customFields: mapped.customFields, + }) + .onConflictDoUpdate({ + target: [org.workspaceId, org.name], + set: { updatedAt: new Date() }, + }) + .returning({ id: org.id }); + + if (!inserted) return "skipped" as const; + + if (mapped.domain) lookups.orgByDomain.set(mapped.domain, inserted.id); + if (mapped.normalizedName) lookups.orgByName.set(mapped.normalizedName, inserted.id); + await upsertIdentity(tx, job, record.externalId, null, inserted.id); + + return "created" as const; +} + +/** + * Resolves or creates the organization a person belongs to. Only runs when the job opts in, + * because inventing orgs from a low-quality list produces more cleanup than value. + */ +async function resolveOrgId( + tx: Tx, + job: { workspaceId: string; options: ImportJobOptionsSnapshot }, + mapped: MappedPerson, + lookups: Lookups, +) { + const match = resolveOrgMatch( + { domain: mapped.orgDomain, normalizedName: mapped.orgNormalizedName }, + { byDomain: lookups.orgByDomain, byNormalizedName: lookups.orgByName }, + ); + + if (match.orgId) return match.orgId; + if (!job.options.createMissingOrgs || !mapped.orgName) return null; + + const [inserted] = await tx + .insert(org) + .values({ + workspaceId: job.workspaceId, + ownerId: job.options.defaultOwnerId, + name: mapped.orgName, + domain: mapped.orgDomain, + }) + .onConflictDoUpdate({ + target: [org.workspaceId, org.name], + set: { updatedAt: new Date() }, + }) + .returning({ id: org.id }); + + if (!inserted) return null; + + if (mapped.orgDomain) lookups.orgByDomain.set(mapped.orgDomain, inserted.id); + if (mapped.orgNormalizedName) lookups.orgByName.set(mapped.orgNormalizedName, inserted.id); + + return inserted.id; +} + +async function upsertIdentity( + tx: Tx, + job: { workspaceId: string; provider: ImportProvider }, + externalId: string | null, + personId: string | null, + orgId: string | null, +) { + if (!externalId) return; + + await tx + .insert(externalIdentities) + .values({ + workspaceId: job.workspaceId, + provider: job.provider, + externalId, + entityType: personId ? "person" : "org", + personId, + orgId, + lastSeenAt: new Date(), + }) + .onConflictDoUpdate({ + target: [ + externalIdentities.workspaceId, + externalIdentities.provider, + externalIdentities.externalId, + ], + set: { personId, orgId, lastSeenAt: new Date(), updatedAt: new Date() }, + }); +} + +function pickLatestDate(existing: Date | null, incoming: Date | null) { + if (!incoming) return existing; + if (!existing) return incoming; + + return incoming > existing ? incoming : existing; +} diff --git a/apps/api/src/workers/import-worker.ts b/apps/api/src/workers/import-worker.ts new file mode 100644 index 0000000..183b8fc --- /dev/null +++ b/apps/api/src/workers/import-worker.ts @@ -0,0 +1,32 @@ +import { logger } from "@/config/logger.config.js"; +import { executeDueImportWork } from "@/services/import.service.js"; + +const POLL_INTERVAL_MS = 15_000; +let running = false; + +export async function runImportWorkerTick() { + if (running) { + logger.warn("Import worker tick skipped because the previous tick is still running"); + return []; + } + + running = true; + try { + const results = await executeDueImportWork(new Date()); + logger.info({ processed: results.length }, "Import worker tick completed"); + return results; + } catch (error) { + logger.error({ error }, "Import worker tick failed"); + throw error; + } finally { + running = false; + } +} + +if (import.meta.main) { + logger.info({ intervalMs: POLL_INTERVAL_MS }, "Import worker started"); + setInterval(() => { + void runImportWorkerTick(); + }, POLL_INTERVAL_MS); + void runImportWorkerTick(); +} diff --git a/apps/api/test/routes/routes.test.ts b/apps/api/test/routes/routes.test.ts index 971a98e..d90a91b 100644 --- a/apps/api/test/routes/routes.test.ts +++ b/apps/api/test/routes/routes.test.ts @@ -53,6 +53,22 @@ const controller = vi.hoisted(() => { generateEmailContentController: respond, previewUnsubscribeController: respond, confirmUnsubscribeController: respond, + listConnectionsController: respond, + createConnectionController: respond, + updateConnectionController: respond, + deleteConnectionController: respond, + previewConnectionFieldsController: respond, + listApiKeysController: respond, + createApiKeyController: respond, + revokeApiKeyController: respond, + startImportJobController: respond, + listImportJobsController: respond, + getImportJobController: respond, + listImportRecordsController: respond, + updateImportJobController: respond, + commitImportJobController: respond, + cancelImportJobController: respond, + ingestWebhookController: respond, authHandler: vi.fn(() => new Response("auth")), }; }); @@ -118,6 +134,24 @@ vi.mock("@/controllers/sequences.controller.js", () => ({ previewUnsubscribeController: controller.previewUnsubscribeController, confirmUnsubscribeController: controller.confirmUnsubscribeController, })); +vi.mock("@/controllers/imports.controller.js", () => ({ + listConnectionsController: controller.listConnectionsController, + createConnectionController: controller.createConnectionController, + updateConnectionController: controller.updateConnectionController, + deleteConnectionController: controller.deleteConnectionController, + previewConnectionFieldsController: controller.previewConnectionFieldsController, + listApiKeysController: controller.listApiKeysController, + createApiKeyController: controller.createApiKeyController, + revokeApiKeyController: controller.revokeApiKeyController, + startImportJobController: controller.startImportJobController, + listImportJobsController: controller.listImportJobsController, + getImportJobController: controller.getImportJobController, + listImportRecordsController: controller.listImportRecordsController, + updateImportJobController: controller.updateImportJobController, + commitImportJobController: controller.commitImportJobController, + cancelImportJobController: controller.cancelImportJobController, + ingestWebhookController: controller.ingestWebhookController, +})); vi.mock("@/middlewares/auth-middleware.js", () => ({ authMiddleware: (_c: Context, next: Next) => next(), })); @@ -141,6 +175,8 @@ import { orgRoutes } from "@/routes/org.route.js"; import { peopleRoutes } from "@/routes/people.route.js"; import { sequenceRoutes } from "@/routes/sequences.route.js"; import { sequenceUnsubscribeRoutes } from "@/routes/sequence-unsubscribe.route.js"; +import { importRoutes } from "@/routes/imports.route.js"; +import { importWebhookRoutes } from "@/routes/import-webhook.route.js"; import { registerRoutes } from "@/routes/index.js"; const jsonRequest = (method: string, body?: unknown) => ({ @@ -264,6 +300,47 @@ describe("API route wiring without database infrastructure", () => { ); }); + it("dispatches every import route with validated values", async () => { + const id = faker.string.uuid(); + const cases: Array<[string, RequestInit | undefined]> = [ + ["/connections", undefined], + [ + "/connections", + jsonRequest("POST", { + provider: "posthog", + displayName: "PostHog", + accessToken: "phx_test", + }), + ], + [`/connections/${id}/fields`, undefined], + [`/connections/${id}`, jsonRequest("PATCH", { displayName: "Renamed" })], + [`/connections/${id}`, { method: "DELETE" }], + ["/api-keys", undefined], + ["/api-keys", jsonRequest("POST", { name: "Zapier" })], + [`/api-keys/${id}`, { method: "DELETE" }], + ["/jobs", undefined], + ["/jobs", jsonRequest("POST", { provider: "csv", csvContent: "Name\nDana\n" })], + [`/jobs/${id}`, undefined], + [`/jobs/${id}/records`, undefined], + [`/jobs/${id}`, jsonRequest("PATCH", { mapping: { fields: [] } })], + [`/jobs/${id}/commit`, { method: "POST" }], + [`/jobs/${id}/cancel`, { method: "POST" }], + ]; + + for (const [path, init] of cases) { + expect((await importRoutes.request(path, init)).status).toBe(200); + } + }); + + it("accepts an inbound webhook push outside the session middleware", async () => { + const response = await importWebhookRoutes.request( + "/", + jsonRequest("POST", { records: [{ email: "dana@northwind.example", name: "Dana" }] }), + ); + + expect(response.status).toBe(200); + }); + it("registers the complete API and constructs the server export", async () => { const app = new Hono(); registerRoutes(app); diff --git a/apps/api/test/services/import-connectors.test.ts b/apps/api/test/services/import-connectors.test.ts new file mode 100644 index 0000000..7421d6b --- /dev/null +++ b/apps/api/test/services/import-connectors.test.ts @@ -0,0 +1,450 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/config/env.config.js", () => ({ + env: { + INTEGRATION_LIVE_FETCH_ENABLED: false, + POSTHOG_API_HOST: "https://us.posthog.com", + CALENDLY_API_HOST: "https://api.calendly.com", + }, +})); + +import { AppError } from "@/lib/app-error.js"; +import { calendlyConnector } from "@/services/import-connectors/calendly.js"; +import { getFixtureFields, getFixturePage } from "@/services/import-connectors/fixtures.js"; +import { + gmailConnector, + googleCalendarConnector, + googleSheetsConnector, +} from "@/services/import-connectors/google.js"; +import { + ConnectionAuthError, + parseRetryAfter, + RateLimitError, + requestJson, +} from "@/services/import-connectors/http.js"; +import { outlookConnector } from "@/services/import-connectors/outlook.js"; +import { posthogConnector } from "@/services/import-connectors/posthog.js"; +import type { ConnectorContext } from "@/services/import-connectors/types.js"; + +const jsonResponse = (body: unknown, init?: ResponseInit) => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + ...init, + }); + +function contextWith( + responses: unknown[], + config: Record = {}, + cursor: Record | null = null, +): ConnectorContext & { calls: string[] } { + const calls: string[] = []; + let index = 0; + + return { + auth: { accessToken: "token", refreshToken: null }, + config, + cursor, + pageSize: 10, + calls, + fetchImpl: (async (url: string | URL) => { + calls.push(String(url)); + const body = responses[Math.min(index, responses.length - 1)]; + index += 1; + return body instanceof Response ? body : jsonResponse(body); + }) as unknown as ConnectorContext["fetchImpl"], + }; +} + +describe("connector http handling", () => { + it("converts a 429 into a reschedule instruction rather than a failure", async () => { + const ctx = contextWith([ + new Response("{}", { status: 429, headers: { "retry-after": "120" } }), + ]); + + await expect( + requestJson({ url: "https://example.test/x", accessToken: "t", fetchImpl: ctx.fetchImpl }), + ).rejects.toBeInstanceOf(RateLimitError); + }); + + it("treats 401 and 403 as credential failures that retrying cannot fix", async () => { + for (const status of [401, 403]) { + const ctx = contextWith([new Response("{}", { status })]); + await expect( + requestJson({ url: "https://example.test/x", accessToken: "t", fetchImpl: ctx.fetchImpl }), + ).rejects.toBeInstanceOf(ConnectionAuthError); + } + }); + + it("surfaces other failures as an operational error", async () => { + const ctx = contextWith([new Response("{}", { status: 500 })]); + + await expect( + requestJson({ url: "https://example.test/x", accessToken: "t", fetchImpl: ctx.fetchImpl }), + ).rejects.toBeInstanceOf(AppError); + }); + + it("parses Retry-After as seconds or as an HTTP date", () => { + const now = new Date("2026-07-20T12:00:00.000Z"); + + expect(parseRetryAfter("30", now)).toBe(30_000); + expect(parseRetryAfter("Mon, 20 Jul 2026 12:02:00 GMT", now)).toBe(120_000); + expect(parseRetryAfter(null, now)).toBe(60_000); + expect(parseRetryAfter("gibberish", now)).toBe(60_000); + // A date in the past must not produce a negative delay. + expect(parseRetryAfter("Mon, 20 Jul 2026 11:00:00 GMT", now)).toBe(0); + }); +}); + +describe("gmail connector", () => { + it("aggregates recipients across messages and counts repeats", async () => { + const ctx = contextWith([ + { messages: [{ id: "m1" }, { id: "m2" }], nextPageToken: "next" }, + { + id: "m1", + internalDate: "1780000000000", + payload: { headers: [{ name: "To", value: "Dana " }] }, + }, + { + id: "m2", + internalDate: "1790000000000", + payload: { headers: [{ name: "To", value: "dana@northwind.example" }] }, + }, + ]); + + const page = await gmailConnector.extract(ctx); + + expect(page.records).toHaveLength(1); + expect(page.records[0]!.data.messageCount).toBe(2); + expect(page.records[0]!.externalId).toBe("dana@northwind.example"); + // The most recent send wins as the contact date. + expect(page.records[0]!.data.lastContactedAt).toBe(new Date(1790000000000).toISOString()); + expect(page.nextCursor).toEqual({ pageToken: "next" }); + }); + + it("drops automated senders", async () => { + const ctx = contextWith([ + { messages: [{ id: "m1" }] }, + { id: "m1", payload: { headers: [{ name: "To", value: "noreply@stripe.com" }] } }, + ]); + + expect((await gmailConnector.extract(ctx)).records).toEqual([]); + }); + + it("returns an empty page and no cursor when the mailbox has nothing left", async () => { + const page = await gmailConnector.extract(contextWith([{}])); + + expect(page.records).toEqual([]); + expect(page.nextCursor).toBeNull(); + }); +}); + +describe("google calendar connector", () => { + it("excludes the connected user and room resources", async () => { + const ctx = contextWith([ + { + items: [ + { + id: "e1", + summary: "Intro", + start: { dateTime: "2026-06-10T14:00:00.000Z" }, + attendees: [ + { email: "me@own.example", self: true }, + { email: "room-a@resource.calendar.google.com", resource: true }, + { email: "Dana@Northwind.example", displayName: "Dana Reeves" }, + ], + }, + ], + }, + ]); + + const page = await googleCalendarConnector.extract(ctx); + + expect(page.records).toHaveLength(1); + expect(page.records[0]!.data.email).toBe("dana@northwind.example"); + expect(page.records[0]!.data.lastMeetingTitle).toBe("Intro"); + }); + + it("counts repeat attendees and keeps the latest meeting", async () => { + const ctx = contextWith([ + { + items: [ + { + id: "e1", + summary: "First", + start: { dateTime: "2026-06-01T10:00:00.000Z" }, + attendees: [{ email: "dana@northwind.example" }], + }, + { + id: "e2", + summary: "Second", + start: { dateTime: "2026-06-09T10:00:00.000Z" }, + attendees: [{ email: "dana@northwind.example" }], + }, + ], + nextPageToken: "p2", + }, + ]); + + const page = await googleCalendarConnector.extract(ctx); + + expect(page.records[0]!.data.meetingCount).toBe(2); + expect(page.records[0]!.data.lastMeetingTitle).toBe("Second"); + expect(page.nextCursor).toEqual({ pageToken: "p2" }); + }); +}); + +describe("google sheets connector", () => { + it("maps a header row onto data rows and paginates by row offset", async () => { + const ctx = contextWith( + [ + { values: [["Name", "Email"]] }, + { + values: Array.from({ length: 10 }, (_, index) => [ + `Person ${index}`, + `p${index}@x.example`, + ]), + }, + ], + { spreadsheetId: "sheet-1" }, + ); + + const page = await googleSheetsConnector.extract(ctx); + + expect(page.records).toHaveLength(10); + expect(page.records[0]!.data).toEqual({ Name: "Person 0", Email: "p0@x.example" }); + expect(page.records[0]!.externalId).toBe("sheet-1:2"); + // A full page implies there may be more rows. + expect(page.nextCursor).toEqual({ startRow: 12 }); + }); + + it("stops paginating on a short page and skips fully blank rows", async () => { + const ctx = contextWith( + [{ values: [["Name", "Email"]] }, { values: [["Dana", "dana@x.example"], ["", ""]] }], + { spreadsheetId: "sheet-1" }, + ); + + const page = await googleSheetsConnector.extract(ctx); + + expect(page.records).toHaveLength(1); + expect(page.nextCursor).toBeNull(); + }); + + it("requires a spreadsheet id", async () => { + await expect(googleSheetsConnector.extract(contextWith([{}]))).rejects.toThrow( + /spreadsheetId/, + ); + }); + + it("lists header fields for the mapping UI", async () => { + const ctx = contextWith([{ values: [["Name", " Email ", ""]] }], { spreadsheetId: "s" }); + + expect(await googleSheetsConnector.listFields(ctx)).toEqual(["Name", "Email"]); + }); +}); + +describe("calendly connector", () => { + it("flattens booking questions onto the record", async () => { + const ctx = contextWith( + [ + { + collection: [ + { uri: "https://api.calendly.com/scheduled_events/abc", name: "Demo", start_time: "2026-06-14T16:00:00.000Z" }, + ], + pagination: { next_page_token: null }, + }, + { + collection: [ + { + uri: "https://api.calendly.com/scheduled_events/abc/invitees/001", + email: "Priya@Vantage.example", + name: "Priya Raman", + questions_and_answers: [ + { question: "Company", answer: "Vantage Systems" }, + { question: "Budget", answer: " " }, + ], + }, + ], + }, + ], + { organization: "https://api.calendly.com/organizations/org-1" }, + ); + + const page = await calendlyConnector.extract(ctx); + + expect(page.records).toHaveLength(1); + expect(page.records[0]!.data.email).toBe("priya@vantage.example"); + expect(page.records[0]!.data["question:Company"]).toBe("Vantage Systems"); + // A blank answer is not a field. + expect(page.records[0]!.data["question:Budget"]).toBeUndefined(); + expect(page.nextCursor).toBeNull(); + }); + + it("requires an organization uri", async () => { + await expect(calendlyConnector.extract(contextWith([{}]))).rejects.toThrow(/organization/); + }); +}); + +describe("posthog connector", () => { + it("reads identity from properties and surfaces custom properties", async () => { + const ctx = contextWith( + [ + { + results: [ + { + id: 42, + distinct_ids: ["user_8812"], + properties: { + email: "Marcus@Lumen-Labs.example", + first_name: "Marcus", + last_name: "Oyelaran", + company: "Lumen Labs", + plan: "trial", + $device_type: "Desktop", + }, + }, + ], + next: "https://us.posthog.com/api/projects/1/persons/?cursor=abc", + }, + ], + { projectId: "1" }, + ); + + const page = await posthogConnector.extract(ctx); + const record = page.records[0]!; + + expect(record.externalId).toBe("42"); + expect(record.data.email).toBe("marcus@lumen-labs.example"); + expect(record.data.name).toBe("Marcus Oyelaran"); + expect(record.data.company).toBe("Lumen Labs"); + expect(record.data["property:plan"]).toBe("trial"); + // PostHog internal properties are noise, not CRM fields. + expect(record.data["property:$device_type"]).toBeUndefined(); + expect(page.nextCursor).toEqual({ + next: "https://us.posthog.com/api/projects/1/persons/?cursor=abc", + }); + }); + + it("follows a stored cursor url verbatim", async () => { + const ctx = contextWith([{ results: [] }], { projectId: "1" }, { + next: "https://us.posthog.com/api/projects/1/persons/?cursor=abc", + }); + + await posthogConnector.extract(ctx); + + expect(ctx.calls[0]).toBe("https://us.posthog.com/api/projects/1/persons/?cursor=abc"); + }); + + it("requires a project id", async () => { + await expect(posthogConnector.extract(contextWith([{}]))).rejects.toThrow(/projectId/); + }); +}); + +describe("outlook connector", () => { + it("reads contacts by default", async () => { + const ctx = contextWith([ + { + value: [ + { + id: "c1", + displayName: "Rosa Iglesias", + jobTitle: "VP Revenue", + companyName: "Meridian Group", + businessPhones: ["+1 415 555 0142"], + emailAddresses: [{ address: "Rosa@Meridian.example" }], + }, + ], + "@odata.nextLink": "https://graph.microsoft.com/next", + }, + ]); + + const page = await outlookConnector.extract(ctx); + + expect(page.records[0]!.data.email).toBe("rosa@meridian.example"); + expect(page.records[0]!.data.orgName).toBe("Meridian Group"); + expect(page.nextCursor).toEqual({ nextLink: "https://graph.microsoft.com/next" }); + }); + + it("aggregates sent-mail recipients when mode is sent_mail", async () => { + const ctx = contextWith( + [ + { + value: [ + { + id: "m1", + sentDateTime: "2026-06-01T10:00:00.000Z", + toRecipients: [{ emailAddress: { address: "dana@northwind.example", name: "Dana" } }], + ccRecipients: [{ emailAddress: { address: "noreply@x.example" } }], + }, + { + id: "m2", + sentDateTime: "2026-06-05T10:00:00.000Z", + toRecipients: [{ emailAddress: { address: "dana@northwind.example" } }], + }, + ], + }, + ], + { mode: "sent_mail" }, + ); + + const page = await outlookConnector.extract(ctx); + + expect(page.records).toHaveLength(1); + expect(page.records[0]!.data.messageCount).toBe(2); + expect(page.records[0]!.data.lastContactedAt).toBe("2026-06-05T10:00:00.000Z"); + }); + + it("excludes resource attendees when mode is calendar", async () => { + const ctx = contextWith( + [ + { + value: [ + { + id: "e1", + subject: "Review", + start: { dateTime: "2026-06-10T14:00:00.000Z" }, + attendees: [ + { emailAddress: { address: "room@meridian.example" }, type: "resource" }, + { emailAddress: { address: "rosa@meridian.example" }, type: "required" }, + ], + }, + ], + }, + ], + { mode: "calendar" }, + ); + + const page = await outlookConnector.extract(ctx); + + expect(page.records).toHaveLength(1); + expect(page.records[0]!.data.email).toBe("rosa@meridian.example"); + }); + + it("falls back to contacts for an unknown mode and lists mode-specific fields", async () => { + const contactsCtx = contextWith([{ value: [] }], { mode: "nonsense" }); + expect((await outlookConnector.extract(contactsCtx)).records).toEqual([]); + + expect(await outlookConnector.listFields(contextWith([], { mode: "calendar" }))).toContain( + "meetingCount", + ); + expect(await outlookConnector.listFields(contextWith([], { mode: "sent_mail" }))).toContain( + "messageCount", + ); + expect(await outlookConnector.listFields(contextWith([], {}))).toContain("orgName"); + }); +}); + +describe("fixtures", () => { + it("serves a deterministic page per provider without touching the network", () => { + expect(getFixturePage("gmail").records).toHaveLength(2); + expect(getFixturePage("csv").records).toEqual([]); + expect(getFixtureFields("posthog")).toContain("distinctId"); + }); + + it("clones records so a caller cannot corrupt later runs", () => { + const first = getFixturePage("gmail"); + first.records[0]!.data.name = "mutated"; + + expect(getFixturePage("gmail").records[0]!.data.name).toBe("Dana Reeves"); + }); +}); diff --git a/apps/api/test/services/import-csv.test.ts b/apps/api/test/services/import-csv.test.ts new file mode 100644 index 0000000..d0bed2a --- /dev/null +++ b/apps/api/test/services/import-csv.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { AppError } from "@/lib/app-error.js"; +import { parseCsv } from "@/services/import-csv.js"; + +describe("csv parsing", () => { + it("parses headers and rows with trimming", () => { + const parsed = parseCsv("Name, Email\nDana Reeves, dana@northwind.example\n"); + + expect(parsed.headers).toEqual(["Name", "Email"]); + expect(parsed.rows).toEqual([{ Name: "Dana Reeves", Email: "dana@northwind.example" }]); + }); + + it("handles quoted fields containing commas, newlines, and escaped quotes", () => { + const parsed = parseCsv( + 'Name,Notes\n"Reeves, Dana","Said ""yes"" on the call\nFollow up Monday"\n', + ); + + expect(parsed.rows).toEqual([ + { Name: "Reeves, Dana", Notes: 'Said "yes" on the call\nFollow up Monday' }, + ]); + }); + + it("supports CRLF endings, a BOM, and a missing trailing newline", () => { + const parsed = parseCsv("Name,Email\r\nDana,dana@northwind.example"); + + expect(parsed.headers).toEqual(["Name", "Email"]); + expect(parsed.rows).toEqual([{ Name: "Dana", Email: "dana@northwind.example" }]); + }); + + it("skips blank rows and tolerates short rows", () => { + const parsed = parseCsv("Name,Email\nDana\n\n \nMarcus,marcus@lumen-labs.example\n"); + + expect(parsed.rows).toEqual([ + { Name: "Dana", Email: "" }, + { Name: "Marcus", Email: "marcus@lumen-labs.example" }, + ]); + }); + + it("suffixes duplicate headers so every column stays addressable", () => { + const parsed = parseCsv("Email,Email\na@x.example,b@x.example\n"); + + expect(parsed.headers).toEqual(["Email", "Email (2)"]); + expect(parsed.rows[0]).toEqual({ Email: "a@x.example", "Email (2)": "b@x.example" }); + }); + + it("ignores unnamed columns rather than creating an empty key", () => { + const parsed = parseCsv("Name,,Email\nDana,junk,dana@northwind.example\n"); + + expect(parsed.headers).toEqual(["Name", "Email"]); + expect(parsed.rows[0]).toEqual({ Name: "Dana", Email: "dana@northwind.example" }); + }); + + it("rejects an empty file and a header-less file", () => { + expect(() => parseCsv("")).toThrow(AppError); + expect(() => parseCsv(",,\n")).toThrow(AppError); + }); +}); diff --git a/apps/api/test/services/import-engine.test.ts b/apps/api/test/services/import-engine.test.ts new file mode 100644 index 0000000..8f51d31 --- /dev/null +++ b/apps/api/test/services/import-engine.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; +import { + deriveNameFromEmail, + deriveRecordFingerprint, + fieldPolicy, + getEmailDomain, + getEmailLocalPart, + isAutomatedEmail, + isFreeEmailDomain, + isValidEmail, + normalizeDomain, + normalizeEmail, + normalizeEmailForMatching, + normalizeLinkedinUrl, + normalizeOrgName, + normalizePersonName, + normalizePhone, + parseAddressList, + resolveFieldValue, + resolveOrgMatch, + resolvePersonMatch, + shouldSkipExistingRecord, +} from "@/services/import-engine.js"; + +describe("import engine email handling", () => { + it("normalizes and validates addresses", () => { + expect(normalizeEmail(" Dana@Northwind.EXAMPLE ")).toBe("dana@northwind.example"); + expect(isValidEmail("dana@northwind.example")).toBe(true); + expect(isValidEmail("not-an-email")).toBe(false); + expect(isValidEmail("missing@domain")).toBe(false); + expect(getEmailDomain("dana@northwind.example")).toBe("northwind.example"); + expect(getEmailDomain("broken")).toBeNull(); + expect(getEmailLocalPart("dana@northwind.example")).toBe("dana"); + }); + + it("recognizes automated senders so they never become CRM records", () => { + expect(isAutomatedEmail("noreply@stripe.com")).toBe(true); + expect(isAutomatedEmail("no-reply@stripe.com")).toBe(true); + expect(isAutomatedEmail("bounces+tag@sendgrid.net")).toBe(true); + expect(isAutomatedEmail("dana@northwind.example")).toBe(false); + expect(isAutomatedEmail("broken")).toBe(false); + }); + + it("collapses subaddressing only for providers that treat it as equivalent", () => { + expect(normalizeEmailForMatching("Da.Na+crm@gmail.com", true)).toBe("dana@gmail.com"); + // Off by default: some domains route dots and tags to distinct mailboxes. + expect(normalizeEmailForMatching("da.na+crm@gmail.com", false)).toBe("da.na+crm@gmail.com"); + // Opting in must not rewrite addresses on domains that do not support it. + expect(normalizeEmailForMatching("da.na+crm@northwind.example", true)).toBe( + "da.na+crm@northwind.example", + ); + expect(normalizeEmailForMatching("broken", true)).toBe("broken"); + }); + + it("derives a readable name when a source supplies only an address", () => { + expect(deriveNameFromEmail("dana.reeves@northwind.example")).toBe("Dana Reeves"); + expect(deriveNameFromEmail("s_patel+news@brightline.example")).toBe("S Patel"); + expect(deriveNameFromEmail("123@northwind.example")).toBeNull(); + expect(deriveNameFromEmail("broken")).toBeNull(); + }); +}); + +describe("import engine domain and name normalization", () => { + it("reduces urls and hostnames to a comparable domain", () => { + expect(normalizeDomain("https://www.Acme.com/pricing?ref=x")).toBe("acme.com"); + expect(normalizeDomain("acme.com:8080")).toBe("acme.com"); + expect(normalizeDomain("dana@acme.com")).toBe("acme.com"); + expect(normalizeDomain(" ")).toBeNull(); + expect(normalizeDomain("localhost")).toBeNull(); + }); + + it("refuses to treat a free mailbox provider as an organization domain", () => { + // Otherwise every personal Gmail address would invent an org called "Gmail". + expect(normalizeDomain("gmail.com")).toBeNull(); + expect(normalizeDomain("https://outlook.com")).toBeNull(); + expect(isFreeEmailDomain("proton.me")).toBe(true); + expect(isFreeEmailDomain("northwind.example")).toBe(false); + expect(isFreeEmailDomain(null)).toBe(false); + }); + + it("collapses legal suffixes and casing so one company resolves to one org", () => { + expect(normalizeOrgName("Acme, Inc.")).toBe("acme"); + expect(normalizeOrgName("ACME")).toBe("acme"); + expect(normalizeOrgName("Acme Corporation")).toBe("acme"); + expect(normalizeOrgName("Bright Line LLC")).toBe("bright line"); + // A name that is only a suffix must survive rather than normalize to nothing. + expect(normalizeOrgName("Inc")).toBe("inc"); + expect(normalizeOrgName(" ")).toBeNull(); + expect(normalizeOrgName("!!!")).toBeNull(); + }); + + it("normalizes people names, phones, and linkedin urls", () => { + expect(normalizePersonName(" Dana Reeves ")).toBe("Dana Reeves"); + expect(normalizePhone("+1 (415) 555-0142")).toBe("+14155550142"); + expect(normalizePhone("415-555-0142")).toBe("4155550142"); + expect(normalizePhone("123")).toBeNull(); + expect(normalizePhone(" ")).toBeNull(); + expect(normalizeLinkedinUrl("linkedin.com/in/dana-reeves/")).toBe( + "https://www.linkedin.com/in/dana-reeves", + ); + expect(normalizeLinkedinUrl("https://LinkedIn.com/company/acme?trk=x")).toBe( + "https://www.linkedin.com/company/acme", + ); + expect(normalizeLinkedinUrl("https://twitter.com/dana")).toBeNull(); + expect(normalizeLinkedinUrl(" ")).toBeNull(); + }); +}); + +describe("import engine address list parsing", () => { + it("parses display names, bare addresses, and quoted names containing commas", () => { + const parsed = parseAddressList( + '"Reeves, Dana" , marcus@lumen-labs.example', + ); + + expect(parsed).toEqual([ + { name: "Reeves, Dana", email: "dana@northwind.example" }, + { name: null, email: "marcus@lumen-labs.example" }, + ]); + }); + + it("drops unparseable entries instead of producing junk records", () => { + expect(parseAddressList("")).toEqual([]); + expect(parseAddressList("not-an-address, , dana@northwind.example")).toEqual([ + { name: null, email: "dana@northwind.example" }, + ]); + }); +}); + +describe("import engine identity resolution", () => { + const lookups = { + byExternalId: new Map([["ext-1", "person-from-identity"]]), + byEmail: new Map([["dana@northwind.example", "person-from-email"]]), + }; + + it("prefers external identity over email so a changed address still matches", () => { + expect( + resolvePersonMatch( + { externalId: "ext-1", email: "new@northwind.example", normalizedEmail: "new@northwind.example" }, + lookups, + ), + ).toEqual({ personId: "person-from-identity", reason: "external_identity" }); + }); + + it("falls back to email, then reports no match", () => { + expect( + resolvePersonMatch( + { externalId: null, email: "dana@northwind.example", normalizedEmail: "dana@northwind.example" }, + lookups, + ), + ).toEqual({ personId: "person-from-email", reason: "email" }); + + expect( + resolvePersonMatch({ externalId: "unknown", email: null, normalizedEmail: null }, lookups), + ).toEqual({ personId: null, reason: "none" }); + }); + + it("matches organizations on domain before name", () => { + const orgLookups = { + byDomain: new Map([["acme.com", "org-by-domain"]]), + byNormalizedName: new Map([["acme", "org-by-name"]]), + }; + + expect(resolveOrgMatch({ domain: "acme.com", normalizedName: "acme" }, orgLookups)).toEqual({ + orgId: "org-by-domain", + reason: "domain", + }); + expect(resolveOrgMatch({ domain: null, normalizedName: "acme" }, orgLookups)).toEqual({ + orgId: "org-by-name", + reason: "name", + }); + expect(resolveOrgMatch({ domain: null, normalizedName: null }, orgLookups)).toEqual({ + orgId: null, + reason: "none", + }); + }); +}); + +describe("import engine record fingerprint", () => { + const person = { + name: "Dana Reeves", + email: "dana@northwind.example", + phone: null, + orgName: "Acme", + }; + + it("is deterministic so a re-uploaded row resolves to the same identity", () => { + // This is what stops an emailless CSV row from duplicating on every re-import: with no + // real external id and a nullable-email unique constraint, the fingerprint is the only + // stable handle the record has. + expect(deriveRecordFingerprint(person)).toBe(deriveRecordFingerprint({ ...person })); + expect(deriveRecordFingerprint(person)).toMatch(/^fp_[0-9a-f]{32}$/); + }); + + it("is insensitive to name and org casing but sensitive to identity changes", () => { + expect(deriveRecordFingerprint(person)).toBe( + deriveRecordFingerprint({ ...person, name: "DANA REEVES", orgName: "ACME" }), + ); + expect(deriveRecordFingerprint(person)).not.toBe( + deriveRecordFingerprint({ ...person, email: "dana@elsewhere.example" }), + ); + expect(deriveRecordFingerprint(person)).not.toBe( + deriveRecordFingerprint({ ...person, phone: "+14155550142" }), + ); + }); + + it("distinguishes an absent field from an empty one without collision", () => { + const withOrg = deriveRecordFingerprint({ name: "A", email: null, phone: null, orgName: "X" }); + const withName = deriveRecordFingerprint({ name: "AX", email: null, phone: null, orgName: null }); + + // A naive concatenation would let "A"+"X" collide with "AX"+""; the delimiter prevents it. + expect(withOrg).not.toBe(withName); + }); +}); + +describe("import engine conflict policy", () => { + it("never overwrites with an empty incoming value", () => { + expect(resolveFieldValue("Existing", null, "source_wins")).toBe("Existing"); + expect(resolveFieldValue("Existing", "", "source_wins")).toBe("Existing"); + expect(resolveFieldValue("Existing", undefined, "fill_empty")).toBe("Existing"); + }); + + it("distinguishes source_wins from fill_empty", () => { + expect(resolveFieldValue("Existing", "Incoming", "source_wins")).toBe("Incoming"); + expect(resolveFieldValue("Existing", "Incoming", "fill_empty")).toBe("Existing"); + expect(resolveFieldValue(null, "Incoming", "fill_empty")).toBe("Incoming"); + expect(resolveFieldValue("", "Incoming", "fill_empty")).toBe("Incoming"); + }); + + it("treats crm_wins as a record-level skip rather than a field rule", () => { + // Collapsing it into a field rule would make it identical to fill_empty. + expect(shouldSkipExistingRecord("crm_wins")).toBe(true); + expect(shouldSkipExistingRecord("fill_empty")).toBe(false); + expect(shouldSkipExistingRecord("source_wins")).toBe(false); + + expect(fieldPolicy("crm_wins")).toBe("fill_empty"); + expect(fieldPolicy("fill_empty")).toBe("fill_empty"); + expect(fieldPolicy("source_wins")).toBe("source_wins"); + }); +}); diff --git a/apps/api/test/services/import-mapping.test.ts b/apps/api/test/services/import-mapping.test.ts new file mode 100644 index 0000000..c30262c --- /dev/null +++ b/apps/api/test/services/import-mapping.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, it } from "vitest"; +import { + applyOrgMapping, + applyPersonMapping, + autoDetectMapping, + isKnownOrgTarget, + isKnownPersonTarget, + validateMappedOrg, + validateMappedPerson, +} from "@/services/import-mapping.js"; + +const mapping = [ + { sourceField: "Name", targetField: "name", customFieldId: null }, + { sourceField: "Email", targetField: "email", customFieldId: null }, + { sourceField: "Company", targetField: "orgName", customFieldId: null }, +]; + +const defaultOptions = { normalizeSubaddressing: false }; + +describe("mapping auto-detection", () => { + it("matches common header spellings for people", () => { + const detected = autoDetectMapping( + ["Full Name", "Email Address", "Phone Number", "Job Title", "Company"], + "person", + ); + + expect(detected.map((field) => field.targetField)).toEqual([ + "name", + "email", + "phone", + "jobTitle", + "orgName", + ]); + }); + + it("leaves unrecognized headers unmapped instead of dropping them", () => { + const detected = autoDetectMapping(["Email", "Favourite Colour"], "person"); + + expect(detected).toEqual([ + { sourceField: "Email", targetField: "email", customFieldId: null }, + { sourceField: "Favourite Colour", targetField: null, customFieldId: null }, + ]); + }); + + it("gives a target to the first claiming header only", () => { + const detected = autoDetectMapping(["Email", "Work Email"], "person"); + + expect(detected[0]!.targetField).toBe("email"); + expect(detected[1]!.targetField).toBeNull(); + }); + + it("uses a different alias table for organizations", () => { + const detected = autoDetectMapping(["Company Name", "Website", "Headcount"], "org"); + + expect(detected.map((field) => field.targetField)).toEqual(["name", "domain", "size"]); + }); +}); + +describe("person mapping", () => { + it("normalizes mapped values and infers the org domain from a corporate address", () => { + const mapped = applyPersonMapping( + { Name: " Dana Reeves ", Email: "Dana@Northwind.EXAMPLE", Company: "Acme, Inc." }, + mapping, + defaultOptions, + ); + + expect(mapped.name).toBe("Dana Reeves"); + expect(mapped.email).toBe("dana@northwind.example"); + expect(mapped.orgName).toBe("Acme, Inc."); + expect(mapped.orgNormalizedName).toBe("acme"); + expect(mapped.orgDomain).toBe("northwind.example"); + }); + + it("does not invent an org domain from a free mailbox provider", () => { + const mapped = applyPersonMapping( + { Name: "Fen Zhao", Email: "fen.zhao@gmail.com", Company: "" }, + mapping, + defaultOptions, + ); + + expect(mapped.orgDomain).toBeNull(); + expect(mapped.orgName).toBeNull(); + }); + + it("derives a name when the source has only an address", () => { + const mapped = applyPersonMapping({ Email: "s.patel@brightline.example" }, mapping, defaultOptions); + + expect(mapped.name).toBe("S Patel"); + }); + + it("rejects an invalid address rather than storing it", () => { + const mapped = applyPersonMapping({ Name: "Dana", Email: "not-an-email" }, mapping, defaultOptions); + + expect(mapped.email).toBeNull(); + expect(mapped.normalizedEmail).toBeNull(); + }); + + it("routes unmapped columns into custom fields", () => { + const mapped = applyPersonMapping( + { Region: "EMEA", Email: "dana@northwind.example" }, + [ + { sourceField: "Region", targetField: null, customFieldId: "field-1" }, + { sourceField: "Email", targetField: "email", customFieldId: null }, + ], + defaultOptions, + ); + + expect(mapped.customFields).toEqual({ "field-1": "EMEA" }); + }); + + it("coerces numeric and boolean cells and ignores blanks", () => { + const mapped = applyPersonMapping( + { Name: "Dana", Email: "dana@northwind.example", Phone: 4155550142, Blank: " " }, + [ + ...mapping, + { sourceField: "Phone", targetField: "phone", customFieldId: null }, + { sourceField: "Blank", targetField: "jobTitle", customFieldId: null }, + ], + defaultOptions, + ); + + expect(mapped.phone).toBe("4155550142"); + expect(mapped.jobTitle).toBeNull(); + }); + + it("parses a last-contacted date and ignores an unparseable one", () => { + const dateMapping = [{ sourceField: "Last", targetField: "lastContactedAt", customFieldId: null }]; + + expect( + applyPersonMapping({ Last: "2026-06-02T10:15:00.000Z" }, dateMapping, defaultOptions) + .lastContactedAt, + ).toEqual(new Date("2026-06-02T10:15:00.000Z")); + expect( + applyPersonMapping({ Last: "not a date" }, dateMapping, defaultOptions).lastContactedAt, + ).toBeNull(); + }); +}); + +describe("org mapping", () => { + it("normalizes name and domain", () => { + const mapped = applyOrgMapping( + { Company: "Acme Corporation", Website: "https://www.acme.com/about" }, + [ + { sourceField: "Company", targetField: "name", customFieldId: null }, + { sourceField: "Website", targetField: "domain", customFieldId: null }, + ], + ); + + expect(mapped.name).toBe("Acme Corporation"); + expect(mapped.normalizedName).toBe("acme"); + expect(mapped.domain).toBe("acme.com"); + }); +}); + +describe("record validation", () => { + it("accepts a record with either a name or an email", () => { + const mapped = applyPersonMapping({ Name: "Dana" }, mapping, defaultOptions); + + expect(validateMappedPerson(mapped, { skipRecordsWithoutEmail: false }, null)).toEqual([]); + }); + + it("reports a malformed address using the original value", () => { + const mapped = applyPersonMapping({ Name: "Dana", Email: "nope" }, mapping, defaultOptions); + const errors = validateMappedPerson(mapped, { skipRecordsWithoutEmail: false }, "nope"); + + expect(errors).toEqual([ + { field: "email", message: '"nope" is not a valid email address' }, + ]); + }); + + it("rejects a record with neither a name nor an address", () => { + const mapped = applyPersonMapping({}, mapping, defaultOptions); + + expect(validateMappedPerson(mapped, { skipRecordsWithoutEmail: false }, null)).toEqual([ + { field: "name", message: "Record has neither a name nor an email address" }, + ]); + }); + + it("honours the skip-without-email option", () => { + const mapped = applyPersonMapping({ Name: "Dana" }, mapping, defaultOptions); + + expect(validateMappedPerson(mapped, { skipRecordsWithoutEmail: true }, null)).toEqual([ + { field: "email", message: "Record has no email address" }, + ]); + }); + + it("rejects automated senders picked up by mailbox mining", () => { + const mapped = applyPersonMapping( + { Name: "Notifications", Email: "noreply@stripe.com" }, + mapping, + defaultOptions, + ); + + expect(validateMappedPerson(mapped, { skipRecordsWithoutEmail: false }, "noreply@stripe.com")).toEqual([ + { field: "email", message: "Address belongs to an automated sender" }, + ]); + }); + + it("requires organizations to have a name", () => { + expect(validateMappedOrg(applyOrgMapping({}, []))).toEqual([ + { field: "name", message: "Organization has no name" }, + ]); + expect( + validateMappedOrg( + applyOrgMapping({ Company: "Acme" }, [ + { sourceField: "Company", targetField: "name", customFieldId: null }, + ]), + ), + ).toEqual([]); + }); + + it("recognizes known target fields", () => { + expect(isKnownPersonTarget("email")).toBe(true); + expect(isKnownPersonTarget("nope")).toBe(false); + expect(isKnownOrgTarget("domain")).toBe(true); + expect(isKnownOrgTarget("nope")).toBe(false); + }); +}); diff --git a/apps/api/test/workers/import-worker.test.ts b/apps/api/test/workers/import-worker.test.ts new file mode 100644 index 0000000..8218816 --- /dev/null +++ b/apps/api/test/workers/import-worker.test.ts @@ -0,0 +1,62 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + executeDueImportWork: vi.fn(), + logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn() }, +})); + +vi.mock("@/services/import.service.js", () => ({ + executeDueImportWork: mocks.executeDueImportWork, +})); +vi.mock("@/config/logger.config.js", () => ({ + logger: mocks.logger, +})); + +import { runImportWorkerTick } from "@/workers/import-worker.js"; + +describe("import worker", () => { + beforeEach(() => { + mocks.executeDueImportWork.mockResolvedValue([{ jobId: "job", outcome: "loaded_batch" }]); + }); + + it("runs one due-work tick and logs the processed count", async () => { + await expect(runImportWorkerTick()).resolves.toEqual([ + { jobId: "job", outcome: "loaded_batch" }, + ]); + + expect(mocks.executeDueImportWork).toHaveBeenCalledWith(expect.any(Date)); + expect(mocks.logger.info).toHaveBeenCalledWith({ processed: 1 }, "Import worker tick completed"); + }); + + it("does not overlap concurrent ticks", async () => { + let resolveWork: (value: never[]) => void = () => {}; + mocks.executeDueImportWork.mockReturnValueOnce( + new Promise((resolve) => { + resolveWork = resolve; + }), + ); + + const firstTick = runImportWorkerTick(); + await expect(runImportWorkerTick()).resolves.toEqual([]); + resolveWork([]); + await firstTick; + + expect(mocks.executeDueImportWork).toHaveBeenCalledTimes(1); + expect(mocks.logger.warn).toHaveBeenCalledWith( + "Import worker tick skipped because the previous tick is still running", + ); + }); + + it("logs and rethrows a failing tick so the interval surfaces the error", async () => { + const failure = new Error("extraction exploded"); + mocks.executeDueImportWork.mockRejectedValueOnce(failure); + + await expect(runImportWorkerTick()).rejects.toThrow("extraction exploded"); + expect(mocks.logger.error).toHaveBeenCalledWith({ error: failure }, "Import worker tick failed"); + + // The running guard must be released even when the tick throws. + await expect(runImportWorkerTick()).resolves.toEqual([ + { jobId: "job", outcome: "loaded_batch" }, + ]); + }); +}); diff --git a/packages/validators/package.json b/packages/validators/package.json index d93387f..5961d00 100644 --- a/packages/validators/package.json +++ b/packages/validators/package.json @@ -28,6 +28,18 @@ "types": "./src/schemas/workspace.validator.ts", "import": "./dist/schemas/workspace.validator.js" }, + "./schemas/import": { + "types": "./src/schemas/import.validator.ts", + "import": "./dist/schemas/import.validator.js" + }, + "./types/import": { + "types": "./src/types/import.types.ts", + "import": "./dist/types/import.types.js" + }, + "./types/crm": { + "types": "./src/types/crm.types.ts", + "import": "./dist/types/crm.types.js" + }, "./types/auth": { "types": "./src/types/auth.types.ts", "import": "./dist/types/auth.types.js" diff --git a/packages/validators/src/schemas/env.validator.ts b/packages/validators/src/schemas/env.validator.ts index 7919eb0..d55aa63 100644 --- a/packages/validators/src/schemas/env.validator.ts +++ b/packages/validators/src/schemas/env.validator.ts @@ -18,6 +18,22 @@ export const apiEnvSchema = z .default("false") .transform((value) => value === "true"), GROQ_API_KEY: z.string().optional(), + /** + * Mirrors SEQUENCE_LIVE_SEND_ENABLED. When false, connectors resolve from fixtures + * instead of reaching the network, so imports are exercisable without credentials. + */ + INTEGRATION_LIVE_FETCH_ENABLED: z + .enum(["true", "false"]) + .optional() + .default("false") + .transform((value) => value === "true"), + /** Falls back to BETTER_AUTH_SECRET when unset, matching the sequence token handling. */ + INTEGRATION_TOKEN_SECRET: z.string().optional(), + MICROSOFT_CLIENT_ID: z.string().optional(), + MICROSOFT_CLIENT_SECRET: z.string().optional(), + MICROSOFT_TENANT_ID: z.string().optional().default("common"), + POSTHOG_API_HOST: z.string().url().optional().default("https://us.posthog.com"), + CALENDLY_API_HOST: z.string().url().optional().default("https://api.calendly.com"), }) .strict(); diff --git a/packages/validators/src/schemas/import.validator.ts b/packages/validators/src/schemas/import.validator.ts new file mode 100644 index 0000000..2ef346f --- /dev/null +++ b/packages/validators/src/schemas/import.validator.ts @@ -0,0 +1,145 @@ +import { z } from "zod"; +import { idSchema } from "./common.validator.js"; +import { + IMPORT_CONFLICT_POLICY_VALUES, + IMPORT_CONNECTION_STATUS_VALUES, + IMPORT_ENTITY_TYPE_VALUES, + IMPORT_JOB_STATUS_VALUES, + IMPORT_MAX_RECORDS_PER_JOB, + IMPORT_ORG_TARGET_FIELD_VALUES, + IMPORT_PERSON_TARGET_FIELD_VALUES, + IMPORT_PROVIDER_VALUES, + IMPORT_RECORD_STATUS_VALUES, +} from "../types/import.types.js"; +import { personStatusSchema } from "./crm.validator.js"; + +export const importProviderSchema = z.enum(IMPORT_PROVIDER_VALUES); +export const importEntityTypeSchema = z.enum(IMPORT_ENTITY_TYPE_VALUES); +export const importJobStatusSchema = z.enum(IMPORT_JOB_STATUS_VALUES); +export const importRecordStatusSchema = z.enum(IMPORT_RECORD_STATUS_VALUES); +export const importConnectionStatusSchema = z.enum(IMPORT_CONNECTION_STATUS_VALUES); +export const importConflictPolicySchema = z.enum(IMPORT_CONFLICT_POLICY_VALUES); +export const importPersonTargetFieldSchema = z.enum(IMPORT_PERSON_TARGET_FIELD_VALUES); +export const importOrgTargetFieldSchema = z.enum(IMPORT_ORG_TARGET_FIELD_VALUES); + +/** + * One source column bound to one CRM destination. `customFieldId` routes the value into + * `people.custom_fields` / `org.custom_fields` instead of a column, and is mutually + * exclusive with `targetField`. + */ +export const importFieldMappingSchema = z + .object({ + sourceField: z.string().trim().min(1).max(255), + targetField: z.string().trim().min(1).max(64).nullable().default(null), + customFieldId: idSchema.nullable().default(null), + }) + .refine((value) => !(value.targetField && value.customFieldId), { + message: "A mapping targets either a CRM field or a custom field, not both", + }); + +export const importMappingSchema = z.object({ + fields: z.array(importFieldMappingSchema).max(200).default([]), +}); + +export const importJobOptionsSchema = z.object({ + conflictPolicy: importConflictPolicySchema.default("fill_empty"), + defaultStatus: personStatusSchema.optional(), + defaultOwnerId: idSchema.nullable().default(null), + /** + * Collapses Gmail-style dots and plus-addressing before matching. Off by default because + * it is wrong for domains that treat those as distinct mailboxes. + */ + normalizeSubaddressing: z.boolean().default(false), + createMissingOrgs: z.boolean().default(true), + skipRecordsWithoutEmail: z.boolean().default(false), +}); + +export const startImportJobSchema = z.object({ + provider: importProviderSchema, + connectionId: idSchema.nullable().default(null), + entityType: importEntityTypeSchema.default("person"), + mapping: importMappingSchema.default({ fields: [] }), + options: importJobOptionsSchema.default({ + conflictPolicy: "fill_empty", + defaultOwnerId: null, + normalizeSubaddressing: false, + createMissingOrgs: true, + skipRecordsWithoutEmail: false, + }), + /** Raw file contents for `csv`. Ignored by every other provider. */ + csvContent: z.string().max(20_000_000).optional(), + /** Provider-scoped extraction settings, e.g. spreadsheet id or Calendly organization. */ + sourceConfig: z.record(z.string(), z.unknown()).default({}), +}); + +export const updateImportJobSchema = z.object({ + mapping: importMappingSchema.optional(), + options: importJobOptionsSchema.partial().optional(), +}); + +export const listImportJobsQuerySchema = z.object({ + page: z.coerce.number().int().positive().default(1), + pageSize: z.coerce.number().int().positive().max(100).default(25), + provider: importProviderSchema.optional(), + status: importJobStatusSchema.optional(), +}); + +export const listImportRecordsQuerySchema = z.object({ + page: z.coerce.number().int().positive().default(1), + pageSize: z.coerce.number().int().positive().max(100).default(50), + status: importRecordStatusSchema.optional(), +}); + +export const importJobParamsSchema = z.object({ id: idSchema }); +export const connectionParamsSchema = z.object({ id: idSchema }); + +export const createConnectionSchema = z.object({ + provider: importProviderSchema, + displayName: z.string().trim().min(1).max(255), + externalAccountId: z.string().trim().max(255).nullable().default(null), + accessToken: z.string().min(1), + refreshToken: z.string().min(1).nullable().default(null), + tokenExpiresAt: z.coerce.date().nullable().default(null), + grantedScopes: z.array(z.string()).default([]), + config: z.record(z.string(), z.unknown()).default({}), +}); + +export const updateConnectionSchema = z.object({ + displayName: z.string().trim().min(1).max(255).optional(), + status: importConnectionStatusSchema.optional(), + config: z.record(z.string(), z.unknown()).optional(), +}); + +/** + * Inbound webhook payload. Deliberately permissive on extra keys — unmapped fields land in + * the staging record's raw payload and can be mapped to custom fields later. + */ +export const webhookPersonSchema = z + .object({ + externalId: z.string().trim().max(255).optional(), + name: z.string().trim().max(255).optional(), + email: z.string().trim().max(255).optional(), + phone: z.string().trim().max(50).optional(), + jobTitle: z.string().trim().max(255).optional(), + linkedinUrl: z.string().trim().max(500).optional(), + orgName: z.string().trim().max(255).optional(), + orgDomain: z.string().trim().max(255).optional(), + status: personStatusSchema.optional(), + }) + .passthrough(); + +export const webhookIngestSchema = z.object({ + entityType: importEntityTypeSchema.default("person"), + records: z.array(webhookPersonSchema).min(1).max(IMPORT_MAX_RECORDS_PER_JOB), +}); + +export type ImportFieldMapping = z.infer; +export type ImportMapping = z.infer; +export type ImportJobOptions = z.infer; +export type StartImportJobInput = z.infer; +export type UpdateImportJobInput = z.infer; +export type ListImportJobsQuery = z.infer; +export type ListImportRecordsQuery = z.infer; +export type CreateConnectionInput = z.infer; +export type UpdateConnectionInput = z.infer; +export type WebhookIngestInput = z.infer; diff --git a/packages/validators/src/types/crm.types.ts b/packages/validators/src/types/crm.types.ts index a405847..a69f81d 100644 --- a/packages/validators/src/types/crm.types.ts +++ b/packages/validators/src/types/crm.types.ts @@ -6,7 +6,7 @@ export const PERSON_STATUS_VALUES = [ "churned", ] as const; -export const PERSON_SOURCE_VALUES = ["manual", "csv", "api"] as const; +export const PERSON_SOURCE_VALUES = ["manual", "csv", "api", "import"] as const; export const DEAL_STAGE_VALUES = ["new", "contacted", "demo", "proposal", "won", "lost"] as const; diff --git a/packages/validators/src/types/import.types.ts b/packages/validators/src/types/import.types.ts new file mode 100644 index 0000000..a73d07e --- /dev/null +++ b/packages/validators/src/types/import.types.ts @@ -0,0 +1,135 @@ +export const IMPORT_PROVIDER_VALUES = [ + "csv", + "webhook", + "gmail", + "google_calendar", + "calendly", + "google_sheets", + "posthog", + "outlook", +] as const; + +export type ImportProvider = (typeof IMPORT_PROVIDER_VALUES)[number]; + +export const IMPORT_ENTITY_TYPE_VALUES = ["person", "org"] as const; + +export type ImportEntityType = (typeof IMPORT_ENTITY_TYPE_VALUES)[number]; + +export const IMPORT_JOB_STATUS_VALUES = [ + "pending", + "extracting", + "ready_for_review", + "loading", + "completed", + "failed", + "canceled", +] as const; + +export type ImportJobStatus = (typeof IMPORT_JOB_STATUS_VALUES)[number]; + +export const IMPORT_RECORD_STATUS_VALUES = [ + "pending", + "valid", + "invalid", + "loaded", + "skipped", + "duplicate", +] as const; + +export type ImportRecordStatus = (typeof IMPORT_RECORD_STATUS_VALUES)[number]; + +export const IMPORT_CONNECTION_STATUS_VALUES = [ + "connected", + "reconnect_required", + "disconnected", +] as const; + +export type ImportConnectionStatus = (typeof IMPORT_CONNECTION_STATUS_VALUES)[number]; + +/** + * Applied when an import matches an existing CRM record. + * `fill_empty` only writes fields the CRM has left blank, which is the safe default + * for low-trust sources. + */ +export const IMPORT_CONFLICT_POLICY_VALUES = ["source_wins", "crm_wins", "fill_empty"] as const; + +export type ImportConflictPolicy = (typeof IMPORT_CONFLICT_POLICY_VALUES)[number]; + +export const IMPORT_MATCH_REASON_VALUES = [ + "external_identity", + "email", + "domain", + "name", + "none", +] as const; + +export type ImportMatchReason = (typeof IMPORT_MATCH_REASON_VALUES)[number]; + +/** + * CRM person fields an import is allowed to write. `customFields` is handled separately + * because it targets `crm_custom_field_definitions` rather than a column. + */ +export const IMPORT_PERSON_TARGET_FIELD_VALUES = [ + "name", + "email", + "phone", + "jobTitle", + "linkedinUrl", + "status", + "orgName", + "orgDomain", + "lastContactedAt", +] as const; + +export type ImportPersonTargetField = (typeof IMPORT_PERSON_TARGET_FIELD_VALUES)[number]; + +export const IMPORT_ORG_TARGET_FIELD_VALUES = [ + "name", + "domain", + "industry", + "size", + "location", +] as const; + +export type ImportOrgTargetField = (typeof IMPORT_ORG_TARGET_FIELD_VALUES)[number]; + +/** + * Providers that authenticate with a long-lived token pasted by the user rather than + * an OAuth redirect. These never carry a refresh token. + */ +export const IMPORT_API_KEY_PROVIDERS: readonly ImportProvider[] = ["posthog", "calendly"] as const; + +export const IMPORT_OAUTH_PROVIDERS: readonly ImportProvider[] = [ + "gmail", + "google_calendar", + "google_sheets", + "outlook", +] as const; + +/** + * Providers with no connection at all — records arrive by upload or push. + */ +export const IMPORT_PUSH_PROVIDERS: readonly ImportProvider[] = ["csv", "webhook"] as const; + +/** + * Seeds `people.status` on create. A source that proves payment or conversation should not + * land next to a newsletter subscriber. Never applied on update — status is CRM-owned + * once the record exists. + */ +export const IMPORT_PROVIDER_DEFAULT_STATUS: Record< + ImportProvider, + "lead" | "prospect" | "qualified" | "customer" | "churned" +> = { + gmail: "qualified", + google_calendar: "qualified", + calendly: "qualified", + outlook: "qualified", + posthog: "lead", + csv: "lead", + google_sheets: "lead", + webhook: "lead", +}; + +export const IMPORT_MAX_RECORDS_PER_JOB = 50_000; +export const IMPORT_LOAD_BATCH_SIZE = 200; +export const IMPORT_EXTRACT_PAGE_SIZE = 100; diff --git a/turbo.json b/turbo.json index 64ea5d7..9ae6a07 100644 --- a/turbo.json +++ b/turbo.json @@ -1,6 +1,15 @@ { "$schema": "https://turbo.build/schema.json", "ui": "tui", + "globalEnv": [ + "INTEGRATION_LIVE_FETCH_ENABLED", + "INTEGRATION_TOKEN_SECRET", + "MICROSOFT_CLIENT_ID", + "MICROSOFT_CLIENT_SECRET", + "MICROSOFT_TENANT_ID", + "POSTHOG_API_HOST", + "CALENDLY_API_HOST" + ], "tasks": { "build": { "dependsOn": ["^build"],