OUT-3528: notify IUs when QBO sync fails with manual-fix errors

src/app/api/core/types/notification.ts

@@ -1,3 +1,32 @@
 export enum NotificationActions {
   AUTH_RECONNECT = 'auth_reconnect',
+  QB_DUPLICATE_NAME = 'qb_duplicate_name',
+  QB_CLOSED_PERIOD = 'qb_closed_period',
+  QB_DEPOSITED_TXN_LOCKED = 'qb_deposited_txn_locked',
+  QB_INACTIVE_REFERENCE = 'qb_inactive_reference',
+  QB_SUBSCRIPTION_INVALID = 'qb_subscription_invalid',
+  QB_VALIDATION_FAILED = 'qb_validation_failed',
+  QB_STALE_OBJECT = 'qb_stale_object',
+  QB_TXN_LINK_FAILED = 'qb_txn_link_failed',
+  QB_ITEM_INCOME_ACCOUNT_MISSING = 'qb_item_income_account_missing',
+  QB_INVALID_ACCOUNT_TYPE = 'qb_invalid_account_type',
+}
+
+/**
+ * Optional context passed alongside a NotificationActions value when dispatching
+ * a sync-failure notification. The notification helper uses these to interpolate
+ * a tailored title/body (e.g. naming the offending invoice number or QB item).
+ *
+ * All fields are optional so callers (like AUTH_RECONNECT) can still dispatch
+ * without context.
+ */
+export interface NotificationContext {
+  entityType?: string
+  eventType?: string
+  entityKey?: string
+  invoiceNumber?: string
+  customerName?: string
+  productName?: string
+  qbItemName?: string
+  errorMessage?: string
 }

src/app/api/notification/notification.service.ts

@@ -1,12 +1,25 @@
 import { BaseService } from '@/app/api/core/services/base.service'
-import { NotificationActions } from '@/app/api/core/types/notification'
 import {
+  NotificationActions,
+  NotificationContext,
+} from '@/app/api/core/types/notification'
+import {
+  NotificationCopy,
   getIEmailNotificationDetail,
   getInProductNotificationDetail,
 } from '@/app/api/notification/notification.helper'
 import { InternalUsersResponse } from '@/type/common'
 import { CopilotAPI } from '@/utils/copilotAPI'
 import CustomLogger from '@/utils/logger'
+import { captureException, captureMessage } from '@sentry/nextjs'
+
+// Notification actions whose recipients are the workspace's IUs. Derived
+// directly from the keys of NotificationCopy so a new action can't accidentally
+// be added to the helper without an IU recipient set — getAllParties returns
+// `null` for any action missing here, which short-circuits dispatch.
+const IU_RECIPIENT_ACTIONS = new Set<NotificationActions>(
+  Object.keys(NotificationCopy) as NotificationActions[],
+)
 
 export class NotificationService extends BaseService {
   async createBulkNotification(
@@ -15,10 +28,12 @@ export class NotificationService extends BaseService {
       disableEmail = false,
       disableInProduct = false,
       senderId,
+      context,
     }: {
       disableEmail?: boolean
       disableInProduct?: boolean
       senderId: string
+      context?: NotificationContext
     },
   ): Promise<void> {
     console.info(
@@ -33,11 +48,11 @@ export class NotificationService extends BaseService {
       if (parties) {
         const inProduct = disableInProduct
           ? undefined
-          : getInProductNotificationDetail()[action]
+          : getInProductNotificationDetail(action, context)
 
         const email = disableEmail
           ? undefined
-          : getIEmailNotificationDetail()[action]
+          : getIEmailNotificationDetail(action, context)
 
         for (const party of parties.data) {
           CustomLogger.info({
@@ -56,32 +71,51 @@ export class NotificationService extends BaseService {
             console.error(
               `Failed to trigger notification for IUID: ${party.id}`,
             )
+            captureMessage(
+              `NotificationService#createBulkNotification | Copilot returned no notification`,
+              {
+                level: 'error',
+                tags: {
+                  key: 'notificationDispatchFailed',
+                  action,
+                  portalId: this.user.workspaceId,
+                },
+                extra: { recipientId: party.id, senderId },
+              },
+            )
           }
         }
       }
     } catch (error) {
       console.error(`Failed to send notification for action: ${action}`, {
         error,
       })
+      captureException(error, {
+        tags: {
+          key: 'notificationDispatchFailed',
+          action,
+          portalId: this.user.workspaceId,
+        },
+        extra: { senderId },
+      })
     }
   }
 
   async getAllParties(
     copilot: CopilotAPI,
     action: NotificationActions,
   ): Promise<InternalUsersResponse | null> {
-    switch (action) {
-      case NotificationActions.AUTH_RECONNECT:
-        return await copilot.getInternalUsers()
-      default:
-        return null
+    if (IU_RECIPIENT_ACTIONS.has(action)) {
+      return await copilot.getInternalUsers()
     }
+    return null
   }
 
   async sendNotificationToIU(
     senderId: string,
     action: NotificationActions,
+    context?: NotificationContext,
   ): Promise<void> {
-    await this.createBulkNotification(action, { senderId })
+    await this.createBulkNotification(action, { senderId, context })
   }
 }

src/app/api/quickbooks/payment/payment.service.ts

@@ -139,6 +139,7 @@ export class PaymentService extends BaseService {
           customerName: recipientInfo.displayName,
           customerEmail: recipientInfo.email,
           errorMessage,
+          errorCode: errorWithCode.code?.toString(),
           category: getCategory(errorWithCode),
           deletedAt: getDeletedAtForAuthAccountCategoryLog(errorWithCode),
         },
@@ -272,6 +273,7 @@ export class PaymentService extends BaseService {
       remark?: string
       qbItemName?: string
       errorMessage?: string
+      errorCode?: string
       category?: FailedRecordCategoryType
       deletedAt?: Date
     },

src/app/api/quickbooks/sync/sync.service.ts

@@ -649,6 +649,7 @@ export class SyncService extends BaseService {
       {
         status: LogStatus.FAILED,
         errorMessage,
+        errorCode: error?.code?.toString(),
         deletedAt: getDeletedAtForAuthAccountCategoryLog(error),
         category: getCategory(error),
       },

src/app/api/quickbooks/syncLog/syncErrorNotifier.ts

@@ -0,0 +1,87 @@
+import { BaseService } from '@/app/api/core/services/base.service'
+import { EntityType, LogStatus } from '@/app/api/core/types/log'
+import {
+  NotificationActions,
+  NotificationContext,
+} from '@/app/api/core/types/notification'
+import { NotificationService } from '@/app/api/notification/notification.service'
+import { UserActionableErrorCodes } from '@/constant/intuitErrorCode'
+import { QBSyncLogSelectSchemaType } from '@/db/schema/qbSyncLogs'
+import { getPortalConnection } from '@/db/service/token.service'
+
+/**
+ * Looks up the user-actionable notification action for a given QBO error code.
+ * Returns null when the code is empty, unknown, or refers to a transient/auth
+ * error handled elsewhere (429, 5xx, invalid_grant, etc.).
+ */
+export function getActionForErrorCode(
+  errorCode: string | null | undefined,
+): NotificationActions | null {
+  if (!errorCode) return null
+  return UserActionableErrorCodes[errorCode] ?? null
+}
+
+/**
+ * Picks the strongest available identifier for the offending QBO entity so the
+ * notification body can reference a concrete record (invoice number, item, etc).
+ */
+export function getEntityKey(log: QBSyncLogSelectSchemaType): string {
+  return (
+    log.quickbooksId ||
+    log.invoiceNumber ||
+    log.qbItemName ||
+    log.copilotPriceId ||
+    log.copilotId ||
+    ''
+  )
+}
+
+export class SyncErrorNotifier extends BaseService {
+  /**
+   * Dispatches an IU notification for a freshly written FAILED sync log row
+   * when its errorCode is in the user-actionable registry. One sync_log insert
+   * = one notification — natural dedup comes from sync_log being created once
+   * per failed entity.
+   *
+   * Errors here are caller-suppressed; a notification failure must not undo
+   * the sync log write.
+   */
+  async notify(log: QBSyncLogSelectSchemaType): Promise<void> {
+    if (log.status !== LogStatus.FAILED) return
+
+    const action = getActionForErrorCode(log.errorCode)
+    if (!action) return
+
+    // PRODUCT 5010 auto-recovers via updateProductSyncToken on the next
+    // cron tick. INVOICE/PAYMENT have no equivalent refresh, so their
+    // 5010s stay user-actionable.
+    if (
+      action === NotificationActions.QB_STALE_OBJECT &&
+      log.entityType === EntityType.PRODUCT
+    ) {
+      return
+    }
+
+    const context: NotificationContext = {
+      entityType: log.entityType,
+      eventType: log.eventType,
+      entityKey: getEntityKey(log),
+      invoiceNumber: log.invoiceNumber ?? undefined,
+      customerName: log.customerName ?? undefined,
+      productName: log.productName ?? undefined,
+      qbItemName: log.qbItemName ?? undefined,
+      errorMessage: log.errorMessage ?? undefined,
+    }
+    const portal = await getPortalConnection(this.user.workspaceId)
+
+    const notificationService = new NotificationService(this.user)
+    // Webhook-driven failures have no calling IU. Empty senderId mirrors the
+    // existing AUTH_RECONNECT pattern (auth.service.ts) where `error.intiatedBy
+    // ?? ''` is passed to sendNotificationToIU.
+    await notificationService.sendNotificationToIU(
+      portal?.intiatedBy || '',
+      action,
+      context,
+    )
+  }
+}

src/app/api/quickbooks/syncLog/syncLog.service.ts

@@ -5,6 +5,8 @@ import {
   FailedRecordCategoryType,
   LogStatus,
 } from '@/app/api/core/types/log'
+import { afterIfAvailable } from '@/app/api/core/utils/afterIfAvailable'
+import { SyncErrorNotifier } from '@/app/api/quickbooks/syncLog/syncErrorNotifier'
 import { ConnectionStatus } from '@/db/schema/qbConnectionLogs'
 import {
   QBSyncLog,
@@ -17,8 +19,10 @@ import {
 } from '@/db/schema/qbSyncLogs'
 import { WhereClause } from '@/type/common'
 import { orderMap } from '@/utils/drizzle'
+import CustomLogger from '@/utils/logger'
 import dayjs from 'dayjs'
 import { and, eq, isNull, lt, sql } from 'drizzle-orm'
+import { captureException } from '@sentry/nextjs'
 import { json2csv } from 'json-2-csv'
 
 export const STALE_PENDING_THRESHOLD_MINUTES = 15
@@ -53,24 +57,99 @@ export class SyncLogService extends BaseService {
       .returning()
 
     console.info('SyncLogService#createQBSyncLog | Sync log complete')
+    this.scheduleFailureNotification(log)
     return log
   }
 
   /**
-   * Creates the sync log
+   * Schedules a sync-failure notification when a sync log row enters the
+   * FAILED state. Errors are swallowed and reported to Sentry — a failed
+   * notification must never propagate up and undo the sync log write.
+   *
+   * Hooked at two sites:
+   *  - `createQBSyncLog`, for the rare path that writes a row directly as
+   *    FAILED (legacy / non-claim flows).
+   *  - `updateQBSyncLog`, for the dominant flow today: webhook entry inserts
+   *    a PENDING claim, then a handler updates it to SUCCESS or FAILED.
+   *
+   * The update site passes `priorStatus` so we only fire on a *transition*
+   * into FAILED. FAILED→FAILED retry replays don't re-page IUs.
+   *
+   * `flipStalePendingToFailed` writes FAILED rows without an `errorCode`;
+   * those naturally no-op inside the notifier (`getActionForErrorCode(null)`
+   * returns null) so we don't need to filter them here.
+   */
+  private scheduleFailureNotification(log: QBSyncLogSelectSchemaType): void {
+    if (log.status !== LogStatus.FAILED) return
+
+    // Defer dispatch until after the request scope (and any enclosing
+    // transaction) commits, so the notification can never undo the sync log
+    // write and never reads an uncommitted row. Errors are caller-suppressed
+    // — surface to Sentry and continue.
+    const user = this.user
+    afterIfAvailable(async () => {
+      try {
+        const notifier = new SyncErrorNotifier(user)
+        await notifier.notify(log)
+      } catch (error) {
+        CustomLogger.error({
+          message:
+            'SyncLogService#scheduleFailureNotification | Notifier failed',
+          obj: { error, logId: log.id },
+        })
+        captureException(error, {
+          tags: {
+            key: 'syncFailureNotifierError',
+            portalId: log.portalId,
+            entityType: log.entityType,
+            errorCode: log.errorCode ?? 'unknown',
+          },
+          extra: { logId: log.id, errorMessage: log.errorMessage },
+        })
+      }
+    })
+  }
+
+  /**
+   * Updates an existing sync log row. When the update transitions the row
+   * INTO the FAILED state (i.e. its prior status was not FAILED), schedules
+   * the IU sync-failure notification. Callers that already know the prior
+   * status (e.g. `updateOrCreateQBSyncLog`) can pass it to skip the lookup;
+   * otherwise we fetch it before applying the update so retry replays
+   * (FAILED→FAILED) don't re-notify.
    */
   async updateQBSyncLog(
     payload: QBSyncLogUpdateSchemaType,
     conditions: WhereClause,
+    priorStatus?: LogStatus,
   ): Promise<QBSyncLogSelectSchemaType> {
     const parsedPayload = QBSyncLogUpdateSchema.parse(payload)
+    // Only consider notifying when this update is *actively* setting status to
+    // FAILED. Partial updates that don't touch status (e.g. attempt counter
+    // bumps in `checkAndUpdateAttempt`) leave a FAILED row FAILED — they are
+    // not transitions and must not re-page IUs.
+    const settingFailed = parsedPayload.status === LogStatus.FAILED
+
+    if (priorStatus === undefined && settingFailed) {
+      const existing = await this.db.query.QBSyncLog.findFirst({
+        where: conditions,
+        columns: { status: true },
+      })
+      priorStatus = existing?.status as LogStatus | undefined
+    }
+
     const [log] = await this.db
       .update(QBSyncLog)
       .set(parsedPayload)
       .where(conditions)
       .returning()
 
     console.info('SyncLogService#updateQBSyncLog | Sync log updated')
+
+    if (settingFailed && log && priorStatus !== LogStatus.FAILED) {
+      this.scheduleFailureNotification(log)
+    }
+
     return log
   }
 
@@ -134,7 +213,11 @@ export class SyncLogService extends BaseService {
     }
 
     if (existingLog) {
-      await this.updateQBSyncLog(payload, eq(QBSyncLog.id, existingLog.id))
+      await this.updateQBSyncLog(
+        payload,
+        eq(QBSyncLog.id, existingLog.id),
+        existingLog.status as LogStatus,
+      )
     } else {
       await this.createQBSyncLog(payload)
     }
@@ -317,6 +400,7 @@ export class SyncLogService extends BaseService {
           ? parseFloat(log.productPrice) / 100
           : null,
         qb_item_name: log.qbItemName,
+        error_code: log.errorCode,
         error_message: log.errorMessage,
       }
     })