OUT-2193 | Implement retry for Copilot API requests in Client Home

package.json

@@ -49,11 +49,12 @@
     "@types/react-color": "^3.0.10",
     "@vercel/blob": "^0.15.1",
     "@vercel/postgres": "^0.5.1",
-    "copilot-node-sdk": "^3.12.1",
     "copilot-design-system": "^2.1.6",
+    "copilot-node-sdk": "^3.12.1",
     "handlebars": "^4.7.8",
     "http-status": "^1.7.4",
     "next": "latest",
+    "p-retry": "^6.2.1",
     "prisma": "^5.6.0",
     "re-resizable": "^6.9.11",
     "react": "latest",

src/types/error.ts

@@ -0,0 +1,3 @@
+export interface StatusableError extends Error {
+  status: number
+}

src/utils/copilotApiUtils.ts

@@ -18,6 +18,7 @@ import {
 } from '@/types/common'
 import { TASKS_APP_URL } from '@/utils/constants'
 import { encodePayload } from '@/utils/crypto'
+import { withRetry } from '@/utils/withRetry'
 import type { CopilotAPI as SDK } from 'copilot-node-sdk'
 import { copilotApi } from 'copilot-node-sdk'
 import { z } from 'zod'
@@ -29,7 +30,7 @@ export class CopilotAPI {
     this.copilot = copilotApi({ apiKey: copilotAPIKey, token })
   }
 
-  async me(): Promise<MeResponse | null> {
+  async _me(): Promise<MeResponse | null> {
     const tokenPayload = await this.getTokenPayload()
     const id = tokenPayload?.internalUserId || tokenPayload?.clientId
     if (!tokenPayload || !id) return null
@@ -43,45 +44,45 @@ export class CopilotAPI {
   }
 
   // Get parsed payload from token
-  async getTokenPayload(): Promise<Token | undefined> {
+  async _getTokenPayload(): Promise<Token | undefined> {
     return TokenSchema.parse(await this.copilot.getTokenPayload?.())
   }
 
-  async getClient(clientId: string): Promise<ClientResponse> {
+  async _getClient(clientId: string): Promise<ClientResponse> {
     return ClientResponseSchema.parse(
       await this.copilot.retrieveClient({ id: clientId }),
     )
   }
 
-  async getClients() {
+  async _getClients() {
     return ClientsResponseSchema.parse(
       await this.copilot.listClients({ limit: 5000 }),
     )
   }
 
-  async getCompany(companyId: string): Promise<CompanyResponse> {
+  async _getCompany(companyId: string): Promise<CompanyResponse> {
     return CompanyResponseSchema.parse(
       await this.copilot.retrieveCompany({ id: companyId }),
     )
   }
 
-  async getCompanies(): Promise<CompanyResponse[]> {
+  async _getCompanies(): Promise<CompanyResponse[]> {
     return z
       .array(CompanyResponseSchema)
       .parse((await this.copilot.listCompanies({ limit: 100_000 })).data)
   }
 
-  async getWorkspaceInfo(): Promise<WorkspaceInfo> {
+  async _getWorkspaceInfo(): Promise<WorkspaceInfo> {
     return WorkspaceInfoSchema.parse(await this.copilot.retrieveWorkspace())
   }
 
-  async getCustomFields(): Promise<CustomFieldResponse> {
+  async _getCustomFields(): Promise<CustomFieldResponse> {
     return CustomFieldResponseSchema.parse(
       await this.copilot.listCustomFields(),
     )
   }
 
-  async getNotifications(recipientId: string): Promise<Notifications> {
+  async _getNotifications(recipientId: string): Promise<Notifications> {
     const notifications = await this.copilot.listNotifications({
       recipientId,
     })
@@ -129,12 +130,29 @@ export class CopilotAPI {
     return todo + inProgress
   }
 
-  async getAppId(appDeploymentId: string): Promise<string | null> {
+  async _getAppId(appDeploymentId: string): Promise<string | null> {
     const installedApps = AppInstallsResponseSchema.parse(
       await this.copilot.listAppInstalls(),
     )
     return (
       installedApps.find((app) => app.appId === appDeploymentId)?.id || null
     )
   }
+
+  private wrapWithRetry<Args extends unknown[], R>(
+    fn: (...args: Args) => Promise<R>,
+  ): (...args: Args) => Promise<R> {
+    return (...args: Args): Promise<R> => withRetry(fn.bind(this), args)
+  }
+
+  me = this.wrapWithRetry(this._me)
+  getTokenPayload = this.wrapWithRetry(this._getTokenPayload)
+  getClient = this.wrapWithRetry(this._getClient)
+  getClients = this.wrapWithRetry(this._getClients)
+  getCompany = this.wrapWithRetry(this._getCompany)
+  getCompanies = this.wrapWithRetry(this._getCompanies)
+  getWorkspaceInfo = this.wrapWithRetry(this._getWorkspaceInfo)
+  getCustomFields = this.wrapWithRetry(this._getCustomFields)
+  getNotifications = this.wrapWithRetry(this._getNotifications)
+  getAppId = this.wrapWithRetry(this._getAppId)
 }

src/utils/withRetry.ts

@@ -0,0 +1,56 @@
+import pRetry, { FailedAttemptError } from 'p-retry'
+import * as Sentry from '@sentry/nextjs'
+import { StatusableError } from '@/types/error'
+
+export const withRetry = async <A extends unknown[], T>(
+  fn: (...args: A) => Promise<T>,
+  args: A,
+): Promise<T> => {
+  let isEventProcessorRegistered = false
+
+  return await pRetry(
+    async () => {
+      try {
+        return await fn(...args)
+      } catch (error) {
+        // Hopefully now sentry doesn't report retry errors as well. We have enough triage issues as it is
+        Sentry.withScope((scope) => {
+          if (isEventProcessorRegistered) return
+
+          isEventProcessorRegistered = true
+          scope.addEventProcessor((event) => {
+            if (
+              event.level === 'error' &&
+              event.message &&
+              event.message.includes('An error occurred during retry')
+            ) {
+              return null // Discard the event as it occured during retry
+            }
+            return event
+          })
+        })
+        // Rethrow the error so pRetry can rety
+        throw error
+      }
+    },
+
+    {
+      retries: 3,
+      minTimeout: 500,
+      maxTimeout: 2000,
+      factor: 2, // Exponential factor for timeout delay. Tweak this if issues still persist
+      onFailedAttempt: (error: FailedAttemptError) => {
+        console.warn(
+          `CopilotAPI#withRetry | Attempt ${error.attemptNumber} failed. There are ${error.retriesLeft} retries left. Error:`,
+          error,
+        )
+      },
+      shouldRetry: (error: any) => {
+        // Typecasting because Copilot doesn't export an error class
+        const err = error as StatusableError
+        // Retry only if statusCode === 429
+        return err.status === 429
+      },
+    },
+  )
+}

yarn.lock

@@ -3251,6 +3251,11 @@
   dependencies:
     "@types/react" "*"
 
+"@types/retry@0.12.2":
+  version "0.12.2"
+  resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.2.tgz#ed279a64fa438bb69f2480eda44937912bb7480a"
+  integrity sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==
+
 "@types/scheduler@*":
   version "0.16.8"
   resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.8.tgz#ce5ace04cfeabe7ef87c0091e50752e36707deff"
@@ -5406,6 +5411,11 @@ is-negative-zero@^2.0.2:
   resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150"
   integrity sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==
 
+is-network-error@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/is-network-error/-/is-network-error-1.1.0.tgz#d26a760e3770226d11c169052f266a4803d9c997"
+  integrity sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==
+
 is-number-object@^1.0.4:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.7.tgz#59d50ada4c45251784e9904f5246c742f07a42fc"
@@ -6302,6 +6312,15 @@ p-locate@^5.0.0:
   dependencies:
     p-limit "^3.0.2"
 
+p-retry@^6.2.1:
+  version "6.2.1"
+  resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-6.2.1.tgz#81828f8dc61c6ef5a800585491572cc9892703af"
+  integrity sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==
+  dependencies:
+    "@types/retry" "0.12.2"
+    is-network-error "^1.0.0"
+    retry "^0.13.1"
+
 parent-module@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2"
@@ -6992,6 +7011,11 @@ restore-cursor@^4.0.0:
     onetime "^5.1.0"
     signal-exit "^3.0.2"
 
+retry@^0.13.1:
+  version "0.13.1"
+  resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658"
+  integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==
+
 reusify@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"