Retry transient TLS network failures

src/app/api/core/utils/withRetry.test.ts

@@ -0,0 +1,84 @@
+import { isTransientNetworkError, withRetry } from './withRetry'
+
+jest.mock('@sentry/nextjs', () => ({
+  withScope: jest.fn((callback: (scope: { addEventProcessor: jest.Mock }) => void) => {
+    callback({ addEventProcessor: jest.fn() })
+  }),
+}))
+
+jest.mock('p-retry', () => ({
+  __esModule: true,
+  default: jest.fn(
+    async (
+      fn: () => Promise<unknown>,
+      options: {
+        retries?: number
+        onFailedAttempt?: (error: Error & { attemptNumber: number; retriesLeft: number }) => void
+        shouldRetry?: (error: unknown) => boolean
+      },
+    ) => {
+      const maxAttempts = (options.retries ?? 0) + 1
+
+      const runAttempt = async (attemptNumber: number): Promise<unknown> => {
+        try {
+          return await fn()
+        } catch (error) {
+          const retriesLeft = maxAttempts - attemptNumber
+          const failedAttemptError = Object.assign(error instanceof Error ? error : new Error(String(error)), {
+            attemptNumber,
+            retriesLeft,
+          })
+
+          options.onFailedAttempt?.(failedAttemptError)
+
+          if (retriesLeft <= 0 || !options.shouldRetry?.(error)) throw error
+          return runAttempt(attemptNumber + 1)
+        }
+      }
+
+      return runAttempt(1)
+    },
+  ),
+}))
+
+const createTlsDisconnectError = () =>
+  Object.assign(new TypeError('fetch failed'), {
+    cause: Object.assign(new Error('Client network socket disconnected before secure TLS connection was established'), {
+      code: 'ECONNRESET',
+    }),
+  })
+
+describe('isTransientNetworkError', () => {
+  it('detects TLS disconnect errors nested under fetch failures', () => {
+    expect(isTransientNetworkError(createTlsDisconnectError())).toBe(true)
+  })
+
+  it('does not classify unrelated errors as transient network errors', () => {
+    expect(isTransientNetworkError(new Error('validation failed'))).toBe(false)
+  })
+})
+
+describe('withRetry', () => {
+  beforeEach(() => {
+    jest.spyOn(console, 'warn').mockImplementation(() => {})
+  })
+
+  afterEach(() => {
+    jest.restoreAllMocks()
+  })
+
+  it('retries transient network errors that do not have an HTTP status', async () => {
+    const fn = jest.fn().mockRejectedValueOnce(createTlsDisconnectError()).mockResolvedValueOnce('ok')
+
+    await expect(withRetry(fn, [])).resolves.toBe('ok')
+    expect(fn).toHaveBeenCalledTimes(2)
+  })
+
+  it('does not retry errors without retryable status or transient network causes', async () => {
+    const error = new Error('validation failed')
+    const fn = jest.fn().mockRejectedValue(error)
+
+    await expect(withRetry(fn, [])).rejects.toBe(error)
+    expect(fn).toHaveBeenCalledTimes(1)
+  })
+})

src/app/api/core/utils/withRetry.ts

@@ -4,6 +4,48 @@ import * as Sentry from '@sentry/nextjs'
 
 export const RETRY_404_ENABLED = process.env.RETRY_404 === 'true'
 
+const TRANSIENT_NETWORK_ERROR_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'EAI_AGAIN'])
+const TLS_DISCONNECT_MESSAGE = 'Client network socket disconnected before secure TLS connection was established'
+
+const getErrorMessage = (error: unknown): string => {
+  if (error instanceof Error) return error.message
+  if (typeof error === 'string') return error
+  return ''
+}
+
+const getErrorCode = (error: unknown): string | undefined => {
+  if (!error || typeof error !== 'object') return undefined
+
+  const code = (error as { code?: unknown }).code
+  return typeof code === 'string' ? code : undefined
+}
+
+const getErrorCause = (error: unknown): unknown => {
+  if (!error || typeof error !== 'object') return undefined
+
+  return (error as { cause?: unknown }).cause
+}
+
+const isRetryableStatusError = (error: unknown): boolean => {
+  const status = (error as Partial<StatusableError>)?.status
+  if (typeof status !== 'number') return false
+
+  return [408, 429].includes(status) || (status >= 500 && status <= 511) || (RETRY_404_ENABLED && status === 404)
+}
+
+export const isTransientNetworkError = (error: unknown): boolean => {
+  const code = getErrorCode(error)
+  if (code && TRANSIENT_NETWORK_ERROR_CODES.has(code)) return true
+
+  const message = getErrorMessage(error)
+  if (message.includes(TLS_DISCONNECT_MESSAGE)) return true
+
+  const cause = getErrorCause(error)
+  if (!cause) return false
+
+  return isTransientNetworkError(cause)
+}
+
 export const withRetry = async <T>(fn: (...args: any[]) => Promise<T>, args: any[]): Promise<T> => {
   let isEventProcessorRegistered = false
 
@@ -39,15 +81,8 @@ export const withRetry = async <T>(fn: (...args: any[]) => Promise<T>, args: any
           `CopilotAPI#withRetry | Attempt ${error.attemptNumber} failed. There are ${error.retriesLeft} retries left`,
         )
       },
-      shouldRetry: (error: any) => {
-        // Typecasting because Copilot doesn't export an error class
-        const err = error as StatusableError
-        // Retry if statusCode is 429 (ratelimit), 408 (timeouts), or any server related (5xx) error
-        return (
-          [408, 429].includes(err.status) ||
-          (err.status >= 500 && err.status <= 511) ||
-          (RETRY_404_ENABLED && err.status === 404)
-        )
+      shouldRetry: (error: unknown) => {
+        return isRetryableStatusError(error) || isTransientNetworkError(error)
       },
     },
   )

src/utils/CopilotAPI.ts

@@ -370,11 +370,15 @@ export class CopilotAPI {
     console.info('CopilotAPI#dispatchWebhook | Request headers:', headers)
 
     try {
-      await fetch(url, {
-        method: 'POST',
-        headers,
-        body: payload ? JSON.stringify(payload) : null,
-      })
+      await withRetry(
+        () =>
+          fetch(url, {
+            method: 'POST',
+            headers,
+            body: payload ? JSON.stringify(payload) : null,
+          }),
+        [],
+      )
     } catch (e) {
       console.error(`Failed to dispatch webhook for event ${eventName}`, e)
     }