OUT-3542: add Sentry context tags, breadcrumbs, and error categorization

src/app/api/core/types/log.ts

@@ -30,5 +30,9 @@ export enum EventType {
 export enum FailedRecordCategoryType {
   AUTH = 'auth',
   ACCOUNT = 'account',
+  RATE_LIMIT = 'rate_limit',
+  VALIDATION = 'validation',
+  QB_API_ERROR = 'qb_api_error',
+  MAPPING_NOT_FOUND = 'mapping_not_found',
   OTHERS = 'others',
 }

src/app/api/core/utils/withErrorHandler.ts

@@ -4,12 +4,14 @@ import {
   StatusableError,
 } from '@/type/CopilotApiError'
 import APIError from '@/app/api/core/exceptions/api'
+import { FailedRecordCategoryType } from '@/app/api/core/types/log'
 import httpStatus from 'http-status'
 import { NextRequest, NextResponse } from 'next/server'
 import { ZodError, ZodFormattedError } from 'zod'
 import { isAxiosError } from '@/app/api/core/exceptions/custom'
 import * as Sentry from '@sentry/nextjs'
-import { RetryableError } from '@/utils/error'
+import { getMessageAndCodeFromError, RetryableError } from '@/utils/error'
+import { getCategory } from '@/utils/synclog'
 
 type RequestHandler = (req: NextRequest, params: any) => Promise<NextResponse>
 
@@ -74,13 +76,27 @@ export const withErrorHandler = (handler: RequestHandler): RequestHandler => {
         status = error.response.status
       }
 
-      // if error is from Copilot or Intuit API (API error), then send the error message to Sentry
-      if (
+      // Send categorized errors to Sentry with context tags
+      if (error instanceof ZodError) {
+        Sentry.withScope((scope) => {
+          scope.setTag('errorCategory', FailedRecordCategoryType.VALIDATION)
+          Sentry.captureException(error)
+        })
+      } else if (
         error instanceof APIError ||
         error instanceof CopilotApiError ||
         isAxiosError(error)
       ) {
-        Sentry.captureException(error)
+        const errorWithCode = getMessageAndCodeFromError(error)
+        const category = getCategory(errorWithCode)
+
+        Sentry.withScope((scope) => {
+          scope.setTag('errorCategory', category)
+          if (errorWithCode.source && errorWithCode.source !== 'unknown') {
+            scope.setTag('errorSource', errorWithCode.source)
+          }
+          Sentry.captureException(error)
+        })
       }
 
       return NextResponse.json({ error: message, errors }, { status })

src/app/api/quickbooks/cron/cron.service.ts

@@ -10,6 +10,7 @@ import { getAllActivePortalConnections } from '@/db/service/token.service'
 import { CopilotAPI } from '@/utils/copilotAPI'
 import { encodePayload } from '@/utils/crypto'
 import CustomLogger from '@/utils/logger'
+import * as Sentry from '@sentry/nextjs'
 import dayjs from 'dayjs'
 import { and, eq, lt } from 'drizzle-orm'
 
@@ -18,29 +19,34 @@ export default class CronService {
     workspaceId: string,
     qbConnectionTokens: QBConnectionProperties,
   ) {
-    const payload = {
-      workspaceId,
-    }
-    const token = encodePayload(copilotAPIKey, payload)
+    return await Sentry.withScope(async (scope) => {
+      scope.setTag('portalId', workspaceId)
+      scope.setTag('workspaceId', workspaceId)
 
-    // check if token is valid or not
-    const copilot = new CopilotAPI(token)
-    const tokenPayload = await copilot.getTokenPayload()
-    CustomLogger.info({
-      obj: { copilotApiCronToken: token, tokenPayload },
-      message:
-        'CronService#_scheduleSinglePortal | Copilot API token and payload',
-    })
-    if (!tokenPayload) throw new APIError(500, 'Encoded token is not valid') // this should trigger p-retry and re-run the function
+      const payload = {
+        workspaceId,
+      }
+      const token = encodePayload(copilotAPIKey, payload)
 
-    const user = new User(token, tokenPayload)
-    user.qbConnection = qbConnectionTokens
-    const syncService = new SyncService(user)
+      // check if token is valid or not
+      const copilot = new CopilotAPI(token)
+      const tokenPayload = await copilot.getTokenPayload()
+      CustomLogger.info({
+        obj: { copilotApiCronToken: token, tokenPayload },
+        message:
+          'CronService#_scheduleSinglePortal | Copilot API token and payload',
+      })
+      if (!tokenPayload) throw new APIError(500, 'Encoded token is not valid') // this should trigger p-retry and re-run the function
+
+      const user = new User(token, tokenPayload)
+      user.qbConnection = qbConnectionTokens
+      const syncService = new SyncService(user)
 
-    // TODO: update this on QB tech debt milestone
-    // const { suspended } = await syncService.checkAndSuspendAccount()
-    // if (suspended) return
-    return await syncService.syncFailedRecords()
+      // TODO: update this on QB tech debt milestone
+      // const { suspended } = await syncService.checkAndSuspendAccount()
+      // if (suspended) return
+      return await syncService.syncFailedRecords()
+    })
   }
 
   async rerunFailedSync() {

src/app/api/quickbooks/customer/customer.service.ts

@@ -15,6 +15,7 @@ import { QBCustomerCreatePayloadType } from '@/type/dto/intuitAPI.dto'
 import { InvoiceCreatedResponseType } from '@/type/dto/webhook.dto'
 import { CopilotAPI } from '@/utils/copilotAPI'
 import IntuitAPI from '@/utils/intuitAPI'
+import { addSyncBreadcrumb } from '@/utils/sentry'
 import { replaceSpecialCharsForQB } from '@/utils/string'
 import { and, eq, isNull } from 'drizzle-orm'
 import httpStatus from 'http-status'
@@ -365,6 +366,10 @@ export class CustomerService extends BaseService {
       customer = undefined
     }
 
+    addSyncBreadcrumb('Customer search in QBO', {
+      found: !!customer,
+    })
+
     // 3. if not found, create a new client in the QB
     if (!customer) {
       console.info(
@@ -390,6 +395,9 @@ export class CustomerService extends BaseService {
       const customerRes = await intuitApiService.createCustomer(customerPayload)
       customer = customerRes
 
+      addSyncBreadcrumb('Customer created in QBO', {
+        qbCustomerId: customer.Id,
+      })
       console.info(
         `InvoiceService#WebhookInvoiceCreated | Customer created in QB with ID: ${customer.Id}.`,
       )

src/app/api/quickbooks/invoice/invoice.service.ts

@@ -47,6 +47,7 @@ import { and, eq, isNull } from 'drizzle-orm'
 import { convert } from 'html-to-text'
 import httpStatus from 'http-status'
 import { z } from 'zod'
+import { addSyncBreadcrumb } from '@/utils/sentry'
 import { replaceSpecialCharsForQB } from '@/utils/string'
 import { AccountTypeObj } from '@/constant/qbConnection'
 
@@ -538,6 +539,10 @@ export class InvoiceService extends BaseService {
     qbTokenInfo: IntuitAPITokensType,
   ): Promise<void> {
     const invoiceResource = payload.data
+    addSyncBreadcrumb('Invoice creation started', {
+      invoiceNumber: invoiceResource.number,
+      portalId: this.user.workspaceId,
+    })
 
     // Check if the invoice with ID already exists in the db. This check is done in this function as it is also called from re-sync failed function
     const existingInvoice = await this.getInvoiceByNumber(
@@ -575,6 +580,10 @@ export class InvoiceService extends BaseService {
         intuitApiService,
       )
 
+    addSyncBreadcrumb('Customer resolved', {
+      existingMapping: !!existingCustomer,
+    })
+
     let customer,
       existingCustomerMapId = existingCustomer?.id
     if (!existingCustomer) {
@@ -731,6 +740,9 @@ export class InvoiceService extends BaseService {
     }
 
     // 6. create invoice in QB
+    addSyncBreadcrumb('Creating invoice in QBO', {
+      invoiceNumber: invoiceResource.number,
+    })
     const invoiceRes = await intuitApiService.createInvoice(qbInvoicePayload)
 
     const invoicePayload = {
@@ -804,6 +816,9 @@ export class InvoiceService extends BaseService {
     payload: InvoiceResponseType,
     qbTokenInfo: IntuitAPITokensType,
   ): Promise<void> {
+    addSyncBreadcrumb('Invoice paid flow started', {
+      invoiceNumber: payload.data.number,
+    })
     // 1. check if the status of invoice is already paid in sync table
     const invoiceSync = await this.getInvoiceByNumber(payload.data.number, [
       'id',
@@ -926,6 +941,9 @@ export class InvoiceService extends BaseService {
     payload: InvoiceVoidedResponse,
     qbTokenInfo: IntuitAPITokensType,
   ): Promise<void> {
+    addSyncBreadcrumb('Invoice voided flow started', {
+      invoiceNumber: payload.number,
+    })
     // 1. check if the status of invoice is already paid in sync table
     const invoiceSync = await this.getInvoiceByNumber(payload.number, [
       'id',
@@ -1013,6 +1031,9 @@ export class InvoiceService extends BaseService {
     payload: InvoiceDeletedResponse,
     qbTokenInfo: IntuitAPITokensType,
   ): Promise<void> {
+    addSyncBreadcrumb('Invoice deleted flow started', {
+      invoiceNumber: payload.number,
+    })
     const syncedInvoice = await this.getInvoiceByNumber(payload.number, [
       'id',
       'qbInvoiceId',

src/app/api/quickbooks/payment/payment.service.ts

@@ -33,6 +33,7 @@ import {
   getDeletedAtForAuthAccountCategoryLog,
   getCategory,
 } from '@/utils/synclog'
+import { addSyncBreadcrumb } from '@/utils/sentry'
 import dayjs from 'dayjs'
 import { z } from 'zod'
 import httpStatus from 'http-status'
@@ -95,6 +96,9 @@ export class PaymentService extends BaseService {
     },
   ): Promise<boolean> {
     const parsedQbPayload = QBPaymentCreatePayloadSchema.parse(qbPaymentPayload)
+    addSyncBreadcrumb('Creating payment in QBO', {
+      invoiceNumber: invoiceInfo.invoiceNumber,
+    })
     // to save error sync log when payment creation fails in QB
     try {
       const qbPaymentRes = await intuitApi.createPayment(parsedQbPayload)
@@ -152,6 +156,7 @@ export class PaymentService extends BaseService {
   ) {
     const parsedPayload = QBPurchaseCreatePayloadSchema.parse(payload)
 
+    addSyncBreadcrumb('Creating expense for absorbed fees')
     console.info(
       'PaymentService#webhookPaymentSucceeded | Creating expense for absorbed fees',
     )
@@ -192,6 +197,10 @@ export class PaymentService extends BaseService {
     invoice: InvoiceResponse | undefined,
   ): Promise<void> {
     const paymentResource = parsedPaymentSucceedResource.data
+    addSyncBreadcrumb('Payment succeeded flow started', {
+      paymentId: paymentResource.id,
+      invoiceId: paymentResource.invoiceId,
+    })
 
     if (!paymentResource.feeAmount)
       throw new APIError(httpStatus.BAD_REQUEST, 'Fee amount is not found')

src/app/api/quickbooks/product/product.service.ts

@@ -33,6 +33,7 @@ import httpStatus from 'http-status'
 import { QBSyncLog, QBSyncLogWithEntityType } from '@/db/schema/qbSyncLogs'
 import User from '@/app/api/core/models/User.model'
 import { SettingService } from '@/app/api/quickbooks/setting/setting.service'
+import { addSyncBreadcrumb } from '@/utils/sentry'
 import { replaceBeforeParens, replaceSpecialCharsForQB } from '@/utils/string'
 import { AccountTypeObj } from '@/constant/qbConnection'
 import { TokenService } from '@/app/api/quickbooks/token/token.service'
@@ -365,6 +366,9 @@ export class ProductService extends BaseService {
     qbTokenInfo: IntuitAPITokensType,
   ): Promise<void> {
     const productResource = resource.data
+    addSyncBreadcrumb('Product updated flow started', {
+      productId: productResource.id,
+    })
 
     // 01. get all the mapped product ids with qb id
     const mappedConditions =
@@ -506,6 +510,10 @@ export class ProductService extends BaseService {
     qbTokenInfo: IntuitAPITokensType,
   ): Promise<void> {
     const priceResource = resource.data
+    addSyncBreadcrumb('Price created flow started', {
+      priceId: priceResource.id,
+      productId: priceResource.productId,
+    })
     const intuitApi = new IntuitAPI(qbTokenInfo)
 
     await this.db.transaction(async (tx) => {
@@ -527,6 +535,9 @@ export class ProductService extends BaseService {
         (product) => product.priceId === priceResource.id,
       ).length
 
+      addSyncBreadcrumb('Product mapping check', {
+        alreadyMapped: !!productWithPriceCount,
+      })
       if (productWithPriceCount && productWithPriceCount > 0) {
         console.info('Product already mapped with price')
         return

src/app/api/quickbooks/sync/sync.service.ts

@@ -17,7 +17,7 @@
 import CustomLogger from '@/utils/logger'
 import { QBSyncLog, QBSyncLogSelectSchemaType } from '@/db/schema/qbSyncLogs'
 import { z } from 'zod'
 import { and, eq, inArray, max } from 'drizzle-orm'
 import { WhereClause } from '@/type/common'
 import { TokenService } from '@/app/api/quickbooks/token/token.service'
 import { QBPortalConnection } from '@/db/schema/qbPortalConnections'
@@ -430,6 +430,34 @@
           })
           break
       }
+
+      // Report to Sentry after the final attempt so breadcrumbs from processing are included
+      if (resyncAttemtps.isLastAttempt) {
+        const currentLog = await this.syncLogService.getOne(
+          eq(QBSyncLog.id, log.id),
+        )
+        if (currentLog?.status === LogStatus.FAILED) {
+          captureMessage(
+            `SyncService#intiateSync | Records exceeded max retry count. Portal Id: ${this.user.workspaceId}.`,
+            {
+              tags: {
+                key: 'exceedMaxAttempts',
+                portalId: this.user.workspaceId,
+                entityType: log.entityType,
+                eventType: log.eventType,
+                errorCategory: currentLog.category,
+              },
+              extra: {
+                LogId: log.id,
+                invoiceNumber: log.invoiceNumber,
+                errorMessage: currentLog.errorMessage,
+                attempt: MAX_ATTEMPTS,
+              },
+              level: 'error',
+            },
+          )
+        }
+      }
     }
   }
 
@@ -541,7 +569,7 @@
         message: `SyncService#checkAndUpdateAttempt | Reached max attempts. Not syncing the record with assembly id: ${log.copilotId}`,
         obj: { workspaceId: this.user.workspaceId },
       })
-      return { maxAttempts: true }
+      return { maxAttempts: true, isLastAttempt: false }
     }
 
     const attempt = log.attempt + 1
@@ -552,28 +580,12 @@
       eq(QBSyncLog.id, log.id),
     )
 
-    // report to sentry if any records has exceeded max retry count
-    if (attempt == MAX_ATTEMPTS) {
-      captureMessage(
-        `SyncService#checkAndUpdateAttempt | Records exceeded max retry count. Portal Id: ${this.user.workspaceId}.`,
-        {
-          tags: {
-            key: 'exceedMaxAttempts', // can be used to search like "key:exceedMaxAttempts"
-          },
-          extra: {
-            LogId: log.id, // shown in "Additional Data" section in Sentry
-          },
-          level: 'error',
-        },
-      )
-    }
-
     CustomLogger.info({
       message: `SyncService#checkAndUpdateAttempt | Attempt: ${attempt}`,
       obj: { workspaceId: this.user.workspaceId },
     })
 
-    return { maxAttempts: false }
+    return { maxAttempts: false, isLastAttempt: attempt === MAX_ATTEMPTS }
   }
 
   private async updateFailedSyncLog(