Request correlation

app/backend/README.md

@@ -114,6 +114,43 @@ try {
 
 ---
 
+## Request Correlation & Distributed Tracing
+
+Every inbound HTTP request receives a unique **correlation ID** (`x-request-id`). This ID is automatically propagated across all layers:
+
+| Layer | How it works |
+|---|---|
+| **HTTP middleware** | `CorrelationIdMiddleware` extracts or generates a UUID and sets it on the request, response headers, and `AsyncLocalStorage` context. |
+| **Logging** | `LoggingInterceptor` and `MetricsInterceptor` emit structured JSON logs containing `correlationId`. |
+| **Metrics** | Prometheus histograms/counters include the correlation ID in log output for Grafana correlation. |
+| **Job queue** | `JobQueueService.enqueue()` captures the caller's correlation ID and persists it in the `jobs` table. `JobExecutor` restores it into `AsyncLocalStorage` when the job runs. |
+| **Error responses** | `GlobalHttpExceptionFilter` includes `request_id` / `correlationId` in every error envelope. |
+| **External calls** | `@TraceExternalCall` decorator reads the correlation ID from `AsyncLocalStorage` for downstream call tracing. |
+
+### Tracing a request end-to-end
+
+1. **Client sends HTTP request** with `x-request-id: <uuid>` (optional — generated if absent).
+2. **Middleware** stores the ID in `AsyncLocalStorage` and sets response headers.
+3. **Interceptors** log structured JSON with the correlation ID on every request.
+4. **If a background job is enqueued**, the correlation ID is stored in the `jobs.correlation_id` column.
+5. **When the job executor picks up the job**, it restores the original correlation ID into `AsyncLocalStorage`, so all downstream service calls (DB, external APIs) carry the same ID.
+6. **To trace a full request lifecycle**, grep logs for the correlation ID:
+   ```bash
+   # Find all log entries for a specific request
+   grep 'correlationId.*abc-123-...' logs/combined.log
+   ```
+
+### Correlation ID headers
+
+| Header | Direction | Purpose |
+|---|---|---|
+| `x-request-id` | Request → Response | Canonical correlation identifier |
+| `x-correlation-id` | Request → Response | Legacy alias (backward compatible) |
+
+Both headers are set on every response so clients can correlate responses to requests.
+
+---
+
 ## Architecture
 
 ```text

app/backend/src/app.module.ts

@@ -23,6 +23,7 @@ import { PaymentsModule } from "./payments/payments.module";
 import { MetricsMiddleware } from "./metrics/metrics.middleware";
 import { MetricsInterceptor } from "./metrics/metrics.interceptor";
 import { CorrelationIdMiddleware } from "./common/middleware/correlation-id.middleware";
+import { CorrelationContextModule } from "./common/correlation/correlation-context.module";
 import { OrganizationContextMiddleware } from "./common/middleware/organization-context.middleware";
 import { ShadowTrafficMiddleware } from "./environment-parity/shadow-traffic.middleware";
 import { IngestionModule } from "./ingestion/ingestion.module";
@@ -56,6 +57,7 @@ const validatedEnv = validateEnv(process.env);
 
 @Module({
   imports: [
+    CorrelationContextModule,
     SentryModule,
     AppConfigModule,
     // ScheduleModule registered once here — shared by NotificationsModule and ReconciliationModule

app/backend/src/common/correlation/correlation-context.module.ts

@@ -0,0 +1,21 @@
+/**
+ * Correlation Context Module
+ *
+ * Provides the `CorrelationContextService` as a singleton across the
+ * application. Import this module in any feature module that needs to
+ * read or write the current correlation identifier.
+ *
+ * The service is backed by `AsyncLocalStorage`, so it works across
+ * HTTP middleware, background job execution, database calls, and any
+ * other async flow without explicit parameter threading.
+ */
+
+import { Module, Global } from '@nestjs/common';
+import { CorrelationContextService } from './correlation-context.service';
+
+@Global()
+@Module({
+  providers: [CorrelationContextService],
+  exports: [CorrelationContextService],
+})
+export class CorrelationContextModule {}

app/backend/src/common/correlation/correlation-context.service.ts

@@ -0,0 +1,87 @@
+/**
+ * Correlation Context Service
+ *
+ * Provides a single source of truth for the correlation identifier across
+ * all execution contexts: HTTP requests, background job processing, database
+ * operations, and any downstream service calls.
+ *
+ * Uses Node.js `AsyncLocalStorage` to propagate the correlation ID through
+ * the async call stack without explicit parameter threading. This enables
+ * operators to follow a single request or background action across the
+ * entire NestJS application, queue, and realtime services.
+ *
+ * Usage:
+ *   // Set context (typically done in middleware or job executor)
+ *   correlationContext.setCorrelationId('abc-123');
+ *
+ *   // Get current context (works anywhere in the same async call stack)
+ *   const id = correlationContext.getCorrelationId();
+ *
+ *   // Run a callback with a specific correlation ID
+ *   await correlationContext.run('abc-123', async () => {
+ *     await someService.doWork();
+ *   });
+ */
+
+import { Injectable } from '@nestjs/common';
+import { AsyncLocalStorage } from 'async_hooks';
+
+/**
+ * Shape of the correlation context stored per async execution.
+ */
+interface CorrelationStore {
+  correlationId: string;
+}
+
+@Injectable()
+export class CorrelationContextService {
+  private readonly storage = new AsyncLocalStorage<CorrelationStore>();
+
+  /**
+   * Get the current correlation ID from the async context.
+   * Returns `undefined` when called outside any tracked context
+   * (e.g. during module initialization or in non-request code).
+   */
+  getCorrelationId(): string | undefined {
+    return this.storage.getStore()?.correlationId;
+  }
+
+  /**
+   * Set the correlation ID for the current async context.
+   * Typically called from the correlation-id middleware (HTTP) or
+   * the job executor (background jobs).
+   */
+  setCorrelationId(correlationId: string): void {
+    const store = this.storage.getStore();
+    if (store) {
+      store.correlationId = correlationId;
+    } else {
+      // Outside any tracked context — create a new root context.
+      this.storage.enterWith({ correlationId });
+    }
+  }
+
+  /**
+   * Run a callback inside a new async context with the given correlation ID.
+   * The callback can be async — all `await`ed work inherits the context.
+   *
+   * @param correlationId - The correlation ID to propagate
+   * @param fn            - The callback to execute in context
+   * @returns The return value of `fn`
+   */
+  async run<T>(correlationId: string, fn: () => Promise<T>): Promise<T> {
+    return this.storage.run({ correlationId }, fn);
+  }
+
+  /**
+   * Run a synchronous callback inside a new async context with the given
+   * correlation ID. Useful for non-async initialization paths.
+   *
+   * @param correlationId - The correlation ID to propagate
+   * @param fn            - The synchronous callback to execute in context
+   * @returns The return value of `fn`
+   */
+  runSync<T>(correlationId: string, fn: () => T): T {
+    return this.storage.run({ correlationId }, fn);
+  }
+}

app/backend/src/common/decorators/trace-external-call.decorator.ts

@@ -1,5 +1,6 @@
 import { Logger } from '@nestjs/common';
 import { MetricsService } from '../../metrics/metrics.service';
+import { CorrelationContextService } from '../correlation/correlation-context.service';
 
 /**
  * Decorator to trace external API calls with timing and error tracking.
@@ -24,7 +25,13 @@ export function TraceExternalCall(service: string, operation: string) {
     descriptor.value = async function (...args: unknown[]) {
       const metricsService: MetricsService = (this as Record<string, unknown>).metricsService as MetricsService;
       const startTime = Date.now();
-      const correlationId = (this as Record<string, unknown>).correlationId as string || 'N/A';
+      // Resolve correlation ID: try AsyncLocalStorage first, then instance property, then fallback
+      const correlationContext: CorrelationContextService | undefined =
+        (this as Record<string, unknown>).correlationContext as CorrelationContextService | undefined;
+      const correlationId =
+        correlationContext?.getCorrelationId()
+        || (this as Record<string, unknown>).correlationId as string
+        || 'N/A';
 
       logger.debug(
         JSON.stringify({

app/backend/src/common/interceptors/logging.interceptor.ts

@@ -8,6 +8,7 @@ import {
 } from '@nestjs/common';
 import { Observable } from 'rxjs';
 import { tap } from 'rxjs/operators';
+import { CorrelationContextService } from '../correlation/correlation-context.service';
 
 /**
  * LoggingInterceptor logs every HTTP request with full request context
@@ -24,6 +25,8 @@ import { tap } from 'rxjs/operators';
 export class LoggingInterceptor implements NestInterceptor {
   private readonly logger = new Logger(LoggingInterceptor.name);
 
+  constructor(private readonly correlationContext?: CorrelationContextService) {}
+
   /** Fields that must never appear in logs */
   private static readonly SENSITIVE_FIELDS = new Set([
     'password',
@@ -44,7 +47,7 @@ export class LoggingInterceptor implements NestInterceptor {
   intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
     const request = context.switchToHttp().getRequest();
     const { method, url, body, route } = request;
-    const correlationId = request.correlationId || 'N/A';
+    const correlationId = this.correlationContext?.getCorrelationId() || request.correlationId || 'N/A';
     const userId = this.extractUserId(request);
     const routePath = route?.path || url;
     const now = Date.now();

app/backend/src/common/middleware/correlation-id.middleware.ts

@@ -1,15 +1,29 @@
 import { Injectable, NestMiddleware } from '@nestjs/common';
 import { Request, Response, NextFunction } from 'express';
 import { v4 as uuidv4 } from 'uuid';
+import { CorrelationContextService } from '../correlation/correlation-context.service';
 
+/**
+ * Middleware that extracts or generates a correlation ID for every inbound
+ * HTTP request and makes it available in three places:
+ *
+ * 1. Response headers (`x-request-id` / `x-correlation-id`) — echoed back to clients.
+ * 2. Express request object (`req.correlationId`) — used by filters and interceptors.
+ * 3. `AsyncLocalStorage` context — used by any service in the same async call stack
+ *    (DB queries, logging, metrics, downstream calls).
+ */
 @Injectable()
 export class CorrelationIdMiddleware implements NestMiddleware {
+  constructor(private readonly correlationContext: CorrelationContextService) {}
+
   use(req: Request, res: Response, next: NextFunction) {
     const correlationId = req.header('x-request-id') || req.header('x-correlation-id') || uuidv4();
     // Expose as both the legacy header and the canonical request-id header
     res.setHeader('x-request-id', correlationId);
     res.setHeader('x-correlation-id', correlationId);
-    req['correlationId'] = correlationId; 
+    req['correlationId'] = correlationId;
+    // Propagate into AsyncLocalStorage so downstream services can read it
+    this.correlationContext.setCorrelationId(correlationId);
     next();
   }
 }