diff --git a/.env.example b/.env.example index 4760b6e..005bd06 100644 --- a/.env.example +++ b/.env.example @@ -1,67 +1,32 @@ -# ─── Service ─── -SERVICE_NAME=mongo-to-clickhouse-migrator -SERVICE_PORT=8080 -SERVICE_HOST=0.0.0.0 -GRACEFUL_SHUTDOWN_TIMEOUT_MS=60000 -RERUN_MODE=resume - -# ─── MongoDB Source ─── -MONGO_URI=mongodb://localhost:27017/?replicaSet=rs0 -MONGO_DB=countly_drill -MONGO_COLLECTION_PREFIX=drill_events -MONGO_READ_PREFERENCE=primary -MONGO_READ_CONCERN=majority -MONGO_RETRY_READS=true -MONGO_APP_NAME=mongo-to-clickhouse-migrator -MONGO_BATCH_ROWS_TARGET=10000 -MONGO_CURSOR_BATCH_SIZE=2000 -MONGO_MAX_TIME_MS=120000 - -# ─── Transform ─── -TRANSFORM_VERSION=v1 - -# ─── ClickHouse Target ─── +# ─── Required (the only two you must set) ─── +MONGO_URI=mongodb://localhost:27017 CLICKHOUSE_URL=http://localhost:8123 + +# ─── Common ─── +#SERVICE_NAME=drill-migrator +SERVICE_PORT=8080 +MONGO_DB=countly_drill # source database with drill_events* collections +MONGO_COUNTLY_DB=countly # for per-event collection-hash resolution CLICKHOUSE_DB=countly_drill CLICKHOUSE_TABLE=drill_events -CLICKHOUSE_USERNAME=default -CLICKHOUSE_PASSWORD= -CLICKHOUSE_QUERY_TIMEOUT_MS=120000 -CLICKHOUSE_MAX_RETRIES=8 -CLICKHOUSE_RETRY_BASE_DELAY_MS=1000 -CLICKHOUSE_RETRY_MAX_DELAY_MS=30000 -CLICKHOUSE_USE_DEDUP_TOKEN=true - -# ─── Backpressure ─── -BACKPRESSURE_ENABLED=true -BACKPRESSURE_PARTS_TO_THROW_INSERT=300 -BACKPRESSURE_MAX_PARTS_IN_TOTAL=500 -BACKPRESSURE_PARTITION_PCT_HIGH=0.50 -BACKPRESSURE_PARTITION_PCT_LOW=0.35 -BACKPRESSURE_TOTAL_PCT_HIGH=0.50 -BACKPRESSURE_TOTAL_PCT_LOW=0.40 -BACKPRESSURE_POLL_INTERVAL_MS=15000 -BACKPRESSURE_MAX_PAUSE_EPISODE_MS=180000 - -# ─── State ─── -MANIFEST_DB=countly_drill -REDIS_URL=redis://redis:6379 -REDIS_KEY_PREFIX=mig -TIMELINE_SNAPSHOT_INTERVAL=10 - -# ─── Memory / GC ─── -GC_ENABLED=true -GC_RSS_SOFT_LIMIT_MB=1536 -GC_RSS_HARD_LIMIT_MB=2048 -GC_HEAP_USED_RATIO=0.70 -GC_EVERY_N_BATCHES=10 - -# ─── Logging ─── -LOG_LEVEL=info - -# ─── Lifecycle ─── -# When true, the service shuts down cleanly (exit 0) once all collections -# finish migrating. Used by one-shot orchestration (e.g. docker compose -# --abort-on-container-exit). Default false: service stays running with -# dashboard available. -EXIT_ON_COMPLETE=false +MANIFEST_DB=countly_drill # where progress state lives: mig_ranges + mig_dlq_docs + # (recoverable if lost: dashboard → Help & Recovery → Rebuild ledger from data) +# Read preference is AUTO: on a replica set the engine picks +# secondaryPreferred by itself (source is frozen after cutover, so secondary +# reads are exact). Set explicitly only to override: +#MONGO_READ_PREFERENCE=primary + +# ─── Run identity & scaling ─── +LEDGER_RUN_ID=migration-1 # stable resume key — keep it the same across restarts +#POD_ID=pod-1 # unique per instance when running multiple pods +#EXIT_ON_COMPLETE=true # exit 0 when all chunks are done (one-shot orchestration) + +# ─── Sizing (see the UI's Configuration card for guidance) ─── +#LEDGER_CHUNK_DOCS_TARGET=2000000 +#LEDGER_MAX_CHUNK_DAYS=7 +#MONGO_PAGE_SIZE=10000 +#LEDGER_INSERT_INFLIGHT=3 + +# ─── Rehearsal ─── +#DRY_RUN=true # sampled rehearsal, nothing stored (or use the UI button) +#DRY_RUN_SAMPLE_PCT=2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..10d26b4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,78 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit: + name: Typecheck + differential harness + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '25' + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Typecheck + run: npm run typecheck + + # Asserts src/transform/ reproduces countly-platform's live ingestion + # normalization exactly (vendored goldens in tests/differential/). + # Pure unit tests — no MongoDB/ClickHouse/Redis required. + - name: Ingestion-matching differential harness + run: npx vitest run tests/differential + + integration: + name: Integration (MongoDB 7 + ClickHouse 26.4) + runs-on: ubuntu-latest + services: + mongo: + image: mongo:7 + ports: ['27017:27017'] + options: >- + --health-cmd "mongosh --quiet --eval 'db.runCommand({ping:1})'" + --health-interval 5s --health-timeout 5s --health-retries 30 + clickhouse: + # Production ClickHouse version (deploy/compose/images.standard.env + # in countly-platform). 26.4 requires a password for non-localhost + # clients — the tests read TEST_CLICKHOUSE_PASSWORD. + image: clickhouse/clickhouse-server:26.4 + env: + CLICKHOUSE_PASSWORD: ci-pass + ports: ['8123:8123'] + options: >- + --health-cmd "clickhouse-client --password ci-pass --query 'SELECT 1'" + --health-interval 5s --health-timeout 5s --health-retries 30 + env: + TEST_CLICKHOUSE_PASSWORD: ci-pass + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '25' + cache: npm + + - name: Install dependencies + run: npm ci + + # Full engine suite: ledger store semantics, e2e chunk pipeline with + # DLQ/coercions/null-cd sweep, multi-collection scoping regressions, + # ledger rebuild, duplicate attribution, pair-based attach recovery. + - name: Integration tests + run: npx vitest run tests/integration diff --git a/README.md b/README.md index 7a62a91..05045c3 100644 --- a/README.md +++ b/README.md @@ -2,541 +2,66 @@ Migrates Countly `drill_events*` collections from MongoDB into a single ClickHouse table. Supports multi-pod horizontal scaling, range-parallel processing, async writes, pause/resume, crash recovery, backpressure monitoring, and a real-time dashboard. -## Quick Start +## Setup & run -```bash -cp .env.example .env # edit with your connection details -docker compose up --build -curl http://localhost:8080/healthz -``` - -Open the dashboard at [http://localhost:8080/viz](http://localhost:8080/viz). - -## Running - -**Docker Compose (recommended):** +Prerequisites: reachable MongoDB (the source) and ClickHouse (the target with +the `drill_events` table — created by the new Countly stack). Nothing else. ```bash +cp .env.example .env # point MONGO_URI / CLICKHOUSE_URL at your systems docker compose up --build +# or, with Node 25+: npm install && npm start ``` -**From source (Node 25+):** - -```bash -npm install -node --experimental-strip-types --expose-gc --max-old-space-size=2048 src/main.ts -``` - -Required env vars: `SERVICE_NAME`, `MONGO_URI`, `CLICKHOUSE_URL`, `REDIS_URL`. - -## Configuration - -Copy `.env.example` and adjust. All values below show defaults where applicable. - -### Service - -| Variable | Default | Description | -|----------|---------|-------------| -| `SERVICE_NAME` | *(required)* | Service identifier | -| `SERVICE_PORT` | `8080` | HTTP server port | -| `SERVICE_HOST` | `0.0.0.0` | HTTP server bind address | -| `GRACEFUL_SHUTDOWN_TIMEOUT_MS` | `60000` | Max wait for graceful shutdown (ms) | -| `RERUN_MODE` | `resume` | `resume`, `new-run`, or `clone-run` | -| `LOG_LEVEL` | `info` | `fatal`, `error`, `warn`, `info`, `debug`, `trace` | -| `EXIT_ON_COMPLETE` | `false` | When `true`, exit 0 once all collections complete. Used for one-shot orchestration. | - -### MongoDB Source - -| Variable | Default | Description | -|----------|---------|-------------| -| `MONGO_URI` | *(required)* | MongoDB connection string | -| `MONGO_DB` | `countly_drill` | Source database | -| `MONGO_COUNTLY_DB` | `countly` | Countly database (for hash resolution) | -| `MONGO_COLLECTION_PREFIX` | `drill_events` | Prefix to discover collections | -| `MONGO_READ_PREFERENCE` | `primary` | Read preference | -| `MONGO_READ_CONCERN` | `majority` | Read concern level | -| `MONGO_RETRY_READS` | `true` | Enable retry reads | -| `MONGO_APP_NAME` | *(optional)* | Connection app name | -| `MONGO_BATCH_ROWS_TARGET` | `10000` | Target docs per ClickHouse write batch | -| `MONGO_PAGE_SIZE` | `10000` | MongoDB page size per cursor read | -| `MONGO_CURSOR_BATCH_SIZE` | `10000` | MongoDB cursor batch size | -| `MONGO_MAX_TIME_MS` | `600000` | Cursor timeout (ms) | - -### Range-Parallel Processing - -| Variable | Default | Description | -|----------|---------|-------------| -| `RANGE_PARALLEL_THRESHOLD` | `500000` | Estimated doc count to trigger range splitting | -| `RANGE_COUNT` | `100` | Number of time-ranges to split a collection into | -| `RANGE_LEASE_TTL_SEC` | `300` | Range lease TTL for dead-pod reclaim (s) | - -### Transform - -| Variable | Default | Description | -|----------|---------|-------------| -| `TRANSFORM_VERSION` | `v1` | Data transform version tag | - -### ClickHouse Target - -| Variable | Default | Description | -|----------|---------|-------------| -| `CLICKHOUSE_URL` | *(required)* | ClickHouse HTTP endpoint | -| `CLICKHOUSE_DB` | `countly_drill` | Target database | -| `CLICKHOUSE_TABLE` | `drill_events` | Target table | -| `CLICKHOUSE_USERNAME` | `default` | Username | -| `CLICKHOUSE_PASSWORD` | *(empty)* | Password | -| `CLICKHOUSE_QUERY_TIMEOUT_MS` | `120000` | Query timeout (ms) | -| `CLICKHOUSE_MAX_RETRIES` | `8` | Max insert retry attempts | -| `CLICKHOUSE_RETRY_BASE_DELAY_MS` | `1000` | Backoff base delay (ms) | -| `CLICKHOUSE_RETRY_MAX_DELAY_MS` | `30000` | Backoff max delay (ms) | -| `CLICKHOUSE_USE_DEDUP_TOKEN` | `true` | Enable insert dedup tokens | - -### Backpressure - -| Variable | Default | Description | -|----------|---------|-------------| -| `BACKPRESSURE_ENABLED` | `true` | Enable ClickHouse parts monitoring | -| `BACKPRESSURE_PARTS_TO_THROW_INSERT` | `300` | Parts threshold to pause inserts | -| `BACKPRESSURE_MAX_PARTS_IN_TOTAL` | `500` | Max total parts allowed | -| `BACKPRESSURE_PARTITION_PCT_HIGH` | `0.70` | Partition high watermark | -| `BACKPRESSURE_PARTITION_PCT_LOW` | `0.55` | Partition low watermark | -| `BACKPRESSURE_TOTAL_PCT_HIGH` | `0.70` | Total high watermark | -| `BACKPRESSURE_TOTAL_PCT_LOW` | `0.55` | Total low watermark | -| `BACKPRESSURE_POLL_INTERVAL_MS` | `5000` | Pressure polling interval (ms) | -| `BACKPRESSURE_MAX_PAUSE_EPISODE_MS` | `180000` | Max pause before force resume (ms) | - -### State - -| Variable | Default | Description | -|----------|---------|-------------| -| `MANIFEST_DB` | `countly_drill` | MongoDB database for run manifests | -| `REDIS_URL` | *(required)* | Redis connection URL | -| `REDIS_KEY_PREFIX` | `mig` | Redis key namespace | -| `TIMELINE_SNAPSHOT_INTERVAL` | `10` | Timeline snapshot every N batches | - -### Memory / GC - -| Variable | Default | Description | -|----------|---------|-------------| -| `GC_ENABLED` | `true` | Enable manual GC | -| `GC_RSS_SOFT_LIMIT_MB` | `3072` | RSS soft limit to trigger GC | -| `GC_RSS_HARD_LIMIT_MB` | `6144` | RSS hard limit warning | -| `GC_HEAP_USED_RATIO` | `0.70` | Heap usage ratio trigger | -| `GC_EVERY_N_BATCHES` | `50` | Run GC every N batches | - -### Worker / Multi-Pod - -| Variable | Default | Description | -|----------|---------|-------------| -| `POD_ID` | `hostname()` | Unique pod identifier | -| `MULTI_POD_ENABLED` | `true` | Enable distributed locking | -| `LOCK_TTL_SECONDS` | `300` | Collection lock TTL — crash safety net (s) | -| `LOCK_RENEW_MS` | `60000` | Lock renewal interval (ms) | -| `PROGRESS_UPDATE_MS` | `5000` | Progress reporting interval to Redis (ms) | -| `POD_HEARTBEAT_MS` | `30000` | Pod heartbeat interval (ms) | -| `POD_DEAD_AFTER_SEC` | `180` | Pod considered dead after this silence (s) | - -### Async Write - -| Variable | Default | Description | -|----------|---------|-------------| -| `ASYNC_WRITE_FLUSH_INTERVAL_MS` | `5000` | Flush queued batch records to MongoDB every N ms | -| `ASYNC_WRITE_FLUSH_BATCH_SIZE` | `10` | Flush after N batch records queued | - -## Multi-Collection Migration - -The service automatically discovers all MongoDB collections matching `MONGO_COLLECTION_PREFIX*` (e.g. `drill_events`, `drill_events5a2b3c4d...`). - -- Collections are processed in priority order (base collection first, then alphabetical) -- Each collection gets its own run ID and isolated Redis key prefix -- Already-completed collections are skipped on restart -- Missing `{cd: 1, _id: 1}` compound index is created automatically in background -- If a collection fails, the service logs the error and continues to the next - -## Multi-Pod Mode - -Multiple pods can process collections in parallel using Redis-based distributed locking. Each pod autonomously discovers collections, acquires locks, and processes work. - -### How It Works - -1. **Pod heartbeat**: Each pod writes a TTL-based heartbeat key (`mig:pod:{podId}`) to Redis every 30s -2. **Collection locking**: Before processing a collection, a pod acquires a lock via atomic Lua script. If the lock is held by another live pod, it moves to the next collection -3. **Dead pod detection**: If a pod's heartbeat expires (180s default), other pods can steal its locks -4. **Global progress**: Each pod reports per-collection progress to Redis. The `/stats` endpoint and dashboard merge data from all pods - -### Range-Parallel Processing - -Collections exceeding `RANGE_PARALLEL_THRESHOLD` (default 500K docs) are split into `RANGE_COUNT` equal time-ranges based on `[min(cd), max(cd)]`. Multiple pods can process different ranges of the same collection concurrently. - -- **Range initialization**: First pod to reach a large collection divides the time span into N ranges via Redis SETNX -- **Atomic claiming**: Pods claim ranges via a Redis Lua script that atomically marks a pending range as processing -- **Exclusive boundaries**: Ranges use `[start, end)` (exclusive upper bound) to prevent duplicate processing. The final range uses `[start, max]` -- **Stale reclaim**: If a pod dies mid-range, other pods reclaim its ranges after the lease TTL expires -- **Batch sequence isolation**: Each range gets 10,000 batch sequence slots to prevent collisions - -### Kubernetes Deployment - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: migration -spec: - replicas: 3 - selector: - matchLabels: - app: migration - template: - metadata: - labels: - app: migration - spec: - terminationGracePeriodSeconds: 120 - containers: - - name: migration - image: countly-migration:latest - ports: - - containerPort: 8080 - env: - - name: POD_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - livenessProbe: - httpGet: - path: /healthz - port: 8080 - initialDelaySeconds: 10 - readinessProbe: - httpGet: - path: /readyz - port: 8080 - initialDelaySeconds: 15 - lifecycle: - preStop: - httpGet: - path: /control/drain - port: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: migration -spec: - selector: - app: migration - ports: - - port: 8080 ---- -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: migration -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: migration - minReplicas: 1 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 -``` - -K8s lifecycle: -- **Scale up**: New pods start, discover collections, acquire locks on unclaimed work -- **Scale down**: `preStop` hook calls `/control/drain`, pod finishes current batch, releases locks, exits -- **Pod crash**: SIGTERM triggers graceful shutdown. On `kill -9`, heartbeat expires after 180s, locks are stolen by live pods - -## Dashboard - -The `/viz` endpoint serves a self-contained real-time HTML dashboard. No external dependencies. - -**Features:** -- Overall migration progress bar with ETA -- Current collection progress and batch sequence -- Per-pod progress bars with docs/rows/throughput stats -- Live batches panel showing batch phase (READING, TRANSFORMING, WRITING, COMMITTING) -- Range heatmap for range-parallel collections -- Active locks table with release buttons -- Stale pod detection with remove action -- Global cluster controls (pause/resume/stop all pods) -- Index build status -- Skip reason breakdown -- Memory and GC metrics -- Infrastructure connection status - -## API Endpoints - -### Health - -| Method | Path | Description | -|--------|------|-------------| -| `GET` | `/healthz` | Liveness probe (always 200) | -| `GET` | `/readyz` | Readiness check (mongo, clickhouse, redis, manifest, runner) | - -### Stats - -| Method | Path | Description | -|--------|------|-------------| -| `GET` | `/stats` | Full dashboard JSON: progress, throughput, integrity, cluster, live batches | -| `GET` | `/viz` | Real-time HTML dashboard | - -### Control (Single Pod) - -| Method | Path | Description | -|--------|------|-------------| -| `POST` | `/control/pause` | Pause after current batch | -| `POST` | `/control/resume` | Resume from pause | -| `POST` | `/control/stop-after-batch` | Stop cleanly after current batch | -| `POST` | `/control/gc` | Trigger manual GC (body: `{"mode":"now"}`) | -| `POST` | `/control/reindex/:collection` | Trigger index creation for a collection | -| `POST` | `/control/retry-collection/:collection` | Re-queue a failed/skipped collection | -| `POST` | `/control/retry-batch/:runId/:batchSeq` | Retry a specific failed/skipped batch | -| `POST` | `/control/retry-skipped-batches/:runId` | Retry all skipped_empty batches in a run | - -### Control (Multi-Pod) - -| Method | Path | Description | -|--------|------|-------------| -| `POST` | `/control/global/pause` | Pause all pods | -| `POST` | `/control/global/resume` | Resume all pods | -| `POST` | `/control/global/stop` | Stop all pods after current batch | -| `GET` | `/control/locks` | List all active collection locks | -| `POST` | `/control/locks/release/:collection` | Force-release a lock (admin) | -| `GET` | `/control/pods` | List all pods with status and locks held | -| `POST` | `/control/pods/remove/:podId` | Remove dead pod and release its locks | -| `POST` | `/control/drain` | Graceful drain for K8s scale-down | - -### Runs - -| Method | Path | Description | -|--------|------|-------------| -| `GET` | `/runs` | List all runs (`?status=`, `?limit=`, `?offset=`) | -| `GET` | `/runs/current` | Current active run with coverage | -| `GET` | `/runs/current/batches` | Batches for current run (`?status=`, `?limit=`) | -| `GET` | `/runs/:id` | Run details by ID | -| `GET` | `/runs/:id/batches` | Batch list (`?status=`, `?limit=`) | -| `GET` | `/runs/:id/failures` | Failed batches, retry errors, digest mismatches | -| `GET` | `/runs/:id/timeline` | Performance snapshots over time | -| `GET` | `/runs/:id/coverage` | Document range coverage and completion % | -| `DELETE` | `/runs/:id/cache` | Clean up Redis cache for a run | - -## Operations - -### Pause and Resume - -```bash -# Single pod -curl -X POST http://localhost:8080/control/pause -curl -X POST http://localhost:8080/control/resume - -# All pods (multi-pod mode) -curl -X POST http://localhost:8080/control/global/pause -curl -X POST http://localhost:8080/control/global/resume -``` - -### Graceful Stop +Then open **http://localhost:8080** — from here the dashboard takes over: +the **Migration Guide** tab walks the whole procedure (preflight checks, +index building, dry run, cutover checklist, live progress, verification and +sign-off gates), and **Help & Recovery** covers every failure scenario with +the fix one click away. -```bash -curl -X POST http://localhost:8080/control/stop-after-batch -``` +To scale: start more instances with the same `.env` and a unique `POD_ID` +each, on separate machines (see Scaling with pods below). -Or send `SIGTERM` / `SIGINT`. The service will: -1. Finish the current batch -2. Flush async write queue to MongoDB -3. Release all collection locks -4. Close all connections -5. Exit cleanly - -### Lock Management +**Docker (no Kubernetes)**: scaling works the same way — pods are just +processes coordinating through chunk leases in MongoDB, and `POD_ID` +defaults to the container hostname (unique automatically). Run one +container per machine with the same `.env`: ```bash -# List all active locks -curl http://localhost:8080/control/locks - -# Force-release a stuck lock -curl -X POST http://localhost:8080/control/locks/release/drill_events5a2b... - -# List pods and their status -curl http://localhost:8080/control/pods - -# Remove a dead pod and release its locks -curl -X POST http://localhost:8080/control/pods/remove/pod-3 +docker run -d --env-file .env --name drill-migrator \ + -p 8080:8080 europe-docker.pkg.dev//drill-migrator: ``` -### Retry Failed Work - -```bash -# Re-queue a failed collection -curl -X POST http://localhost:8080/control/retry-collection/drill_events5a2b... - -# Retry a specific failed batch -curl -X POST http://localhost:8080/control/retry-batch/{runId}/{batchSeq} - -# Retry all skipped batches in a run -curl -X POST http://localhost:8080/control/retry-skipped-batches/{runId} -``` - -### Crash Recovery - -On restart after a crash, the service automatically: -1. Checks Redis for the last committed cursor (hot-path authority) -2. Falls back to MongoDB manifest if Redis is empty -3. Replays any inflight batches with digest verification -4. Resumes from the last committed position -5. ClickHouse dedup tokens (`mig:{runId}:{batchSeq}`) prevent duplicate inserts - -### Monitor Progress - -```bash -# JSON stats -curl -s http://localhost:8080/stats | jq '.summary' - -# Cluster overview -curl -s http://localhost:8080/stats | jq '.cluster.pods' - -# Or use the dashboard -open http://localhost:8080/viz -``` - -## Architecture - -``` -MongoDB (drill_events*) - | - | cursor pagination on (cd, _id) compound index - v -+------------------------------------------------------+ -| CollectionOrchestrator | -| |-- discovers collections, index-aware scheduling | -| |-- multi-pod: Redis lock per collection | -| | | -| +-- Small collections (< 500K docs): | -| | BatchRunner --> ClickHouse | -| | | -| +-- Large collections (>= 500K docs): | -| RangeCoordinator | -| |-- splits [min(cd), max(cd)] into N ranges | -| |-- atomic range claiming via Redis Lua script | -| +-- per-range BatchRunner --> ClickHouse | -+------------------------------------------------------+ - | | | - | batch inserts | hot-path commit | async flush - | (dedup tokens) | | - v v v - ClickHouse Redis MongoDB manifest - (drill_events) (cursor, bitmap, (batch records, - live phases, run metadata, - range stats, audit trail) - pod heartbeats, - collection locks) -``` - -- **Redis (hot-path authority)**: Committed cursor, batch completion bitmap, live batch phases, range stats, pod heartbeats, collection locks, global commands -- **MongoDB manifest (async, authoritative for audit)**: Batch records flushed asynchronously every 5s or 10 batches. Run metadata, skip samples, error history -- **Async batch writer**: Redis is the commit point for each batch. MongoDB writes are queued and bulk-flushed in the background. On graceful shutdown, the queue is drained -- **Backpressure**: Monitors ClickHouse active parts count; pauses inserts when thresholds exceeded (hysteresis with high/low watermarks) -- **Dedup tokens**: Each batch carries `mig:{runId}:{batchSeq}` for idempotent retries - -## Redis Key Schema - -| Key Pattern | Type | TTL | Description | -|-------------|------|-----|-------------| -| `mig:active_run` | STRING | - | Current active run ID | -| `mig:run:{runId}:state` | STRING | - | Run state JSON blob | -| `mig:run:{runId}:cursor` | STRING | - | Last committed cursor (hot-path authority) | -| `mig:run:{runId}:done_bitmap` | STRING | - | Batch completion bitmap (SETBIT/GETBIT) | -| `mig:run:{runId}:stats:latest` | STRING | - | Latest run stats snapshot | -| `mig:run:{runId}:commands` | HASH | - | Operator command flags (pause, abort, etc.) | -| `mig:run:{runId}:recent_errors` | LIST | - | Last 100 batch errors | -| `mig:run:{runId}:timeline` | LIST | - | Last 1000 timeline snapshots | -| `mig:run:{runId}:batch:{seq}:errors` | LIST | - | Verbose errors per batch (last 20) | -| `mig:liveBatch:{collection}` | STRING | 30s | Live batch phase data (heartbeat-refreshed) | -| `mig:rangeLive:{collection}:{idx}` | STRING | 60s | Per-range live stats (heartbeat-refreshed) | -| `mig:lock:{collection}` | STRING | 300s | Collection lock with pod owner | -| `mig:pod:{podId}` | STRING | 180s | Pod heartbeat/liveness key | -| `mig:progress:{collection}` | STRING | 300s | Per-collection progress for cluster aggregation | -| `mig:cmd:global` | HASH | - | Global commands (pause, stop) | -| `mig:ranges:{collection}` | HASH | - | Range entries (idx, status, podId, claimedAt) | -| `mig:ranges:{collection}:init` | STRING | 60s | SETNX coordinator election for range init | -| `mig:ranges:{collection}:runId` | STRING | - | Shared run ID for range-parallel processing | -| `mig:ranges:{collection}:meta` | STRING | - | Range metadata (minCd, maxCd, count) | -| `mig:ranges:{collection}:finalized` | STRING | 60s | SETNX for run finalization (one pod only) | - -## Project Structure - -``` -src/ - main.ts # Entry point - config/ - schema.ts # Zod config schema with defaults - loader.ts # Env var loader and validator - http/ - health-route.ts # /healthz, /readyz - stats-route.ts # /stats (JSON dashboard) - control-route.ts # /control/* (pause, resume, locks, pods) - run-route.ts # /runs/* (history, batches, coverage) - viz-route.ts # /viz (HTML dashboard) - source/ - mongo-reader.ts # Cursor pagination on (cd, _id) - discover-collections.ts # Collection discovery - target/ - clickhouse-writer.ts # Batch inserts with dedup - clickhouse-pressure.ts # Backpressure monitoring - transform/ - normalize.ts # Document normalization - skip-reasons.ts # Skip tracking - validators.ts # Data validation - hash-resolver.ts # Collection hash defaults - runtime/ - collection-orchestrator.ts # Multi-collection + multi-pod orchestration - batch-runner.ts # Core batch lifecycle (read, transform, write) - range-coordinator.ts # Range-parallel processing - retry-policy.ts # Exponential backoff - gc-controller.ts # Manual GC management - process-metrics.ts # Memory/CPU metrics - resolve-run.ts # Run resolution on startup - state/ - manifest-store.ts # MongoDB manifest (authoritative audit trail) - redis-hot-state.ts # Redis hot state (bitmap, timeline, phases) - async-batch-writer.ts # Async MongoDB write queue - global-progress.ts # Multi-pod progress aggregation - collection-lock.ts # Distributed collection locking - coverage.ts # Coverage interval tracking - types/ - cursor.ts # Cursor serialization -``` - -## CI/CD - -A GitHub Actions workflow automatically builds and pushes the Docker image to Docker Hub when a GitHub Release is published. - -### Tag Behavior - -| Release Tag | Docker Tags | `latest`? | -|-------------|-------------|-----------| -| `v1.0.0` | `1.0.0`, `1.0`, `1`, `latest` | Yes | -| `v2.3.1` | `2.3.1`, `2.3`, `2`, `latest` | Yes | -| `v1.0.0-rc.1` | `1.0.0-rc.1` | No | -| `v2.0.0-beta` | `2.0.0-beta` | No | - -### Setting Up Docker Hub Credentials - -1. Go to [Docker Hub](https://hub.docker.com) > Account Settings > Security > New Access Token -2. Add `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` as GitHub repo secrets -3. Create a release (e.g. tag `v1.0.0`) and the workflow builds and pushes - -### Using a Pre-Built Image - -```yaml -services: - migration: - image: /countly-migration:1.0.0 - # ... rest of config unchanged -``` +Add machines by running the same command there — nothing to configure, +each container's dashboard shows the whole run. Scale across MACHINES, +not on one host: a single container saturates ~4 cores on BSON decode, +so `docker compose --scale migration=N` on one box only makes sense for +testing (and requires dropping the fixed published port). Set +`EXIT_ON_COMPLETE=true` for fire-and-forget runs — containers exit 0 +when every chunk is done. + +**Kubernetes**: ready-to-apply manifests live in `k8s/` — +`k8s/migration.yaml` (Deployment + Service: pods keep serving the dashboard +after completion for verification and sign-off; scale with +`kubectl scale deployment/drill-migrator --replicas=N`) and `k8s/job.yaml` +(fire-and-forget Job using `EXIT_ON_COMPLETE`). Pods coordinate through +chunk leases in MongoDB, `POD_ID` defaults to the pod name, and abrupt +kills/evictions are safe by design (chunk redo). Reach the dashboard with +`kubectl port-forward svc/drill-migrator 8080:8080` — any pod shows the +whole run. + +No schema changes are made to the live table. Migrated and live-ingested +rows are distinguished by construction: migrated rows carry their historical +`cd`, live rows are stamped at post-cutover insert time. Where an `_id` alone +would be ambiguous (an SDK retry across the cutover lands the same event in +both stacks, in the same partition), checks match `(_id, cd)` pairs — the +retry copy's cd can never equal the migrated copy's. Preflight verifies the +boundary is trustworthy (source frozen, clocks sane) before anything runs. + +This README covers what you need BEFORE the dashboard exists (installing, +env vars, starting the service, automation reference). Everything after — +running, monitoring, troubleshooting, verifying — lives in the dashboard, +with `docs/RUNBOOK.md` as the cross-system procedure (cutover choreography, +Kafka retention, incident tables) for operators. + +## Architecture \ No newline at end of file diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..27dcf81 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,39 @@ +# Validation harness + +Seed a synthetic dataset and prove the migrator's two core guarantees: +exactness (every doc lands exactly once) and crash safety (SIGKILL at any +moment, restart, converge — zero loss, zero duplicates). + +## 1. Seed + +```bash +AB_DOCS=250000 node --experimental-strip-types bench/setup.ts +``` + +Creates `mig_ab.drill_events` in MongoDB (with the `{cd,_id}` index) and a +clone of the production `drill_events` DDL in ClickHouse db `mig_ab`. + +## 2. Straight run + +```bash +SERVICE_NAME=validate MONGO_URI=mongodb://localhost:27017 MONGO_DB=mig_ab \ +MANIFEST_DB=mig_ab_manifest CLICKHOUSE_URL=http://localhost:8123 CLICKHOUSE_DB=mig_ab \ +LEDGER_CHUNK_DOCS_TARGET=50000 EXIT_ON_COMPLETE=true SERVICE_PORT=18081 npm start +``` + +Then verify (both numbers must equal the seeded doc count): + +```sql +SELECT count() AS total, uniqExact(_id) AS distinct_ids FROM mig_ab.drill_events; +``` + +## 3. Crash drill + +Repeatedly SIGKILLs the service at random points and restarts it until the +migration completes, then verifies zero loss and zero duplicates: + +```bash +node --experimental-strip-types bench/kill-drill.ts +``` + +Watch progress on the dashboard: `http://localhost:18081`. diff --git a/bench/gen.ts b/bench/gen.ts new file mode 100644 index 0000000..d95e238 --- /dev/null +++ b/bench/gen.ts @@ -0,0 +1,93 @@ +/** Synthetic drill_events doc generator — realistic shape/size mix (A/B test data). */ + +const APPS = Array.from({ length: 3 }, (_, i) => `5f${i}b2c3d4e5f6a7b8c9d0e1f${i}`); +const EVENTS = ['purchase_completed', 'level_up', 'item_viewed', 'search', 'cart_add', 'video_played', 'settings_changed', 'share']; +const PLATFORMS = ['Android', 'iOS', 'Windows', 'Macintosh']; +const COUNTRIES = ['US', 'DE', 'TR', 'GB', 'FR', 'JP', 'BR', 'IN']; +const VIEWS = ['/home', '/product/detail', '/checkout', '/profile', '/search/results', '/settings']; + +let seed = 0x9e3779b9; +function rnd(): number { + // mulberry32 — full 32-bit period, no float precision loss + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; +} +function pick(arr: T[]): T { return arr[Math.floor(rnd() * arr.length)]; } +function hex(n: number): string { + let s = ''; + for (let i = 0; i < n; i++) s += Math.floor(rnd() * 16).toString(16); + return s; +} + +export interface GenOptions { days?: number; startTs?: number } + +export function generateDoc(i: number, opts: GenOptions = {}): Record { + const days = opts.days ?? 30; + const start = opts.startTs ?? Date.UTC(2026, 5, 1); + const ts = start + Math.floor(rnd() * days * 86400_000); + const uid = String(1 + Math.floor(rnd() * 50000)); + const did = `${hex(8)}-${hex(4)}-${hex(4)}-${hex(4)}-${hex(12)}`; + const r = rnd(); + const a = pick(APPS); + + let e: string; let sg: Record; + if (r < 0.6) { + e = pick(EVENTS); + sg = { + category: pick(['electronics', 'clothing', 'books', 'toys', 'garden']), + price: Math.round(rnd() * 20000) / 100, + currency: pick(['USD', 'EUR', 'TRY']), + quantity: 1 + Math.floor(rnd() * 5), + source: pick(['organic', 'push', 'deeplink', 'widget']), + logged_in: rnd() > 0.5, + ab_variant: pick(['control', 'variant_a', 'variant_b']), + item_id: `SKU-${Math.floor(rnd() * 100000)}`, + }; + } else if (r < 0.75) { + e = '[CLY]_view'; + sg = { name: pick(VIEWS), visit: 1, start: rnd() > 0.8 ? 1 : 0, bounce: rnd() > 0.9 ? 1 : 0, segment: pick(PLATFORMS) }; + } else if (r < 0.85) { + e = '[CLY]_session'; + sg = { session_id: hex(16) }; + } else if (r < 0.93) { + e = '[CLY]_action'; + sg = { type: 'click', x: Math.floor(rnd() * 1080), y: Math.floor(rnd() * 1920), width: 1080, height: 1920, view: pick(VIEWS) }; + } else if (r < 0.97) { + e = '[CLY]_crash'; + sg = { group: hex(32), fatal: rnd() > 0.5, os: pick(PLATFORMS), app_version: `4.${Math.floor(rnd() * 10)}.0` }; + } else { + e = '[CLY]_nps'; + sg = { widget_id: hex(24), rating: Math.floor(rnd() * 11), platform: pick(PLATFORMS) }; + } + + const doc: Record = { + _id: `${hex(40)}_${uid}_${ts}_${i}`, + a, e, uid, did, + ts, + cd: new Date(ts + Math.floor(rnd() * 30000)), + lu: new Date(ts + Math.floor(rnd() * 60000)), + up: { + p: pick(PLATFORMS), + pv: `p${Math.floor(rnd() * 15)}.${Math.floor(rnd() * 9)}`, + d: pick(['iPhone14,2', 'SM-G991B', 'Pixel 7', 'iPad13,4']), + av: `4.${Math.floor(rnd() * 12)}.${Math.floor(rnd() * 9)}`, + cc: pick(COUNTRIES), + cty: pick(['Istanbul', 'Berlin', 'London', 'Tokyo', 'Austin', 'Unknown']), + la: pick(['en', 'de', 'tr', 'ja', 'pt']), + src: pick(['com.android.vending', 'App Store', 'web']), + dnst: pick(['wifi', '4g', '5g']), + brw: pick(['Chrome', 'Safari', 'Firefox']), + fs: ts - Math.floor(rnd() * 300 * 86400_000), + ls: ts - Math.floor(rnd() * 30 * 86400_000), + sc: Math.floor(rnd() * 500), + }, + c: 1, + }; + if (rnd() > 0.7) doc.s = Math.round(rnd() * 10000) / 100; + if (rnd() > 0.6) doc.dur = Math.round(rnd() * 600000) / 1000; + if (rnd() > 0.8) doc.custom = { plan: pick(['free', 'pro', 'enterprise']), seats: 1 + Math.floor(rnd() * 50) }; + if (rnd() > 0.9) doc.cmp = { c: pick(['summer_sale', 'onboarding_v2']), m: pick(['email', 'push']) }; + return doc; +} diff --git a/bench/kill-drill.ts b/bench/kill-drill.ts new file mode 100644 index 0000000..d48ca96 --- /dev/null +++ b/bench/kill-drill.ts @@ -0,0 +1,114 @@ +/** + * Kill drill for the ledger engine: repeatedly SIGKILL the service at random + * points mid-migration and restart it, until the run completes. Then verify: + * 1. every source doc is accounted for (rows in CH == mongo docs - skipped) + * 2. zero duplicates (count() == uniqExact(_id)) + * + * Env: same AB_* vars as setup.ts, plus KILL_MIN_MS / KILL_MAX_MS (5000/20000). + */ +import { spawn } from 'node:child_process'; +import { MongoClient } from 'mongodb'; +import { createClient } from '@clickhouse/client'; + +const MONGO_URI = process.env.AB_MONGO_URI ?? 'mongodb://localhost:27017'; +const MONGO_DB = process.env.AB_MONGO_DB ?? 'mig_ab'; +const CH_URL = process.env.AB_CH_URL ?? 'http://localhost:8123'; +const CH_DB = process.env.AB_CH_DB ?? 'mig_ab'; +const PORT = Number(process.env.AB_PORT ?? 18081); +const KILL_MIN = Number(process.env.KILL_MIN_MS ?? 5_000); +const KILL_MAX = Number(process.env.KILL_MAX_MS ?? 20_000); +const MAX_ROUNDS = Number(process.env.KILL_MAX_ROUNDS ?? 60); + +const env: NodeJS.ProcessEnv = { + ...process.env, + SERVICE_NAME: 'kill-drill', + SERVICE_PORT: String(PORT), + MONGO_URI, + MONGO_DB, + MANIFEST_DB: `${MONGO_DB}_manifest`, + CLICKHOUSE_URL: CH_URL, + CLICKHOUSE_DB: CH_DB, + EXIT_ON_COMPLETE: 'true', + LOG_LEVEL: 'warn', + NODE_ENV: 'production', +}; +env.LEDGER_RUN_ID = process.env.LEDGER_RUN_ID ?? 'kill-drill-1'; +env.LEDGER_CHUNK_DOCS_TARGET = process.env.LEDGER_CHUNK_DOCS_TARGET ?? '25000'; +env.MULTI_POD_ENABLED = 'false'; + +function sleep(ms: number) { return new Promise((r) => setTimeout(r, ms)); } + +async function isComplete(): Promise { + try { + const res = await fetch(`http://localhost:${PORT}/stats`); + const stats = (await res.json()) as { status?: string }; + return stats.status === 'completed'; + } catch { return false; } +} + +async function main() { + let round = 0; + let completed = false; + + while (!completed && round < MAX_ROUNDS) { + round++; + const child = spawn('node', ['--experimental-strip-types', 'src/main.ts'], { + env, stdio: ['ignore', 'inherit', 'inherit'], + }); + const exited = new Promise((resolve) => child.on('exit', resolve)); + + const killAfter = KILL_MIN + Math.random() * (KILL_MAX - KILL_MIN); + const timer = setTimeout(() => { + console.log(`\n[drill] round ${round}: SIGKILL after ${(killAfter / 1000).toFixed(1)}s`); + child.kill('SIGKILL'); + }, killAfter); + + const code = await exited; + clearTimeout(timer); + + if (code === 0) { + completed = true; + console.log(`[drill] round ${round}: service completed and exited cleanly`); + } else { + // Killed (or crashed) — brief pause, then restart-resume. + await sleep(500); + } + void isComplete; + } + + if (!completed) { + console.error(`[drill] did not complete within ${MAX_ROUNDS} rounds`); + process.exit(1); + } + + // ── Verification ── + const mc = new MongoClient(MONGO_URI); + await mc.connect(); + const mongoDocs = await mc.db(MONGO_DB).collection('drill_events').countDocuments(); + await mc.close(); + + const ch = createClient({ url: CH_URL, database: CH_DB }); + const res = await ch.query({ + query: `SELECT count() AS total, uniqExact(_id) AS distinct_ids FROM ${CH_DB}.drill_events`, + format: 'JSONEachRow', + }); + const [row] = await res.json<{ total: string; distinct_ids: string }>(); + await ch.close(); + + const total = Number(row.total); + const distinct = Number(row.distinct_ids); + const dups = total - distinct; + const missing = mongoDocs - total; // generator produces no skippable docs + + console.log(`\n[drill] RESULT after ${round} rounds (${round - 1} kills):`); + console.log(` mongo source docs: ${mongoDocs}`); + console.log(` clickhouse rows: ${total}`); + console.log(` distinct _ids: ${distinct}`); + console.log(` duplicates: ${dups}`); + console.log(` missing: ${missing}`); + const pass = dups === 0 && missing === 0; + console.log(pass ? ' ✅ PASS — zero loss, zero duplicates' : ' ❌ FAIL'); + process.exit(pass ? 0 : 1); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/bench/poison-drill.ts b/bench/poison-drill.ts new file mode 100644 index 0000000..0173cc9 --- /dev/null +++ b/bench/poison-drill.ts @@ -0,0 +1,131 @@ +/** + * Poison-pill drill: a document that CRASHES the process every time it is + * touched (simulated via the LEDGER_TEST_CRASH_ID chaos hook). + * + * Expected behavior: crash → restart → after 3 crash-retries the chunk is + * SPLIT into sub-chunks instead of retried — repeated splitting converges on + * a ≤1-minute window around the poison doc, which is then quarantined as a + * tiny failed chunk. Everything else migrates. Phase 2 removes the "poison" + * (operator fixed the doc / a patched build) and retries the failed chunks → + * exact convergence. + * + * Env: AB_* as setup.ts. Uses its own scratch db mig_poison. + */ +import { spawn } from 'node:child_process'; +import { MongoClient } from 'mongodb'; +import { createClient } from '@clickhouse/client'; + +const MONGO_URI = process.env.AB_MONGO_URI ?? 'mongodb://localhost:27017'; +const DB = 'mig_poison'; +const CH_URL = process.env.AB_CH_URL ?? 'http://localhost:8123'; +const PORT = 18106; +const DOCS = 20_000; +const POISON_ID = 'POISON_PILL_DOC'; +const MAX_ROUNDS = 120; + +const baseEnv = { + ...process.env, + SERVICE_NAME: 'poison-drill', + SERVICE_PORT: String(PORT), + MONGO_URI, + MONGO_DB: DB, + MANIFEST_DB: `${DB}_manifest`, + CLICKHOUSE_URL: CH_URL, + CLICKHOUSE_DB: DB, + LEDGER_RUN_ID: 'poison-1', + LEDGER_CHUNK_DOCS_TARGET: '5000', + MULTI_POD_ENABLED: 'false', + EXIT_ON_COMPLETE: 'true', + LOG_LEVEL: 'error', + NODE_ENV: 'production', +}; + +function runEngine(extra: Record): Promise { + return new Promise((resolve) => { + const child = spawn('node', ['--experimental-strip-types', 'src/main.ts'], { + env: { ...baseEnv, ...extra }, stdio: ['ignore', 'ignore', 'inherit'], + }); + child.on('exit', resolve); + }); +} + +async function main() { + // ── Seed ── + const mc = new MongoClient(MONGO_URI); + await mc.connect(); + await mc.db(DB).dropDatabase(); + await mc.db(`${DB}_manifest`).dropDatabase(); + const coll = mc.db(DB).collection('drill_events'); + const base = Date.UTC(2026, 0, 1); + const spanMs = 30 * 86400_000; + const docs: Record[] = []; + for (let i = 0; i < DOCS; i++) { + const ts = base + Math.floor((spanMs * i) / DOCS); + docs.push({ _id: `d${i}`, a: 'app1', e: 'ev', uid: String(i % 20), did: 'x', ts, cd: new Date(ts), sg: { v: i }, c: 1 }); + } + const poisonTs = base + Math.floor(spanMs * 0.37); + docs.push({ _id: POISON_ID, a: 'app1', e: 'ev', uid: 'p', did: 'x', ts: poisonTs, cd: new Date(poisonTs), sg: { v: -1 }, c: 1 }); + await coll.insertMany(docs as never[]); + await coll.createIndex({ cd: 1, _id: 1 }); + + const ch = createClient({ url: CH_URL }); + await ch.command({ query: `CREATE DATABASE IF NOT EXISTS ${DB}` }); + await ch.command({ query: `DROP TABLE IF EXISTS ${DB}.drill_events` }); + await ch.command({ + query: `CREATE TABLE ${DB}.drill_events ( + \`a\` LowCardinality(String), \`e\` LowCardinality(String), \`n\` String, + \`uid\` String, \`uid_canon\` Nullable(String), \`did\` String, \`lsid\` Nullable(String), + \`_id\` String, \`ts\` DateTime64(3), \`up\` JSON(max_dynamic_paths = 32), + \`custom\` Nullable(JSON(max_dynamic_paths = 0)), \`cmp\` Nullable(JSON(max_dynamic_paths = 0)), + \`sg\` JSON(max_dynamic_paths = 0), \`c\` UInt32, \`s\` Float64, \`dur\` Float64, + \`lu\` Nullable(DateTime64(3)), \`cd\` DateTime64(3) DEFAULT now64(3)) + ENGINE = MergeTree PARTITION BY toYYYYMM(ts, 'UTC') ORDER BY (a, e, n, ts)`, + }); + console.log(`[poison] seeded ${DOCS} clean docs + 1 poison pill (${POISON_ID})`); + + // ── Phase 1: crash-loop until convergence ── + let rounds = 0; + let code: number | null = -1; + while (code !== 0 && rounds < MAX_ROUNDS) { + rounds++; + code = await runEngine({ LEDGER_TEST_CRASH_ID: POISON_ID }); + } + if (code !== 0) { console.error(`[poison] did not converge in ${MAX_ROUNDS} rounds`); process.exit(1); } + + const ledger = mc.db(`${DB}_manifest`).collection('mig_ranges'); + const statuses = await ledger.aggregate<{ _id: string; n: number }>([ + { $match: { run_id: 'poison-1' } }, { $group: { _id: '$status', n: { $sum: 1 } } }, + ]).toArray(); + const failed = await ledger.find({ run_id: 'poison-1', status: 'failed' }).toArray(); + const res1 = await ch.query({ query: `SELECT count() AS t FROM ${DB}.drill_events`, format: 'JSONEachRow' }); + const t1 = Number((await res1.json<{ t: string }>())[0].t); + console.log(`[poison] phase 1 done in ${rounds} rounds (crashes + splits)`); + console.log(`[poison] chunk statuses: ${JSON.stringify(Object.fromEntries(statuses.map(s => [s._id, s.n])))}`); + for (const f of failed) { + const windowMin = ((f.upper_cd - f.lower_cd) / 60_000).toFixed(1); + const inWindow = await coll.countDocuments({ cd: { $gte: new Date(f.lower_cd), $lt: new Date(f.upper_cd) } }); + console.log(`[poison] quarantined chunk #${f.idx}: ${windowMin} min window, ${inWindow} source docs in it`); + } + console.log(`[poison] rows migrated so far: ${t1} / ${DOCS + 1}`); + + // ── Phase 2: "operator fixed the doc" — retry without the poison ── + await ledger.updateMany( + { run_id: 'poison-1', status: 'failed' }, + { $set: { status: 'pending', pod_id: null, staging_table: null, attempts: 0, last_error: null } }, + ); + code = await runEngine({}); // no crash hook + const res2 = await ch.query({ query: `SELECT count() AS t, uniqExact(_id) AS u FROM ${DB}.drill_events`, format: 'JSONEachRow' }); + const [r2] = await res2.json<{ t: string; u: string }>(); + const pass = code === 0 && Number(r2.t) === DOCS + 1 && Number(r2.u) === DOCS + 1; + console.log(`[poison] phase 2 (post-fix retry): rows=${r2.t} uniq=${r2.u} of ${DOCS + 1}`); + console.log(pass ? '[poison] ✅ PASS — poison localized, everything else migrated, exact after fix' : '[poison] ❌ FAIL'); + + await ch.command({ query: `DROP DATABASE IF EXISTS ${DB}` }).catch(() => {}); + await ch.close(); + await mc.db(DB).dropDatabase(); + await mc.db(`${DB}_manifest`).dropDatabase(); + await mc.close(); + process.exit(pass ? 0 : 1); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/bench/seed-failures.ts b/bench/seed-failures.ts new file mode 100644 index 0000000..e06457c --- /dev/null +++ b/bench/seed-failures.ts @@ -0,0 +1,61 @@ +/** + * Seed a failure-scenario dataset on top of bench/setup.ts's clean docs: + * - a burst of unmigratable docs (bad ts) concentrated in ONE chunk's cd + * window → trips the circuit breaker (>5% of that chunk fails) + * - a few scattered unmigratable docs → DLQ capture without a breaker trip + * - a few oversized-integer segmentation values → coercion counters + * + * Run AFTER bench/setup.ts, same env vars. + */ +import { MongoClient } from 'mongodb'; + +const MONGO_URI = process.env.AB_MONGO_URI ?? 'mongodb://localhost:27017'; +const MONGO_DB = process.env.AB_MONGO_DB ?? 'mig_ab'; +const BURST = Number(process.env.FAIL_BURST ?? 800); +const SCATTER = Number(process.env.FAIL_SCATTER ?? 5); + +async function main() { + const mc = new MongoClient(MONGO_URI); + await mc.connect(); + const coll = mc.db(MONGO_DB).collection('drill_events'); + + const bounds = await coll + .aggregate<{ min: Date; max: Date }>([{ $group: { _id: null, min: { $min: '$cd' }, max: { $max: '$cd' } } }]) + .toArray(); + const min = bounds[0].min.getTime(); + const max = bounds[0].max.getTime(); + const span = max - min; + + const docs: Record[] = []; + // Burst: concentrated in the OLDEST 2% of the cd span (the last chunk to be + // processed, since work goes newest-first — the breaker fires at the end, + // after the healthy chunks are already done). + for (let i = 0; i < BURST; i++) { + docs.push({ + _id: `burst_bad_${i}`, a: 'app1', e: 'corrupted_event', uid: `u${i}`, + ts: 'not-a-timestamp', cd: new Date(min + Math.floor((span * 0.02 * i) / BURST)), + }); + } + // Scatter: a handful of bad docs spread across the middle of the span. + for (let i = 0; i < SCATTER; i++) { + docs.push({ + _id: `scatter_bad_${i}`, a: 'app1', e: 'odd_event', uid: `s${i}`, + ts: 'garbage', cd: new Date(min + Math.floor(span * 0.3) + i * 60_000), + }); + } + // Coercion: oversized integers in customer segmentation. + for (let i = 0; i < 3; i++) { + docs.push({ + _id: `coerce_${i}`, a: 'app1', e: 'big_numbers', uid: `c${i}`, did: 'd', + ts: min + Math.floor(span * 0.5) + i * 60_000, + cd: new Date(min + Math.floor(span * 0.5) + i * 60_000), + sg: { order_id: 9.2e25, weird: Number.POSITIVE_INFINITY }, c: 1, + }); + } + + await coll.insertMany(docs as never[], { ordered: false }); + console.log(`Seeded failure scenarios: ${BURST} burst (oldest 2% of span), ${SCATTER} scattered, 3 coercion docs`); + await mc.close(); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/bench/setup.ts b/bench/setup.ts new file mode 100644 index 0000000..f93efd6 --- /dev/null +++ b/bench/setup.ts @@ -0,0 +1,72 @@ +/** + * Seed a scratch A/B environment: synthetic drill docs in MongoDB + a clone + * of the production drill_events DDL in ClickHouse. + * + * Env: AB_DOCS (250000), AB_MONGO_URI, AB_MONGO_DB (mig_ab), AB_CH_URL, AB_CH_DB (mig_ab) + */ +import { MongoClient } from 'mongodb'; +import { createClient } from '@clickhouse/client'; +import { generateDoc } from './gen.ts'; + +const DOCS = Number(process.env.AB_DOCS ?? 250_000); +const MONGO_URI = process.env.AB_MONGO_URI ?? 'mongodb://localhost:27017'; +const MONGO_DB = process.env.AB_MONGO_DB ?? 'mig_ab'; +const CH_URL = process.env.AB_CH_URL ?? 'http://localhost:8123'; +const CH_DB = process.env.AB_CH_DB ?? 'mig_ab'; +const COLL = 'drill_events'; + +const CH_DDL_COLS = ` + \`a\` LowCardinality(String), + \`e\` LowCardinality(String), + \`n\` String, + \`uid\` String, + \`uid_canon\` Nullable(String), + \`did\` String, + \`lsid\` Nullable(String), + \`_id\` String, + \`ts\` DateTime64(3), + \`up\` JSON(max_dynamic_paths = 32), + \`custom\` Nullable(JSON(max_dynamic_paths = 0)), + \`cmp\` Nullable(JSON(max_dynamic_paths = 0)), + \`sg\` JSON(max_dynamic_paths = 0), + \`c\` UInt32, + \`s\` Float64, + \`dur\` Float64, + \`lu\` Nullable(DateTime64(3)) CODEC(Delta(8), LZ4), + \`cd\` DateTime64(3) DEFAULT now64(3) CODEC(Delta(8), LZ4)`; + +async function main() { + const ch = createClient({ url: CH_URL }); + await ch.command({ query: `CREATE DATABASE IF NOT EXISTS ${CH_DB}` }); + await ch.command({ query: `DROP TABLE IF EXISTS ${CH_DB}.${COLL}` }); + await ch.command({ + query: `CREATE TABLE ${CH_DB}.${COLL} (${CH_DDL_COLS}, + INDEX uid_bloom uid TYPE bloom_filter(0.01) GRANULARITY 4, + INDEX cd_minmax cd TYPE minmax GRANULARITY 1) + ENGINE = MergeTree PARTITION BY toYYYYMM(ts, 'UTC') ORDER BY (a, e, n, ts) + SETTINGS index_granularity = 8192`, + }); + await ch.close(); + console.log(`ClickHouse: ${CH_DB}.${COLL} created`); + + const mc = new MongoClient(MONGO_URI); + await mc.connect(); + const coll = mc.db(MONGO_DB).collection(COLL); + await coll.drop().catch(() => {}); + + const t0 = performance.now(); + const CHUNK = 5000; + for (let done = 0; done < DOCS; done += CHUNK) { + const n = Math.min(CHUNK, DOCS - done); + const docs = Array.from({ length: n }, (_, j) => generateDoc(done + j)); + await coll.insertMany(docs as never[], { ordered: false }); + } + console.log(`Mongo: inserted ${DOCS} docs in ${((performance.now() - t0) / 1000).toFixed(1)}s`); + + await coll.createIndex({ cd: 1, _id: 1 }); + const stats = await mc.db(MONGO_DB).command({ collStats: COLL }); + console.log(`Mongo: ${(stats.size / 1e6).toFixed(1)} MB BSON, avg doc ${stats.avgObjSize.toFixed(0)} B, index ready`); + await mc.close(); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/docker-compose.yml b/docker-compose.yml index d659823..3a79f52 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,53 +1,10 @@ +# Runs ONLY the migrator. It connects to YOUR existing MongoDB and ClickHouse +# via env vars — copy .env.example to .env and point it at them. +# (From a container, "localhost" on the host machine is host.docker.internal.) services: - redis: - image: redis:7-alpine - volumes: - - redis-data:/data - command: redis-server --appendonly yes - ports: - - "6379:6379" - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - mongodb: - image: mongo:7 - volumes: - - mongo-data:/data/db - ports: - - "27017:27017" - healthcheck: - test: ["CMD", "mongosh", "--quiet", "--eval", "db.runCommand({ping:1})"] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - migration: build: . + env_file: .env ports: - - "8080:8080" - env_file: - - path: .env - required: false - extra_hosts: - - "host.docker.internal:host-gateway" - depends_on: - redis: - condition: service_healthy - mongodb: - condition: service_healthy + - "${SERVICE_PORT:-8080}:${SERVICE_PORT:-8080}" restart: unless-stopped - healthcheck: - test: ["CMD", "node", "-e", "fetch('http://localhost:8080/healthz').then(r=>{if(!r.ok)throw 1})"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 15s - -volumes: - redis-data: - mongo-data: diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md new file mode 100644 index 0000000..6c2bb9d --- /dev/null +++ b/docs/RUNBOOK.md @@ -0,0 +1,85 @@ +# Migration Runbook + +Operational procedure for migrating a customer's `drill_events` from MongoDB +to ClickHouse with this service. The guiding property: **after cutover, no +failure anywhere in this flow can touch live data** — every incident response +is *restart or resume*, never clean up or restore. Ingestion pauses exactly +once, for minutes, at cutover — never for the migration. + +## The flow + +1. **Prepare** (old cluster still live, no customer impact) + - Deploy the new stack alongside the old. + - Set Kafka `drill-events` retention to cover the migration window + (14 days default). Replication factor is the customer's redundancy + choice — RF≥2 recommended for large instances; if RF=1, record the + accepted risk (one broker disk loss forfeits the replay guarantee). + - Bulk pre-copy the stateful set: apps & app keys, `app_users`, event + definitions, dashboard users, plugin configs, aggregated data. + +2. **Index** — start `{cd:1,_id:1}` builds on all `drill_events*` collections + now (background, throttled, secondaries where possible). ~1–3 days for + 10 TB; this must not sit inside the post-cutover window. The service also + builds missing indexes itself, but starting early overlaps the wait. + No collection consolidation is ever needed. + +3. **Rehearse** — dry run with `DRY_RUN=1` (≤5% stratified sample against a + Null-engine clone; full ClickHouse validation, nothing stored). Review + `GET /report` (skips, coercions per key, DLQ) with the customer, sign off. + +4. **Cutover** — stop old ingestion → sync the stateful-set delta since the + pre-copy (changed users via last-seen; aggregated data must land BEFORE + new ingestion writes current-period docs) → enable ingestion on the new + stack. `app_users` must be complete first or new ingestion mints colliding + uids. SDK offline queues absorb the window (minutes with pre-copy+delta). + The old MongoDB is now FROZEN — which is what makes everything after this + safe to redo. + +5. **Migrate** — start the service (see README env vars; scale with pods — + they claim chunks via leases). Newest data first: the last 30 days are + visible within hours; the full backfill runs for days with zero impact on + live ingestion. Watch `/viz`; the invariant monitor spot-checks + continuously. + +6. **Finish** — all chunks done → final `GET /report` → customer sign-off → + revert Kafka retention → decommission old cluster. + +## Incident responses + +| Incident | What happens | Operator action | +|---|---|---| +| A doc can't be inserted / converted | Isolated automatically (bisection), stored in DLQ with the full raw doc; run continues | Later: fix the transform rule (platform-first, sync goldens) or fix the stored raw doc, then `POST /control/replay-dlq`. Docs that keep failing stay pending with an updated error — terminal outcomes are fix-and-replay or `POST /control/waive-dlq` (explicitly accept non-migration; raw docs are retained as the record). Sign-off requires pending = 0 | +| Systematic failures (>5% of a chunk) | Circuit breaker pauses the engine; DLQ already names the error | Investigate, fix, `POST /control/retry-failed` (purges + redoes failed chunks, resumes) | +| Migrator crashes / pod dies | Nothing else notices. In-flight chunks are redone from their staging tables; a dead pod's lease expires and others reclaim | Restart the pod. No manual cleanup exists in this flow | +| Live-table rows lost/corrupted for a done chunk | Invariant monitor detects the count mismatch, pauses, flags the chunk | `POST /control/retry-failed` — the chunk's cd window is purged and redone | +| A doc CRASHES the process every time (poison pill) | After 3 crash-retries the chunk is auto-split instead of retried; repeated splitting converges on a ≤1-min window quarantined as a tiny failed chunk — everything else migrates (verified: 20k-doc drill localized 1 poison doc to a 2-doc window in 25 restarts) | Inspect the few source docs in the failed chunk's cd window; fix/remove them, then `POST /control/retry-failed` | +| Live ClickHouse itself must be rebuilt | Live events still sit in the Kafka log; history still sits in frozen Mongo | Recreate table → reset ONLY the ClickHouse-sink connector's offsets to earliest (aggregator groups untouched) → re-run the migrator | + +## Verification cheat sheet + +```sql +-- exactness (instant, exact): +SELECT count() AS total, uniqExact(_id) AS distinct_ids FROM countly_drill.drill_events; +-- full re-verification of the whole migration in minutes: +-- grouped count per chunk window vs the ledger's rows_expected (mig_ranges) +``` + +The ledger (`mig_ranges`) and DLQ (`mig_dlq_docs`) live in `MANIFEST_DB`. +Recovery never trusts the ledger blindly — every claim it makes is verified +against actual row counts before anything irreversible happens. + +## In-place upgrades (same cluster, MongoDB stays) + +Phases 1 & 4 collapse to a config flip (no stateful copy, easy rollback while +the old drill collections still exist). Watch instead: resource contention +(throttle the migrator, read from a secondary, build indexes off-peak), peak +disk (Mongo keeps its data while ClickHouse + staging grow beside it — drop +old per-event collections only after their chunks are done and signed off), +and hard memory limits on the new components — an OOM there is a production +incident. + +## Validation before a customer run + +`bench/README.md`: seed → straight run (counts must be exact) → SIGKILL crash +drill → optionally `bench/seed-failures.ts` for a full failure-scenario drill +(breaker, DLQ, monitor, retry-failed). diff --git a/k8s/job.yaml b/k8s/job.yaml new file mode 100644 index 0000000..9140e69 --- /dev/null +++ b/k8s/job.yaml @@ -0,0 +1,39 @@ +# Drill migration on Kubernetes — Job variant (fire-and-forget automation). +# +# Requires the ConfigMap and Secret from k8s/migration.yaml. +# +# EXIT_ON_COMPLETE=true makes each pod exit 0 once every chunk of the run is +# terminal, so the Job completes on its own. A crashed pod restarts +# (OnFailure) and resumes from the ledger — crash-redo is the designed path. +# +# Note: when the Job finishes, the dashboard goes away with the pods. For an +# operator-driven migration (watch progress, verify, sign off in the UI), +# prefer the Deployment in k8s/migration.yaml. +apiVersion: batch/v1 +kind: Job +metadata: + name: drill-migration +spec: + parallelism: 3 # = pods working the run; scale across nodes + completions: 3 # keep equal to parallelism (every pod exits 0 on completion) + backoffLimit: 50 # crash-redo is normal operation, not failure — be generous + template: + metadata: + labels: { app: drill-migrator } + spec: + restartPolicy: OnFailure + terminationGracePeriodSeconds: 30 + containers: + - name: migrator + image: your-registry/drill-migrator:latest + ports: + - containerPort: 8080 + env: + - name: EXIT_ON_COMPLETE + value: "true" + envFrom: + - configMapRef: { name: drill-migrator-config } + - secretRef: { name: drill-migrator-secrets } + resources: + requests: { cpu: "2", memory: 2Gi } + limits: { cpu: "4", memory: 6Gi } diff --git a/k8s/migration.yaml b/k8s/migration.yaml new file mode 100644 index 0000000..33ce35a --- /dev/null +++ b/k8s/migration.yaml @@ -0,0 +1,91 @@ +# Drill migration on Kubernetes — Deployment variant (recommended). +# +# Pods coordinate through chunk leases in MongoDB — no shared service, no +# operator choreography. Each pod claims chunks atomically; a killed pod's +# lease expires and others reclaim its chunk (drop staging, redo). Abrupt +# kills, evictions and OOMs are safe by design, so no preStop hook is needed. +# +# Pod identity: POD_ID defaults to the hostname, which in Kubernetes is the +# (unique) pod name — nothing to configure. +# +# Scale with `kubectl scale deployment/drill-migrator --replicas=N`. +# Pods scale across NODES, not on one machine: a single pod saturates ~4 +# cores on BSON decode; more pods on the same node just contend. +# +# When the run completes, pods stay up serving the dashboard (progress, +# verification, sign-off gates). Scale to 0 / delete when signed off. +# For fire-and-forget automation use k8s/job.yaml instead. +# +# Dashboard: kubectl port-forward svc/drill-migrator 8080:8080 +# then open http://localhost:8080 — chunk state lives in MongoDB, so ANY +# pod's dashboard shows the whole run. +apiVersion: v1 +kind: ConfigMap +metadata: + name: drill-migrator-config +data: + SERVICE_NAME: "drill-migrator" + MONGO_DB: "countly_drill" + MONGO_COUNTLY_DB: "countly" + MANIFEST_DB: "countly_drill" + CLICKHOUSE_DB: "countly_drill" + CLICKHOUSE_TABLE: "drill_events" + # Stable resume key — keep identical across ALL pods and restarts. + LEDGER_RUN_ID: "migration-1" + # On a replica set, offload the primary (exact reads — source is frozen): + #MONGO_READ_PREFERENCE: "secondaryPreferred" +--- +apiVersion: v1 +kind: Secret +metadata: + name: drill-migrator-secrets +type: Opaque +stringData: + MONGO_URI: "mongodb://mongo.example.internal:27017" + CLICKHOUSE_URL: "http://clickhouse.example.internal:8123" + #CLICKHOUSE_USERNAME: "default" + #CLICKHOUSE_PASSWORD: "" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: drill-migrator + labels: { app: drill-migrator } +spec: + replicas: 1 + selector: + matchLabels: { app: drill-migrator } + template: + metadata: + labels: { app: drill-migrator } + spec: + terminationGracePeriodSeconds: 30 + containers: + - name: migrator + image: your-registry/drill-migrator:latest # docker build -t … . && push + ports: + - containerPort: 8080 + envFrom: + - configMapRef: { name: drill-migrator-config } + - secretRef: { name: drill-migrator-secrets } + resources: + requests: { cpu: "2", memory: 2Gi } + limits: { cpu: "4", memory: 6Gi } # node heap is capped at 4G in the image + livenessProbe: + httpGet: { path: /healthz, port: 8080 } + initialDelaySeconds: 10 + periodSeconds: 15 + readinessProbe: + httpGet: { path: /healthz, port: 8080 } + initialDelaySeconds: 5 + periodSeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: drill-migrator +spec: + selector: { app: drill-migrator } + ports: + - port: 8080 + targetPort: 8080 diff --git a/package-lock.json b/package-lock.json index dd7b449..b06a6d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,6 @@ "dependencies": { "@clickhouse/client": "^1.8.0", "fastify": "^5.2.1", - "ioredis": "^5.4.2", "mongodb": "^6.12.0", "pino": "^9.6.0", "zod": "^3.24.2" @@ -188,12 +187,6 @@ "ipaddr.js": "^2.1.0" } }, - "node_modules/@ioredis/commands": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz", - "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==", - "license": "MIT" - }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -783,15 +776,6 @@ "node": ">=18" } }, - "node_modules/cluster-key-slot": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", - "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", @@ -829,32 +813,6 @@ "node": "*" } }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/denque": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.10" - } - }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -1082,30 +1040,6 @@ "dev": true, "license": "MIT" }, - "node_modules/ioredis": { - "version": "5.10.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.0.tgz", - "integrity": "sha512-HVBe9OFuqs+Z6n64q09PQvP1/R4Bm+30PAyyD4wIEqssh3v9L21QjCVk4kRLucMBcDokJTcLjsGeVRlq/nH6DA==", - "license": "MIT", - "dependencies": { - "@ioredis/commands": "1.5.1", - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.4", - "denque": "^2.1.0", - "lodash.defaults": "^4.2.0", - "lodash.isarguments": "^3.1.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - }, - "engines": { - "node": ">=12.22.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" - } - }, "node_modules/ipaddr.js": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.3.0.tgz", @@ -1448,18 +1382,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/lodash.defaults": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", - "license": "MIT" - }, - "node_modules/lodash.isarguments": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", - "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", - "license": "MIT" - }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -1542,12 +1464,6 @@ "whatwg-url": "^14.1.0 || ^13.0.0" } }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -1776,27 +1692,6 @@ "node": ">= 12.13.0" } }, - "node_modules/redis-errors": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/redis-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", - "license": "MIT", - "dependencies": { - "redis-errors": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -1981,12 +1876,6 @@ "dev": true, "license": "MIT" }, - "node_modules/standard-as-callback": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", - "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", - "license": "MIT" - }, "node_modules/std-env": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.0.0.tgz", diff --git a/package.json b/package.json index f996b40..e5f754b 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,6 @@ "dependencies": { "@clickhouse/client": "^1.8.0", "fastify": "^5.2.1", - "ioredis": "^5.4.2", "mongodb": "^6.12.0", "pino": "^9.6.0", "zod": "^3.24.2" diff --git a/src/config/loader.ts b/src/config/loader.ts index 7470ffc..8482642 100644 --- a/src/config/loader.ts +++ b/src/config/loader.ts @@ -12,12 +12,27 @@ import { configSchema, type Config } from "./schema.ts"; */ function envToRawConfig(env: NodeJS.ProcessEnv) { return { + ledger: { + runId: env.LEDGER_RUN_ID, + chunkDocsTarget: env.LEDGER_CHUNK_DOCS_TARGET, + insertInflight: env.LEDGER_INSERT_INFLIGHT, + leaseSec: env.LEDGER_LEASE_SEC, + breakerPct: env.LEDGER_BREAKER_PCT, + dlqPauseThreshold: env.LEDGER_DLQ_PAUSE_THRESHOLD, + sourceCountCheck: env.LEDGER_SOURCE_COUNT_CHECK, + breakerConsecutive: env.LEDGER_BREAKER_CONSECUTIVE, + monitorIntervalMs: env.LEDGER_MONITOR_INTERVAL_MS, + maxChunkDays: env.LEDGER_MAX_CHUNK_DAYS, + captureTransformErrors: env.LEDGER_CAPTURE_TRANSFORM_ERRORS, + dryRun: env.DRY_RUN, + dryRunSamplePct: env.DRY_RUN_SAMPLE_PCT, + }, + service: { name: env.SERVICE_NAME, port: env.SERVICE_PORT, host: env.SERVICE_HOST, gracefulShutdownTimeoutMs: env.GRACEFUL_SHUTDOWN_TIMEOUT_MS, - rerunMode: env.RERUN_MODE, exitOnComplete: env.EXIT_ON_COMPLETE, }, @@ -30,13 +45,9 @@ function envToRawConfig(env: NodeJS.ProcessEnv) { readConcern: env.MONGO_READ_CONCERN, retryReads: env.MONGO_RETRY_READS, appName: env.MONGO_APP_NAME, - batchRowsTarget: env.MONGO_BATCH_ROWS_TARGET, mongoPageSize: env.MONGO_PAGE_SIZE, cursorBatchSize: env.MONGO_CURSOR_BATCH_SIZE, maxTimeMs: env.MONGO_MAX_TIME_MS, - rangeParallelThreshold: env.RANGE_PARALLEL_THRESHOLD, - rangeCount: env.RANGE_COUNT, - rangeLeaseTtlSec: env.RANGE_LEASE_TTL_SEC, }, transform: { @@ -53,7 +64,6 @@ function envToRawConfig(env: NodeJS.ProcessEnv) { maxRetries: env.CLICKHOUSE_MAX_RETRIES, retryBaseDelayMs: env.CLICKHOUSE_RETRY_BASE_DELAY_MS, retryMaxDelayMs: env.CLICKHOUSE_RETRY_MAX_DELAY_MS, - useDedupToken: env.CLICKHOUSE_USE_DEDUP_TOKEN, }, backpressure: { @@ -70,32 +80,11 @@ function envToRawConfig(env: NodeJS.ProcessEnv) { state: { manifestDb: env.MANIFEST_DB, - redisUrl: env.REDIS_URL, - redisKeyPrefix: env.REDIS_KEY_PREFIX, - timelineSnapshotInterval: env.TIMELINE_SNAPSHOT_INTERVAL, - }, - - memory: { - gcEnabled: env.GC_ENABLED, - gcRssSoftLimitMb: env.GC_RSS_SOFT_LIMIT_MB, - gcRssHardLimitMb: env.GC_RSS_HARD_LIMIT_MB, - gcHeapUsedRatio: env.GC_HEAP_USED_RATIO, - gcEveryNBatches: env.GC_EVERY_N_BATCHES, - }, - - asyncWrite: { - flushIntervalMs: env.ASYNC_WRITE_FLUSH_INTERVAL_MS, - flushBatchSize: env.ASYNC_WRITE_FLUSH_BATCH_SIZE, }, worker: { podId: env.POD_ID, enabled: env.MULTI_POD_ENABLED, - lockTtlSec: env.LOCK_TTL_SECONDS, - lockRenewMs: env.LOCK_RENEW_MS, - progressUpdateMs: env.PROGRESS_UPDATE_MS, - podHeartbeatMs: env.POD_HEARTBEAT_MS, - podDeadAfterSec: env.POD_DEAD_AFTER_SEC, }, log: { @@ -145,15 +134,7 @@ export function loadConfig(): Config { } // Semantic validation - const { memory, target } = config; - - const rssSoft = memory.gcRssSoftLimitMb * 1024 * 1024; - const rssHard = memory.gcRssHardLimitMb * 1024 * 1024; - if (rssSoft > rssHard) { - throw new Error( - `GC_RSS_SOFT_LIMIT_MB (${memory.gcRssSoftLimitMb}) must be <= GC_RSS_HARD_LIMIT_MB (${memory.gcRssHardLimitMb})`, - ); - } + const { target } = config; if (target.retryBaseDelayMs > target.retryMaxDelayMs) { throw new Error( diff --git a/src/config/schema.ts b/src/config/schema.ts index 77d9925..1394272 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -36,13 +36,44 @@ const positiveIntFromEnv = z // --------------------------------------------------------------------------- export const configSchema = z.object({ + // ── Ledger engine ──────────────────────────────────────────────────── + ledger: z + .object({ + runId: z.string().default("ledger-v1"), + chunkDocsTarget: positiveIntFromEnv.default(2_000_000), + insertInflight: positiveIntFromEnv.default(3), + leaseSec: positiveIntFromEnv.default(600), + // Circuit breaker: pause when >pct% of a chunk's docs fail, or + // after N consecutive failed chunks (systematic-bug detection). + breakerPct: numberFromEnv.default(5).pipe(z.number().min(0).max(100)), + // Per-chunk breakers miss EVENLY-SPREAD failure (1% of every + // chunk never trips 5%-of-one-chunk) — this is the global guard. + dlqPauseThreshold: numberFromEnv.default(1_000_000).pipe(z.number().min(0)), + // Tally-independent per-commit guard: after each chunk promotes, + // ask the SOURCE how many docs its window holds. ~1% overhead. + sourceCountCheck: booleanFromEnv.default(true), + breakerConsecutive: positiveIntFromEnv.default(3), + // Background invariant spot checks (0 disables). + monitorIntervalMs: intFromEnv.default(900_000), + // Capture full raw docs of transform failures into the DLQ. + captureTransformErrors: booleanFromEnv.default(true), + // Upper bound on a chunk's time span. Guards chunk sizing against + // bad estimatedDocumentCount (e.g. metadata fastcount reset after + // an unclean mongod shutdown) — a wrong estimate can never produce + // a whole-collection mega-chunk. + maxChunkDays: numberFromEnv.default(7).pipe(z.number().positive()), + // Dry run: sampled rehearsal against a Null-engine clone. + dryRun: booleanFromEnv.default(false), + dryRunSamplePct: numberFromEnv.default(2).pipe(z.number().min(0.1).max(5)), + }) + .default({}), + // ── Service ────────────────────────────────────────────────────────── service: z.object({ - name: z.string().min(1), + name: z.string().min(1).default("drill-migrator"), port: positiveIntFromEnv.default(8080), host: z.string().default("0.0.0.0"), gracefulShutdownTimeoutMs: intFromEnv.default(60_000), - rerunMode: z.enum(['resume', 'clone-run', 'new-run']).default('resume'), exitOnComplete: booleanFromEnv.default(false), }), @@ -52,17 +83,14 @@ export const configSchema = z.object({ db: z.string().default("countly_drill"), countlyDb: z.string().default("countly"), collectionPrefix: z.string().default("drill_events"), - readPreference: z.string().default("primary"), + readPreference: z.string().default("auto"), + readPreferenceAuto: z.boolean().default(false), readConcern: z.string().default("majority"), retryReads: booleanFromEnv.default(true), appName: z.string().optional(), - batchRowsTarget: positiveIntFromEnv.default(10_000), mongoPageSize: positiveIntFromEnv.default(10_000), cursorBatchSize: positiveIntFromEnv.default(10_000), maxTimeMs: positiveIntFromEnv.default(600_000), - rangeParallelThreshold: intFromEnv.default(500_000), - rangeCount: positiveIntFromEnv.default(100), - rangeLeaseTtlSec: positiveIntFromEnv.default(300), }), // ── Transform ──────────────────────────────────────────────────────── @@ -81,7 +109,6 @@ export const configSchema = z.object({ maxRetries: intFromEnv.default(8), retryBaseDelayMs: positiveIntFromEnv.default(1_000), retryMaxDelayMs: positiveIntFromEnv.default(30_000), - useDedupToken: booleanFromEnv.default(true), }), // ── Backpressure ───────────────────────────────────────────────────── @@ -97,38 +124,15 @@ export const configSchema = z.object({ maxPauseEpisodeMs: intFromEnv.default(180_000), }), - // ── State ──────────────────────────────────────────────────────────── + // ── State (chunk ledger + DLQ live here) ───────────────────────────── state: z.object({ manifestDb: z.string().default("countly_drill"), - redisUrl: z.string().min(1), - redisKeyPrefix: z.string().default("mig"), - timelineSnapshotInterval: positiveIntFromEnv.default(10), - }), - - // ── Memory / GC ───────────────────────────────────────────────────── - memory: z.object({ - gcEnabled: booleanFromEnv.default(true), - gcRssSoftLimitMb: intFromEnv.default(3_072), - gcRssHardLimitMb: intFromEnv.default(6_144), - gcHeapUsedRatio: numberFromEnv.default(0.70), - gcEveryNBatches: intFromEnv.default(50), }), // ── Worker / Multi-Pod ────────────────────────────────────────────── worker: z.object({ podId: z.string().default(""), enabled: booleanFromEnv.default(true), - lockTtlSec: positiveIntFromEnv.default(300), - lockRenewMs: positiveIntFromEnv.default(60_000), - progressUpdateMs: positiveIntFromEnv.default(5_000), - podHeartbeatMs: positiveIntFromEnv.default(30_000), - podDeadAfterSec: positiveIntFromEnv.default(180), - }), - - // ── Async Write ────────────────────────────────────────────────────── - asyncWrite: z.object({ - flushIntervalMs: positiveIntFromEnv.default(5_000), - flushBatchSize: positiveIntFromEnv.default(10), }), // ── Logging ────────────────────────────────────────────────────────── diff --git a/src/http/control-route.ts b/src/http/control-route.ts deleted file mode 100644 index 42d0201..0000000 --- a/src/http/control-route.ts +++ /dev/null @@ -1,376 +0,0 @@ -import type { FastifyInstance } from 'fastify'; -import type { RunnerStatus } from '../runtime/batch-runner.ts'; -import type { GcMode } from '../runtime/gc-controller.ts'; -import type { Batch, ManifestStore } from '../state/manifest-store.ts'; -import type { MongoReader, PageResult } from '../source/mongo-reader.ts'; -import type { ClickHouseWriter } from '../target/clickhouse-writer.ts'; -import type { GlobalProgress } from '../state/global-progress.ts'; -import type { CollectionLock } from '../state/collection-lock.ts'; -import type { RedisHotState } from '../state/redis-hot-state.ts'; -import { transformBatch } from '../transform/normalize.ts'; -import { SkipCounter } from '../transform/skip-reasons.ts'; -import { deserializeCursor } from '../types/cursor.ts'; - -export interface ControlDeps { - orchestrator: { - pause(): void; - resume(): void; - stopAfterBatch(): void; - getStatus(): RunnerStatus; - triggerReindex(collectionName: string): void; - retryCollection(collectionName: string): void; - }; - gcController: { - runGc(mode: GcMode, reason: string): Promise; - isAvailable: boolean; - }; - manifestStore: ManifestStore; - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - globalProgress?: GlobalProgress; - collectionLock?: CollectionLock; - redisState?: RedisHotState; -} - -interface GcRequestBody { - mode: GcMode; -} - -export function registerControlRoutes(app: FastifyInstance, deps: ControlDeps): void { - const { orchestrator, gcController, manifestStore, mongoReader, chWriter } = deps; - - // POST /control/pause - pause after current batch - app.post('/control/pause', async (_request, reply) => { - orchestrator.pause(); - return reply.status(200).send({ - ok: true, - status: orchestrator.getStatus(), - }); - }); - - // POST /control/resume - resume batch scheduling - app.post('/control/resume', async (_request, reply) => { - orchestrator.resume(); - return reply.status(200).send({ - ok: true, - status: orchestrator.getStatus(), - }); - }); - - // POST /control/stop-after-batch - stop cleanly after current batch - app.post('/control/stop-after-batch', async (_request, reply) => { - orchestrator.stopAfterBatch(); - return reply.status(200).send({ - ok: true, - status: orchestrator.getStatus(), - }); - }); - - // POST /control/reindex/:collection - trigger index creation for a collection - app.post('/control/reindex/:collection', async (request, reply) => { - const { collection } = request.params as { collection: string }; - orchestrator.triggerReindex(collection); - return reply.status(200).send({ - ok: true, - collection, - message: 'Index build triggered', - }); - }); - - // POST /control/retry-collection/:collection - re-queue a failed/skipped collection - app.post('/control/retry-collection/:collection', async (request, reply) => { - const { collection } = request.params as { collection: string }; - orchestrator.retryCollection(collection); - return reply.status(200).send({ - ok: true, - collection, - message: 'Collection queued for retry', - }); - }); - - // POST /control/retry-batch/:runId/:batchSeq - re-read, re-transform, re-insert a skipped/failed batch - app.post('/control/retry-batch/:runId/:batchSeq', async (request, reply) => { - const { runId, batchSeq } = request.params as { runId: string; batchSeq: string }; - const seq = Number(batchSeq); - - try { - const batches = await manifestStore.getBatches(runId, {}); - const batch = batches.find(b => b.batch_seq === seq); - if (!batch) { - return reply.status(404).send({ ok: false, error: 'Batch not found' }); - } - if (batch.status !== 'skipped_empty' && batch.status !== 'failed') { - return reply.status(400).send({ ok: false, error: `Batch status is "${batch.status}", only skipped_empty/failed can be retried` }); - } - - // Re-read the exact MongoDB range - const lowerCursor = batch.lower_exclusive_cursor ? deserializeCursor(batch.lower_exclusive_cursor) : null; - const upperCursor = deserializeCursor(batch.upper_inclusive_cursor); - const batchPhase = (batch as any).phase ?? "cursor"; - let page: PageResult; - if (batchPhase === "null_cd") { - const lowerId = lowerCursor ? lowerCursor.id : null; - const upperId = upperCursor.id; - page = await mongoReader.readNullCdPage(lowerId, upperId); - } else { - page = await mongoReader.readPage(lowerCursor, upperCursor); - } - - if (page.docs.length === 0) { - return reply.status(200).send({ ok: true, message: 'No documents in range', docsRead: 0, rowsInserted: 0 }); - } - - // Re-transform - const skipCounter = new SkipCounter(); - const { rows } = transformBatch(page.docs, skipCounter); - - if (rows.length === 0) { - return reply.status(200).send({ ok: true, message: 'All documents skipped again', docsRead: page.docs.length, rowsInserted: 0 }); - } - - // Insert into ClickHouse - const result = await chWriter.insertBatch({ runId, batchSeq: seq, rows }); - - // Update batch status in manifest - await manifestStore.updateBatchStatus(runId, seq, 'done'); - - return reply.status(200).send({ - ok: true, - message: 'Batch retried successfully', - docsRead: page.docs.length, - rowsInserted: result.rowsInserted, - }); - } catch (err) { - const error = err instanceof Error ? err.message : String(err); - return reply.status(500).send({ ok: false, error }); - } - }); - - // POST /control/retry-skipped-batches/:runId - retry all skipped_empty batches in a run - app.post('/control/retry-skipped-batches/:runId', async (request, reply) => { - const { runId } = request.params as { runId: string }; - - try { - const batches = await manifestStore.getBatches(runId, { status: 'skipped_empty' as any }); - if (batches.length === 0) { - return reply.status(200).send({ ok: true, message: 'No skipped batches found', retried: 0 }); - } - - let retried = 0; - let totalInserted = 0; - - for (const batch of batches) { - const lowerCursor = batch.lower_exclusive_cursor ? deserializeCursor(batch.lower_exclusive_cursor) : null; - const upperCursor = deserializeCursor(batch.upper_inclusive_cursor); - const batchPhase = (batch as any).phase ?? "cursor"; - let page: PageResult; - if (batchPhase === "null_cd") { - const lowerId = lowerCursor ? lowerCursor.id : null; - const upperId = upperCursor.id; - page = await mongoReader.readNullCdPage(lowerId, upperId); - } else { - page = await mongoReader.readPage(lowerCursor, upperCursor); - } - if (page.docs.length === 0) continue; - - const skipCounter = new SkipCounter(); - const { rows } = transformBatch(page.docs, skipCounter); - if (rows.length === 0) continue; - - const result = await chWriter.insertBatch({ runId, batchSeq: batch.batch_seq, rows }); - await manifestStore.updateBatchStatus(runId, batch.batch_seq, 'done'); - retried++; - totalInserted += result.rowsInserted; - } - - return reply.status(200).send({ ok: true, message: `Retried ${retried} batches`, retried, totalInserted }); - } catch (err) { - const error = err instanceof Error ? err.message : String(err); - return reply.status(500).send({ ok: false, error }); - } - }); - - // POST /control/gc - trigger manual GC - app.post<{ Body: GcRequestBody }>('/control/gc', async (request, reply) => { - if (!gcController.isAvailable) { - return reply.status(400).send({ - ok: false, - error: 'GC not available (--expose-gc not set)', - }); - } - - const { mode } = request.body ?? { mode: 'now' }; - const triggered = await gcController.runGc(mode, 'manual-http-request'); - - return reply.status(200).send({ - ok: true, - mode, - triggered, - }); - }); - - // ── Global control endpoints (multi-pod mode) ───────────────────────── - const { globalProgress } = deps; - - // POST /control/global/pause - pause all pods - app.post('/control/global/pause', async (_request, reply) => { - if (!globalProgress) { - return reply.status(400).send({ ok: false, error: 'Multi-pod mode not enabled' }); - } - await globalProgress.setGlobalCommand('pause', true); - orchestrator.pause(); // Also pause this pod - return reply.status(200).send({ ok: true, message: 'Global pause issued' }); - }); - - // POST /control/global/resume - resume all pods - app.post('/control/global/resume', async (_request, reply) => { - if (!globalProgress) { - return reply.status(400).send({ ok: false, error: 'Multi-pod mode not enabled' }); - } - await globalProgress.setGlobalCommand('pause', false); - orchestrator.resume(); - return reply.status(200).send({ ok: true, message: 'Global resume issued' }); - }); - - // POST /control/global/stop - stop all pods - app.post('/control/global/stop', async (_request, reply) => { - if (!globalProgress) { - return reply.status(400).send({ ok: false, error: 'Multi-pod mode not enabled' }); - } - await globalProgress.setGlobalCommand('stop', true); - orchestrator.stopAfterBatch(); - return reply.status(200).send({ ok: true, message: 'Global stop issued' }); - }); - - // ── Lock + Pod management endpoints ─────────────────────────────────── - const { collectionLock } = deps; - - // GET /control/locks - list all active locks - app.get('/control/locks', async (_request, reply) => { - if (!collectionLock) { - return reply.status(400).send({ ok: false, error: 'Multi-pod mode not enabled' }); - } - const locks = await collectionLock.listAllLocks(); - return reply.status(200).send({ ok: true, locks }); - }); - - // POST /control/locks/release/:collection - force-release a lock (admin) - app.post('/control/locks/release/:collection', async (request, reply) => { - if (!collectionLock) { - return reply.status(400).send({ ok: false, error: 'Multi-pod mode not enabled' }); - } - const { collection } = request.params as { collection: string }; - await collectionLock.forceRelease(collection); - return reply.status(200).send({ ok: true, collection, message: 'Lock force-released' }); - }); - - // GET /control/pods - list all pods with alive status and locks - app.get('/control/pods', async (_request, reply) => { - if (!collectionLock) { - return reply.status(400).send({ ok: false, error: 'Multi-pod mode not enabled' }); - } - const [podKeys, locks] = await Promise.all([ - collectionLock.listAllPodKeys(), - collectionLock.listAllLocks(), - ]); - const alivePodIds = new Set(podKeys.map(p => p.podId)); - // Find pods referenced in locks but not alive - const lockPodIds = [...new Set(locks.map(l => l.podId))]; - const pods = lockPodIds.map(podId => { - const alive = alivePodIds.has(podId); - const podLocks = locks.filter(l => l.podId === podId); - const podInfo = podKeys.find(p => p.podId === podId); - return { - podId, - alive, - lastHeartbeat: podInfo?.lastHeartbeat ?? null, - collectionsActive: podInfo?.collectionsActive ?? [], - locks: podLocks.map(l => l.collectionName), - lockCount: podLocks.length, - }; - }); - // Add alive pods with no locks - for (const pk of podKeys) { - if (!lockPodIds.includes(pk.podId)) { - pods.push({ - podId: pk.podId, - alive: true, - lastHeartbeat: pk.lastHeartbeat, - collectionsActive: pk.collectionsActive, - locks: [], - lockCount: 0, - }); - } - } - return reply.status(200).send({ ok: true, pods }); - }); - - // POST /control/pods/remove/:podId - remove a dead pod's keys and release its locks - app.post('/control/pods/remove/:podId', async (request, reply) => { - if (!collectionLock) { - return reply.status(400).send({ ok: false, error: 'Multi-pod mode not enabled' }); - } - const { podId } = request.params as { podId: string }; - await collectionLock.deletePodKey(podId); - const released = await collectionLock.releaseLocksForPod(podId); - return reply.status(200).send({ ok: true, podId, releasedLocks: released, message: `Pod removed, ${released.length} locks released` }); - }); - - // POST /control/drain - graceful drain for K8s scale-down - app.post('/control/drain', async (_request, reply) => { - orchestrator.stopAfterBatch(); - return reply.status(200).send({ ok: true, message: 'Drain initiated — finishing current batch then releasing locks' }); - }); - - // ── DANGER ZONE ──────────────────────────────────────────────────── - - // POST /control/danger/clear-mongodb - drop all migration state from MongoDB - app.post('/control/danger/clear-mongodb', async (_request, reply) => { - const { manifestStore: ms } = deps; - try { - const runs = await ms.listRuns({ limit: 1000 }); - let totalDeleted = 0; - for (const run of runs.runs) { - totalDeleted += await ms.deleteRunData(run.run_id); - } - // Also delete the run records themselves - const db = (ms as any).client.db((ms as any).dbName); - const runResult = await db.collection('mig_runs').deleteMany({}); - totalDeleted += runResult.deletedCount ?? 0; - return reply.status(200).send({ - ok: true, - message: `Cleared MongoDB migration state: ${totalDeleted} records deleted`, - deletedRecords: totalDeleted, - }); - } catch (err) { - return reply.status(500).send({ ok: false, error: err instanceof Error ? err.message : String(err) }); - } - }); - - // POST /control/danger/clear-redis - flush all migration keys from Redis - app.post('/control/danger/clear-redis', async (_request, reply) => { - const { redisState: rs } = deps; - if (!rs) { - return reply.status(400).send({ ok: false, error: 'Redis not available' }); - } - try { - const redis = rs.getRedisClient(); - const keys = await (async () => { - const found: string[] = []; - const stream = redis.scanStream({ match: 'mig:*', count: 500 }); - for await (const batch of stream) found.push(...(batch as string[])); - return found; - })(); - if (keys.length > 0) { - await redis.unlink(...keys); - } - return reply.status(200).send({ - ok: true, - message: `Cleared Redis migration state: ${keys.length} keys deleted`, - deletedKeys: keys.length, - }); - } catch (err) { - return reply.status(500).send({ ok: false, error: err instanceof Error ? err.message : String(err) }); - } - }); -} diff --git a/src/http/health-route.ts b/src/http/health-route.ts deleted file mode 100644 index 31dbf84..0000000 --- a/src/http/health-route.ts +++ /dev/null @@ -1,53 +0,0 @@ -import type { FastifyInstance } from 'fastify'; -import type { RunnerStatus } from '../runtime/batch-runner.ts'; - -export interface HealthDeps { - mongoReader: { isConnected(): boolean }; - chWriter: { isConnected(): boolean }; - redisState: { isHealthy(): Promise }; - manifestStore: { isWritable(): Promise }; - orchestrator: { getStatus(): RunnerStatus }; -} - -export function registerHealthRoutes(app: FastifyInstance, deps: HealthDeps): void { - const { mongoReader, chWriter, redisState, manifestStore, orchestrator } = deps; - - // GET /healthz - simple liveness (always 200 if server is up) - app.get('/healthz', async (_request, reply) => { - return reply.status(200).send({ status: 'alive' }); - }); - - // GET /readyz - readiness check - app.get('/readyz', async (_request, reply) => { - const checks: Record = { - mongo: false, - clickhouse: false, - redis: false, - manifestStore: false, - batchRunner: false, - }; - - checks.mongo = mongoReader.isConnected(); - checks.clickhouse = chWriter.isConnected(); - - try { - checks.redis = await redisState.isHealthy(); - } catch { - checks.redis = false; - } - - try { - checks.manifestStore = await manifestStore.isWritable(); - } catch { - checks.manifestStore = false; - } - checks.batchRunner = orchestrator.getStatus() !== 'failed'; - - const allHealthy = Object.values(checks).every(Boolean); - - return reply.status(allHealthy ? 200 : 503).send({ - ready: allHealthy, - checks, - }); - }); -} diff --git a/src/http/ledger-viz-route.ts b/src/http/ledger-viz-route.ts new file mode 100644 index 0000000..3e79aa0 --- /dev/null +++ b/src/http/ledger-viz-route.ts @@ -0,0 +1,866 @@ +/** + * Countly-branded operator console for the ledger engine. + * + * Three tabs: + * - Overview: live state (counters, chunk map, failed chunks, DLQ, coercions) + * plus one-click actions with receipts. + * - Guide: the migration runbook as a guided checklist — automated phases + * report their own status (preflight, index, dry run, progress, sign-off + * gates); manual phases are persistent checkboxes (localStorage) so a + * self-hosted customer can walk the whole cutover themselves. + * - Help: the incident-response scenarios from docs/RUNBOOK.md, each with + * the relevant action inline. + * + * Data source: the chunk ledger + DLQ (MongoDB) + in-process engine stats. + * No Redis. Brand tokens sampled from countly.com. + */ + +import type { FastifyInstance } from 'fastify'; +import type { ChunkOrchestrator } from '../runtime/chunk-orchestrator.ts'; +import type { LedgerStore } from '../state/ledger-store.ts'; +import type { DlqStore } from '../state/dlq-store.ts'; +import type { Config } from '../config/schema.ts'; + +export interface LedgerVizDeps { + orchestrator: ChunkOrchestrator; + ledger: LedgerStore; + dlq: DlqStore; + config: Config; +} + +export function registerLedgerVizRoutes(app: FastifyInstance, deps: LedgerVizDeps): void { + const runId = () => deps.config.ledger.dryRun ? `${deps.config.ledger.runId}-dry` : deps.config.ledger.runId; + + app.get('/api/chunks', async () => { + // Summary is O(collections); full chunk details only while they're small + // enough to render (a 10TB run can have tens of thousands of chunks). + const summary = await deps.ledger.summarize(runId()); + const truncated = summary.total > 2_000; + const chunks = truncated + ? await deps.ledger.listActive(runId(), 500) + : await deps.ledger.listAll(runId()); + return { runId: runId(), summary, chunks, truncated }; + }); + + // Counts + grouped errors are aggregations — cheap at thousands, real + // work at a hundred million. Cache 15s so the 2s poll stays harmless. + let dlqAggCache: { at: number; byStatus: Record; topErrors: unknown[]; storage: { dlqBytes: number; dlqDocs: number; diskFreePct: number | null } | null } | null = null; + app.get<{ Querystring: { offset?: string } }>('/api/dlq', async (req) => { + const offset = Math.max(0, parseInt(req.query.offset ?? '0', 10) || 0); + const pending = await deps.dlq.listPending(runId(), 8, offset); + if (!dlqAggCache || Date.now() - dlqAggCache.at > 15_000) { + dlqAggCache = { + at: Date.now(), + byStatus: await deps.dlq.countByStatus(runId()), + topErrors: await deps.dlq.topErrors(runId(), 8), + storage: await deps.dlq.storageStats().catch(() => null), + }; + } + return { + byStatus: dlqAggCache.byStatus, + topErrors: dlqAggCache.topErrors, + storage: dlqAggCache.storage, + // Where fixes go: Replay re-transforms raw_doc FROM THIS COLLECTION — + // never from the source. The source stays the untouched record. + fixLocation: { db: deps.config.state.manifestDb, collection: 'mig_dlq_docs' }, + sourceDb: deps.config.source.db, + offset, + samples: pending.map((p) => ({ + dlq_id: p._id, + source_id: p.source_id, + collection: p.collection, + reason: p.reason, + error: p.error, + raw_doc: JSON.stringify(p.raw_doc).slice(0, 2_000), + })), + }; + }); + + app.get('/api/preflight', async () => deps.orchestrator.preflight()); + + // Verify runs as a background task: a 10TB run recounts tens of thousands + // of windows — minutes of work that must not sit inside one HTTP request. + const verifyState: { status: string; result: Record | null; error: string | null } = + { status: 'not_run', result: null, error: null }; + app.post('/control/verify', async () => { + if (verifyState.status === 'running') return { started: false, reason: 'already running' }; + verifyState.status = 'running'; verifyState.result = null; verifyState.error = null; + void deps.orchestrator.verifyMigration() + .then((r) => { verifyState.result = r; verifyState.status = 'completed'; }) + .catch((e) => { verifyState.error = (e as Error).message; verifyState.status = 'failed'; }); + return { started: true }; + }); + app.get('/api/verify', async () => ({ + status: verifyState.status, + progress: deps.orchestrator.verifyProgress, + result: verifyState.result, + error: verifyState.error, + })); + + // The console IS the product's front door — serve it at the root. + // /viz stays as an alias (docs, bookmarks, muscle memory). + app.get('/', async (_req, reply) => { + reply.type('text/html').send(PAGE); + }); + app.get('/viz', async (_req, reply) => { + reply.type('text/html').send(PAGE); + }); +} + +const PAGE = ` + + + + +Countly Data Migration + + + + + +
+ +
+
Data Migration
+
+ ledger engine · no redis + + starting… +
+ +
+ +
+ + +
+
+
Docs migrated–
+
Docs / second–
+
Chunks done–
+
Skipped–
+
Failed chunks–
+
ETA–
+
+ +
+ + + + + + Destructive actions ask for a second click. Every action shows a receipt. +
+ +
+

Collections

+
Waiting for first chunk…
+
+ +
+

Chunk map (newest data first — chunks are processed right to left)

+
+
+ pending + copying + verifying / merging + done + failed + split (poison hunt) +
+
+ +
+

Failed chunks

+
None 🎉
+
+ +
+

Dead-letter queue (unmigratable docs, stored with their full raw source — replay after a fix, or waive)

+
+
+
+
+
+
+ +
+

Coercions (values the transform had to alter — the data-quality report)

+
None
+
+ +
+

Pods (scale by starting more instances with the same env + unique POD_ID — they claim chunks via leases; add machines, not processes, once one machine's CPU saturates)

+
–
+
+
+ + +
+
+

Preflight checks — run anytime; read-only

+ +
+
+ +
1 Prepare manual +
+

Old cluster stays live — nothing changes for users yet.

+ + + +
+
+ +
2 Index auto +
+

Every drill_events* collection needs the {cd:1,_id:1} index. The migrator builds missing ones itself, but pre-building avoids a long pause at start (~1–3 days for 10 TB). No collection consolidation is ever needed.

+

+ — server-side, background; safe to leave running.

+
+
+
+ +
3 Rehearse (dry run) auto +
+

A ≤5% sample goes through the full pipeline with real ClickHouse validation and nothing stored. Review the Overview tab's DLQ and Coercions panels afterwards — that is the data-quality report to sign off before the real run.

+

+ — available while the main migration is not running.

+
+
+ +
4 Cutover manual +
+

The only ingestion pause in the whole flow — minutes, absorbed by SDK offline queues.

+ + + +
+
+ +
5 Migrate – +
+

Start the service (scale with pods — they share work via leases). Newest data first: recent dashboards fill within hours. Watch the Overview tab; the invariant monitor spot-checks continuously. Live progress: –

+
+
+ +
+

Configuration — current values; change via env vars + restart

+
–
+
+ +
6 Verify & sign off gated +
+

Three gates, all must be green:

+
…All chunks done–
+
…DLQ pending = 0every unmigrated doc explicitly fixed or waived
+
…Full verification passedrun it below
+

+ + + + — Verify recounts chunks vs their tallies (exact). Audit vs source recounts every window against MongoDB itself (catches a self-consistent under-read). Content audit re-transforms random source docs and compares them field-by-field with their live rows (catches right-count-wrong-content). +

+
+
+

Then: final report (/report), customer sign-off, revert Kafka retention, decommission the old cluster.

+
+
+
+ + +
+

Every situation below ends in restart or resume — never restore, never wipe. Live data cannot be touched by a migration failure: history sits in the frozen source, in-flight work sits in disposable staging tables, and the live table only ever receives whole verified chunks.

+ +
🔌 The migrator crashed / a pod died +

What happened: nothing, to your data. In-flight chunks will be redone; a dead pod's lease expires and others reclaim its work.

+

Do: restart the process (or let your orchestrator do it). There is no manual cleanup step in this flow.

+
+ +
📄 Some documents won't migrate (DLQ pending > 0) +

What happened: documents ClickHouse or the transform rejected were isolated automatically and stored in the dead-letter queue with their full raw source — inspect them in the Overview tab.

+

Do: after a transform fix (or after editing the stored raw docs):

+ + +

Waiving is the explicit decision that they will not migrate — raw docs are kept as the record.

+

Always replay here, in the tool — replaying historical documents through Countly's own ingestion would re-stamp their cd to today and duplicate history at the wrong date.

+
+ +
⛔ The engine paused itself (circuit breaker) +

What happened: too many documents in one chunk failed — that pattern means a systematic problem, not dirty data. The DLQ already names the error.

+

Do: read the top error in the Overview tab, fix the cause, then:

+ +
+
+ +
🧨 A chunk keeps failing / keeps crashing the process +

What happened: a poison-pill document. After repeated crashes the chunk is automatically split into smaller pieces — the quarantine converges on a window of minutes around the offending doc(s) while everything else migrates. Split chunks show as hatched cells in the chunk map.

+

Do: when a tiny chunk ends up failed, inspect the few source documents in its time range (shown in Failed chunks), fix or remove them, then:

+
+
+ +
🔍 Counts look wrong / trust is in question +

What happened: maybe nothing — but never guess. The invariant monitor spot-checks continuously; a violation pauses the engine and flags the chunk.

+

Do: run the full check — every completed chunk recounted against the live table:

+ +

A flagged chunk is healed with Retry failed chunks (its live window is purged and redone cleanly).

+
+ +
🗂️ Migration progress state lost (mig_ranges gone or corrupted) +

What happened: the chunk ledger in MongoDB was dropped, restored from an old backup, or otherwise no longer matches reality. Your data is intact — only the notes about which windows were already copied are gone.

+

Do: rebuild the ledger from the data itself. The source is frozen, so every chunk window can be recounted in MongoDB and compared against the live ClickHouse rows for the same collection and time window: equal → done, empty → pending, partial → failed (its redo purges the window first). New events ingested since cutover carry newer timestamps than any migrated window, so they are never counted or touched.

+

Requires: engine not copying (pause or restart into a lost-ledger state is fine) and no other pods active. Windows that had DLQ'd or skipped documents re-run and re-capture them — that is expected. Afterwards, resume the run: it finishes only what the rebuild marked pending or failed.

+ + +
+
+
+ +
🔥 Live ClickHouse itself must be rebuilt (worst case) +

What happened: catastrophic loss of the target. Your data still exists twice: live events since cutover sit in the Kafka log; history sits in the frozen source MongoDB.

+

Do: recreate the table → reset ONLY the ClickHouse-sink connector's offsets to earliest (Kafka replays the live window; aggregator groups untouched) → re-run this migrator for history. Zero data loss. See docs/RUNBOOK.md.

+
+
+ +
State source: chunk ledger (MongoDB) + live engine counters — refreshed every 2s. No Redis involved.
+
+ + + +`; diff --git a/src/http/run-route.ts b/src/http/run-route.ts deleted file mode 100644 index db74eb4..0000000 --- a/src/http/run-route.ts +++ /dev/null @@ -1,255 +0,0 @@ -import type { FastifyInstance } from 'fastify'; -import { buildCoverageFromBatches, compactIntervals } from '../state/coverage.ts'; -import type { Run, Batch, BatchStatus, RunStatus, GetBatchesOptions } from '../state/manifest-store.ts'; - -export interface RunDeps { - manifestStore: { - getActiveRun(sourceNs?: string, targetTable?: string): Promise; - getRun(runId: string): Promise; - listRuns(opts: { status?: RunStatus; limit?: number; offset?: number }): Promise<{ runs: Run[]; total: number }>; - getBatches(runId: string, opts: GetBatchesOptions): Promise; - getFailedBatches(runId: string): Promise; - countEvents(runId: string, eventType?: string): Promise; - }; - redisState: { - getTimeline(runId: string): Promise; - getRecentErrors(runId: string): Promise; - getVerboseErrors(runId: string, batchSeq: number): Promise; - cleanupRun(runId: string): Promise; - isHealthy(): Promise; - }; - orchestrator?: { - getCurrentRunId(): string | null; - }; -} - -interface BatchesQuerystring { - status?: string; - limit?: string; -} - -export function registerRunRoutes(app: FastifyInstance, deps: RunDeps): void { - const { manifestStore, redisState } = deps; - - // GET /runs/current - returns current run header from manifest store - app.get('/runs/current', async (_request, reply) => { - const activeRun = await manifestStore.getActiveRun(); - - if (activeRun == null) { - return reply.status(404).send({ - error: 'No active run found', - }); - } - - const runId = activeRun.run_id; - const doneBatches = await manifestStore.getBatches(runId, { status: 'done' }); - const coverage = buildCoverageFromBatches(doneBatches); - const compactedIntervals = compactIntervals(coverage); - - return reply.status(200).send({ ...activeRun, coverage: compactedIntervals }); - }); - - // GET /runs/current/batches?status=inflight|failed|done&limit=N - app.get<{ Querystring: BatchesQuerystring }>( - '/runs/current/batches', - async (request, reply) => { - const activeRun = await manifestStore.getActiveRun(); - - if (activeRun == null) { - return reply.status(404).send({ - error: 'No active run found', - }); - } - - const runId = activeRun.run_id; - const { status, limit } = request.query; - const parsedLimit = limit != null ? Math.max(1, parseInt(limit, 10) || 50) : 50; - - const batches = await manifestStore.getBatches(runId, { - status: status as BatchStatus | undefined, - limit: parsedLimit, - }); - - return reply.status(200).send({ - runId, - status: status ?? 'all', - limit: parsedLimit, - count: batches.length, - batches, - }); - }, - ); - - // GET /runs - list runs - app.get<{ Querystring: { status?: string; limit?: string; offset?: string } }>( - '/runs', - async (request, reply) => { - const { status, limit, offset } = request.query; - const parsedLimit = limit != null ? Math.max(1, parseInt(limit, 10) || 20) : 20; - const parsedOffset = offset != null ? Math.max(0, parseInt(offset, 10) || 0) : 0; - - const result = await manifestStore.listRuns({ - status: status as RunStatus | undefined, - limit: parsedLimit, - offset: parsedOffset, - }); - - return reply.status(200).send({ - ...result, - limit: parsedLimit, - offset: parsedOffset, - }); - }, - ); - - // GET /runs/:runId - app.get<{ Params: { runId: string } }>( - '/runs/:runId', - async (request, reply) => { - const run = await manifestStore.getRun(request.params.runId); - if (!run) { - return reply.status(404).send({ error: 'Run not found' }); - } - return reply.status(200).send(run); - }, - ); - - // GET /runs/:runId/batches - app.get<{ Params: { runId: string }; Querystring: BatchesQuerystring }>( - '/runs/:runId/batches', - async (request, reply) => { - const { runId } = request.params; - const { status, limit } = request.query; - const parsedLimit = limit != null ? Math.max(1, parseInt(limit, 10) || 50) : 50; - - const batches = await manifestStore.getBatches(runId, { - status: status as BatchStatus | undefined, - limit: parsedLimit, - }); - - return reply.status(200).send({ - runId, - status: status ?? 'all', - limit: parsedLimit, - count: batches.length, - batches, - }); - }, - ); - - // GET /runs/:runId/failures - app.get<{ Params: { runId: string } }>( - '/runs/:runId/failures', - async (request, reply) => { - const { runId } = request.params; - - const [run, failedBatches, totalRetryErrors, digestMismatches] = await Promise.all([ - manifestStore.getRun(runId), - manifestStore.getFailedBatches(runId), - manifestStore.countEvents(runId, 'batch_retry_error'), - manifestStore.countEvents(runId, 'digest_mismatch'), - ]); - - if (!run) { - return reply.status(404).send({ error: 'Run not found' }); - } - - let recentErrors: unknown[] = []; - let verboseErrors: Record = {}; - let redisDataAvailable = false; - - try { - const healthy = await redisState.isHealthy(); - if (healthy) { - redisDataAvailable = true; - recentErrors = await redisState.getRecentErrors(runId); - const verboseResults = await Promise.all( - failedBatches.map(batch => - redisState.getVerboseErrors(runId, batch.batch_seq) - .then(v => ({ seq: batch.batch_seq, errors: v })) - ) - ); - const errorEntries: Record = {}; - for (const { seq, errors } of verboseResults) { - if (errors.length > 0) { - errorEntries[seq] = errors; - } - } - verboseErrors = errorEntries; - } - } catch { - redisDataAvailable = false; - } - - return reply.status(200).send({ - runId, - total_failed_batches: failedBatches.length, - total_retry_errors: totalRetryErrors, - digest_mismatches: digestMismatches, - estimated_duplicate_rows: run.summary?.estimated_duplicate_rows ?? 0, - failed_batches: failedBatches, - recent_errors: recentErrors, - verbose_errors: verboseErrors, - redis_data_available: redisDataAvailable, - }); - }, - ); - - // GET /runs/:runId/timeline - app.get<{ Params: { runId: string } }>( - '/runs/:runId/timeline', - async (request, reply) => { - const { runId } = request.params; - try { - const snapshots = await redisState.getTimeline(runId); - return reply.status(200).send({ runId, snapshots }); - } catch { - return reply.status(200).send({ runId, snapshots: [], redis_available: false }); - } - }, - ); - - // GET /runs/:runId/coverage - app.get<{ Params: { runId: string } }>( - '/runs/:runId/coverage', - async (request, reply) => { - const { runId } = request.params; - - const run = await manifestStore.getRun(runId); - if (!run) { - return reply.status(404).send({ error: 'Run not found' }); - } - - const doneBatches = await manifestStore.getBatches(runId, { status: 'done' }); - const allBatches = await manifestStore.getBatches(runId, {}); - const coverage = buildCoverageFromBatches(doneBatches); - const compactedIntervals = compactIntervals(coverage); - - return reply.status(200).send({ - runId, - intervals: compactedIntervals, - total_batches_done: doneBatches.length, - total_batches: allBatches.length, - coverage_pct: allBatches.length > 0 - ? (doneBatches.length / allBatches.length) * 100 - : 0, - }); - }, - ); - - // DELETE /runs/:runId/cache - app.delete<{ Params: { runId: string } }>( - '/runs/:runId/cache', - async (request, reply) => { - const { runId } = request.params; - const currentRunId = deps.orchestrator?.getCurrentRunId?.(); - if (currentRunId && currentRunId === runId) { - return reply.status(409).send({ - error: 'Cannot delete cache for the currently active run', - }); - } - const keysDeleted = await redisState.cleanupRun(runId); - return reply.status(200).send({ runId, keys_deleted: keysDeleted }); - }, - ); -} diff --git a/src/http/stats-route.ts b/src/http/stats-route.ts deleted file mode 100644 index d0af3ea..0000000 --- a/src/http/stats-route.ts +++ /dev/null @@ -1,566 +0,0 @@ -import type { FastifyInstance } from 'fastify'; -import { hostname } from 'node:os'; -import type { Run } from '../state/manifest-store.ts'; -import type { RunnerStatus } from '../runtime/batch-runner.ts'; -import type { GcTelemetry } from '../runtime/gc-controller.ts'; -import type { ProcessMetricsSnapshot } from '../runtime/process-metrics.ts'; -import type { CommandFlags, LiveBatchData, RangeLiveStats } from '../state/redis-hot-state.ts'; -import type { Config } from '../config/schema.ts'; -import type { CollectionOrchestrator, OrchestratorProgress } from '../runtime/collection-orchestrator.ts'; -import type { GlobalProgress, CollectionProgress, PodInfo } from '../state/global-progress.ts'; -import type { CollectionLock, LockInfo } from '../state/collection-lock.ts'; - -// ───────────────────────────────────────────────────────────────────────────── -// Formatting helpers -// ───────────────────────────────────────────────────────────────────────────── - -function progressBar(pct: number, width = 30): string { - const filled = Math.round((pct / 100) * width); - const arrow = filled < width ? '>' : ''; - const empty = Math.max(0, width - filled - (arrow ? 1 : 0)); - return '[' + '='.repeat(filled) + arrow + ' '.repeat(empty) + '] ' + pct + '%'; -} - -function fmtDuration(ms: number): string { - const s = Math.floor(ms / 1000); - const h = Math.floor(s / 3600); - const m = Math.floor((s % 3600) / 60); - const sec = s % 60; - if (h > 0) return `${h}h ${m}m ${sec}s`; - if (m > 0) return `${m}m ${sec}s`; - return `${sec}s`; -} - -function fmtNum(n: number): string { - return n.toLocaleString('en-US'); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Route -// ───────────────────────────────────────────────────────────────────────────── - -export interface StatsDeps { - orchestrator: CollectionOrchestrator; - mongoReader: { isConnected(): boolean }; - chWriter: { isConnected(): boolean }; - redisState: { - isHealthy(): Promise; - getBitmapCount(runId: string): Promise; - getCommands(runId: string): Promise; - getMetrics(): { lastStateWriteMs: number; lastError: string | null }; - getAllLiveBatches(): Promise; - getRangeLiveStats(collection: string): Promise; - getThroughputWindow(runId: string): Promise>; - getAllCollectionCompleted(): Promise>; - }; - gcController: { getTelemetry(): GcTelemetry }; - processMetrics: { snapshot(): ProcessMetricsSnapshot }; - manifestStore: { getRun(runId: string): Promise }; - config: Config; - startedAt: Date; - version: string; - globalProgress?: GlobalProgress; - collectionLock?: CollectionLock; -} - -export function registerStatsRoute(app: FastifyInstance, deps: StatsDeps): void { - const { - orchestrator, - redisState, - gcController, - processMetrics, - manifestStore, - config, - startedAt, - version, - } = deps; - - let frozenElapsedMs: number | null = null; - - app.get('/stats', async (_request, reply) => { - const now = new Date(); - const uptimeSec = Math.floor((now.getTime() - startedAt.getTime()) / 1000); - - const progress = orchestrator.getProgress(); - const batchStats = orchestrator.getStats(); - const runnerStatus = orchestrator.getStatus(); - const currentBatchSeq = orchestrator.getCurrentBatchSeq(); - const estimatedCounts = orchestrator.getEstimatedCounts(); - const indexStatus = orchestrator.getIndexStatus(); - - // Find the current run ID from the orchestrator's progress - const currentResult = progress.results.find( - (r) => r.collection === progress.currentCollection, - ); - const runId = currentResult?.runId ?? null; - - let redisConnected = false; - try { - redisConnected = await redisState.isHealthy(); - } catch { - redisConnected = false; - } - - let bitmapBitsSet = 0; - if (runId) { - try { - bitmapBitsSet = await redisState.getBitmapCount(runId); - } catch { - bitmapBitsSet = -1; - } - } - - const runRecord = runId ? await manifestStore.getRun(runId) : undefined; - - // Throughput calculations - const totalDocsRead = batchStats?.totalDocsRead ?? 0; - const totalDocsSkipped = batchStats?.totalDocsSkipped ?? 0; - const totalRowsInserted = batchStats?.totalRowsInserted ?? 0; - const elapsedMs = batchStats?.elapsedMs ?? 0; - const elapsedSec = elapsedMs / 1000; - - // ── Progress calculations (local only; cluster merge happens after clusterData fetch) ── - const currentCollEstimate = progress.currentCollection - ? estimatedCounts.get(progress.currentCollection) ?? 0 : 0; - let currentCollDocsRead = totalDocsRead; - let currentCollPct = 0; - - // Overall progress — recomputed after mergedCollectionProgress is built - const totalEstimated = Array.from(estimatedCounts.values()).reduce((a, b) => a + b, 0); - let overallDocsRead = 0; - let overallPct = 0; - - // ETA computed later using cluster-wide data - let etaMs: number | null = null; - - // Commands from Redis (best-effort) - let commands: CommandFlags = {}; - if (runId) { - try { - commands = await redisState.getCommands(runId); - } catch { - // default all false - } - } - - const redisMetrics = redisState.getMetrics(); - - // ── Cluster data (multi-pod mode) ───────────────────────────────── - let clusterData: { - podCount: number; - pods: PodInfo[]; - locks: LockInfo[]; - globalCommands: { pause: boolean; stop: boolean }; - collectionProgress: CollectionProgress[]; - } | null = null; - - if (deps.globalProgress) { - try { - const [allProgress, allPods, allLocks, globalCmds] = await Promise.all([ - deps.globalProgress.getAllCollectionProgress(), - deps.globalProgress.getAllPods(), - deps.collectionLock?.listAllLocks() ?? Promise.resolve([]), - deps.globalProgress.getGlobalCommands(), - ]); - clusterData = { - podCount: allPods.length, - pods: allPods, - locks: allLocks, - globalCommands: globalCmds, - collectionProgress: allProgress, - }; - } catch { - // best-effort - } - } - - // ── Persistent completion aggregates (no TTL) ─────────────────── - let completedAggregates = new Map(); - try { - completedAggregates = await redisState.getAllCollectionCompleted(); - } catch { /* best-effort */ } - - // ── Finalize current collection progress (merge cluster data) ── - if (clusterData && progress.currentCollection) { - const remoteProgress = clusterData.collectionProgress - .filter(p => p.collectionName === progress.currentCollection); - for (const rp of remoteProgress) { - if (rp.podId !== config.worker.podId) { - currentCollDocsRead += (rp.docsRead ?? 0); - } - } - } - currentCollPct = currentCollEstimate > 0 - ? Math.min(100, Math.round((currentCollDocsRead / currentCollEstimate) * 100)) : 0; - - // ── Sliding window throughput (best-effort) ────────────────────── - let slidingThroughput: number | null = null; - if (runId) { - try { - const throughputWindow = await redisState.getThroughputWindow(runId); - if (throughputWindow.length >= 2) { - const newest = throughputWindow[0]; - const oldest = throughputWindow[throughputWindow.length - 1]; - const durationSec = (newest.ts - oldest.ts) / 1000; - const delta = newest.docsRead - oldest.docsRead; - slidingThroughput = durationSec > 0 ? Math.round(delta / durationSec) : null; - } - } catch { - // best-effort - } - } - - // ── Live batches from Redis ────────────────────────────────────── - let liveBatches: LiveBatchData[] = []; - try { - liveBatches = await redisState.getAllLiveBatches(); - } catch { - // best-effort - } - - // Merge cluster progress with local data for comprehensive collection status - const mergedCollectionProgress = progress.collections.map(collection => { - const localResult = progress.results.find(r => r.collection === collection); - // Sum ALL pods' progress entries for this collection (per-pod keys) - const remotes = clusterData?.collectionProgress.filter(p => p.collectionName === collection) ?? []; - const remoteDocsRead = remotes.reduce((s, r) => s + (r.docsRead ?? 0), 0); - const remoteRowsInserted = remotes.reduce((s, r) => s + (r.rowsInserted ?? 0), 0); - const remoteAnyCompleted = remotes.some(r => r.status === 'completed'); - const remoteAnyProcessing = remotes.some(r => r.status === 'processing'); - const remoteStatus = remoteAnyProcessing ? 'processing' : remoteAnyCompleted ? 'completed' : remotes[0]?.status; - const remoteRunId = remotes[0]?.runId || null; - const remoteEstimated = remotes[0]?.estimatedTotal ?? null; - const remotePodId = remotes[0]?.podId ?? null; - - // Local "skipped" means another pod completed it — use remote data for attribution - if (localResult && localResult.status === 'skipped' && remotes.length > 0) { - return { - collection, - status: remoteStatus as string, - runId: remoteRunId, - estimated: remoteEstimated ?? estimatedCounts.get(collection) ?? null, - docsRead: remoteDocsRead || null, - rowsInserted: remoteRowsInserted || null, - podId: remotePodId, - }; - } - // Local result — overlay persistent completion and remote aggregate for best counts - if (localResult) { - const completedAgg = completedAggregates.get(collection); - const isCompleted = remoteAnyCompleted || !!completedAgg || localResult.status === 'completed'; - const bestDocsRead = Math.max(localResult.docsRead ?? 0, remoteDocsRead, completedAgg?.docsRead ?? 0); - const bestRowsInserted = Math.max(localResult.rowsInserted ?? 0, remoteRowsInserted, completedAgg?.rowsInserted ?? 0); - return { - collection, - status: isCompleted ? 'completed' : localResult.status, - runId: localResult.runId || completedAgg?.runId || remoteRunId || null, - estimated: estimatedCounts.get(collection) ?? null, - docsRead: isCompleted ? (bestDocsRead || null) : (localResult.docsRead ?? null), - rowsInserted: isCompleted ? (bestRowsInserted || null) : (localResult.rowsInserted ?? null), - podId: remoteAnyCompleted ? remotePodId : config.worker.podId, - }; - } - // Check cluster progress from other pods - if (remotes.length > 0) { - return { - collection, - status: remoteStatus as string, - runId: remoteRunId, - estimated: remoteEstimated ?? null, - docsRead: remoteDocsRead || null, - rowsInserted: remoteRowsInserted || null, - podId: remotePodId, - }; - } - // Check persistent completion data (survives TTL expiry of progress:* keys) - const completedAgg = completedAggregates.get(collection); - if (completedAgg) { - return { - collection, - status: "completed" as const, - runId: completedAgg.runId || null, - estimated: estimatedCounts.get(collection) ?? null, - docsRead: completedAgg.docsRead, - rowsInserted: completedAgg.rowsInserted, - podId: null, - }; - } - // No data yet - return { - collection, - status: "pending" as const, - runId: null, - estimated: estimatedCounts.get(collection) ?? null, - docsRead: null, - rowsInserted: null, - podId: null, - }; - }); - - // ── Recompute overall progress from merged data ───────────────── - { - const completedDocsRead = mergedCollectionProgress - .filter(c => c.status === 'completed' || c.status === 'skipped') - .reduce((sum, c) => sum + (c.docsRead ?? 0), 0); - const processingDocsRead = progress.currentCollection - ? (mergedCollectionProgress.find(c => c.collection === progress.currentCollection && c.status === 'processing')?.docsRead - ?? currentCollDocsRead) - : 0; - overallDocsRead = completedDocsRead + processingDocsRead; - overallPct = totalEstimated > 0 - ? Math.min(100, Math.round((overallDocsRead / totalEstimated) * 100)) : 0; - } - - // ── Compute cluster-wide aggregates for summary ───────────────── - // Exclude collections with null docsRead (not yet started) from both numerator AND denominator - const activeMerged = clusterData - ? mergedCollectionProgress.filter(c => c.docsRead !== null) - : []; - const clusterDocsRead = clusterData - ? activeMerged.reduce((s, c) => s + (c.docsRead ?? 0), 0) - : overallDocsRead; - const clusterEstimated = clusterData - ? (activeMerged.reduce((s, c) => s + (c.estimated ?? 0), 0) || totalEstimated) - : totalEstimated; - const clusterPct = clusterEstimated > 0 - ? Math.min(100, Math.round((clusterDocsRead / clusterEstimated) * 100)) - : overallPct; - const clusterDone = clusterData - ? mergedCollectionProgress.filter(c => c.status === 'completed' || c.status === 'skipped').length - : progress.completedCollections + progress.skippedCollections; - const clusterFailed = clusterData - ? mergedCollectionProgress.filter(c => c.status === 'failed').length - : progress.failedCollections; - const clusterProcessing = clusterData - ? mergedCollectionProgress.filter(c => c.status === 'processing').length - : (progress.currentCollection ? 1 : 0); - const clusterTotal = clusterData - ? (progress.totalCollections || mergedCollectionProgress.length) - : progress.totalCollections; - - // ETA based on cluster-wide progress — use earliest collection start for accuracy - const earliestProcessingStart = clusterData - ? activeMerged.reduce((earliest, c) => { - const started = (c as Record)['startedAt']; - if (typeof started === 'string') { - const ms = new Date(started).getTime(); - return ms > 0 && ms < earliest ? ms : earliest; - } - return earliest; - }, startedAt.getTime()) - : (orchestrator.getFirstCollectionStartedAt() ?? startedAt.getTime()); - const processingElapsedMs = Date.now() - earliestProcessingStart; - etaMs = clusterPct > 0 && clusterPct < 100 && processingElapsedMs > 0 - ? Math.round((processingElapsedMs / clusterPct) * (100 - clusterPct)) - : null; - - // ── Range live stats for current collection ──────────────────── - let rangeLiveStats: RangeLiveStats[] = []; - if (progress.currentCollection) { - try { - rangeLiveStats = await redisState.getRangeLiveStats(progress.currentCollection); - } catch { /* best-effort */ } - } - - const payload = { - // ── Quick-glance summary (cluster-wide when multi-pod) ────────── - summary: { - overall: progressBar(clusterPct), - overallPct: clusterPct, - currentCollection: progress.currentCollection - ? `${progress.currentCollection} ${progressBar(currentCollPct)}` - : 'idle', - currentCollectionPct: currentCollPct, - collections: `${clusterDone}/${clusterTotal} done` - + (clusterFailed > 0 ? `, ${clusterFailed} failed` : '') - + (clusterProcessing > 0 ? `, ${clusterProcessing} processing` : ''), - docsProgress: `${fmtNum(clusterDocsRead)} / ~${fmtNum(clusterEstimated)} docs`, - throughput: (() => { - // Prefer sliding window throughput when available - if (slidingThroughput !== null && slidingThroughput > 0) { - return `${fmtNum(slidingThroughput)} docs/s`; - } - // Fallback: cluster-wide lifetime throughput - const clusterDocs = clusterData - ? activeMerged.reduce((s, c) => s + (c.docsRead ?? 0), 0) - : 0; - if (clusterDocs > 0 && uptimeSec > 0) { - return `${fmtNum(Math.round(clusterDocs / uptimeSec))} docs/s`; - } - return `${fmtNum(Math.round(batchStats?.docsPerSecond ?? 0))} docs/s`; - })(), - elapsed: (() => { - const processingStart = orchestrator.getFirstCollectionStartedAt() ?? startedAt.getTime(); - const isTerminal = runnerStatus === 'completed' || runnerStatus === 'failed' || runnerStatus === 'stopped'; - if (isTerminal && frozenElapsedMs === null) { - frozenElapsedMs = Date.now() - processingStart; - } - return fmtDuration(frozenElapsedMs ?? (Date.now() - processingStart)); - })(), - eta: (runnerStatus === 'completed' || runnerStatus === 'failed' || runnerStatus === 'stopped') - ? 'done' - : (etaMs !== null ? `~${fmtDuration(etaMs)}` : 'calculating...'), - status: (() => { - if (!clusterData) return runnerStatus; - if (clusterProcessing > 0) return "running"; - const clusterPending = clusterTotal - clusterDone - clusterFailed; - if (clusterPending > 0) return "running"; - if (clusterTotal > 0 && clusterDone + clusterFailed >= clusterTotal) { - return clusterFailed > 0 ? "completed_with_errors" : "completed"; - } - return runnerStatus; - })(), - }, - - // ── Current collection detail ───────────────────────────────────── - currentCollectionProgress: progress.currentCollection ? { - collection: progress.currentCollection, - estimated: currentCollEstimate, - docsRead: currentCollDocsRead, - rowsInserted: totalRowsInserted, - pct: currentCollPct, - bar: progressBar(currentCollPct), - batchSeq: currentBatchSeq, - skipRate: currentCollDocsRead > 0 - ? `${((totalDocsSkipped / currentCollDocsRead) * 100).toFixed(1)}%` : '0%', - rangeBreakdown: rangeLiveStats.length > 0 ? rangeLiveStats : undefined, - } : null, - - indexStatus, - - service: { - name: config.service.name, - version, - runId, - status: runnerStatus, - uptimeSec, - pid: process.pid, - hostname: hostname(), - }, - orchestrator: { - totalCollections: progress.totalCollections, - completedCollections: progress.completedCollections, - failedCollections: progress.failedCollections, - skippedCollections: progress.skippedCollections, - currentCollection: progress.currentCollection, - collections: progress.collections, - collectionProgress: mergedCollectionProgress, - }, - run: { - sourceNs: runRecord?.source_ns ?? null, - targetTable: runRecord?.target_table ?? null, - upperBoundCursor: runRecord?.upper_bound_cursor ?? null, - lastCommittedCursor: runRecord?.last_committed_cursor ?? null, - batchSeqCommitted: currentBatchSeq, - batchSeqInFlight: null, - transformVersion: runRecord?.transform_version ?? null, - pauseReason: null, - stopAfterBatch: false, - catchupMode: false, - }, - throughput: { - sourceDocsReadTotal: totalDocsRead, - docsSkippedTotal: totalDocsSkipped, - rowsInsertedTotal: totalRowsInserted, - avgSourceDocsPerSec: elapsedSec > 0 ? Math.round((totalDocsRead / elapsedSec) * 100) / 100 : 0, - avgRowsInsertedPerSec: elapsedSec > 0 ? Math.round((totalRowsInserted / elapsedSec) * 100) / 100 : 0, - }, - skipReasons: batchStats?.skipsByReason ?? {}, - integrity: { - digestMismatches: batchStats?.digestMismatches ?? 0, - estimatedDuplicateRows: batchStats?.estimatedDuplicateRows ?? 0, - batchesFailed: batchStats?.batchesFailed ?? 0, - }, - batch: null, - mongo: { - connected: deps.mongoReader.isConnected(), - readPreference: config.source.readPreference, - readConcern: config.source.readConcern, - batchRowsTarget: config.source.batchRowsTarget, - cursorBatchSize: config.source.cursorBatchSize, - }, - clickhouse: { - connected: deps.chWriter.isConnected(), - target: `${config.target.db}.${config.target.table}`, - compression: 'gzip', - partsToThrowInsert: config.backpressure.partsToThrowInsert, - maxPartsInTotal: config.backpressure.maxPartsInTotal, - }, - redis: { - connected: redisConnected, - lastStateWriteMs: redisMetrics.lastStateWriteMs, - bitmapBitsSet, - lastError: redisMetrics.lastError, - }, - manifest: { - db: config.state.manifestDb, - lastCheckpointTime: runRecord?.updated_at ?? null, - }, - gc: gcController.getTelemetry(), - process: processMetrics.snapshot(), - commands: { - pauseRequested: !!(commands.pause), - resumeRequested: !!(commands.resume), - stopAfterBatchRequested: !!(commands.abort ?? commands.stopAfterBatch), - gcRequested: !!(commands.gc), - }, - // ── Live batches + range progress ─────────────────────────────── - liveBatches, - cluster: clusterData ? { - ...clusterData, - pods: clusterData.pods.map(pod => { - // Aggregate per-pod stats from merged collection progress - const podCollections = mergedCollectionProgress.filter(c => c.podId === pod.podId); - return { - ...pod, - stats: { - collectionsCompleted: podCollections.filter(c => c.status === 'completed' || c.status === 'skipped').length, - docsRead: podCollections.reduce((s, c) => s + (c.docsRead ?? 0), 0), - rowsInserted: podCollections.reduce((s, c) => s + (c.rowsInserted ?? 0), 0), - }, - }; - }), - stalePods: clusterData.locks - .map(l => l.podId) - .filter(podId => !clusterData!.pods.some(p => p.podId === podId)) - .filter((v, i, a) => a.indexOf(v) === i), - lockSummary: { - total: clusterData.locks.length, - byPod: Object.fromEntries( - [...new Set(clusterData.locks.map(l => l.podId))].map(pid => - [pid, clusterData!.locks.filter(l => l.podId === pid).length] - ) - ), - stale: clusterData.locks.filter(l => { - const pod = clusterData!.pods.find(p => p.podId === l.podId); - if (!pod) return true; - // Pod key exists but heartbeat is stale (>180s old) - const heartbeatAge = Date.now() - new Date(pod.lastHeartbeat).getTime(); - return heartbeatAge > 180_000; - }).length, - }, - } : null, - clusterProgress: clusterData ? (() => { - const done = mergedCollectionProgress.filter(c => c.status === 'completed' || c.status === 'skipped').length; - const failed = mergedCollectionProgress.filter(c => c.status === 'failed').length; - const processing = mergedCollectionProgress.filter(c => c.status === 'processing').length; - const pending = mergedCollectionProgress.filter(c => c.status === 'pending').length; - const total = progress.totalCollections || mergedCollectionProgress.length; - const pct = total > 0 ? Math.min(100, Math.round((done / total) * 100)) : 0; - let docsRead = mergedCollectionProgress.reduce((s, c) => s + (c.docsRead ?? 0), 0); - let rowsInserted = mergedCollectionProgress.reduce((s, c) => s + (c.rowsInserted ?? 0), 0); - // Include completed collections whose volatile progress:* TTL expired - for (const [collection, agg] of completedAggregates) { - if (!mergedCollectionProgress.some(c => c.collection === collection)) { - docsRead += agg.docsRead; - rowsInserted += agg.rowsInserted; - } - } - const estimated = mergedCollectionProgress.reduce((s, c) => s + (c.estimated ?? 0), 0); - return { total, done, failed, processing, pending, pct, docsRead, rowsInserted, estimated }; - })() : null, - }; - - return reply.status(200).send(payload); - }); -} diff --git a/src/http/viz-route.ts b/src/http/viz-route.ts deleted file mode 100644 index bc9efa8..0000000 --- a/src/http/viz-route.ts +++ /dev/null @@ -1,1074 +0,0 @@ -import type { FastifyInstance } from 'fastify'; - -export function registerVizRoute(app: FastifyInstance): void { - app.get('/viz', async (_request, reply) => { - return reply.type('text/html').send(DASHBOARD_HTML); - }); -} - -// --------------------------------------------------------------------------- -// Self-contained HTML dashboard -// -// Security note: This dashboard renders only data from its own trusted API -// endpoints (/stats, /readyz) which return structured JSON from the migration -// service. All user-visible text is set via textContent (safe). The only -// innerHTML usage is for rendering collection table rows and skip reason grids -// from the service's own structured data (collection names, numeric values, -// status enums) — these are not user-supplied and do not contain executable -// content. The dashboard has no user input fields or URL-sourced data. -// --------------------------------------------------------------------------- - -const DASHBOARD_HTML = /* html */ ` - - - - -Migration Dashboard - - - - - -
-
-

Migration Dashboard

- idle -
-
- - - v- - -
-
-
- - -
-

Overall Progress

-
-
- 0% -
-
-
-
-
Docs Progress
-
-
Throughput
-
-
Elapsed
-
-
ETA
-
-
Collections
-
-
- -
- -
-

Current Collection

-
-
-
-
- 0% -
-
-
-
0
Read
-
0
Inserted
-
0
Batch
-
0%
Skip Rate
-
-
- - -
-

Controls

-
- - - - -
-
-

Health

-
-
MongoDB
-
ClickHouse
-
Redis
-
-
-
- -
- -
-

Throughput & Integrity

-
-
0
Docs Read
-
0
Rows Inserted
-
0
Skipped
-
-
-
0
Digest Mismatch
-
0
Est. Duplicates
-
0
Batches Failed
-
-
- - -
-

Skip Reasons

-
-

System

-
- RSS -
- - -
-
- Heap -
- - -
-
-
-
-
- - -
-
-

Run Details

-
-
-
-

Infrastructure

-
-
-
- - - - - - - - - - - -
-

Index Status

-
-
- - -
-

Collections

-
-
Active & Done
-
Failed
-
Skipped
-
All
-
-
- - - -
CollectionStatusPodEstimatedReadInsertedAction
-
-
- - -
-

Danger Zone

-

These actions are irreversible and will destroy migration state. Use with caution.

-
- - -
-
-
- - - -`; diff --git a/src/main.ts b/src/main.ts index 9aad61b..96f5d33 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,337 +1,27 @@ // src/main.ts // MongoDB -> ClickHouse Migration Service Entry Point +// +// Architecture: chunk-checklist engine — work is cut into cd-bounded chunks +// tracked in a MongoDB ledger; each chunk is stream-copied into its own +// staging table, verified (read tally vs exact ClickHouse count), then +// promoted into the live table via verify-then-attach. Dependencies: +// MongoDB + ClickHouse. No Redis. import { loadConfig, createLogger } from './config/loader.ts'; -import { ManifestStore } from './state/manifest-store.ts'; -import { RedisHotState } from './state/redis-hot-state.ts'; -import { MongoReader } from './source/mongo-reader.ts'; -import { ClickHouseWriter } from './target/clickhouse-writer.ts'; -import { ClickHousePressure } from './target/clickhouse-pressure.ts'; -import { CollectionOrchestrator } from './runtime/collection-orchestrator.ts'; -import { HashResolver } from './transform/hash-resolver.ts'; -import { CollectionLock } from './state/collection-lock.ts'; -import { GlobalProgress } from './state/global-progress.ts'; -import { AsyncBatchWriter } from './state/async-batch-writer.ts'; -import { RetryPolicy } from './runtime/retry-policy.ts'; -import { GcController, type GcConfig } from './runtime/gc-controller.ts'; -import { ProcessMetricsCollector } from './runtime/process-metrics.ts'; -import { wireExitOnComplete } from './runtime/exit-on-complete.ts'; -import { registerHealthRoutes } from './http/health-route.ts'; -import { registerStatsRoute } from './http/stats-route.ts'; -import { registerControlRoutes } from './http/control-route.ts'; -import { registerRunRoutes } from './http/run-route.ts'; -import { registerVizRoute } from './http/viz-route.ts'; -import Fastify from 'fastify'; -import { createClient as createClickHouseClient } from '@clickhouse/client'; - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -const SERVICE_VERSION = '1.0.0'; - -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- +import { runLedgerEngine } from './runtime/ledger-engine.ts'; async function main(): Promise { - // ── 1. Load and validate config ───────────────────────────────────── const config = loadConfig(); - - // ── 2. Create logger ──────────────────────────────────────────────── const logger = createLogger(config); logger.info({ service: config.service.name }, 'Starting migration service'); - - // ── 3. Initialize components ──────────────────────────────────────── - - // State stores - const manifestStore = new ManifestStore(config.source.uri, config.state.manifestDb); - await manifestStore.connect(); - logger.info({ db: config.state.manifestDb }, 'ManifestStore initialized'); - - const redisState = new RedisHotState(config.state.redisUrl, config.state.redisKeyPrefix); - logger.info('RedisHotState initialized'); - - // Source (no collection binding — orchestrator handles switchCollection per collection) - const mongoReader = new MongoReader( - { - uri: config.source.uri, - database: config.source.db, - readPreference: config.source.readPreference, - readConcern: config.source.readConcern, - retryReads: config.source.retryReads, - appName: config.source.appName ?? config.service.name, - batchRowsTarget: config.source.batchRowsTarget, - cursorBatchSize: config.source.cursorBatchSize, - maxTimeMs: config.source.maxTimeMs, - }, - logger, - ); - - // Target - const chWriter = new ClickHouseWriter( - { - url: config.target.url, - database: config.target.db, - table: config.target.table, - username: config.target.username, - password: config.target.password, - queryTimeoutMs: config.target.queryTimeoutMs, - useDedupToken: config.target.useDedupToken, - }, - logger, - ); - - // Create a separate ClickHouse client for pressure monitoring queries - const pressureClient = createClickHouseClient({ - url: config.target.url, - database: config.target.db, - username: config.target.username, - password: config.target.password, - request_timeout: config.target.queryTimeoutMs, - }); - - // Fetch actual ClickHouse MergeTree settings and override config defaults - const serverLimits = await ClickHousePressure.fetchServerLimits(pressureClient, logger); - config.backpressure.partsToThrowInsert = serverLimits.partsToThrowInsert; - config.backpressure.maxPartsInTotal = serverLimits.maxPartsInTotal; - - const chPressure = new ClickHousePressure(pressureClient, config.backpressure, logger); - - // Map config.memory to GcConfig shape - const gcConfig: GcConfig = { - enabled: config.memory.gcEnabled, - rssSoftLimitBytes: config.memory.gcRssSoftLimitMb * 1024 * 1024, - rssHardLimitBytes: config.memory.gcRssHardLimitMb * 1024 * 1024, - heapUsedRatio: config.memory.gcHeapUsedRatio, - everyNBatches: config.memory.gcEveryNBatches, - }; - const gcController = new GcController(gcConfig, logger); - - const processMetrics = new ProcessMetricsCollector(); - - const retryPolicy = new RetryPolicy({ - maxRetries: config.target.maxRetries, - baseDelayMs: config.target.retryBaseDelayMs, - maxDelayMs: config.target.retryMaxDelayMs, - }); - - // ── 4. Connect to external services ───────────────────────────────── - logger.info('Connecting to external services...'); - - await mongoReader.connect(); - await chWriter.connect(); - await redisState.connect(); - - // Verify Redis is reachable - const redisHealthy = await redisState.isHealthy(); - if (!redisHealthy) { - throw new Error('Redis is not reachable. Cannot start migration service.'); - } - logger.info('All external services connected'); - - // ── 4b. Multi-pod coordination (optional) ──────────────────────────── - let collectionLock: CollectionLock | undefined; - let globalProgress: GlobalProgress | undefined; - - if (config.worker.enabled) { - const redisClient = redisState.getRedisClient(); - const podId = config.worker.podId; - - collectionLock = new CollectionLock( - redisClient, - podId, - { - lockTtlSec: config.worker.lockTtlSec, - renewIntervalMs: config.worker.lockRenewMs, - podHeartbeatMs: config.worker.podHeartbeatMs, - podDeadAfterSec: config.worker.podDeadAfterSec, - keyPrefix: config.state.redisKeyPrefix, - }, - logger, - ); - - globalProgress = new GlobalProgress( - redisClient, - podId, - config.state.redisKeyPrefix, - logger, - ); - - logger.info({ podId, multiPod: true }, 'Multi-pod mode enabled'); - } - - // ── 4b2. Async batch writer ────────────────────────────────────────── - const asyncBatchWriter = new AsyncBatchWriter( - manifestStore, - config.asyncWrite, - logger, - ); - asyncBatchWriter.startPeriodicFlush(); - logger.info({ flushIntervalMs: config.asyncWrite.flushIntervalMs, flushBatchSize: config.asyncWrite.flushBatchSize }, 'AsyncBatchWriter started'); - - // ── 4c. Build hash resolver for drill_events* collection defaults ──── - const hashResolver = new HashResolver( - { uri: config.source.uri, countlyDb: config.source.countlyDb }, - logger, - ); - await hashResolver.build(); - logger.info({ hashEntries: hashResolver.size }, 'Hash resolver built'); - - // ── 5. Create CollectionOrchestrator ──────────────────────────────── - const orchestrator = new CollectionOrchestrator({ - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - hashResolver, - logger, - config, - collectionLock, - globalProgress, - asyncBatchWriter, - }); - - // ── 6. Create Fastify HTTP server and register routes ─────────────── - const app = Fastify({ logger: false }); - const startedAt = new Date(); - - registerHealthRoutes(app, { - mongoReader, - chWriter, - redisState, - manifestStore, - orchestrator, - }); - - registerStatsRoute(app, { - orchestrator, - mongoReader, - chWriter, - redisState, - gcController, - processMetrics, - manifestStore, - config, - startedAt, - version: SERVICE_VERSION, - globalProgress, - collectionLock, - }); - - registerControlRoutes(app, { - orchestrator, - gcController, - manifestStore, - mongoReader, - chWriter, - globalProgress, - collectionLock, - redisState, - }); - - registerRunRoutes(app, { - manifestStore, - redisState, - orchestrator, - }); - - registerVizRoute(app); - - // ── 7. Start HTTP server ──────────────────────────────────────────── - await app.listen({ port: config.service.port, host: config.service.host }); - logger.info( - { port: config.service.port, host: config.service.host }, - 'HTTP server listening', - ); - - // ── 8. Start orchestrator (background) ────────────────────────────── - processMetrics.start(); - gcController.start(); - - const runPromise = orchestrator.run(); - runPromise.catch((err) => { - logger.fatal({ err }, 'CollectionOrchestrator crashed unexpectedly'); - process.exit(1); - }); - wireExitOnComplete(runPromise, config.service.exitOnComplete, logger); - - // ── 9. Log startup complete ───────────────────────────────────────── - logger.info( - { service: config.service.name, version: SERVICE_VERSION }, - 'Migration service started successfully', - ); - - // ── Graceful shutdown ─────────────────────────────────────────────── - let shuttingDown = false; - - async function shutdown(signal: string): Promise { - if (shuttingDown) return; - shuttingDown = true; - - logger.info({ signal }, 'Shutdown signal received, starting graceful shutdown'); - - // 1. Signal orchestrator to stop after current batch - orchestrator.stopAfterBatch(); - - // 2. Wait for orchestrator to stop (with timeout) - const timeoutMs = config.service.gracefulShutdownTimeoutMs; - try { - await Promise.race([ - orchestrator.waitForStop(), - new Promise((_, reject) => - setTimeout(() => reject(new Error('timeout')), timeoutMs), - ), - ]); - logger.info('Orchestrator stopped gracefully'); - } catch { - logger.warn('Orchestrator did not stop within timeout, forcing shutdown'); - } - - // 3. Close all resources - async function closeResource(name: string, fn: () => Promise): Promise { - try { await fn(); logger.info(`${name} closed`); } - catch (err) { logger.warn({ err }, `Error closing ${name}`); } - } - - if (collectionLock) { - collectionLock.stopHeartbeat(); - await closeResource('CollectionLocks', () => collectionLock!.releaseAll()); - } - await closeResource('AsyncBatchWriter', () => asyncBatchWriter.drainAndStop()); - await closeResource('HTTP server', () => app.close()); - await closeResource('MongoDB', () => mongoReader.close()); - await closeResource('ClickHouse', async () => { await chWriter.close(); await pressureClient.close(); }); - await closeResource('Redis', () => redisState.close()); - await closeResource('ManifestStore', () => manifestStore.close()); - await closeResource('HashResolver', () => hashResolver.close()); - processMetrics.stop(); - gcController.dispose(); - - logger.info('Graceful shutdown complete'); - process.exit(0); - } - - process.on('SIGTERM', () => shutdown('SIGTERM')); - process.on('SIGINT', () => shutdown('SIGINT')); + await runLedgerEngine(config, logger); } -// --------------------------------------------------------------------------- -// Unhandled rejection handler -// --------------------------------------------------------------------------- - process.on('unhandledRejection', (reason) => { console.error('Unhandled rejection:', reason); process.exit(1); }); -// --------------------------------------------------------------------------- -// Run -// --------------------------------------------------------------------------- - main().catch((err) => { console.error('Fatal error during startup:', err); process.exit(1); diff --git a/src/runtime/batch-runner.ts b/src/runtime/batch-runner.ts deleted file mode 100644 index d4826fc..0000000 --- a/src/runtime/batch-runner.ts +++ /dev/null @@ -1,1179 +0,0 @@ -import { EventEmitter } from "node:events"; -import { setTimeout as sleep } from "node:timers/promises"; -import type { Logger } from "pino"; - -import type { ManifestStore, Batch, BatchStatus, RunSummary, CompactError, BatchSeqRange } from "../state/manifest-store.ts"; -import type { RedisHotState, VerboseError, BatchPhase, LiveBatchData } from "../state/redis-hot-state.ts"; -import type { AsyncBatchWriter } from "../state/async-batch-writer.ts"; -import type { MongoReader } from "../source/mongo-reader.ts"; -import type { ClickHouseWriter } from "../target/clickhouse-writer.ts"; -import type { ClickHousePressure, BackpressureConfig } from "../target/clickhouse-pressure.ts"; -import type { GcController } from "./gc-controller.ts"; -import type { RetryPolicy } from "./retry-policy.ts"; -import { SkipCounter, type SkipReason } from "../transform/skip-reasons.ts"; -import { transformBatch, type OutputRow } from "../transform/normalize.ts"; -import type { CollectionDefaults } from "../transform/hash-resolver.ts"; -import { type Cursor, deserializeCursor, serializeCursor } from "../types/cursor.ts"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface BatchRunnerConfig { - runId: string; - transformVersion: string; - sourceNs: string; - targetTable: string; - upperBoundId: string; - batchRowsTarget: number; - mongoPageSize: number; - backpressure: BackpressureConfig; - useDedupToken: boolean; - database: string; - table: string; - snapshotInterval: number; - collectionDefaults?: CollectionDefaults; - batchSeqOffset?: number; - collectionName?: string; - podId?: string; - rangeIdx?: number; - batchSeqMax?: number; - nullCdMode?: boolean; - nullCdUpperBound?: string; -} - -export interface BatchRunnerDeps { - manifestStore: ManifestStore; - redisState: RedisHotState; - globalRedisState?: RedisHotState; - asyncBatchWriter?: AsyncBatchWriter; - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - chPressure: ClickHousePressure; - gcController: GcController; - retryPolicy: RetryPolicy; - logger: Logger; - config: BatchRunnerConfig; -} - -export type RunnerStatus = - | "idle" - | "running" - | "waiting_for_index" - | "paused" - | "stopping" - | "stopped" - | "completed" - | "failed"; - -export interface BatchRunnerStats { - status: RunnerStatus; - batchSeq: number; - lastCommittedId: string | null; - totalDocsRead: number; - totalRowsInserted: number; - totalDocsSkipped: number; - skipsByReason: Record; - elapsedMs: number; - docsPerSecond: number; - rowsPerSecond: number; - batchesFailed: number; - digestMismatches: number; - estimatedDuplicateRows: number; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/** Normalize unknown catch values to Error instances. */ -function toError(err: unknown): Error { - return err instanceof Error ? err : new Error(String(err)); -} - -/** Build a Batch record with common defaults, overridden by specifics. */ -function buildBatch( - runId: string, - batchSeq: number, - lowerExclusiveId: string, - upperInclusiveId: string, - docsRead: number, - docsSkipped: number, - phase: "cursor" | "null_cd", - overrides: Partial, -): Batch { - return { - run_id: runId, - batch_seq: batchSeq, - lower_exclusive_cursor: lowerExclusiveId, - upper_inclusive_cursor: upperInclusiveId, - source_docs_read: docsRead, - docs_skipped: docsSkipped, - rows_to_insert: 0, - payload_digest: "", - insert_dedup_token: "", - query_id: "", - status: "prepared", - retry_count: 0, - last_error: null, - started_at: new Date().toISOString(), - finished_at: null, - error_history: [], - digest_match: null, - phase, - ...overrides, - }; -} - -/** Lightweight digest: row count as string. ClickHouse dedup tokens handle actual deduplication. */ -function computePayloadDigest(rows: OutputRow[]): string { - return String(rows.length); -} - -// --------------------------------------------------------------------------- -// BatchRunner -// --------------------------------------------------------------------------- - -/** - * Core batch lifecycle orchestrator. - * - * The runner implements an idempotent, crash-safe processing loop: - * - * 1. Read a page from MongoDB. - * 2. Transform the documents. - * 3. Persist a batch manifest (MongoDB) in `prepared` state. - * 4. Mark `inflight`, insert into ClickHouse. - * 5. On success: mark `done` in manifest first, then update Redis. - * - * MongoDB manifest is authoritative; Redis is rebuildable. - */ -export class BatchRunner { - private status: RunnerStatus = "idle"; - private batchSeq: number; - private lastCommittedId: string | null = null; - private skipCounter: SkipCounter; - private totalRowsInserted = 0; - private totalDocsRead = 0; - private startedAt: number = 0; - private readonly emitter = new EventEmitter(); - - private batchesFailed = 0; - private batchesSkippedEmpty = 0; - private digestMismatches = 0; - private estimatedDuplicateRows = 0; - private nullCdPhaseActive = false; - private nullCdUpperBound: string | null = null; - - private readonly deps: BatchRunnerDeps; - private readonly logger: Logger; - private readonly isRangeMode: boolean; - private phaseHeartbeatTimer: ReturnType | null = null; - private currentPhaseData: LiveBatchData | null = null; - - constructor(deps: BatchRunnerDeps) { - this.deps = deps; - this.logger = deps.logger.child({ component: "BatchRunner" }); - this.skipCounter = new SkipCounter(); - this.batchSeq = deps.config.batchSeqOffset ?? 0; - this.isRangeMode = deps.config.rangeIdx !== undefined; - if (deps.config.nullCdMode) { - this.nullCdPhaseActive = true; - this.nullCdUpperBound = deps.config.nullCdUpperBound ?? null; - } - } - - // ----------------------------------------------------------------------- - // Public API - // ----------------------------------------------------------------------- - - /** Start the batch processing loop. Optional startCursor for range-parallel mode. */ - async run(startCursor?: string): Promise { - if (this.status === "running") { - throw new Error("BatchRunner is already running"); - } - - this.status = "running"; - this.startedAt = Date.now(); - const { runId, upperBoundId } = this.deps.config; - - this.logger.info({ runId, upperBoundId }, "Batch runner starting"); - - try { - // ------------------------------------------------------------------ - // 1. Resume check: look for interrupted batches - // ------------------------------------------------------------------ - await this.resumeFromInterruption(startCursor); - - // If a startCursor was provided (range mode) and no resume state exists, use it - if (startCursor && !this.lastCommittedId) { - this.lastCommittedId = startCursor; - } - - // ------------------------------------------------------------------ - // 2. Main loop - // ------------------------------------------------------------------ - while ((this.status as RunnerStatus) === "running" || (this.status as RunnerStatus) === "paused") { - // (a) Check commands (pause / stop-after-batch) - await this.checkCommands(); - - const currentStatus = this.status as RunnerStatus; - if (currentStatus === "paused") { - this.logger.info("Runner paused, waiting for resume"); - await this.waitForResume(); - if ((this.status as RunnerStatus) !== "running") break; - } - - if (currentStatus === "stopping" || currentStatus === "stopped") { - break; - } - - // (b) Sample backpressure; wait if pressured - if (this.deps.config.backpressure.enabled) { - await this.waitForBackpressure(); - if (this.status !== "running") break; - } - - // (c) Accumulate multiple MongoDB pages into one ClickHouse write batch - await this.setPhase("READING", { docsRead: 0, rowsToInsert: 0 }); - const { batchRowsTarget, mongoPageSize } = this.deps.config; - const accRows: OutputRow[] = []; - const accSkipSamples: Array<{ _id: string; reason: SkipReason }> = []; - let accDocsRead = 0; - let pageCursor: Cursor | null = this.lastCommittedId - ? deserializeCursor(this.lastCommittedId) : null; - const upperBoundCursor = deserializeCursor(upperBoundId); - let lastPageCursor: Cursor | null = null; - - while (accRows.length < batchRowsTarget) { - const page = await this.deps.retryPolicy.execute( - () => this.nullCdPhaseActive - ? this.deps.mongoReader.readNullCdPage( - pageCursor?.id ?? null, - this.nullCdUpperBound!, - mongoPageSize, - ) - : this.deps.mongoReader.readPage(pageCursor, upperBoundCursor, mongoPageSize), - `mongo-read-batch-${this.batchSeq + 1}-page`, - this.logger, - ); - - if (page.docs.length === 0) break; - - accDocsRead += page.docs.length; - lastPageCursor = page.lastCursor; - pageCursor = page.lastCursor; - - const { rows: pageRows, skippedSamples } = transformBatch( - page.docs, - this.skipCounter, - this.deps.config.collectionDefaults, - ); - accRows.push(...pageRows); - accSkipSamples.push(...skippedSamples); - - // Stop if MongoDB returned fewer docs than requested (last page) - if (page.docs.length < mongoPageSize) break; - } - - // (d) Empty accumulation = run complete (or transition to null-cd sweep) - if (accDocsRead === 0) { - // Cursor phase exhausted — check for null-cd sweep (standard mode only) - if (!this.nullCdPhaseActive && !this.isRangeMode) { - let bounds: { lower: string; upper: string } | null = null; - try { - bounds = await this.deps.mongoReader.getNullCdBounds(); - } catch (nullCdErr) { - this.logger.error( - { error: toError(nullCdErr).message }, - "Failed to query null-cd bounds — skipping null-cd sweep (cursor-phase docs already migrated)", - ); - } - if (bounds) { - this.nullCdUpperBound = bounds.upper; - await this.deps.manifestStore.updateRunPhase( - runId, "null_cd", bounds.upper, - ); - - await this.bestEffortRedis( - () => this.deps.redisState.setLastCommittedCursor(runId, ""), - "Redis cursor clear failed on phase transition", - ); - - this.nullCdPhaseActive = true; - this.lastCommittedId = null; - this.logger.info( - { nullCdUpperBound: bounds.upper }, - "Transitioning to null-cd sweep phase", - ); - continue; - } - } - - this.logger.info( - { - totalBatches: this.batchSeq, - totalDocsRead: this.totalDocsRead, - totalRowsInserted: this.totalRowsInserted, - }, - "All documents processed, run complete", - ); - - // In range mode, only the RangeCoordinator finalizes the shared run status - const runStatus = this.batchesFailed > 0 ? "completed" : "completed"; - if (this.batchesFailed > 0) { - this.logger.warn( - { batchesFailed: this.batchesFailed, totalBatches: this.batchSeq }, - "Run completed with failed batches — failed batch docs were skipped", - ); - } - if (!this.isRangeMode) { - const summary = this.buildSummary("completed"); - await this.deps.manifestStore.writeSummary(runId, "completed", summary); - } - - await this.bestEffortRedis( - () => this.deps.redisState.setState(runId, { - runId, - status: "completed", - sourceNs: this.deps.config.sourceNs, - targetTable: this.deps.config.targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: this.lastCommittedId, - transformVersion: this.deps.config.transformVersion, - totalBatches: this.batchSeq, - completedBatches: this.batchSeq - this.batchesFailed, - startedAt: new Date(this.startedAt).toISOString(), - }), - "Redis setState failed on completion", - ); - - this.setTerminalStatus("completed"); - break; - } - - await this.setPhase("TRANSFORMING", { docsRead: accDocsRead, rowsToInsert: 0 }); - this.totalDocsRead += accDocsRead; - this.batchSeq++; - - // Record skip samples in manifest (single batch write) - if (accSkipSamples.length > 0) { - const now = new Date().toISOString(); - await this.deps.manifestStore.insertSkipSamples( - accSkipSamples.map(s => ({ - run_id: runId, - batch_seq: this.batchSeq, - doc_id: s._id, - reason: s.reason, - captured_at: now, - })), - ); - } - - const lowerExclusiveId = this.lastCommittedId ?? ""; - const upperInclusiveId = serializeCursor(lastPageCursor!); - const docsSkipped = accDocsRead - accRows.length; - const rows = accRows; - - // (f) All docs skipped -> record skipped_empty batch, advance - if (rows.length === 0) { - this.logger.info( - { batchSeq: this.batchSeq, docsRead: accDocsRead, docsSkipped }, - "All documents in batch were skipped", - ); - - const batch = buildBatch(runId, this.batchSeq, lowerExclusiveId, upperInclusiveId, accDocsRead, docsSkipped, this.nullCdPhaseActive ? "null_cd" : "cursor", { - status: "skipped_empty", - finished_at: new Date().toISOString(), - }); - - await this.deps.manifestStore.insertBatch(batch); - this.lastCommittedId = upperInclusiveId; - - // Advance run cursor - await this.deps.manifestStore.updateRunLastCommittedCursor(runId, upperInclusiveId); - await this.bestEffortRedis( - () => this.deps.redisState.markBatchDone(runId, this.batchSeq), - "Redis markBatchDone failed (continuing)", - ); - - this.batchesSkippedEmpty++; - continue; - } - - // (g) Build batch record (not persisted yet — only written on success or failure) - const payloadDigest = computePayloadDigest(rows); - const queryId = `mig__${runId}__${this.batchSeq}`; - const dedupToken = this.deps.config.useDedupToken - ? `mig:${runId}:${this.batchSeq}` - : ""; - - const batch = buildBatch(runId, this.batchSeq, lowerExclusiveId, upperInclusiveId, accDocsRead, docsSkipped, this.nullCdPhaseActive ? "null_cd" : "cursor", { - rows_to_insert: rows.length, - payload_digest: payloadDigest, - insert_dedup_token: dedupToken, - query_id: queryId, - }); - - // (h) Insert into ClickHouse (no pre-write to manifest — single write on success) - try { - await this.setPhase("WRITING", { docsRead: accDocsRead, rowsToInsert: rows.length }); - this.startPhaseHeartbeat(); - const currentBatchSeq = this.batchSeq; - const result = await this.deps.retryPolicy.execute( - () => - this.deps.chWriter.insertBatch({ - runId, - batchSeq: currentBatchSeq, - rows, - }), - `ch-insert-batch-${currentBatchSeq}`, - this.logger, - async (attempt, err) => { - const now = new Date().toISOString(); - - // Audit event for retry - await this.deps.manifestStore.insertEvent({ - run_id: runId, - event_type: "batch_retry_error", - message: `Batch ${currentBatchSeq} attempt ${attempt} failed: ${err.message.slice(0, 200)}`, - metadata: { batch_seq: currentBatchSeq, attempt }, - created_at: now, - }).catch(() => {}); - - // Verbose error to Redis - await this.bestEffortRedis( - () => this.deps.redisState.pushVerboseError(runId, currentBatchSeq, { - attempt, - error: err.message, - stack: err.stack ?? null, - timestamp: now, - context: { queryId: `mig__${runId}__${currentBatchSeq}`, rowCount: rows.length }, - }), - "Redis pushVerboseError failed", - ); - }, - ); - - // (i) On success: write cursor + bitmap to Redis (commit point), then queue MongoDB - this.stopPhaseHeartbeat(); - await this.setPhase("COMMITTING", { docsRead: accDocsRead, rowsToInsert: rows.length }); - - // Redis commit point — atomic MULTI/EXEC for cursor + bitmap - await this.bestEffortRedis( - () => this.deps.redisState.commitBatch(runId, upperInclusiveId, this.batchSeq), - "Redis cursor/bitmap commit failed", - ); - - // MongoDB: async queue or direct write - if (this.deps.asyncBatchWriter) { - await this.deps.asyncBatchWriter.queueBatch(batch, upperInclusiveId); - } else { - await this.deps.manifestStore.insertCompletedBatch(batch, upperInclusiveId); - } - this.lastCommittedId = upperInclusiveId; - this.totalRowsInserted += result.rowsInserted; - - const batchSeqOffset = this.deps.config.batchSeqOffset ?? 0; - - // Redis stats (rebuildable, continue if fails) - await this.bestEffortRedis( - async () => { - await this.deps.redisState.updateStats(runId, { - docsRead: this.totalDocsRead, - docsSkipped: this.skipCounter.getTotal(), - rowsInserted: this.totalRowsInserted, - batchesDone: (this.batchSeq - batchSeqOffset) - this.batchesFailed, - batchesFailed: this.batchesFailed, - batchesInflight: 0, - elapsedMs: Date.now() - this.startedAt, - docsPerSecond: ((Date.now() - this.startedAt) / 1000) > 0 - ? this.totalDocsRead / ((Date.now() - this.startedAt) / 1000) - : 0, - lastBatchSeq: this.batchSeq, - lastBatchFinishedAt: new Date().toISOString(), - }); - }, - "Redis update failed after batch success (batch is durably committed, continuing)", - { batchSeq: this.batchSeq }, - ); - - // Timeline snapshot (every N batches) - if (this.batchSeq % this.deps.config.snapshotInterval === 0) { - const elapsedSec = (Date.now() - this.startedAt) / 1000; - await this.bestEffortRedis( - () => this.deps.redisState.pushTimelineSnapshot(runId, { - timestamp: new Date().toISOString(), - batch_seq: this.batchSeq, - docs_read: this.totalDocsRead, - rows_inserted: this.totalRowsInserted, - docs_skipped: this.skipCounter.getTotal(), - docs_per_second: elapsedSec > 0 ? this.totalDocsRead / elapsedSec : 0, - rows_per_second: elapsedSec > 0 ? this.totalRowsInserted / elapsedSec : 0, - skip_reasons: this.skipCounter.getCounts(), - digest_mismatches: this.digestMismatches, - estimated_duplicate_rows: this.estimatedDuplicateRows, - batches_failed: this.batchesFailed, - heap_used_mb: Math.round(process.memoryUsage().heapUsed / 1024 / 1024), - rss_mb: Math.round(process.memoryUsage().rss / 1024 / 1024), - }), - "Redis timeline snapshot failed (continuing)", - ); - } - - // Throughput sliding window sample (lightweight LPUSH per batch) - await this.bestEffortRedis( - () => this.deps.redisState.pushThroughputSample(runId, { - ts: Date.now(), - docsRead: this.totalDocsRead, - }), - "Redis throughput sample failed", - ); - - // Clear live batch phase - if (this.deps.config.collectionName) { - const liveRedis = this.deps.globalRedisState ?? this.deps.redisState; - await this.bestEffortRedis( - () => liveRedis.clearLiveBatch(this.deps.config.collectionName!), - "Redis clearLiveBatch failed", - ); - } - - this.logger.info( - { - batchSeq: this.batchSeq, - rowsInserted: result.rowsInserted, - insertMs: Math.round(result.insertMs), - lastId: upperInclusiveId, - }, - "Batch completed", - ); - } catch (err) { - // Insert failed after all retries — write batch record as failed - this.stopPhaseHeartbeat(); - const error = toError(err); - batch.status = "failed" as any; - batch.last_error = error.message; - batch.finished_at = new Date().toISOString(); - await this.deps.manifestStore.insertBatch(batch); - - await this.bestEffortRedis( - () => this.deps.redisState.pushError(runId, { - batchSeq: this.batchSeq, - error: error.message, - timestamp: new Date().toISOString(), - retryCount: this.deps.retryPolicy.maxRetries, - }), - "Redis pushError failed (continuing)", - ); - - this.logger.error( - { batchSeq: this.batchSeq, error: error.message }, - "Batch failed after all retries", - ); - - this.batchesFailed++; - - // Advance cursor past the failed batch so migration continues - // with the next batch. The failed batch is recorded in manifest - // and can be investigated/retried by an operator. - this.lastCommittedId = upperInclusiveId; - await this.deps.manifestStore.updateRunLastCommittedCursor(runId, upperInclusiveId); - await this.bestEffortRedis( - () => this.deps.redisState.setLastCommittedCursor(runId, upperInclusiveId), - "Redis cursor advance after batch failure", - ); - this.logger.warn( - { batchSeq: this.batchSeq, docsInBatch: accDocsRead }, - "Batch failed — cursor advanced, continuing with next batch", - ); - } - - // (l) Release batch data references (let V8 collect them) - // Rows array and page docs go out of scope naturally here. - - // (m) Conditional GC check - if (this.deps.gcController.shouldRunAfterBatch(this.batchSeq)) { - await this.deps.gcController.runGc( - "after-batch", - `post-batch-${this.batchSeq}`, - ); - } else if (this.deps.gcController.isPending) { - await this.deps.gcController.runGc( - "after-batch", - `pending-gc-after-batch-${this.batchSeq}`, - ); - } - - // (n) Next batch (loop continues) - } - - // Handle graceful stop - if ((this.status as RunnerStatus) === "stopping") { - if (!this.isRangeMode) { - const summary = this.buildSummary("stopped"); - await this.deps.manifestStore.writeSummary(runId, "stopped", summary); - } - await this.bestEffortRedis( - () => this.deps.redisState.setState(runId, { - runId, - status: "stopped", - sourceNs: this.deps.config.sourceNs, - targetTable: this.deps.config.targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: this.lastCommittedId, - transformVersion: this.deps.config.transformVersion, - totalBatches: this.batchSeq, - completedBatches: this.batchSeq - this.batchesFailed, - startedAt: new Date(this.startedAt).toISOString(), - }), - "Redis setState failed on stop", - ); - this.setTerminalStatus("stopped"); - } - } catch (err) { - this.stopPhaseHeartbeat(); - const error = toError(err); - this.logger.error({ error: error.message }, "BatchRunner fatal error"); - this.setTerminalStatus("failed"); - - if (!this.isRangeMode) { - try { - const summary = this.buildSummary("failed"); - await this.deps.manifestStore.writeSummary( - this.deps.config.runId, "failed", summary, - ); - } catch (summaryErr) { - this.logger.warn( - { error: toError(summaryErr).message }, - "Failed to write run summary on fatal error (continuing with throw)", - ); - } - } - - try { - await this.deps.manifestStore.insertEvent({ - run_id: this.deps.config.runId, - event_type: "fatal_error", - message: error.message, - metadata: { stack: error.stack }, - created_at: new Date().toISOString(), - }); - } catch (eventErr) { - this.logger.warn( - { error: toError(eventErr).message }, - "Failed to insert fatal error event (continuing with throw)", - ); - } - - throw error; - } - } - - /** Pause after current batch completes. */ - pause(): void { - if (this.status === "running") { - this.logger.info("Pause requested"); - this.status = "paused"; - this.deps.manifestStore.updateRunStatus(this.deps.config.runId, "paused").catch(() => {}); - } - } - - /** Resume from pause. */ - resume(): void { - if (this.status === "paused") { - this.logger.info("Resume requested"); - this.status = "running"; - this.emitter.emit("resumed"); - this.deps.manifestStore.updateRunStatus(this.deps.config.runId, "active").catch(() => {}); - } - } - - /** Stop after current batch completes. */ - stopAfterBatch(): void { - if (this.status === "running" || this.status === "paused") { - this.logger.info("Stop-after-batch requested"); - this.status = "stopping"; - this.emitter.emit("resumed"); // unblock waitForResume if paused - } - } - - /** Get current status. */ - getStatus(): RunnerStatus { - return this.status; - } - - /** Returns a promise that resolves when the runner reaches stopped or failed state. */ - waitForStop(): Promise { - if (this.status === "stopped" || this.status === "failed" || this.status === "completed") { - return Promise.resolve(); - } - return new Promise((resolve) => { - this.emitter.once("stopped", resolve); - }); - } - - /** Get current batch sequence number. */ - getCurrentBatchSeq(): number { - return this.batchSeq; - } - - /** Get throughput stats. */ - getStats(): BatchRunnerStats { - const elapsedMs = this.startedAt > 0 ? Date.now() - this.startedAt : 0; - const elapsedSec = elapsedMs / 1000; - - return { - status: this.status, - batchSeq: this.batchSeq, - lastCommittedId: this.lastCommittedId, - totalDocsRead: this.totalDocsRead, - totalRowsInserted: this.totalRowsInserted, - totalDocsSkipped: this.skipCounter.getTotal(), - skipsByReason: this.skipCounter.getCounts(), - elapsedMs, - docsPerSecond: elapsedSec > 0 ? this.totalDocsRead / elapsedSec : 0, - rowsPerSecond: - elapsedSec > 0 ? this.totalRowsInserted / elapsedSec : 0, - batchesFailed: this.batchesFailed, - digestMismatches: this.digestMismatches, - estimatedDuplicateRows: this.estimatedDuplicateRows, - }; - } - - // ----------------------------------------------------------------------- - // Private helpers - // ----------------------------------------------------------------------- - - /** Set terminal status and notify waiters. */ - private setTerminalStatus(status: "stopped" | "failed" | "completed"): void { - this.status = status as RunnerStatus; - this.emitter.emit("stopped"); - } - - /** Best-effort Redis call — logs warning and continues on failure. */ - private async bestEffortRedis( - fn: () => Promise, - msg: string, - ctx?: Record, - ): Promise { - try { await fn(); } - catch (err) { this.logger.warn({ error: toError(err).message, ...ctx }, msg); } - } - - private buildSummary( - _status: "completed" | "failed" | "stopped", - ): RunSummary { - const elapsedMs = this.startedAt > 0 ? Date.now() - this.startedAt : 0; - const elapsedSec = elapsedMs / 1000; - const batchesDone = this.batchSeq - this.batchesFailed; - - return { - finished_at: new Date().toISOString(), - duration_ms: elapsedMs, - total_docs_read: this.totalDocsRead, - total_rows_inserted: this.totalRowsInserted, - total_docs_skipped: this.skipCounter.getTotal(), - avg_docs_per_second: elapsedSec > 0 ? this.totalDocsRead / elapsedSec : 0, - avg_rows_per_second: elapsedSec > 0 ? this.totalRowsInserted / elapsedSec : 0, - total_batches: this.batchSeq, - batches_done: batchesDone, - batches_failed: this.batchesFailed, - batches_skipped_empty: this.batchesSkippedEmpty, - skip_reasons: this.skipCounter.getCounts(), - total_errors: this.batchesFailed, - failed_batch_seqs: [], - digest_mismatches: this.digestMismatches, - estimated_duplicate_rows: this.estimatedDuplicateRows, - coverage_pct: this.batchSeq > 0 ? (batchesDone / this.batchSeq) * 100 : 0, - }; - } - - /** - * Resume from an interrupted run (spec §15.1, §15.2). - * - * For each non-done batch (inflight or prepared): - * 1. Re-read the exact _id range from MongoDB - * 2. Re-transform and recompute SHA-256 digest - * 3. If digest matches manifest → retry insert (CH dedup handles idempotency) - * 4. If digest doesn't match → log warning and continue (lenient mode) - * - * This handles both: - * - §15.1: Crash before insert (batch is prepared or inflight, insert never ack'd) - * - §15.2: Crash after CH success but before checkpoint (CH dedup ignores the retry) - */ - private async resumeFromInterruption(startCursor?: string): Promise { - const { runId, upperBoundId } = this.deps.config; - const run = await this.deps.manifestStore.getRun(runId); - - if (!run) { - this.logger.info({ runId }, "No existing run found, starting fresh"); - return; - } - - // Compute range-scoped batch filter (undefined in standard mode) - const { rangeIdx, batchSeqOffset, batchSeqMax } = this.deps.config; - const batchSeqRange: BatchSeqRange | undefined = - rangeIdx !== undefined && batchSeqOffset !== undefined && batchSeqMax !== undefined - ? { min: batchSeqOffset, max: batchSeqMax } - : undefined; - - // ── Cursor recovery ────────────────────────────────────────────────── - // Redis per-range cursor (hot-path authority, already isolated by prefix) - const redisCursor = await this.deps.redisState.getLastCommittedCursor(runId).catch(() => null); - - if (redisCursor) { - this.lastCommittedId = redisCursor; - } else if (rangeIdx === undefined) { - // Standard (non-range) mode: fall back to run-level cursor - this.lastCommittedId = run.last_committed_cursor; - } - // Range mode with no Redis cursor: lastCommittedId stays null - // (will be set from batch-derived cursor or startCursor in run()) - - // ── Batch-derived cursor (scoped to this range's slot) ─────────────── - const lastDoneBatch = await this.deps.manifestStore.getLastDoneBatch(runId, batchSeqRange); - if (lastDoneBatch?.upper_inclusive_cursor) { - this.lastCommittedId = lastDoneBatch.upper_inclusive_cursor; - } - - // ── Bounds guard (Layer 2) ─────────────────────────────────────────── - if (this.lastCommittedId && rangeIdx !== undefined && startCursor) { - const recovered = deserializeCursor(this.lastCommittedId); - const rangeStart = deserializeCursor(startCursor); - const rangeEnd = deserializeCursor(upperBoundId); - - if (recovered.cd < rangeStart.cd || recovered.cd > rangeEnd.cd) { - this.logger.warn( - { recoveredCd: recovered.cd, rangeStartCd: rangeStart.cd, rangeEndCd: rangeEnd.cd, rangeIdx }, - "Recovered cursor outside range bounds — discarding, will use startCursor", - ); - this.lastCommittedId = null; - } - } - - // ── Null-cd phase recovery ────────────────────────────────────────── - if (run?.phase === "null_cd") { - this.nullCdPhaseActive = true; - this.nullCdUpperBound = run.null_cd_upper_bound ?? null; - - // Discard stale cursor-phase cursors — only trust null-cd phase batches - this.lastCommittedId = null; - - const lastDoneNullCd = await this.deps.manifestStore.getLastDoneBatch(runId, batchSeqRange); - if (lastDoneNullCd?.upper_inclusive_cursor - && (lastDoneNullCd.phase ?? "cursor") === "null_cd") { - this.lastCommittedId = lastDoneNullCd.upper_inclusive_cursor; - } - } - - // ── Recover interrupted batches (scoped to range slot) ─────────────── - const inflightBatches = await this.deps.manifestStore.getBatches(runId, { - status: "inflight", - batchSeqRange, - }); - const preparedBatches = await this.deps.manifestStore.getBatches(runId, { - status: "prepared", - batchSeqRange, - }); - const recoverableBatches = [...preparedBatches, ...inflightBatches] - .sort((a, b) => a.batch_seq - b.batch_seq); - - for (const batch of recoverableBatches) { - this.logger.info( - { batchSeq: batch.batch_seq, status: batch.status }, - "Attempting to recover interrupted batch", - ); - - try { - // Re-read the exact source range — use batch.phase to pick reader - const batchPhase = batch.phase ?? "cursor"; - let page; - if (batchPhase === "null_cd") { - const lowerId = batch.lower_exclusive_cursor - ? deserializeCursor(batch.lower_exclusive_cursor).id - : null; - const upperId = deserializeCursor(batch.upper_inclusive_cursor).id; - page = await this.deps.mongoReader.readNullCdPage(lowerId, upperId); - } else { - page = await this.deps.mongoReader.readPage( - batch.lower_exclusive_cursor ? deserializeCursor(batch.lower_exclusive_cursor) : null, - deserializeCursor(batch.upper_inclusive_cursor), - this.deps.config.batchRowsTarget, - ); - } - - // Re-transform - const { rows } = transformBatch(page.docs, this.skipCounter, this.deps.config.collectionDefaults); - - // Recompute digest - const newDigest = computePayloadDigest(rows); - - // Compare with stored digest (lenient mode) - const digestMatched = newDigest === batch.payload_digest; - - if (newDigest !== batch.payload_digest) { - this.digestMismatches++; - this.estimatedDuplicateRows += batch.rows_to_insert; - - await this.deps.manifestStore.insertEvent({ - run_id: runId, - event_type: "digest_mismatch", - message: `Batch ${batch.batch_seq}: source data changed, continuing (lenient mode)`, - metadata: { - batch_seq: batch.batch_seq, - stored_digest: batch.payload_digest, - computed_digest: newDigest, - original_rows: batch.rows_to_insert, - new_rows: rows.length, - estimated_duplicates: batch.rows_to_insert, - }, - created_at: new Date().toISOString(), - }); - - await this.bestEffortRedis( - () => this.deps.redisState.pushVerboseError(runId, batch.batch_seq, { - attempt: 0, - error: `Digest mismatch: stored=${batch.payload_digest} computed=${newDigest}`, - stack: null, - timestamp: new Date().toISOString(), - context: { - stored_digest: batch.payload_digest, - computed_digest: newDigest, - original_rows: batch.rows_to_insert, - new_rows: rows.length, - }, - }), - "Redis pushVerboseError failed on digest mismatch", - ); - - this.logger.warn( - { - batchSeq: batch.batch_seq, - storedDigest: batch.payload_digest, - computedDigest: newDigest, - estimatedDuplicates: batch.rows_to_insert, - }, - "Digest mismatch on recovery — continuing (lenient mode, CH dedup will handle)", - ); - } - - // Retry the insert with the same dedup token (CH dedup ignores duplicates) - if (rows.length > 0) { - await this.deps.manifestStore.updateBatchStatus(runId, batch.batch_seq, "inflight"); - - await this.deps.retryPolicy.execute( - () => - this.deps.chWriter.insertBatch({ - runId, - batchSeq: batch.batch_seq, - rows, - }), - `ch-recovery-batch-${batch.batch_seq}`, - this.logger, - ); - } - - // Mark done + advance cursor, then set digest_match - await this.deps.manifestStore.completeBatch(runId, batch.batch_seq, batch.upper_inclusive_cursor); - await this.deps.manifestStore.updateBatchDigestMatch(runId, batch.batch_seq, digestMatched); - this.lastCommittedId = batch.upper_inclusive_cursor; - this.totalDocsRead += page.docs.length; - this.totalRowsInserted += rows.length; - - try { - await this.deps.redisState.markBatchDone(runId, batch.batch_seq); - } catch { - this.logger.warn("Redis markBatchDone failed during recovery (continuing)"); - } - - this.logger.info( - { batchSeq: batch.batch_seq }, - "Successfully recovered interrupted batch", - ); - - await this.deps.manifestStore.insertEvent({ - run_id: runId, - event_type: "batch_recovered", - message: `Batch ${batch.batch_seq} recovered from ${batch.status}`, - metadata: { - batch_seq: batch.batch_seq, - prior_status: batch.status, - digest_matched: digestMatched, - }, - created_at: new Date().toISOString(), - }); - } catch (err) { - const error = toError(err); - this.logger.error( - { batchSeq: batch.batch_seq, error: error.message }, - "Failed to recover interrupted batch — marked as failed, continuing with remaining batches", - ); - await this.deps.manifestStore.updateBatchStatus(runId, batch.batch_seq, "failed", error.message).catch(() => {}); - this.batchesFailed++; - } - } - - // ── Restore batchSeq (scoped to this range's slot) ─────────────────── - const lastBatch = await this.deps.manifestStore.getLastBatch(runId, batchSeqRange); - if (lastBatch) { - this.batchSeq = lastBatch.batch_seq; - } - - // ── Recover accumulated counters ───────────────────────────────────── - const lastStats = await this.deps.redisState.getStats(runId).catch(() => null); - if (lastStats && lastStats.docsRead > 0) { - this.totalDocsRead = lastStats.docsRead; - this.totalRowsInserted = lastStats.rowsInserted; - this.logger.info( - { docsRead: lastStats.docsRead, rowsInserted: lastStats.rowsInserted, source: "redis" }, - "Recovered accumulated counters from Redis", - ); - } else { - // Redis stats missing (flushed) — fallback to manifest aggregate (scoped) - const aggregate = await this.deps.manifestStore.sumCompletedBatchStats(runId, batchSeqRange); - if (aggregate.docsRead > 0) { - this.totalDocsRead = aggregate.docsRead; - this.totalRowsInserted = aggregate.rowsInserted; - this.logger.info( - { docsRead: aggregate.docsRead, rowsInserted: aggregate.rowsInserted, source: "manifest" }, - "Recovered accumulated counters from manifest", - ); - } - } - - this.logger.info( - { - runId, - resumeFromId: this.lastCommittedId, - resumeFromBatchSeq: this.batchSeq, - inflightRecovered: inflightBatches.length, - rangeIdx, - batchSeqRange: batchSeqRange ? `[${batchSeqRange.min}, ${batchSeqRange.max})` : "global", - }, - "Resuming from last committed position", - ); - } - - /** Update the live batch phase in Redis with heartbeat. */ - private async setPhase(phase: BatchPhase, stats: { docsRead: number; rowsToInsert: number }): Promise { - const { collectionName, podId, rangeIdx } = this.deps.config; - if (!collectionName) return; - - this.currentPhaseData = { - collection: collectionName, - podId: podId ?? "unknown", - batchSeq: this.batchSeq, - phase, - docsRead: stats.docsRead, - rowsToInsert: stats.rowsToInsert, - startedAt: Date.now(), - rangeIdx, - }; - - const liveRedis = this.deps.globalRedisState ?? this.deps.redisState; - await this.bestEffortRedis( - () => liveRedis.setLiveBatch(collectionName, this.currentPhaseData!), - "Redis setLiveBatch failed", - ); - } - - private startPhaseHeartbeat(): void { - this.stopPhaseHeartbeat(); - const liveRedis = this.deps.globalRedisState ?? this.deps.redisState; - this.phaseHeartbeatTimer = setInterval(() => { - if (this.currentPhaseData && this.deps.config.collectionName) { - liveRedis.setLiveBatch(this.deps.config.collectionName, this.currentPhaseData).catch(() => {}); - } - }, 10_000); - } - - private stopPhaseHeartbeat(): void { - if (this.phaseHeartbeatTimer) { - clearInterval(this.phaseHeartbeatTimer); - this.phaseHeartbeatTimer = null; - } - } - - /** - * Poll Redis for operator commands (pause, stop-after-batch). - */ - private async checkCommands(): Promise { - const { runId } = this.deps.config; - - try { - const commands = await this.deps.redisState.getCommands(runId); - - if (commands.pause) { - this.pause(); - // Clear the command flag - await this.deps.redisState.setCommand(runId, "pause", false); - } - - if (commands.abort) { - this.stopAfterBatch(); - await this.deps.redisState.setCommand(runId, "abort", false); - } - } catch (err) { - this.logger.warn({ error: toError(err).message }, "Failed to check Redis commands, continuing"); - } - } - - /** - * Block until the runner is resumed via the public API or Redis command. - * Uses event-driven wait with periodic Redis command polling. - */ - private async waitForResume(): Promise { - while (this.status === "paused") { - await new Promise((resolve) => { - const onResume = () => { clearTimeout(timer); resolve(); }; - const timer = setTimeout(() => { - this.emitter.off("resumed", onResume); - resolve(); - }, 2_000); - this.emitter.once("resumed", onResume); - }); - if (this.status === "paused") { - await this.checkCommands(); - } - } - } - - /** - * Poll ClickHouse backpressure and wait until pressure subsides. - * - * Polls at `backpressure.pollIntervalMs`, up to `maxPauseEpisodeMs`. - * If the deadline is reached the runner continues regardless (operator - * must intervene if merges are truly stuck). - */ - private async waitForBackpressure(): Promise { - const { backpressure, database, table } = this.deps.config; - - const pressure = await this.deps.chPressure.sample(database, table); - - if (!pressure.shouldPause) return; - - this.logger.warn( - { reason: pressure.pauseReason }, - "Backpressure detected, pausing inserts", - ); - - const deadline = Date.now() + backpressure.maxPauseEpisodeMs; - - while (Date.now() < deadline && this.status === "running") { - await sleep(backpressure.pollIntervalMs); - - const sample = await this.deps.chPressure.sample(database, table); - if (sample.canResume) { - this.logger.info("Backpressure cleared, resuming inserts"); - return; - } - } - - if (this.status === "running") { - this.logger.warn( - { maxPauseEpisodeMs: backpressure.maxPauseEpisodeMs }, - "Backpressure wait deadline reached, resuming anyway", - ); - } - } -} diff --git a/src/runtime/chunk-orchestrator.ts b/src/runtime/chunk-orchestrator.ts new file mode 100644 index 0000000..2255ed2 --- /dev/null +++ b/src/runtime/chunk-orchestrator.ts @@ -0,0 +1,1695 @@ +/** + * ChunkOrchestrator — the `ledger` engine. + * + * Work model: each collection is split into cd-bounded chunks. Per chunk: + * claim (atomic, leased, newest-first) → stream-copy into a per-chunk staging + * table (one long-lived cursor; concurrent synchronous inserts) → verify + * (read tally vs exact ClickHouse count) → promote (verify-then-ATTACH per + * partition, INSERT SELECT fallback) → drop staging → done. + * + * Failure model: + * - permanent insert errors are BISECTED down to the offending documents, + * which land in the DLQ with their full raw source doc (replayable); + * - a circuit breaker pauses the engine when failures look systematic; + * - ClickHouse parts pressure is respected via a TTL-cached sampler; + * - crash recovery never trusts the ledger: in_progress → drop + redo, + * written → recount, attaching → verify-then-attach per partition; + * - an invariant monitor spot-checks done chunks against the live table. + * + * No Redis anywhere; MongoDB (ledger + DLQ) + ClickHouse are the only + * dependencies. Dry-run mode targets a Null-engine clone with ≤5% sampling. + */ + +import type { Logger } from 'pino'; +import type { Config } from '../config/schema.ts'; +import type { MongoReader } from '../source/mongo-reader.ts'; +import type { HashResolver, CollectionDefaults } from '../transform/hash-resolver.ts'; +import { chScopeOf, type ChScope } from '../transform/hash-resolver.ts'; +import { toEpochMillis, clampDateTime64 } from '../transform/validators.ts'; +import type { RetryPolicy } from './retry-policy.ts'; +import type { ClickHousePressure, PressureState } from '../target/clickhouse-pressure.ts'; +import { LedgerStore, type ChunkDoc } from '../state/ledger-store.ts'; +import { DlqStore } from '../state/dlq-store.ts'; +import { StagingManager } from '../target/staging-manager.ts'; +import { transformDocument, type OutputRow, type SourceDocument } from '../transform/normalize.ts'; +import { SkipCounter, SkipReason } from '../transform/skip-reasons.ts'; +import { CoercionCounter } from '../transform/coercions.ts'; +import { classifyError } from './error-classifier.ts'; +import { discoverCollections } from '../source/discover-collections.ts'; +import type { Cursor } from '../types/cursor.ts'; +import { createHash } from 'node:crypto'; +import { setTimeout as sleep } from 'node:timers/promises'; + +export interface ChunkOrchestratorDeps { + config: Config; + logger: Logger; + mongoReader: MongoReader; + ledger: LedgerStore; + dlq: DlqStore; + staging: StagingManager; + retryPolicy: RetryPolicy; + hashResolver: HashResolver; + chPressure?: ClickHousePressure; +} + +export interface LedgerEngineStats { + engine: 'ledger'; + runId: string; + podId: string; + status: string; + fatalError: string | null; + dryRun: boolean; + currentCollection: string | null; + currentChunk: string | null; + totalDocsRead: number; + totalDocsSkipped: number; + totalRowsInserted: number; + totalDocsDlq: number; + totalCoercions: number; + chunksDone: number; + chunksFailed: number; + docsPerSecond: number; + stageMs: { read: number; transform: number; insert: number; verify: number; attach: number; pressureWait: number }; + dedupWorks: boolean | null; + chunkStatusCounts: Record; +} + +/** + * Chunk grid from cd span + doc estimate. Two independent sizing signals: + * doc estimate AND time span — the span floor protects against a corrupted + * estimate (metadata fastcount resets after unclean mongod shutdowns) + * producing a whole-collection chunk. Pure; shared with ledger rebuild. + */ +export function computeChunkBounds( + lowerCd: number, + upperCd: number, + estimated: number, + chunkDocsTarget: number, + maxChunkDays: number, +): Array<{ lowerCd: number; upperCd: number }> { + const spanMs = upperCd + 1 - lowerCd; + const byDocs = Math.ceil(estimated / chunkDocsTarget); + const bySpan = Math.ceil(spanMs / (maxChunkDays * 86_400_000)); + const chunkCount = Math.max(1, Math.min(50_000, Math.max(byDocs, bySpan))); + const bounds: Array<{ lowerCd: number; upperCd: number }> = []; + for (let i = 0; i < chunkCount; i++) { + const lo = lowerCd + Math.floor((spanMs * i) / chunkCount); + const hi = i === chunkCount - 1 ? upperCd + 1 : lowerCd + Math.floor((spanMs * (i + 1)) / chunkCount); + if (hi > lo) bounds.push({ lowerCd: lo, upperCd: hi }); + } + return bounds; +} + +const MAX_CHUNK_ATTEMPTS = 3; +const BISECT_LOG_THRESHOLD = 1; + +function shortHash(s: string): string { + return createHash('sha1').update(s).digest('hex').slice(0, 8); +} + +export class ChunkOrchestrator { + private readonly d: ChunkOrchestratorDeps; + private readonly logger: Logger; + private readonly runId: string; + private readonly podId: string; + private readonly dryRun: boolean; + + private status = 'idle'; + private fatalError: string | null = null; + private multiCollection = false; + private finishedAt = 0; + /** Live progress of a running verify (billion-scale runs take minutes). */ + readonly verifyProgress = { running: false, checked: 0, total: 0, phase: '' }; + private stopping = false; + private paused = false; + private currentCollection: string | null = null; + private currentChunk: string | null = null; + private startedAt = 0; + private consecutiveFailed = 0; + private lastReclaimAt = 0; + private monitorTimer: ReturnType | null = null; + + private readonly coercions = new CoercionCounter(); + private readonly skips = new SkipCounter(); + private totalDocsRead = 0; + private totalDocsSkipped = 0; + private totalRowsInserted = 0; + private totalDocsDlq = 0; + private chunksDone = 0; + private chunksFailed = 0; + private stageMs = { read: 0, transform: 0, insert: 0, verify: 0, attach: 0, pressureWait: 0 }; + private lastStatusCounts: Record = {}; + + private lastPressure: { state: PressureState; at: number } | null = null; + + constructor(deps: ChunkOrchestratorDeps) { + this.d = deps; + this.logger = deps.logger.child({ component: 'ChunkOrchestrator' }); + this.dryRun = deps.config.ledger.dryRun; + this.runId = this.dryRun ? `${deps.config.ledger.runId}-dry` : deps.config.ledger.runId; + this.podId = deps.config.worker.podId; + } + + // ------------------------------------------------------------------------- + // Controls + // ------------------------------------------------------------------------- + + stopAfterChunk(): void { this.stopping = true; } + pause(): void { this.paused = true; if (this.status === 'running') this.status = 'paused'; } + resume(): void { this.paused = false; if (this.status === 'paused') this.status = 'running'; } + getStatus(): string { return this.status; } + + /** ClickHouse row-identity scope of a chunk's collection; null when unresolvable. */ + private scopeOf(chunk: ChunkDoc): ChScope | null { + if (!chunk.scope_a || !chunk.scope_e) return null; + return { a: chunk.scope_a, e: chunk.scope_e, ...(chunk.scope_n ? { n: chunk.scope_n } : {}) }; + } + + /** A startup/config failure should not kill the console — surface it instead. */ + markFatal(message: string): void { + this.status = 'failed'; + this.fatalError = message; + this.finishedAt = Date.now(); + } + + // ------------------------------------------------------------------------- + // Main + // ------------------------------------------------------------------------- + + async run(): Promise { + this.status = 'running'; + this.startedAt = Date.now(); + this.finishedAt = 0; + const { config } = this.d; + + if (this.dryRun) { + await this.d.staging.createDryRunTable(); + this.logger.warn( + { samplePct: config.ledger.dryRunSamplePct }, + 'DRY RUN: sampled rehearsal against a Null-engine clone — nothing is stored, nothing is promoted', + ); + } else { + await this.d.staging.runDedupCanary(); + await this.checkDlqPressure(this.logger, true); // inherited mass-DLQ pauses a resumed run too + this.startInvariantMonitor(); + } + + const db = this.d.mongoReader.getDatabase(); + let collections = await discoverCollections(db, config.source.collectionPrefix, this.logger); + + const skipEventNames = new Set(['[CLY]_apm_device', '[CLY]_apm_network']); + collections = collections.filter((name) => { + const defaults = this.d.hashResolver.resolveCollectionName(name, config.source.collectionPrefix); + return !(defaults && skipEventNames.has(defaults.e)); + }); + + this.multiCollection = collections.length > 1; + this.logger.info({ collections: collections.length, runId: this.runId, dryRun: this.dryRun }, 'Ledger engine starting'); + + for (const collection of collections) { + if (this.stopping) break; + await this.processCollection(collection); + } + + if (this.monitorTimer) clearInterval(this.monitorTimer); + this.status = this.stopping ? 'stopped' : 'completed'; + this.finishedAt = Date.now(); + this.logger.info( + { + status: this.status, + chunksDone: this.chunksDone, + chunksFailed: this.chunksFailed, + totalDocsRead: this.totalDocsRead, + totalRowsInserted: this.totalRowsInserted, + totalDocsDlq: this.totalDocsDlq, + totalCoercions: this.coercions.getTotal(), + elapsedSec: Math.round((Date.now() - this.startedAt) / 1000), + }, + 'Ledger engine finished', + ); + } + + // ------------------------------------------------------------------------- + // Per-collection flow + // ------------------------------------------------------------------------- + + private async processCollection(collection: string): Promise { + const { config, mongoReader, ledger } = this.d; + this.currentCollection = collection; + const log = this.logger.child({ collection }); + + await mongoReader.switchCollection(collection); + + if (!(await mongoReader.hasRequiredIndex(collection))) { + log.info('Building {cd:1,_id:1} index'); + await mongoReader.startIndexCreation(collection); + } + + const lower = await mongoReader.getLowerBound(); + const upper = await mongoReader.getUpperBound(); + if (!lower || !upper) { + log.info('Collection empty (no cd-bearing docs), skipping'); + return; + } + + const estimated = await mongoReader.getEstimatedCount(); + let bounds = computeChunkBounds(lower.cd, upper.cd, estimated, config.ledger.chunkDocsTarget, config.ledger.maxChunkDays); + + // Dry run: keep every k-th chunk so old and new data shapes are both covered. + if (this.dryRun) { + const k = Math.max(1, Math.ceil(100 / config.ledger.dryRunSamplePct)); + bounds = bounds.filter((_, i) => i % k === 0); + } + + // Null-cd sweep chunk: documents with no `cd` value are invisible to the + // cd-bounded chunks — they get one dedicated chunk, paged by `_id`. + // Sentinel bounds {-1, 0} mark it. + if (!this.dryRun && (await mongoReader.hasNullCdDocuments())) { + bounds.push({ lowerCd: -1, upperCd: 0 }); + log.info('Collection has null-cd documents — added null-cd sweep chunk'); + } + + const defaults = this.d.hashResolver.resolveCollectionName(collection, config.source.collectionPrefix) ?? undefined; + const created = await ledger.initChunks(this.runId, collection, bounds, config.transform.version, defaults ? chScopeOf(defaults) : null); + log.info({ estimated, chunks: bounds.length, created, scoped: !!defaults, dryRun: this.dryRun }, 'Chunk list ready'); + + await this.recoverChunks(collection, log); + + for (;;) { + if (this.stopping) return; + while (this.paused && !this.stopping) await sleep(1_000); + await this.reclaimExpiredLeases(collection, log); + + // The null-cd sweep must run strictly after every regular chunk is + // terminal — its rows land inside regular chunks' cd windows and would + // poison their verify-then-attach checks. + const regularsRemaining = await ledger.countRegularNonTerminal(this.runId, collection); + const chunk = await ledger.claimNext(this.runId, collection, this.podId, config.ledger.leaseSec, regularsRemaining > 0); + if (chunk && chunk.attempts > MAX_CHUNK_ATTEMPTS && this.isSplittable(chunk)) { + // Poison-pill quarantine: this chunk keeps killing the process (a + // clean data error would have failed it long before exhausting + // crash-retries). Don't retry the same span again — bisect it, so + // repeated splitting converges on a tiny window around the poison + // document instead of quarantining millions of docs. + const parts = await ledger.splitChunk(chunk, 4); + log.warn( + { chunk: chunk._id, attempts: chunk.attempts, parts }, + 'Chunk exhausted crash-retries — split into sub-chunks (poison-pill hunt)', + ); + continue; + } + if (!chunk) { + // Nothing pending — but "complete" means NO non-terminal chunks. + // Chunks may still be leased by another pod (or orphaned by a dead + // one); wait for their leases instead of declaring a hole "done". + const nonTerminal = await ledger.findRecoverable(this.runId, collection, true); + if (nonTerminal.length === 0) break; + if (!config.worker.enabled) { + // Single-pod: no other pod can own these — recover immediately. + for (const orphan of nonTerminal) await this.recoverOne(orphan, log); + continue; + } + log.info( + { waitingOn: nonTerminal.length }, + 'No pending chunks; waiting on leased in-flight chunks (reclaim on lease expiry)', + ); + await sleep(Math.min((config.ledger.leaseSec * 1000) / 2, 15_000)); + continue; + } + if (chunk.attempts > MAX_CHUNK_ATTEMPTS) { + // Too small to split further (or the null-cd sentinel): quarantine. + // The window is now tiny — the offending doc(s) are inspectable + // directly in the source between lower_cd and upper_cd. + await ledger.transition(chunk._id, 'in_progress', 'failed', { + last_error: `exceeded ${MAX_CHUNK_ATTEMPTS} attempts (crash quarantine — inspect source docs in this cd window)`, + }); + this.noteChunkFailure(log); + continue; + } + await this.processChunk(chunk, defaults, log); + this.lastStatusCounts = await ledger.statusCounts(this.runId, collection); + } + + await this.sweepOrphanStaging(collection); + this.lastStatusCounts = await ledger.statusCounts(this.runId, collection); + log.info({ statusCounts: this.lastStatusCounts }, 'Collection complete'); + this.currentCollection = null; + } + + // ------------------------------------------------------------------------- + // Recovery & multi-pod lease reclaim + // ------------------------------------------------------------------------- + + private async recoverChunks(collection: string, log: Logger): Promise { + const includeAll = !this.d.config.worker.enabled; + const recoverable = await this.d.ledger.findRecoverable(this.runId, collection, includeAll); + for (const chunk of recoverable) { + await this.recoverOne(chunk, log); + } + } + + /** Periodic tick (multi-pod): reclaim chunks whose owner's lease expired. */ + private async reclaimExpiredLeases(collection: string, log: Logger): Promise { + if (!this.d.config.worker.enabled) return; + const intervalMs = Math.min((this.d.config.ledger.leaseSec * 1000) / 2, 30_000); + if (Date.now() - this.lastReclaimAt < intervalMs) return; + this.lastReclaimAt = Date.now(); + const expired = await this.d.ledger.findRecoverable(this.runId, collection, false); + for (const chunk of expired) { + log.warn({ chunk: chunk._id, pod: chunk.pod_id }, 'Reclaiming chunk from expired lease'); + await this.recoverOne(chunk, log); + } + } + + private async recoverOne(chunk: ChunkDoc, log: Logger): Promise { + const { ledger, staging } = this.d; + const stagingTable = chunk.staging_table ?? this.stagingName(chunk.collection, chunk.idx); + log.info({ chunk: chunk._id, status: chunk.status }, 'Recovering chunk'); + + if (chunk.status === 'in_progress') { + // Mid-copy crash: never reconstruct — drop and redo. + await staging.dropStaging(stagingTable); + await ledger.transition(chunk._id, 'in_progress', 'pending', { staging_table: null, pod_id: null }); + return; + } + if (chunk.status === 'written') { + const count = await staging.countRows(stagingTable).catch(() => -1); + if (count === chunk.rows_expected && count >= 0) { + await this.promoteChunk({ ...chunk, staging_table: stagingTable }, log); + } else { + await staging.dropStaging(stagingTable); + await ledger.transition(chunk._id, 'written', 'pending', { staging_table: null, pod_id: null }); + } + return; + } + if (chunk.status === 'attaching') { + // The one state where blind retry is unsafe (double-attach duplicates). + await this.finishAttaching({ ...chunk, staging_table: stagingTable }, log); + } + } + + // ------------------------------------------------------------------------- + // Backpressure (TTL-cached — never 3 system queries per batch) + // ------------------------------------------------------------------------- + + private async respectBackpressure(): Promise { + const { chPressure, config } = this.d; + if (!chPressure || !config.backpressure.enabled || this.dryRun) return; + + const now = Date.now(); + if (this.lastPressure && now - this.lastPressure.at < config.backpressure.pollIntervalMs) { + if (!this.lastPressure.state.shouldPause) return; + } + + const t0 = performance.now(); + let state = await chPressure.sample(config.target.db, config.target.table); + this.lastPressure = { state, at: Date.now() }; + + if (state.shouldPause) { + this.logger.warn({ reason: state.pauseReason }, 'ClickHouse backpressure — pausing inserts'); + const deadline = Date.now() + config.backpressure.maxPauseEpisodeMs; + while (Date.now() < deadline && !this.stopping) { + await sleep(config.backpressure.pollIntervalMs); + state = await chPressure.sample(config.target.db, config.target.table); + this.lastPressure = { state, at: Date.now() }; + if (state.canResume) break; + } + } + this.stageMs.pressureWait += performance.now() - t0; + } + + // ------------------------------------------------------------------------- + // Chunk processing + // ------------------------------------------------------------------------- + + private stagingName(collection: string, idx: number): string { + return `${this.d.config.target.table}__stg_${shortHash(`${this.runId}:${collection}`)}_${idx}`; + } + + private async processChunk( + chunk: ChunkDoc, + defaults: CollectionDefaults | undefined, + log: Logger, + ): Promise { + const { config, mongoReader, ledger, staging } = this.d; + this.currentChunk = chunk._id; + const stagingTable = this.dryRun ? staging.dryRunTable : this.stagingName(chunk.collection, chunk.idx); + const clog = log.child({ chunk: chunk.idx, staging: stagingTable }); + + const heartbeat = setInterval(() => { + ledger.heartbeat(chunk._id, this.podId, config.ledger.leaseSec).catch(() => {}); + }, Math.max(10_000, (config.ledger.leaseSec * 1000) / 3)); + + try { + if (!this.dryRun) { + await staging.createStaging(stagingTable); + } + await ledger.transition(chunk._id, 'in_progress', 'in_progress', { staging_table: stagingTable }); + + const result = await this.copyChunk(chunk, stagingTable, defaults, clog); + + this.totalDocsRead += result.docsRead; + this.totalDocsSkipped += result.docsSkipped; + this.totalDocsDlq += result.docsDlq; + + const rowsExpected = result.docsRead - result.docsSkipped - result.docsDlq; + await ledger.transition(chunk._id, 'in_progress', 'written', { + docs_read: result.docsRead, + docs_skipped: result.docsSkipped, + rows_expected: rowsExpected, + }); + + // Circuit breaker: a high in-chunk failure rate is a systematic bug, + // not dirty data — halt before the DLQ balloons into a dataset copy. + const failRate = result.docsRead > 1_000 + ? (result.docsDlq + result.transformErrors) / result.docsRead : 0; + if (failRate > config.ledger.breakerPct / 100) { + clog.error( + { failRate: (failRate * 100).toFixed(1) + '%', dlq: result.docsDlq, transformErrors: result.transformErrors }, + 'Circuit breaker tripped — pausing engine (systematic failure suspected)', + ); + if (!this.dryRun) await staging.dropStaging(stagingTable).catch(() => {}); + await ledger.transition(chunk._id, 'written', 'failed', { + staging_table: null, + last_error: `circuit breaker: ${(failRate * 100).toFixed(1)}% of docs failed`, + }); + this.noteChunkFailure(clog); + this.pause(); + return; + } + + if (this.dryRun) { + // Null-engine target: nothing stored, nothing to verify or promote. + await ledger.transition(chunk._id, 'written', 'done', { attach_method: null }); + this.chunksDone++; + this.consecutiveFailed = 0; + clog.info({ docsRead: result.docsRead, dlq: result.docsDlq }, 'Dry-run chunk done'); + return; + } + + // Verify: read tally vs exact ClickHouse count. + const vStart = performance.now(); + const landed = await staging.countRows(stagingTable); + this.stageMs.verify += performance.now() - vStart; + + if (landed !== rowsExpected) { + clog.warn({ landed, rowsExpected }, 'Verification mismatch — dropping chunk for redo'); + await staging.dropStaging(stagingTable); + await ledger.transition(chunk._id, 'written', 'pending', { + staging_table: null, + pod_id: null, + last_error: `verify mismatch: expected ${rowsExpected}, landed ${landed}`, + }); + return; + } + + await this.promoteChunk( + { ...chunk, staging_table: stagingTable, rows_expected: rowsExpected, docs_read: result.docsRead, docs_skipped: result.docsSkipped }, + clog, + ); + this.consecutiveFailed = 0; + + // Per-commit under-read guard: the tally-independent check. Everything + // else compares against docs READ; this one asks the SOURCE how many + // docs the window holds. A silently truncated cursor (tally == staging + // == live, all short) is caught here, at commit time, for the price of + // one indexed count (~1% of chunk duration). + if (config.ledger.sourceCountCheck && !this.isNullCdChunk(chunk)) { + const srcCount = await this.d.mongoReader.getDatabase().collection(chunk.collection) + .countDocuments({ cd: { $gte: new Date(chunk.lower_cd), $lt: new Date(chunk.upper_cd) } }); + if (srcCount !== result.docsRead) { + clog.error( + { sourceCount: srcCount, docsRead: result.docsRead }, + 'SOURCE-COUNT MISMATCH: window holds more/fewer docs than were read — flagging chunk for redo', + ); + await ledger.transition(chunk._id, 'done', 'failed', { + last_error: `source-count mismatch: source=${srcCount} read=${result.docsRead} — under/over-read; retry redoes the window`, + }); + this.noteChunkFailure(clog); + } + } + + clog.info( + { docsRead: result.docsRead, docsSkipped: result.docsSkipped, dlq: result.docsDlq, rowsExpected }, + 'Chunk done', + ); + + await this.checkDlqPressure(clog); + } catch (err) { + const error = err as Error; + const isPermanent = classifyError(err) === 'permanent'; + clog.error({ error: error.message, isPermanent }, 'Chunk failed'); + if (!this.dryRun) await staging.dropStaging(stagingTable).catch(() => {}); + const target = isPermanent || chunk.attempts >= MAX_CHUNK_ATTEMPTS ? 'failed' : 'pending'; + await ledger.transition(chunk._id, ['in_progress', 'written'], target, { + staging_table: null, + pod_id: null, + last_error: error.message.slice(0, 500), + }); + if (target === 'failed') this.noteChunkFailure(clog); + } finally { + clearInterval(heartbeat); + this.currentChunk = null; + } + } + + /** + * Stream-copy one chunk into its staging table. + * One long-lived cursor (reopened from the last committed position on + * cursor death), per-doc transform that keeps the raw doc paired with its + * row (for DLQ), and a bounded window of concurrent inserts with + * bisection on permanent errors. + */ + private isNullCdChunk(chunk: ChunkDoc): boolean { + return chunk.lower_cd === -1 && chunk.upper_cd === 0; + } + + /** Splittable for the poison-pill hunt: cd-bounded and wider than 1 minute. */ + private isSplittable(chunk: ChunkDoc): boolean { + return !this.isNullCdChunk(chunk) && chunk.upper_cd - chunk.lower_cd > 60_000; + } + + private async copyChunk( + chunk: ChunkDoc, + stagingTable: string, + defaults: CollectionDefaults | undefined, + clog: Logger, + ): Promise<{ docsRead: number; docsSkipped: number; docsDlq: number; transformErrors: number }> { + const { config, mongoReader } = this.d; + + if (this.isNullCdChunk(chunk)) { + return this.copyNullCdChunk(chunk, stagingTable, defaults, clog); + } + const upperBound: Cursor = { cd: chunk.upper_cd, id: '' }; + let resumeFrom: Cursor | null = { cd: chunk.lower_cd, id: '' }; + let skipFirstId: string | null = null; + + const state = { docsRead: 0, docsSkipped: 0, docsDlq: 0, transformErrors: 0, batchSeq: 0, firstError: null as Error | null }; + const inflight: Promise[] = []; + + for (let attempt = 0; attempt < 5 && !state.firstError; attempt++) { + try { + const stream = mongoReader.readStream(resumeFrom, upperBound, config.source.mongoPageSize); + for await (const page of stream) { + this.stageMs.read += page.fetchMs; + + let docs = page.docs; + // min() is inclusive: on (re)open, drop the already-processed boundary doc. + if (skipFirstId !== null && String(docs[0]?._id) === skipFirstId) docs = docs.slice(1); + skipFirstId = null; + + if (docs.length === 0) { resumeFrom = page.lastCursor; continue; } + await this.processPage(docs, chunk, stagingTable, defaults, state, inflight, clog); + + resumeFrom = page.lastCursor; + if (state.firstError) break; + } + break; // stream exhausted cleanly + } catch (err) { + // Cursor died — reopen from the last committed position. + clog.warn({ error: (err as Error).message, attempt }, 'Read stream failed — reopening from last cursor'); + skipFirstId = resumeFrom && resumeFrom.id !== '' ? resumeFrom.id : null; + if (attempt === 4) throw err; + await sleep(1_000 * (attempt + 1)); + } + } + + const iStart = performance.now(); + await Promise.all(inflight); + this.stageMs.insert += performance.now() - iStart; + + if (state.firstError) throw state.firstError; + + // DLQ'd transform errors are already counted in docsSkipped; insert-DLQ'd + // docs are not skipped (they were readable and transformable). + return { docsRead: state.docsRead, docsSkipped: state.docsSkipped, docsDlq: state.docsDlq, transformErrors: state.transformErrors }; + } + + /** Null-cd sweep: page by `_id` over docs with no cd value. */ + private async copyNullCdChunk( + chunk: ChunkDoc, + stagingTable: string, + defaults: CollectionDefaults | undefined, + clog: Logger, + ): Promise<{ docsRead: number; docsSkipped: number; docsDlq: number; transformErrors: number }> { + const { config, mongoReader } = this.d; + const bounds = await mongoReader.getNullCdBounds(); + const state = { docsRead: 0, docsSkipped: 0, docsDlq: 0, transformErrors: 0, batchSeq: 0, firstError: null as Error | null }; + if (!bounds) return state; + + const inflight: Promise[] = []; + let lastId: string | null = null; + for (;;) { + const page = await mongoReader.readNullCdPage(lastId, bounds.upper, config.source.mongoPageSize); + this.stageMs.read += page.fetchMs; + if (page.docs.length === 0) break; + await this.processPage(page.docs, chunk, stagingTable, defaults, state, inflight, clog); + lastId = page.lastCursor!.id; + if (state.firstError || page.docs.length < config.source.mongoPageSize) break; + } + + const iStart = performance.now(); + await Promise.all(inflight); + this.stageMs.insert += performance.now() - iStart; + if (state.firstError) throw state.firstError; + return state; + } + + /** Shared per-page pipeline: transform (with raw-doc pairing), DLQ capture, + * backpressure, and windowed insert-or-bisect. */ + private async processPage( + docs: SourceDocument[], + chunk: ChunkDoc, + stagingTable: string, + defaults: CollectionDefaults | undefined, + state: { docsRead: number; docsSkipped: number; docsDlq: number; transformErrors: number; batchSeq: number; firstError: Error | null }, + inflight: Promise[], + clog: Logger, + ): Promise { + const { config } = this.d; + + // Chaos hook for poison-pill drills (bench/poison-drill.ts): hard-kills + // the process when a specific doc is touched. Inert unless the test-only + // env var is set — simulates a doc that OOMs/crashes the transform. + if (process.env.LEDGER_TEST_CRASH_ID + && docs.some((d) => String(d._id) === process.env.LEDGER_TEST_CRASH_ID)) { + this.logger.fatal({ poison: process.env.LEDGER_TEST_CRASH_ID }, 'CHAOS: simulated poison-pill crash'); + process.exit(137); + } + + state.docsRead += docs.length; + + const tfStart = performance.now(); + const rows: OutputRow[] = []; + const srcs: SourceDocument[] = []; + const dlqBatch: Parameters[0] = []; + for (const doc of docs) { + const { row, skipReason } = transformDocument(doc, defaults, this.coercions); + if (row !== null) { + rows.push(row); + srcs.push(doc); + } else if (skipReason !== null) { + this.skips.increment(skipReason); + state.docsSkipped++; + // Every unmigratable doc (except already-migrated) is captured with + // its raw source doc — accounted for and replayable, never dropped. + if (skipReason !== SkipReason.ALREADY_MARKED_MIGRATED) { + state.transformErrors++; + if (config.ledger.captureTransformErrors) { + dlqBatch.push({ + run_id: this.runId, + collection: chunk.collection, + chunk_id: chunk._id, + source_id: String(doc._id ?? `unknown_${state.docsRead}`), + raw_doc: doc as Record, + reason: skipReason === SkipReason.TRANSFORM_ERROR ? 'transform_error' : 'skipped', + error: `skip:${skipReason}`, + transform_version: config.transform.version, + }); + } + } + } + } + this.stageMs.transform += performance.now() - tfStart; + if (dlqBatch.length > 0) await this.d.dlq.add(dlqBatch); + + await this.respectBackpressure(); + + if (rows.length > 0) { + const seq = state.batchSeq++; + const p = this.insertOrBisect(chunk, stagingTable, rows, srcs, seq, clog) + .then((r) => { state.docsDlq += r.dlqd; }) + .catch((err) => { if (!state.firstError) state.firstError = err as Error; }); + inflight.push(p); + if (inflight.length >= config.ledger.insertInflight) { + const iStart = performance.now(); + await inflight.shift(); + this.stageMs.insert += performance.now() - iStart; + } + } + } + + /** + * Insert a batch; on a PERMANENT error, bisect (halve and retry each half, + * still with transient-retry protection) until the offending documents are + * isolated, then DLQ them with their raw source docs. Transient errors + * exhaust the retry policy and propagate (chunk redo). + */ + private async insertOrBisect( + chunk: ChunkDoc, + stagingTable: string, + rows: OutputRow[], + srcs: SourceDocument[], + seq: number, + clog: Logger, + depth = 0, + ): Promise<{ inserted: number; dlqd: number }> { + const { retryPolicy, staging, dlq, config } = this.d; + try { + await retryPolicy.execute( + () => staging.insertBatch( + stagingTable, + rows, + `mig:${this.runId}:${chunk._id}:${seq}:${depth}:${rows.length}`, + `mig__${shortHash(`${chunk._id}:${seq}:${depth}:${rows.length}`)}`, + ), + `chunk-${chunk.idx}-batch-${seq}-d${depth}`, + clog, + undefined, + classifyError, + ); + this.totalRowsInserted += rows.length; + return { inserted: rows.length, dlqd: 0 }; + } catch (err) { + if (classifyError(err) !== 'permanent') throw err; + + if (rows.length === 1) { + await dlq.add([{ + run_id: this.runId, + collection: chunk.collection, + chunk_id: chunk._id, + source_id: String(srcs[0]._id), + raw_doc: srcs[0] as Record, + reason: 'insert_rejected', + error: (err as Error).message.slice(0, 1_000), + transform_version: config.transform.version, + }]); + return { inserted: 0, dlqd: 1 }; + } + + if (depth === BISECT_LOG_THRESHOLD) { + clog.warn({ batch: seq, size: rows.length }, 'Permanent insert error — bisecting to isolate offending docs'); + } + const mid = Math.ceil(rows.length / 2); + const left = await this.insertOrBisect(chunk, stagingTable, rows.slice(0, mid), srcs.slice(0, mid), seq, clog, depth + 1); + const right = await this.insertOrBisect(chunk, stagingTable, rows.slice(mid), srcs.slice(mid), seq, clog, depth + 1); + return { inserted: left.inserted + right.inserted, dlqd: left.dlqd + right.dlqd }; + } + } + + // ------------------------------------------------------------------------- + // Promotion + // ------------------------------------------------------------------------- + + private async promoteChunk(chunk: ChunkDoc, log: Logger): Promise { + const { ledger, staging } = this.d; + const stagingTable = chunk.staging_table!; + + const aStart = performance.now(); + const partitions = await staging.listPartitions(stagingTable); + await ledger.transition(chunk._id, ['written', 'attaching'], 'attaching', { partitions, staging_table: stagingTable }); + + await this.finishAttaching({ ...chunk, partitions, attached: chunk.attached ?? [] }, log); + this.stageMs.attach += performance.now() - aStart; + } + + private async finishAttaching(chunk: ChunkDoc, log: Logger): Promise { + const { ledger, staging } = this.d; + const stagingTable = chunk.staging_table!; + const attachedSet = new Set(chunk.attached); + let method: 'attach' | 'insert_select' = chunk.attach_method ?? 'attach'; + + const remaining = chunk.partitions.filter((p) => !attachedSet.has(p)); + for (const partitionId of remaining) { + // Verify-then-attach: never attach a partition whose rows are already + // live. Checked by staged row ids — precise for THIS chunk even when + // sibling collections share the month partition and cd window (a + // window-count here once skipped attaches on multi-collection runs). + const already = await staging.countLiveByStagedIds(stagingTable, partitionId); + if (already > 0) { + await ledger.recordAttached(chunk._id, partitionId); + continue; + } + try { + await staging.attachPartition(stagingTable, partitionId); + } catch (err) { + if (attachedSet.size === 0 && remaining[0] === partitionId) { + log.warn({ err: (err as Error).message }, 'ATTACH unavailable — falling back to INSERT SELECT'); + await staging.insertSelect(stagingTable); + method = 'insert_select'; + for (const p of chunk.partitions) await ledger.recordAttached(chunk._id, p); + break; + } + throw err; // partial attach + failure → stays 'attaching', recovery resumes + } + await ledger.recordAttached(chunk._id, partitionId); + } + + await ledger.transition(chunk._id, 'attaching', 'done', { attach_method: method }); + await staging.dropStaging(stagingTable); + this.chunksDone++; + } + + /** Delete the live rows of a collection's null-cd docs, precisely by id. */ + /** + * Purge a chunk's live rows by their Mongo ids (paged). Fallback for + * unresolvable collections in multi-collection runs, where a cd-window + * DELETE would also hit sibling collections' rows. + */ + private async purgeWindowByIds(collection: string, lowerCd: number, upperCd: number): Promise { + const db = this.d.mongoReader.getDatabase(); + const cursor = db.collection(collection).find( + { cd: { $gte: new Date(lowerCd), $lt: new Date(upperCd) } }, + { projection: { _id: 1, cd: 1 } }, + ).batchSize(10_000); + let pairs: Array<{ id: string; cdMs: number }> = []; + for await (const doc of cursor) { + pairs.push({ id: String(doc._id), cdMs: (doc.cd as Date).getTime() }); + if (pairs.length >= 10_000) { await this.d.staging.deleteLiveByPairs(pairs); pairs = []; } + } + await this.d.staging.deleteLiveByPairs(pairs); + } + + private async purgeNullCdRows(collection: string): Promise { + const { MongoClient } = await import('mongodb'); + const mc = new MongoClient(this.d.config.source.uri); + try { + await mc.connect(); + const coll = mc.db(this.d.config.source.db).collection(collection); + // The sweep wrote these rows with cd derived from ts (the transform's + // fallback) — reconstruct the same pairs so the purge is provenance- + // exact and can never touch a live row sharing an id. + const cursor = coll.find( + { $or: [{ cd: null }, { cd: { $exists: false } }] }, + { projection: { _id: 1, ts: 1 } }, + ).batchSize(10_000); + let batch: Array<{ id: string; cdMs: number }> = []; + for await (const doc of cursor) { + const tsMillis = toEpochMillis(doc.ts); + if (tsMillis === null || tsMillis <= 0) continue; // never transformed → never inserted + batch.push({ id: String(doc._id), cdMs: clampDateTime64(tsMillis) }); + if (batch.length >= 10_000) { await this.d.staging.deleteLiveByPairs(batch); batch = []; } + } + if (batch.length > 0) await this.d.staging.deleteLiveByPairs(batch); + } finally { + await mc.close().catch(() => {}); + } + } + + /** + * Double-probe the source for writes: snapshot per-collection newest cd + + * estimated count, wait, snapshot again. Any advance means old ingestion + * is still running. Public-ish for tests (delay injectable). + */ + async probeSourceFrozen( + db: ReturnType, + collections: string[], + probeMs: number, + wait: (ms: number) => Promise = (ms) => sleep(ms), + ): Promise<{ frozen: boolean; grew: string[]; probeMs: number }> { + const snapshot = async (): Promise> => { + const out = new Map(); + for (const name of collections) { + const [top] = await db.collection(name).find({ cd: { $type: 'date' } }) + .sort({ cd: -1 }).limit(1).project({ cd: 1 }).toArray(); + out.set(name, { + maxCd: top?.cd instanceof Date ? top.cd.getTime() : 0, + est: await db.collection(name).estimatedDocumentCount(), + }); + } + return out; + }; + const before = await snapshot(); + await wait(probeMs); + const after = await snapshot(); + const grew: string[] = []; + for (const [name, b] of before) { + const a = after.get(name)!; + if (a.maxCd > b.maxCd || a.est > b.est) grew.push(name); + } + return { frozen: grew.length === 0, grew, probeMs }; + } + + /** Drop staging tables orphaned by crash-between-done-and-drop. */ + private async sweepOrphanStaging(collection: string): Promise { + if (this.dryRun) return; + const prefix = `${this.d.config.target.table}__stg_${shortHash(`${this.runId}:${collection}`)}_`; + const orphans = await this.d.staging.listStagingTables(prefix).catch(() => [] as string[]); + for (const t of orphans) { + await this.d.staging.dropStaging(t).catch(() => {}); + } + if (orphans.length > 0) this.logger.info({ orphans: orphans.length }, 'Dropped orphaned staging tables'); + } + + // ------------------------------------------------------------------------- + // Circuit breaker bookkeeping + // ------------------------------------------------------------------------- + + private noteChunkFailure(log: Logger): void { + this.chunksFailed++; + this.consecutiveFailed++; + if (this.consecutiveFailed >= this.d.config.ledger.breakerConsecutive) { + log.error( + { consecutiveFailed: this.consecutiveFailed }, + 'Circuit breaker: consecutive chunk failures — pausing engine', + ); + this.pause(); + } + } + + // ------------------------------------------------------------------------- + // Invariant monitor (background spot checks — never on the hot path) + // ------------------------------------------------------------------------- + + private startInvariantMonitor(): void { + const intervalMs = this.d.config.ledger.monitorIntervalMs; + if (intervalMs <= 0) return; + this.monitorTimer = setInterval(() => { + this.runInvariantCheck().catch((err) => + this.logger.warn({ err: (err as Error).message }, 'Invariant check failed to run')); + }, intervalMs); + this.monitorTimer.unref?.(); + } + + private async runInvariantCheck(): Promise { + if (!this.currentCollection) return; + const done = await this.d.ledger.listByStatus(this.runId, this.currentCollection, 'done'); + if (done.length === 0) return; + // Null-cd rows carry cd values derived from ts, which land inside regular + // chunks' cd windows — with a null-cd sweep present, exact equality per + // window is not a valid invariant; missing rows still are. + const hasNullCd = done.some((c) => this.isNullCdChunk(c)); + const samples = done + .filter((c) => !this.isNullCdChunk(c)) + .sort(() => Math.random() - 0.5) + .slice(0, 5); + for (const chunk of samples) { + const scope = this.scopeOf(chunk); + // Sibling collections overlap in cd — an unscoped window count is only + // meaningful when this collection is the whole table. + if (!scope && this.multiCollection) continue; + const live = await this.d.staging.countLiveInCdRange(chunk.lower_cd, chunk.upper_cd, scope); + const violated = hasNullCd ? live < chunk.rows_expected : live !== chunk.rows_expected; + if (violated) { + this.logger.error( + { chunk: chunk._id, live, expected: chunk.rows_expected }, + 'INVARIANT VIOLATION: live-table count disagrees with verified chunk — pausing engine', + ); + await this.d.ledger.transition(chunk._id, 'done', 'failed', { + last_error: `invariant violation: live=${live} expected=${chunk.rows_expected}`, + }); + this.pause(); + return; + } + } + this.logger.debug({ sampled: samples.length }, 'Invariant spot check passed'); + } + + // ------------------------------------------------------------------------- + // Operator: retry failed chunks + // ------------------------------------------------------------------------- + + /** + * Reset all failed chunks of this run back to pending and resume. + * If a failed chunk had already been (partially) promoted — e.g. flagged by + * the invariant monitor — its live-table cd window is purged first so the + * redo starts clean and verify-then-attach behaves correctly. + * (Null-cd sweep chunks have no cd window and are reset without a purge — + * their id-based attach check tolerates partial presence.) + */ + async retryFailed(): Promise<{ retried: number }> { + const { ledger, staging } = this.d; + let retried = 0; + const collections = new Set(); + const failedAll: ChunkDoc[] = []; + // Failed chunks may span collections; statusCounts is global, listByStatus per collection. + const counts = await ledger.statusCounts(this.runId); + if ((counts.failed ?? 0) > 0) { + const all = await ledger.listAll(this.runId); + for (const c of all) if (c.status === 'failed') { collections.add(c.collection); failedAll.push(c as ChunkDoc); } + } + const collectionsNeedingSweepReset = new Set(); + for (const chunk of failedAll) { + if (!this.isNullCdChunk(chunk as ChunkDoc) && !this.dryRun) { + // Purge must never touch sibling collections' rows in the same cd + // window: scope by (a, e) when the collection resolves; otherwise + // purge precisely by the window's Mongo ids. + const scope = this.scopeOf(chunk as ChunkDoc); + if (scope || !this.multiCollection) { + await staging.deleteLiveCdRange(chunk.lower_cd, chunk.upper_cd, scope); + } else { + await this.purgeWindowByIds(chunk.collection, chunk.lower_cd, chunk.upper_cd); + } + collectionsNeedingSweepReset.add(chunk.collection); + } + const reset = await ledger.transition(chunk._id, 'failed', 'pending', { + pod_id: null, + staging_table: null, + partitions: [], + attached: [], + attach_method: null, + attempts: 0, + last_error: null, + }); + if (reset) retried++; + } + + // A regular chunk's cd-window purge also deletes any null-cd sweep rows + // whose derived cd fell inside that window — reset the sweep too, purging + // its remaining rows precisely by id (it has no cd window of its own). + for (const collection of collectionsNeedingSweepReset) { + const sentinel = await ledger.getSentinel(this.runId, collection); + if (!sentinel || sentinel.status !== 'done') continue; + await this.purgeNullCdRows(collection); + await ledger.transition(sentinel._id, 'done', 'pending', { + pod_id: null, staging_table: null, partitions: [], attached: [], + attach_method: null, attempts: 0, last_error: 'reset alongside regular-chunk retry (cd-window purge overlaps sweep rows)', + }); + this.logger.info({ collection }, 'Null-cd sweep reset alongside regular-chunk retry'); + } + this.consecutiveFailed = 0; + this.resume(); + this.logger.info({ retried }, 'Failed chunks reset to pending'); + return { retried }; + } + + // ------------------------------------------------------------------------- + // DLQ replay + // ------------------------------------------------------------------------- + + /** + * Global DLQ mass guard. The per-chunk circuit breaker (5% of one chunk) + * never trips on EVENLY-SPREAD failure — 1% of every chunk on a 10B-doc + * migration would silently accumulate ~100M raw docs. When total pending + * crosses the threshold, pause: that scale of DLQ means a systematic + * problem to fix in the transform/source, not data to collect. + */ + async checkDlqPressure(log: Logger, force = false): Promise { + const threshold = this.d.config.ledger.dlqPauseThreshold; + if (threshold <= 0 || this.paused || this.dryRun) return false; + if (!force && this.totalDocsDlq + this.totalDocsSkipped < threshold) return false; // cheap in-process pre-filter + const counts = await this.d.dlq.countByStatus(this.runId); + if ((counts.pending ?? 0) >= threshold) { + const storage = await this.d.dlq.storageStats().catch(() => null); + log.error( + { + pending: counts.pending, + threshold, + dlqStorageMB: storage ? Math.round(storage.dlqBytes / 1e6) : null, + manifestDbDiskFreePct: storage?.diskFreePct ?? null, + }, + 'DLQ MASS GUARD: pending dead-letter docs crossed the threshold — pausing. ' + + 'This is a systematic problem: fix the cause, then either Retry failed chunks ' + + '(redo re-reads the source) or Replay DLQ. Raise LEDGER_DLQ_PAUSE_THRESHOLD only ' + + 'if the disk numbers above say you can afford to keep collecting.', + ); + this.pause(); + return true; + } + return false; + } + + /** Live progress of a running sampled content audit. */ + readonly contentAuditProgress = { + running: false, sampled: 0, matched: 0, + mismatches: [] as Array<{ _id: string; collection: string; kind: string; fields?: string[] }>, + }; + + /** + * Sampled doc-per-doc audit — the answer to what count-based verification + * cannot see: the right NUMBER of wrong rows. Random source documents are + * re-transformed and compared field-by-field against their live rows. + * Scalar columns compare exactly; JSON columns (sg/up/custom/cmp) compare + * by top-level key set (ClickHouse's JSON type normalizes value encodings, + * so value-level equality there belongs to the differential harness, which + * pins the transform itself). + */ + async contentAudit(samplesPerCollection = 500): Promise<{ + sampled: number; matched: number; missing: number; different: number; + mismatches: Array<{ _id: string; collection: string; kind: string; fields?: string[] }>; + }> { + const { config, staging } = this.d; + const p = this.contentAuditProgress; + p.running = true; p.sampled = 0; p.matched = 0; p.mismatches = []; + try { + const db = this.d.mongoReader.getDatabase(); + let collections = await discoverCollections(db, config.source.collectionPrefix, this.logger); + const skipEventNames = new Set(['[CLY]_apm_device', '[CLY]_apm_network']); + collections = collections.filter((name) => { + const defaults = this.d.hashResolver.resolveCollectionName(name, config.source.collectionPrefix); + return !(defaults && skipEventNames.has(defaults.e)); + }); + + let missing = 0, different = 0; + for (const collection of collections) { + const defaults = this.d.hashResolver.resolveCollectionName(collection, config.source.collectionPrefix) ?? undefined; + const coll = db.collection(collection); + const [lowDoc] = await coll.find({ cd: { $type: 'date' } }).sort({ cd: 1 }).limit(1).project({ cd: 1 }).toArray(); + const [highDoc] = await coll.find({ cd: { $type: 'date' } }).sort({ cd: -1 }).limit(1).project({ cd: 1 }).toArray(); + if (!lowDoc || !highDoc) continue; + const lo = (lowDoc.cd as Date).getTime(), hi = (highDoc.cd as Date).getTime(); + + // K random cd probe points, a small run of docs from each — cheap + // index-served sampling without $sample's whole-collection scan. + const RUN_LEN = 25; + const probes = Math.max(1, Math.ceil(samplesPerCollection / RUN_LEN)); + const docs: Record[] = []; + for (let k = 0; k < probes; k++) { + const at = new Date(lo + Math.floor(((k + 0.5) / probes) * (hi - lo))); + const page = await coll.find({ cd: { $gte: at } }).sort({ cd: 1, _id: 1 }).limit(RUN_LEN).toArray(); + docs.push(...(page as Record[])); + } + + const expected = new Map(); + for (const doc of docs) { + const { row } = transformDocument(doc as SourceDocument, defaults); + if (row) expected.set(row._id, row); + } + if (expected.size === 0) continue; + const expCds = [...expected.values()].map((r) => Date.parse(r.cd.replace(' ', 'T') + 'Z')); + const live = await staging.fetchRowsByIds( + [...expected.keys()], + { loMs: Math.min(...expCds), hiMs: Math.max(...expCds) }, + ); + + for (const [id, exp] of expected) { + p.sampled++; + const got = live.get(id); + if (!got || String(got.cd_txt) !== exp.cd) { + missing++; + if (p.mismatches.length < 100) p.mismatches.push({ _id: id, collection, kind: 'missing (no live row with this (_id, cd))' }); + continue; + } + const bad: string[] = []; + const eq = (a: unknown, b: unknown): boolean => (a ?? null) === (b ?? null); + if (!eq(got.a, exp.a)) bad.push('a'); + if (!eq(got.e, exp.e)) bad.push('e'); + if (!eq(got.n, exp.n)) bad.push('n'); + if (!eq(got.uid, exp.uid)) bad.push('uid'); + if (!eq(got.uid_canon, exp.uid_canon)) bad.push('uid_canon'); + if (!eq(got.did, exp.did)) bad.push('did'); + if (!eq(got.lsid, exp.lsid)) bad.push('lsid'); + if (String(got.ts_txt) !== exp.ts) bad.push('ts'); + if (Number(got.c) !== exp.c) bad.push('c'); + if (Number(got.s) !== exp.s) bad.push('s'); + if (Number(got.dur) !== exp.dur) bad.push('dur'); + for (const jf of ['sg', 'up', 'custom', 'cmp'] as const) { + const g = got[jf], x = exp[jf]; + const gKeys = g && typeof g === 'object' ? Object.keys(g as object).sort().join(',') : ''; + const xKeys = x && typeof x === 'object' ? Object.keys(x as object).sort().join(',') : ''; + if (gKeys !== xKeys) bad.push(jf + ':keys'); + } + if (bad.length > 0) { + different++; + if (p.mismatches.length < 100) p.mismatches.push({ _id: id, collection, kind: 'field mismatch', fields: bad }); + } else { + p.matched++; + } + } + } + return { sampled: p.sampled, matched: p.matched, missing, different, mismatches: p.mismatches }; + } finally { + p.running = false; + } + } + + /** Live progress of a running DLQ replay (large queues take a while). */ + readonly replayProgress = { running: false, processed: 0, replayed: 0, stillFailing: 0, alreadyLive: 0 }; + + /** + * Replay pending DLQ entries: re-transform the stored raw docs under the + * CURRENT transform version and insert them directly into the live table. + * Safe to run anytime after the affected chunks are done — entries whose + * rows are ALREADY live (e.g. a chunk redo with a fixed transform migrated + * them from the source first) are marked resolved without inserting, so + * redo-then-replay cannot duplicate. + */ + async replayDlq(): Promise<{ replayed: number; stillFailing: number; alreadyLive: number }> { + const { dlq, staging, retryPolicy, config } = this.d; + let replayed = 0; + let stillFailing = 0; + let alreadyLive = 0; + this.replayProgress.running = true; + Object.assign(this.replayProgress, { processed: 0, replayed: 0, stillFailing: 0, alreadyLive: 0 }); + try { + + // Keyset drain: pages of 500 by _id so a large DLQ is fully processed + // (a plain limited fetch silently replayed only the first page). Entries + // that fail again keep status pending but sort behind the advancing + // cursor, so the loop always terminates. + let afterId: string | null = null; + for (;;) { + const batch = await dlq.listPendingAfter(this.runId, afterId, 500); + if (batch.length === 0) break; + afterId = batch[batch.length - 1]._id; + const batchKey = batch[0]._id; + this.replayProgress.processed += batch.length; + const rows: OutputRow[] = []; + const ids: string[] = []; + for (const entry of batch) { + const defaults = this.d.hashResolver.resolveCollectionName(entry.collection, config.source.collectionPrefix) ?? undefined; + const { row } = transformDocument(entry.raw_doc as SourceDocument, defaults, this.coercions); + if (row) { rows.push(row); ids.push(entry._id); } + else { + await dlq.recordRetryError(entry._id, 'still fails transform under ' + config.transform.version); + stillFailing++; + } + } + + // Skip rows already live as (_id, cd) pairs — a chunk redo with a + // fixed transform migrates DLQ'd docs from the source; replaying them + // on top would duplicate. Marked resolved: the doc IS migrated. + if (rows.length > 0) { + const cdMsOf = (r: OutputRow): number => Date.parse(r.cd.replace(' ', 'T') + 'Z'); + const cdVals = rows.map(cdMsOf); + const liveCd = await staging.fetchLiveCdByIds( + rows.map((r) => r._id), + { loMs: Math.min(...cdVals), hiMs: Math.max(...cdVals) }, + ); + const keep: OutputRow[] = []; + const keepIds: string[] = []; + const resolvedIds: string[] = []; + for (let j = 0; j < rows.length; j++) { + const cdMs = Date.parse(rows[j].cd.replace(' ', 'T') + 'Z'); + if (liveCd.get(rows[j]._id) === cdMs) { resolvedIds.push(ids[j]); } + else { keep.push(rows[j]); keepIds.push(ids[j]); } + } + if (resolvedIds.length > 0) { + await dlq.markResolved(resolvedIds, config.transform.version + ' (already live — no insert)'); + alreadyLive += resolvedIds.length; + } + rows.length = 0; rows.push(...keep); + ids.length = 0; ids.push(...keepIds); + } + if (rows.length === 0) { this.syncReplayProgress(replayed, stillFailing, alreadyLive); continue; } + try { + await retryPolicy.execute( + () => staging.insertIntoLive(rows, `dlqreplay:${batchKey}`), + `dlq-replay-${batchKey}`, + this.logger, + undefined, + classifyError, + ); + await dlq.markResolved(ids, config.transform.version); + replayed += rows.length; + } catch (err) { + // Isolate row-level failures within the replay batch too. + for (let j = 0; j < rows.length; j++) { + try { + await staging.insertIntoLive([rows[j]], `dlqreplay:${batchKey}:${j}`); + await dlq.markResolved([ids[j]], config.transform.version); + replayed++; + } catch (rowErr) { + await dlq.recordRetryError(ids[j], (rowErr as Error).message.slice(0, 1_000)); + stillFailing++; + } + } + void err; + } + this.syncReplayProgress(replayed, stillFailing, alreadyLive); + } + + this.logger.info({ replayed, stillFailing, alreadyLive }, 'DLQ replay complete'); + return { replayed, stillFailing, alreadyLive }; + } finally { + this.replayProgress.running = false; + } + } + + private syncReplayProgress(replayed: number, stillFailing: number, alreadyLive: number): void { + this.replayProgress.replayed = replayed; + this.replayProgress.stillFailing = stillFailing; + this.replayProgress.alreadyLive = alreadyLive; + } + + // ------------------------------------------------------------------------- + // Self-service: index building (UI action with progress) + // ------------------------------------------------------------------------- + + private indexBuild: { running: boolean; total: number; done: string[]; current: string | null; error: string | null } = + { running: false, total: 0, done: [], current: null, error: null }; + + /** Kick off {cd,_id} index builds on all collections missing it (background). */ + async startIndexBuilds(): Promise<{ started: boolean; missing: number }> { + if (this.indexBuild.running) return { started: false, missing: this.indexBuild.total - this.indexBuild.done.length }; + const db = this.d.mongoReader.getDatabase(); + const collections = await discoverCollections(db, this.d.config.source.collectionPrefix, this.logger); + const missing: string[] = []; + for (const name of collections) { + const idx = await db.collection(name).indexes().catch(() => []); + const has = idx.some((i) => (i.key as Record).cd !== undefined && (i.key as Record)._id !== undefined); + if (!has) missing.push(name); + } + if (missing.length === 0) return { started: false, missing: 0 }; + + this.indexBuild = { running: true, total: missing.length, done: [], current: null, error: null }; + void (async () => { + for (const name of missing) { + this.indexBuild.current = name; + try { + await db.collection(name).createIndex({ cd: 1, _id: 1 }); + this.indexBuild.done.push(name); + } catch (err) { + this.indexBuild.error = `${name}: ${(err as Error).message}`; + break; + } + } + this.indexBuild.running = false; + this.indexBuild.current = null; + })(); + return { started: true, missing: missing.length }; + } + + /** Index-build progress incl. live server-side build progress via $currentOp. */ + async indexBuildProgress(): Promise> { + let serverOps: Array<{ collection: string; pct: number | null; msg: string }> = []; + try { + const adminDb = this.d.mongoReader.getDatabase().client.db('admin'); + const ops = await adminDb + .aggregate([ + { $currentOp: { allUsers: true } }, + { $match: { 'command.createIndexes': { $exists: true } } }, + { $project: { command: 1, progress: 1, msg: 1 } }, + ]) + .toArray(); + serverOps = ops.map((o: Record) => ({ + collection: String((o.command as Record)?.createIndexes ?? '?'), + pct: o.progress && (o.progress as Record).total + ? Math.round(((o.progress as Record).done / (o.progress as Record).total) * 100) + : null, + msg: String(o.msg ?? ''), + })); + } catch { /* $currentOp may need privileges — progress is then best-effort */ } + return { ...this.indexBuild, serverOps }; + } + + // ------------------------------------------------------------------------- + // Self-service: preflight & verification + // ------------------------------------------------------------------------- + + /** + * Environment/readiness checks for the guided UI. Read-only; safe to run + * anytime (uses the raw Db handle, never mutates reader state). + */ + async preflight(): Promise> { + const { config, mongoReader, staging, ledger } = this.d; + const checks: Array<{ id: string; label: string; status: 'pass' | 'warn' | 'fail'; detail: string }> = []; + + // MongoDB source + let collections: string[] = []; + try { + const db = mongoReader.getDatabase(); + collections = await discoverCollections(db, config.source.collectionPrefix, this.logger); + checks.push({ id: 'mongo', label: 'MongoDB source reachable', status: 'pass', detail: `${collections.length} drill collection(s) found` }); + let indexed = 0; + let totalDocs = 0; + for (const name of collections) { + const idx = await db.collection(name).indexes().catch(() => []); + const has = idx.some((i) => (i.key as Record).cd !== undefined && (i.key as Record)._id !== undefined); + if (has) indexed++; + totalDocs += await db.collection(name).estimatedDocumentCount().catch(() => 0); + } + checks.push({ + id: 'index', + label: '{cd,_id} index on all collections', + status: indexed === collections.length ? 'pass' : 'warn', + detail: indexed === collections.length + ? 'all indexed' + : `${collections.length - indexed} collection(s) missing the index — the service builds it automatically, but pre-building avoids a long pause (see Guide step 2)`, + }); + checks.push({ id: 'docs', label: 'Estimated documents to migrate', status: 'pass', detail: `${totalDocs.toLocaleString('en-US')} (estimate; can be off after an unclean mongod shutdown — chunk sizing is span-guarded)` }); + // cd should essentially always exist; the sweep handles outliers, last. + let nullCd = 0; + for (const name of collections) { + nullCd += await db.collection(name).countDocuments({ cd: null }).catch(() => 0); + } + // Clock sanity: every provenance decision rests on live cd (insert + // time) being newer than all source cd. A source containing cd values + // at/over ClickHouse's present means skewed clocks or a source that is + // still ingesting — both invalidate the boundary. + let maxSourceCd = 0; + for (const name of collections) { + const [top] = await db.collection(name).find({ cd: { $type: 'date' } }) + .sort({ cd: -1 }).limit(1).project({ cd: 1 }).toArray(); + if (top?.cd instanceof Date) maxSourceCd = Math.max(maxSourceCd, top.cd.getTime()); + } + const chNow = await this.d.staging.serverNowMs().catch(() => 0); + const skewOk = maxSourceCd === 0 || chNow === 0 || maxSourceCd < chNow - 60_000; + checks.push({ + id: 'clock', + label: 'Source frozen & clocks sane (cd boundary)', + status: skewOk ? 'pass' : 'fail', + detail: skewOk + ? `newest source cd ${maxSourceCd ? new Date(maxSourceCd).toISOString() : 'n/a'} is safely behind ClickHouse server time` + : `newest source cd ${new Date(maxSourceCd).toISOString()} is within 60s of ClickHouse server time (${new Date(chNow).toISOString()}) — source still ingesting or clocks skewed; the migrated/live cd boundary is NOT trustworthy yet`, + }); + // Old ingestion stopped? Probe twice: if any collection's newest cd or + // estimated count advances between probes, the source is still being + // written — cutover step 'stop old ingestion' has not happened. + const frozen = await this.probeSourceFrozen(db, collections, 4_000); + checks.push({ + id: 'frozen', + label: 'Old ingestion stopped (source frozen)', + status: frozen.frozen ? 'pass' : 'fail', + detail: frozen.frozen + ? `no writes observed during a ${Math.round(frozen.probeMs / 1000)}s probe` + : `STILL RECEIVING WRITES: ${frozen.grew.join(', ')} — stop old ingestion before migrating (works the same for new-cluster and same-cluster setups)`, + }); + + // New ingestion flowing? Post-cutover rows carry recent cd. Zero recent + // rows is a warning, not a failure — traffic may legitimately be zero, + // or preflight may be running before the SDK flip (both topologies). + try { + const recent = await this.d.staging.countRecentLive(15); + checks.push({ + id: 'live-ingest', + label: 'New ingestion flowing into ClickHouse', + status: recent > 0 ? 'pass' : 'warn', + detail: recent > 0 + ? `${recent.toLocaleString('en-US')} rows ingested in the last 15 min` + : 'no rows with recent cd in the last 15 min — either the SDK flip has not happened yet or traffic is zero; fine to migrate, but verify live ingestion separately', + }); + } catch { /* table missing — already reported by the table check */ } + + checks.push({ + id: 'nullcd', + label: 'Documents without cd (outliers)', + status: nullCd === 0 ? 'pass' : 'warn', + detail: nullCd === 0 + ? 'none — every document carries cd' + : `${nullCd.toLocaleString('en-US')} — a dedicated sweep chunk migrates them, strictly after all regular chunks`, + }); + } catch (err) { + checks.push({ id: 'mongo', label: 'MongoDB source reachable', status: 'fail', detail: (err as Error).message }); + } + + // Replica set: reading from secondaries offloads the primary during the + // days-long scan. (After cutover the source is frozen, so secondary reads + // are exact.) + try { + const hello = await mongoReader.getDatabase().admin().command({ hello: 1 }); + if (hello.setName) { + const onPrimary = config.source.readPreference === 'primary'; + checks.push({ + id: 'replicaset', + label: `Replica set detected (${hello.setName})`, + status: onPrimary ? 'warn' : 'pass', + detail: onPrimary + ? `MONGO_READ_PREFERENCE=primary was forced by env — remove it to let the engine auto-select secondaryPreferred (it does this by default on replica sets; source is frozen, so secondary reads are exact)` + : `read preference: ${config.source.readPreference}${config.source.readPreferenceAuto ? ' (auto-selected — replica set detected)' : ''}`, + }); + } + } catch { /* standalone or no permission — nothing to suggest */ } + + // Disk headroom — the #1 preventable mid-migration incident. + try { + const dbStats = await mongoReader.getDatabase().stats(); + if (dbStats.fsTotalSize) { + const freePct = Math.round(((dbStats.fsTotalSize - dbStats.fsUsedSize) / dbStats.fsTotalSize) * 100); + checks.push({ + id: 'mongo-disk', + label: 'MongoDB disk headroom', + status: freePct < 10 ? 'fail' : freePct < 20 ? 'warn' : 'pass', + detail: `${freePct}% free`, + }); + } + } catch { /* stats not available */ } + const chDisk = await staging.diskSpace(); + if (chDisk && chDisk.totalBytes > 0) { + const freePct = Math.round((chDisk.freeBytes / chDisk.totalBytes) * 100); + checks.push({ + id: 'ch-disk', + label: 'ClickHouse disk headroom', + status: freePct < 10 ? 'fail' : freePct < 20 ? 'warn' : 'pass', + detail: `${freePct}% free (${(chDisk.freeBytes / 1e9).toFixed(1)} GB) — needs room for staging + the migrated data (~10-20% of the Mongo size after compression)`, + }); + } + + // ClickHouse target + const target = await staging.targetTableInfo(); + checks.push({ + id: 'clickhouse', + label: `ClickHouse target table (${config.target.db}.${config.target.table})`, + status: target.exists ? 'pass' : 'fail', + detail: target.exists ? `exists, ${target.rows.toLocaleString('en-US')} rows` : 'table not found — create it (or start the new stack) before migrating', + }); + + // Dedup canary + checks.push({ + id: 'dedup', + label: 'Insert-dedup canary', + status: staging.dedupWorks === null ? 'warn' : staging.dedupWorks ? 'pass' : 'warn', + detail: staging.dedupWorks === null + ? 'not probed yet (runs at migration start)' + : staging.dedupWorks ? 'dedup token verified working on this target' + : 'dedup token inert on this target — safe (chunk redo covers it), but ambiguous insert retries may need chunk redo', + }); + + // Dry run + const dryCounts = await ledger.statusCounts(`${this.d.config.ledger.runId}-dry`); + const dryTotal = Object.values(dryCounts).reduce((a, b) => a + b, 0); + checks.push({ + id: 'dryrun', + label: 'Dry run (sampled rehearsal)', + status: dryTotal > 0 && (dryCounts.done ?? 0) === dryTotal ? 'pass' : 'warn', + detail: dryTotal === 0 + ? 'not run yet — start once with DRY_RUN=1 and review the report (Guide step 3)' + : `${dryCounts.done ?? 0}/${dryTotal} sampled chunks done`, + }); + + return { engineStatus: this.status, checks }; + } + + /** + * Full migration verification: every done chunk's live-table count checked + * against its verified expectation, plus table totals. Exact, minutes at + * most — run before sign-off or any time trust is in question. + */ + async verifyMigration(): Promise> { + const { ledger, staging } = this.d; + const all = await ledger.listAll(this.runId); + const byCollection = new Map(); + for (const c of all) { + if (this.isNullCdChunk(c as ChunkDoc)) byCollection.set(c.collection, true); + } + + let checked = 0; + let unscopedSkipped = 0; + const collectionCount = new Set(all.map((c) => c.collection)).size; + const mismatches: Array<{ chunk: string; expected: number; live: number }> = []; + const targets = all.filter((chunk) => chunk.status === 'done' && !this.isNullCdChunk(chunk as ChunkDoc)); + this.verifyProgress.running = true; + this.verifyProgress.total = targets.length; + this.verifyProgress.checked = 0; + this.verifyProgress.phase = 'recounting chunk windows'; + try { + // Bounded concurrency: each window count is minmax-pruned and cheap, + // but a 10TB run has tens of thousands of them — sequential would take + // hours, unbounded would hammer ClickHouse. + const CONCURRENCY = 8; + let cursor = 0; + await Promise.all(Array.from({ length: CONCURRENCY }, async () => { + for (;;) { + const i = cursor++; + if (i >= targets.length) return; + const chunk = targets[i]; + const scope = this.scopeOf(chunk as ChunkDoc); + if (!scope && collectionCount > 1) { unscopedSkipped++; continue; } + const live = await staging.countLiveInCdRange(chunk.lower_cd, chunk.upper_cd, scope); + const relaxed = byCollection.get(chunk.collection) === true; + const bad = relaxed ? live < chunk.rows_expected : live !== chunk.rows_expected; + checked++; + this.verifyProgress.checked = checked; + if (bad) mismatches.push({ chunk: chunk._id, expected: chunk.rows_expected, live }); + } + })); + + this.verifyProgress.phase = 'scanning for duplicates (per partition)'; + + // Duplicate detection + attribution, partition by partition — exact for + // every duplicate class we act on (copies of the same document share + // their ts month) and memory-bounded on billion-row tables, unlike a + // global uniqExact/GROUP BY. Attribution by cd against the ledger's + // end-of-migrated-data boundary: + // 0 copies below → live at-least-once artifact (nightly job cleans) + // 1 copy below → cross-cutover SDK retry (benign, reported) + // 2+ copies below → migration defect; verification fails. + const boundaryMs = all.reduce((m, c) => Math.max(m, c.upper_cd), 0); + const dup = await staging.duplicateStats(boundaryMs); + let migrationDuplicates = 0; + const duplicateSample = dup.sample.map((d) => { + if (d.migratedCopies >= 2) migrationDuplicates++; + return { + _id: d._id, + copies: d.copies, + migratedCopies: d.migratedCopies, + minCd: new Date(d.min_cd_ms).toISOString(), + maxCd: new Date(d.max_cd_ms).toISOString(), + verdict: d.migratedCopies === 0 + ? 'live at-least-once artifact (nightly dedup job cleans these)' + : d.migratedCopies === 1 + ? 'cross-cutover retry duplicate (event reached both stacks — one benign live copy)' + : 'MIGRATION DEFECT: same document migrated more than once — investigate', + }; + }); + + return { + ok: mismatches.length === 0 && migrationDuplicates === 0, + checkedChunks: checked, + unscopedSkipped, + mismatches, + table: { rows: dup.rows, distinctIds: dup.rows - dup.duplicates, duplicates: dup.duplicates }, + duplicateSample, + migrationDuplicates, + }; + } finally { + this.verifyProgress.running = false; + this.verifyProgress.phase = ''; + } + } + + // ------------------------------------------------------------------------- + // Stats & report + // ------------------------------------------------------------------------- + + getStats(): LedgerEngineStats { + // Freeze the clock at completion: docs/second is the run's average + // afterwards, not a number decaying while the finished engine idles. + const endMs = this.finishedAt > 0 ? this.finishedAt : Date.now(); + const elapsedSec = this.startedAt > 0 ? (endMs - this.startedAt) / 1000 : 0; + return { + engine: 'ledger', + runId: this.runId, + podId: this.podId, + status: this.status, + fatalError: this.fatalError, + dryRun: this.dryRun, + currentCollection: this.currentCollection, + currentChunk: this.currentChunk, + totalDocsRead: this.totalDocsRead, + totalDocsSkipped: this.totalDocsSkipped, + totalRowsInserted: this.totalRowsInserted, + totalDocsDlq: this.totalDocsDlq, + totalCoercions: this.coercions.getTotal(), + chunksDone: this.chunksDone, + chunksFailed: this.chunksFailed, + docsPerSecond: elapsedSec > 0 ? this.totalDocsRead / elapsedSec : 0, + stageMs: { + read: Math.round(this.stageMs.read), + transform: Math.round(this.stageMs.transform), + insert: Math.round(this.stageMs.insert), + verify: Math.round(this.stageMs.verify), + attach: Math.round(this.stageMs.attach), + pressureWait: Math.round(this.stageMs.pressureWait), + }, + dedupWorks: this.d.staging.dedupWorks, + chunkStatusCounts: this.lastStatusCounts, + }; + } + + /** Data-quality report: coercions, skips, DLQ — the dry-run/final artifact. */ + async getReport(): Promise> { + return { + runId: this.runId, + dryRun: this.dryRun, + status: this.status, + chunkStatusCounts: await this.d.ledger.statusCounts(this.runId), + skipsByReason: this.skips.getCounts(), + coercions: this.coercions.getReport(), + dlq: { + byStatus: await this.d.dlq.countByStatus(this.runId), + topErrors: await this.d.dlq.topErrors(this.runId), + }, + }; + } +} diff --git a/src/runtime/collection-orchestrator.ts b/src/runtime/collection-orchestrator.ts deleted file mode 100644 index c7cc45a..0000000 --- a/src/runtime/collection-orchestrator.ts +++ /dev/null @@ -1,1020 +0,0 @@ -import type { Logger } from "pino"; -import type { ManifestStore } from "../state/manifest-store.ts"; -import { RedisHotState } from "../state/redis-hot-state.ts"; -import type { MongoReader } from "../source/mongo-reader.ts"; -import type { ClickHouseWriter } from "../target/clickhouse-writer.ts"; -import type { ClickHousePressure, BackpressureConfig } from "../target/clickhouse-pressure.ts"; -import type { GcController } from "./gc-controller.ts"; -import type { RetryPolicy } from "./retry-policy.ts"; -import { BatchRunner, type RunnerStatus, type BatchRunnerStats } from "./batch-runner.ts"; -import { resolveRun } from "./resolve-run.ts"; -import { discoverCollections } from "../source/discover-collections.ts"; -import type { HashResolver } from "../transform/hash-resolver.ts"; -import type { Config } from "../config/schema.ts"; -import type { CollectionLock } from "../state/collection-lock.ts"; -import type { GlobalProgress, CollectionProgress } from "../state/global-progress.ts"; -import type { AsyncBatchWriter } from "../state/async-batch-writer.ts"; -import { RangeCoordinator } from "./range-coordinator.ts"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export type IndexBuildStatus = "checking" | "building" | "ready" | "failed"; - -export interface IndexStatusSummary { - ready: number; - building: number; - checking: number; - failed: number; - details: Array<{ collection: string; status: IndexBuildStatus; elapsedSec?: number }>; -} - -export interface CollectionResult { - collection: string; - sourceNs: string; - runId: string; - status: "completed" | "failed" | "skipped"; - error?: string; - docsRead?: number; - rowsInserted?: number; -} - -export interface OrchestratorResult { - collections: CollectionResult[]; - totalCompleted: number; - totalFailed: number; - totalSkipped: number; -} - -export interface OrchestratorProgress { - totalCollections: number; - completedCollections: number; - failedCollections: number; - skippedCollections: number; - currentCollection: string | null; - collections: string[]; - results: CollectionResult[]; -} - -export interface OrchestratorDeps { - manifestStore: ManifestStore; - redisState: RedisHotState; - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - chPressure: ClickHousePressure; - gcController: GcController; - retryPolicy: RetryPolicy; - hashResolver: HashResolver; - logger: Logger; - config: Config; - collectionLock?: CollectionLock; - globalProgress?: GlobalProgress; - asyncBatchWriter?: AsyncBatchWriter; -} - -// --------------------------------------------------------------------------- -// CollectionOrchestrator -// --------------------------------------------------------------------------- - -export class CollectionOrchestrator { - private readonly deps: OrchestratorDeps; - private readonly logger: Logger; - - private discoveredCollections: string[] = []; - private skippedApmCollections: Set = new Set(); - private results: CollectionResult[] = []; - private estimatedCounts: Map = new Map(); - private indexStatus: Map = new Map(); - private indexBuildStarted: Map = new Map(); - private currentCollection: string | null = null; - private currentRunId: string | null = null; - private currentBatchRunner: BatchRunner | null = null; - private lastBatchStats: BatchRunnerStats | null = null; - private currentRangeCoordinator: RangeCoordinator | null = null; - private currentRedisState: RedisHotState | null = null; - private orchestratorStatus: "idle" | "running" | "waiting_for_index" | "completed" = "idle"; - private stopping = false; - private firstCollectionStartedAt: number | null = null; - - constructor(deps: OrchestratorDeps) { - this.deps = deps; - this.logger = deps.logger.child({ component: "CollectionOrchestrator" }); - } - - // ----------------------------------------------------------------------- - // Public API - // ----------------------------------------------------------------------- - - async run(): Promise { - const { mongoReader, manifestStore, config, collectionLock, globalProgress } = this.deps; - - // 1. Discover collections - const db = mongoReader.getDatabase(); - this.discoveredCollections = await discoverCollections(db, config.source.collectionPrefix, this.logger); - - // Filter out APM collections by resolved event name - const skipEventNames = new Set(['[CLY]_apm_device', '[CLY]_apm_network']); - this.discoveredCollections = this.discoveredCollections.filter(name => { - const defaults = this.deps.hashResolver.resolveCollectionName( - name, config.source.collectionPrefix, - ); - if (defaults && skipEventNames.has(defaults.e)) { - this.skippedApmCollections.add(name); - this.results.push({ - collection: name, - sourceNs: `${config.source.db}.${name}`, - runId: '', - status: 'skipped', - error: 'apm_collections', - }); - this.logger.info({ collection: name, event: defaults.e }, 'Skipping APM collection'); - return false; - } - return true; - }); - if (this.skippedApmCollections.size > 0) { - this.logger.info( - { skipped: this.skippedApmCollections.size, remaining: this.discoveredCollections.length }, - 'Filtered APM collections from discovery', - ); - } - - this.logger.info( - { totalCollections: this.discoveredCollections.length, multiPod: !!collectionLock }, - "Starting migration with index-aware scheduling", - ); - - // 2a. Upfront: estimate ALL collections in parallel and persist to Redis - await this.populateAllEstimates(); - - // 2b. Recover completed collection aggregates from Redis (+ manifest fallback) - await this.recoverCompletedCollections(); - - // 3. Start lock heartbeat (multi-pod mode) - if (collectionLock) { - collectionLock.onLockLost = (name) => { - if (this.currentCollection === name) { - this.logger.warn({ collection: name }, "Lock lost — stopping current runner"); - this.currentRangeCoordinator?.stop(); - this.currentBatchRunner?.stopAfterBatch(); - } - }; - collectionLock.startHeartbeat(); - } - - // 3. Start all index checks in background (non-blocking) - this.startBackgroundIndexInit(); - - this.orchestratorStatus = "running"; - - // 4. Process loop: pick ready collections, wait for building ones - while (!this.stopping) { - // Check global commands (multi-pod mode) - if (globalProgress) { - try { - const globalCmds = await globalProgress.getGlobalCommands(); - if (globalCmds.stop) { - this.logger.info("Global stop command received"); - this.stopping = true; - break; - } - if (globalCmds.pause) { - this.currentBatchRunner?.pause(); - this.currentRangeCoordinator?.pause(); - } - } catch { - // best-effort - } - } - - const completed = new Set(this.results.map(r => r.collection)); - - // Gate: if heartbeat is unhealthy, don't acquire new collections - if (collectionLock && !collectionLock.heartbeatHealthy) { - this.logger.warn("Pod heartbeat unhealthy — skipping collection acquisition, waiting 10s"); - await this.interruptibleSleep(10_000); - continue; - } - - // Find next collection with index ready, not yet processed, and not locked by another pod - const next = await this.findNextReadyCollection(completed); - - if (next) { - this.orchestratorStatus = "running"; - const sourceNs = `${config.source.db}.${next}`; - const targetTable = `${config.target.db}.${config.target.table}`; - - // Check if range-parallel (uses cached estimate from isRangeParallelCandidate or fresh query) - const isRangeParallel = await this.isRangeParallelCandidate(next); - - // Already recovered from Redis in recoverCompletedCollections? Skip. - if (this.results.some(r => r.collection === next)) { - await collectionLock?.release(next).catch(() => {}); - continue; - } - - // For range-parallel: check if ranges are all done (not existsCompletedRun) - // For standard: check if already completed - if (!isRangeParallel) { - const alreadyCompleted = await manifestStore.existsCompletedRun(sourceNs, targetTable); - if (alreadyCompleted) { - // Manifest says completed but Redis had no data (flushed) — recover from manifest - const completedRun = await manifestStore.getCompletedRun(sourceNs, targetTable); - const docsRead = completedRun?.summary?.total_docs_read ?? 0; - const rowsInserted = completedRun?.summary?.total_rows_inserted ?? 0; - const completedRunId = completedRun?.run_id ?? ""; - this.logger.info({ collection: next, sourceNs, docsRead, rowsInserted }, "Collection already migrated, skipping (manifest fallback)"); - this.results.push({ collection: next, sourceNs, runId: completedRunId, status: "completed", docsRead, rowsInserted }); - - // Backfill Redis for next restart - await this.deps.redisState.setCollectionCompleted(next, config.worker.podId, { - docsRead, rowsInserted, runId: completedRunId, - completedAt: new Date().toISOString(), - }).catch(() => {}); - await collectionLock?.release(next).catch(() => {}); - continue; - } - } - - // Process this collection - try { - const result = await this.processCollection(next, sourceNs, targetTable); - this.results.push(result); - - if (result.status === "completed") { - this.logger.info({ collection: next, runId: result.runId }, "Collection migration completed"); - // Persist this pod's completion counts (per-pod key, aggregated on read) - await this.deps.redisState.setCollectionCompleted(next, config.worker.podId, { - docsRead: result.docsRead ?? 0, - rowsInserted: result.rowsInserted ?? 0, - runId: result.runId, - completedAt: new Date().toISOString(), - }).catch(() => {}); - } else if (result.status === "failed") { - this.logger.error( - { collection: next, runId: result.runId, error: result.error }, - "Collection migration failed, continuing to next", - ); - } - } catch (err) { - const error = err instanceof Error ? err.message : String(err); - this.logger.error({ collection: next, error }, "Unexpected error migrating collection"); - this.results.push({ collection: next, sourceNs, runId: "", status: "failed", error }); - } finally { - // Release collection lock — always, including range-parallel - await collectionLock?.release(next).catch(() => {}); - } - continue; - } - - // No ready collection — are some still checking or building? - const counts = this.getIndexStatusCounts(); - const pendingIndexWork = counts.checking + counts.building; - - if (pendingIndexWork > 0) { - this.orchestratorStatus = "waiting_for_index"; - this.logger.info( - { checking: counts.checking, building: counts.building, ready: counts.ready }, - "Waiting for index creation — rechecking in 10s", - ); - await this.interruptibleSleep(10_000); - await this.recheckBuildingIndexes(); - continue; - } - - // In multi-pod mode, remaining collections may be locked by other pods — wait and retry - if (collectionLock) { - const remaining = this.discoveredCollections.filter( - name => !completed.has(name) && this.indexStatus.get(name) === "ready", - ); - if (remaining.length > 0) { - this.logger.info( - { lockedByOthers: remaining.length }, - "Remaining collections locked by other pods, waiting 10s", - ); - await this.interruptibleSleep(10_000); - continue; - } - } - - // All done (or all failed/skipped) - break; - } - - this.currentCollection = null; - this.currentRunId = null; - this.currentBatchRunner = null; - this.currentRangeCoordinator = null; - this.orchestratorStatus = "completed"; - - // Cleanup multi-pod resources - collectionLock?.stopHeartbeat(); - await collectionLock?.releaseAll().catch(() => {}); - - // Summary - const summary: OrchestratorResult = { - collections: this.results, - totalCompleted: this.results.filter((r) => r.status === "completed").length, - totalFailed: this.results.filter((r) => r.status === "failed").length, - totalSkipped: this.results.filter((r) => r.status === "skipped").length, - }; - - this.logger.info( - { - totalCollections: this.discoveredCollections.length + this.skippedApmCollections.size, - completed: summary.totalCompleted, - failed: summary.totalFailed, - skipped: summary.totalSkipped, - skippedApm: this.skippedApmCollections.size, - }, - "Migration orchestration complete", - ); - - return summary; - } - - getProgress(): OrchestratorProgress { - return { - totalCollections: this.discoveredCollections.length + this.skippedApmCollections.size, - completedCollections: this.results.filter((r) => r.status === "completed").length, - failedCollections: this.results.filter((r) => r.status === "failed").length, - skippedCollections: this.results.filter((r) => r.status === "skipped").length, - currentCollection: this.currentCollection, - collections: this.discoveredCollections, - results: this.results, - }; - } - - getCurrentBatchRunner(): BatchRunner | null { - return this.currentBatchRunner; - } - - getCurrentRunId(): string | null { - return this.currentRunId; - } - - getFirstCollectionStartedAt(): number | null { - return this.firstCollectionStartedAt; - } - - pause(): void { - this.currentBatchRunner?.pause(); - this.currentRangeCoordinator?.pause(); - } - - resume(): void { - this.currentBatchRunner?.resume(); - this.currentRangeCoordinator?.resume(); - } - - stopAfterBatch(): void { - this.stopping = true; - this.currentBatchRunner?.stopAfterBatch(); - this.currentRangeCoordinator?.stop(); - } - - async waitForStop(): Promise { - if (this.currentBatchRunner) { - await this.currentBatchRunner.waitForStop(); - } - } - - getStatus(): RunnerStatus { - if (this.orchestratorStatus === "waiting_for_index") { - return "waiting_for_index"; - } - if (this.currentBatchRunner) { - return this.currentBatchRunner.getStatus(); - } - if (this.currentRangeCoordinator) { - return "running"; - } - if (this.orchestratorStatus === "completed") { - return "completed"; - } - if (this.discoveredCollections.length > 0 && this.results.length >= this.discoveredCollections.length) { - return "completed"; - } - return "idle"; - } - - getStats(): BatchRunnerStats | null { - if (this.currentBatchRunner) { - const stats = this.currentBatchRunner.getStats(); - this.lastBatchStats = stats; - return stats; - } - if (this.currentRangeCoordinator) { - const coord = this.currentRangeCoordinator; - const active = coord.activeBatchRunner?.getStats(); - const totalDocsRead = coord.totalDocsRead + (active?.totalDocsRead ?? 0); - const totalRowsInserted = coord.totalRowsInserted + (active?.totalRowsInserted ?? 0); - const totalDocsSkipped = coord.totalDocsSkipped + (active?.totalDocsSkipped ?? 0); - const elapsedMs = coord.startTime > 0 ? Date.now() - coord.startTime : 0; - const elapsedSec = elapsedMs / 1000; - return { - status: "running", - batchSeq: active?.batchSeq ?? 0, - lastCommittedId: null, - totalDocsRead, - totalRowsInserted, - totalDocsSkipped, - skipsByReason: { ...coord.skipsByReason }, - elapsedMs, - docsPerSecond: elapsedSec > 0 ? totalDocsRead / elapsedSec : 0, - rowsPerSecond: elapsedSec > 0 ? totalRowsInserted / elapsedSec : 0, - batchesFailed: 0, - digestMismatches: 0, - estimatedDuplicateRows: 0, - } as BatchRunnerStats; - } - // Return cached stats from the last active runner (preserves skip reasons etc.) - return this.lastBatchStats; - } - - getCurrentBatchSeq(): number { - return this.currentBatchRunner?.getCurrentBatchSeq() ?? 0; - } - - getEstimatedCounts(): Map { - return new Map(this.estimatedCounts); - } - - getIndexStatus(): IndexStatusSummary { - let ready = 0, building = 0, failed = 0, checking = 0; - const details: Array<{ collection: string; status: IndexBuildStatus; elapsedSec?: number }> = []; - - for (const [name, status] of this.indexStatus) { - if (status === "ready") { - ready++; - } else if (status === "building") { - building++; - const elapsed = Math.round((Date.now() - (this.indexBuildStarted.get(name) ?? Date.now())) / 1000); - details.push({ collection: name, status, elapsedSec: elapsed }); - } else if (status === "failed") { - failed++; - details.push({ collection: name, status }); - } else if (status === "checking") { - checking++; - details.push({ collection: name, status }); - } - } - - return { ready, building, checking, failed, details }; - } - - triggerReindex(collectionName: string): void { - this.indexStatus.set(collectionName, "building"); - this.indexBuildStarted.set(collectionName, Date.now()); - this.logger.info({ collection: collectionName }, "Manual reindex triggered"); - this.deps.mongoReader.startIndexCreation(collectionName) - .then(() => { - this.indexStatus.set(collectionName, "ready"); - const elapsed = Math.round((Date.now() - (this.indexBuildStarted.get(collectionName) ?? Date.now())) / 1000); - this.logger.info({ collection: collectionName, durationSec: elapsed }, "Manual reindex completed"); - }) - .catch((err) => { - this.indexStatus.set(collectionName, "failed"); - this.logger.error({ collection: collectionName, err: err instanceof Error ? err.message : String(err) }, "Manual reindex failed"); - }); - } - - retryCollection(collectionName: string): void { - if (this.skippedApmCollections.has(collectionName)) { - this.logger.warn({ collection: collectionName }, "Cannot retry APM collection — permanently excluded"); - return; - } - this.results = this.results.filter(r => r.collection !== collectionName); - this.logger.info({ collection: collectionName }, "Collection queued for retry"); - } - - // ----------------------------------------------------------------------- - // Private: Upfront Estimates & Recovery - // ----------------------------------------------------------------------- - - /** - * Query estimatedDocumentCount() for ALL discovered collections in parallel - * and persist to Redis. This ensures the overall progress denominator is - * correct from the first stats request. - */ - private async populateAllEstimates(): Promise { - const db = this.deps.mongoReader.getDatabase(); - const CHUNK_SIZE = 10; - - for (let i = 0; i < this.discoveredCollections.length; i += CHUNK_SIZE) { - const chunk = this.discoveredCollections.slice(i, i + CHUNK_SIZE); - const results = await Promise.allSettled( - chunk.map(async (name) => { - const est = await db.collection(name).estimatedDocumentCount(); - return { name, est }; - }), - ); - for (const r of results) { - if (r.status === "fulfilled") { - this.estimatedCounts.set(r.value.name, r.value.est); - } - } - } - - // Persist to Redis for other pods and resume - await this.deps.redisState.setCollectionEstimates(this.estimatedCounts).catch((err) => { - this.logger.warn({ error: err instanceof Error ? err.message : String(err) }, "Failed to persist estimates to Redis"); - }); - - this.logger.info( - { collections: this.estimatedCounts.size, totalEstimated: Array.from(this.estimatedCounts.values()).reduce((a, b) => a + b, 0) }, - "Estimated counts for all collections stored in Redis", - ); - } - - /** - * On resume: recover completed collection aggregates from Redis. - * Populates this.results so completed collections aren't re-processed - * and their docsRead/rowsInserted are available for progress calculations. - */ - private async recoverCompletedCollections(): Promise { - const completedFromRedis = await this.deps.redisState.getAllCollectionCompleted().catch(() => new Map()); - if (completedFromRedis.size === 0) return; - - const { config } = this.deps; - let recovered = 0; - - for (const [collection, data] of completedFromRedis) { - // Only recover if this collection is in our discovered list - if (!this.discoveredCollections.includes(collection)) continue; - // Don't double-add (e.g. APM skip results already in this.results) - if (this.results.some(r => r.collection === collection)) continue; - - this.results.push({ - collection, - sourceNs: `${config.source.db}.${collection}`, - runId: data.runId, - status: "completed", - docsRead: data.docsRead, - rowsInserted: data.rowsInserted, - }); - recovered++; - } - - if (recovered > 0) { - this.logger.info( - { recovered, total: completedFromRedis.size }, - "Recovered completed collection aggregates from Redis", - ); - } - } - - // ----------------------------------------------------------------------- - // Private: Index Management - // ----------------------------------------------------------------------- - - /** - * Start all index checks in the background. Does NOT block. - * drill_events is first in the list so it gets checked first, - * but we don't wait for it — migrate whatever is ready first. - */ - private startBackgroundIndexInit(): void { - for (const name of this.discoveredCollections) { - this.indexStatus.set(name, "checking"); - } - - this.runIndexChecks().catch((err) => { - this.logger.error({ err: err instanceof Error ? err.message : String(err) }, "Background index init crashed"); - }); - } - - private async runIndexChecks(): Promise { - const CONCURRENCY = 10; - for (let i = 0; i < this.discoveredCollections.length; i += CONCURRENCY) { - const chunk = this.discoveredCollections.slice(i, i + CONCURRENCY); - await Promise.allSettled(chunk.map(name => this.checkAndBuildIndex(name))); - } - const counts = this.getIndexStatusCounts(); - this.logger.info(counts, "Background index initialization complete"); - } - - private async checkAndBuildIndex(name: string): Promise { - const { mongoReader } = this.deps; - try { - const hasIndex = await mongoReader.hasRequiredIndex(name); - if (hasIndex) { - this.indexStatus.set(name, "ready"); - return; - } - this.indexStatus.set(name, "building"); - this.indexBuildStarted.set(name, Date.now()); - mongoReader.startIndexCreation(name) - .then(() => { - this.indexStatus.set(name, "ready"); - const elapsed = Math.round((Date.now() - (this.indexBuildStarted.get(name) ?? Date.now())) / 1000); - this.logger.info({ collection: name, durationSec: elapsed }, "Background index build completed"); - }) - .catch((err) => { - this.indexStatus.set(name, "failed"); - this.logger.error({ collection: name, err: err instanceof Error ? err.message : String(err) }, "Index creation failed"); - }); - } catch (err) { - this.indexStatus.set(name, "failed"); - this.logger.error({ collection: name, err: err instanceof Error ? err.message : String(err) }, "Failed to check index"); - } - } - - private getIndexStatusCounts(): { ready: number; building: number; failed: number; checking: number; total: number } { - let ready = 0, building = 0, failed = 0, checking = 0; - for (const s of this.indexStatus.values()) { - if (s === "ready") ready++; - else if (s === "building") building++; - else if (s === "failed") failed++; - else checking++; - } - return { ready, building, failed, checking, total: this.discoveredCollections.length }; - } - - private async findNextReadyCollection(completed: Set): Promise { - const { collectionLock } = this.deps; - - // Priority 1: Any unlocked, ready collection (spread pods across different collections) - for (const name of this.discoveredCollections) { - if (completed.has(name)) continue; - if (this.indexStatus.get(name) !== "ready") continue; - if (!collectionLock || (await collectionLock.tryAcquire(name)) !== "locked") { - return name; - } - } - - // Priority 2: All remaining are locked by other pods — join range-parallel - // on any large collection that has pending ranges (no lock needed) - for (const name of this.discoveredCollections) { - if (completed.has(name)) continue; - if (this.indexStatus.get(name) !== "ready") continue; - if (await this.isRangeParallelCandidate(name)) { - return name; - } - } - - return null; - } - - /** Check if a collection qualifies for range-parallel (cached or fresh query). - * Uses getDatabase() directly to avoid mutating the shared MongoReader cursor. */ - private async isRangeParallelCandidate(collectionName: string): Promise { - const rpThreshold = this.deps.config.source.rangeParallelThreshold; - let est = this.estimatedCounts.get(collectionName); - if (est === undefined) { - try { - const db = this.deps.mongoReader.getDatabase(); - est = await db.collection(collectionName).estimatedDocumentCount(); - this.estimatedCounts.set(collectionName, est); - } catch { - return false; - } - } - return est >= rpThreshold; - } - - private getCollectionsWithStatus(status: IndexBuildStatus): string[] { - return [...this.indexStatus.entries()] - .filter(([, s]) => s === status) - .map(([name]) => name); - } - - private async recheckBuildingIndexes(): Promise { - const { mongoReader } = this.deps; - const building = [...this.indexStatus.entries()].filter(([, s]) => s === "building"); - if (building.length === 0) return; - - await Promise.allSettled(building.map(async ([name]) => { - try { - const ready = await mongoReader.hasRequiredIndex(name); - if (ready) { - const elapsed = Math.round((Date.now() - (this.indexBuildStarted.get(name) ?? Date.now())) / 1000); - this.indexStatus.set(name, "ready"); - this.logger.info({ collection: name, durationSec: elapsed }, "Index build detected as complete"); - } - } catch (err) { - this.logger.warn({ collection: name, err: err instanceof Error ? err.message : String(err) }, "Failed to recheck index"); - } - })); - } - - private async interruptibleSleep(ms: number): Promise { - return new Promise((resolve) => { - let resolved = false; - const done = () => { if (resolved) return; resolved = true; clearTimeout(timer); clearInterval(check); resolve(); }; - const timer = setTimeout(done, ms); - const check = setInterval(() => { if (this.stopping) done(); }, 1000); - }); - } - - // ----------------------------------------------------------------------- - // Private: Collection Processing - // ----------------------------------------------------------------------- - - private async processCollection( - collectionName: string, - sourceNs: string, - targetTable: string, - ): Promise { - const { mongoReader, manifestStore, chWriter, chPressure, gcController, retryPolicy, config } = this.deps; - - this.currentCollection = collectionName; - if (!this.firstCollectionStartedAt) { - this.firstCollectionStartedAt = Date.now(); - } - - // Switch MongoReader to this collection (index already confirmed ready) - await mongoReader.switchCollection(collectionName); - - // Use cached estimated count (populated by main loop or isRangeParallelCandidate) - const estimatedCount = this.estimatedCounts.get(collectionName) ?? await mongoReader.getEstimatedCount(); - this.estimatedCounts.set(collectionName, estimatedCount); - this.logger.info({ collection: collectionName, estimatedCount }, "Estimated document count"); - - // Resolve collection defaults from hash (once per collection) - const collectionDefaults = this.deps.hashResolver.resolveCollectionName( - collectionName, - config.source.collectionPrefix, - ); - - if (collectionDefaults) { - this.logger.info( - { collection: collectionName, appId: collectionDefaults.a, event: collectionDefaults.e }, - "Resolved collection hash defaults", - ); - } else if (collectionName !== config.source.collectionPrefix) { - this.logger.warn( - { collection: collectionName }, - "No hash match found for collection — documents missing a/e will be skipped", - ); - } - - // ── Range-parallel mode for large collections ───────────────── - if (estimatedCount >= config.source.rangeParallelThreshold) { - this.logger.info( - { collection: collectionName, estimatedCount, threshold: config.source.rangeParallelThreshold }, - "Collection exceeds range-parallel threshold — using range splitting", - ); - - const redisClient = this.deps.redisState.getRedisClient(); - const coordinator = new RangeCoordinator({ - redis: redisClient, - manifestStore, - redisState: this.deps.redisState, - asyncBatchWriter: this.deps.asyncBatchWriter, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger: this.deps.logger, - config: { - collectionName, - sourceNs, - targetTable, - transformVersion: config.transform.version, - rangeCount: config.source.rangeCount, - rangeLeaseTtlSec: config.source.rangeLeaseTtlSec, - batchRowsTarget: config.source.batchRowsTarget, - mongoPageSize: config.source.mongoPageSize, - backpressure: config.backpressure, - useDedupToken: config.target.useDedupToken, - database: config.target.db, - table: config.target.table, - snapshotInterval: config.state.timelineSnapshotInterval, - collectionDefaults: collectionDefaults ?? undefined, - podId: config.worker.podId, - redisKeyPrefix: config.state.redisKeyPrefix, - }, - }); - - this.currentRangeCoordinator = coordinator; - - // Progress updates for range-parallel mode - const rpStartedAt = new Date().toISOString(); - const rpGlobalProgress = this.deps.globalProgress; - const rpProgressInterval = rpGlobalProgress - ? setInterval(async () => { - const status = await coordinator.getRangeStatus().catch(() => ({ pending: 0, processing: 0, done: 0, failed: 0 })); - rpGlobalProgress.updateCollectionProgress({ - collectionName, - podId: config.worker.podId, - status: "processing", - runId: "", - docsRead: coordinator.totalDocsRead, - rowsInserted: coordinator.totalRowsInserted, - estimatedTotal: estimatedCount, - batchSeq: status.done, - startedAt: rpStartedAt, - updatedAt: new Date().toISOString(), - isRangeParallel: true, - rangeCount: config.source.rangeCount, - }).catch(() => {}); - }, config.worker.progressUpdateMs) - : null; - - try { - const rangeResult = await coordinator.run(); - - if (rpProgressInterval) clearInterval(rpProgressInterval); - - // Use GLOBAL range status (not just this pod's local counters) - // to prevent marking a collection as "completed" while other pods have failed ranges - const globalRangeStatus = await coordinator.getRangeStatus().catch(() => ({ - pending: 0, processing: 0, done: 0, failed: rangeResult.failedRanges, - })); - const allGloballyDone = globalRangeStatus.failed === 0 - && globalRangeStatus.pending === 0 - && globalRangeStatus.processing === 0; - - const rpStatus = allGloballyDone ? "completed" as const : "failed" as const; - await rpGlobalProgress?.updateCollectionProgress({ - collectionName, - podId: config.worker.podId, - status: rpStatus, - runId: rangeResult.runId, - docsRead: rangeResult.totalDocsRead, - rowsInserted: rangeResult.totalRowsInserted, - estimatedTotal: estimatedCount, - batchSeq: rangeResult.completedRanges, - startedAt: rpStartedAt, - updatedAt: new Date().toISOString(), - isRangeParallel: true, - rangeCount: config.source.rangeCount, - }).catch(() => {}); - - return { - collection: collectionName, - sourceNs, - runId: rangeResult.runId, - status: rpStatus, - docsRead: rangeResult.totalDocsRead, - rowsInserted: rangeResult.totalRowsInserted, - error: !allGloballyDone - ? `${globalRangeStatus.failed} ranges failed, ${globalRangeStatus.pending} pending globally` - : undefined, - }; - } finally { - if (rpProgressInterval) clearInterval(rpProgressInterval); - this.currentRangeCoordinator = null; - } - } - - // ── Standard single-pod mode ───────────────────────────────── - - // Create per-collection RedisHotState with isolated key prefix - const redisClient = this.deps.redisState.getRedisClient(); - const collectionPrefix = `${config.state.redisKeyPrefix}:${collectionName}`; - const redisState = RedisHotState.fromExistingConnection(redisClient, collectionPrefix); - this.currentRedisState = redisState; - - // Resolve run (resume or new) - const resolved = await resolveRun({ - rerunMode: config.service.rerunMode, - manifestStore, - redisState, - mongoReader, - sourceNs, - targetTable, - transformVersion: config.transform.version, - logger: this.logger, - }); - - const { runId, upperBoundId } = resolved; - this.currentRunId = runId; - - if (resolved.isEmpty) { - this.currentRunId = null; - this.logger.info({ collection: collectionName }, "Collection is empty, skipping"); - return { collection: collectionName, sourceNs, runId, status: "completed" }; - } - - this.logger.info( - { collection: collectionName, runId, upperBoundId }, - "Starting migration for collection", - ); - - // Create BatchRunner for this collection - const batchRunner = new BatchRunner({ - manifestStore, - redisState, - globalRedisState: this.deps.redisState, - asyncBatchWriter: this.deps.asyncBatchWriter, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger: this.deps.logger, - config: { - runId, - transformVersion: config.transform.version, - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget: config.source.batchRowsTarget, - mongoPageSize: config.source.mongoPageSize, - backpressure: config.backpressure, - useDedupToken: config.target.useDedupToken, - database: config.target.db, - table: config.target.table, - snapshotInterval: config.state.timelineSnapshotInterval, - collectionDefaults: collectionDefaults ?? undefined, - collectionName, - podId: config.worker.podId, - }, - }); - - this.currentBatchRunner = batchRunner; - - // Start periodic progress updates (multi-pod mode) - const { globalProgress } = this.deps; - const progressStartedAt = new Date().toISOString(); - const progressInterval = globalProgress - ? setInterval(() => { - const stats = batchRunner.getStats(); - globalProgress.updateCollectionProgress({ - collectionName, - podId: config.worker.podId, - status: "processing", - runId, - docsRead: stats.totalDocsRead, - rowsInserted: stats.totalRowsInserted, - estimatedTotal: estimatedCount, - batchSeq: stats.batchSeq, - startedAt: progressStartedAt, - updatedAt: new Date().toISOString(), - }).catch(() => {}); // best-effort - }, config.worker.progressUpdateMs) - : null; - - // Run the batch processing - try { - await batchRunner.run(); - - const finalStatus = batchRunner.getStatus(); - const finalStats = batchRunner.getStats(); - - // Write final progress to Redis - const terminalStatus = finalStatus === "completed" ? "completed" as const : "failed" as const; - await globalProgress?.updateCollectionProgress({ - collectionName, - podId: config.worker.podId, - status: terminalStatus, - runId, - docsRead: finalStats.totalDocsRead, - rowsInserted: finalStats.totalRowsInserted, - estimatedTotal: estimatedCount, - batchSeq: finalStats.batchSeq, - startedAt: progressStartedAt, - updatedAt: new Date().toISOString(), - }).catch(() => {}); - - if (finalStatus === "completed") { - return { - collection: collectionName, sourceNs, runId, status: "completed", - docsRead: finalStats.totalDocsRead, - rowsInserted: finalStats.totalRowsInserted, - }; - } - if (finalStatus === "stopped" || finalStatus === "stopping") { - return { - collection: collectionName, sourceNs, runId, status: "failed", error: "Stopped by operator", - docsRead: finalStats.totalDocsRead, - rowsInserted: finalStats.totalRowsInserted, - }; - } - return { - collection: collectionName, sourceNs, runId, status: "failed", - error: `BatchRunner ended with status: ${finalStatus}`, - docsRead: finalStats.totalDocsRead, - rowsInserted: finalStats.totalRowsInserted, - }; - } catch (err) { - const error = err instanceof Error ? err.message : String(err); - await globalProgress?.updateCollectionProgress({ - collectionName, - podId: config.worker.podId, - status: "failed", - runId, - docsRead: 0, - rowsInserted: 0, - estimatedTotal: estimatedCount, - batchSeq: 0, - startedAt: progressStartedAt, - updatedAt: new Date().toISOString(), - error, - }).catch(() => {}); - await manifestStore.insertEvent({ - run_id: runId, - event_type: "collection_migration_failed", - message: `Migration of ${collectionName} failed: ${error}`, - metadata: { collection: collectionName, error }, - created_at: new Date().toISOString(), - }); - return { collection: collectionName, sourceNs, runId, status: "failed", error }; - } finally { - if (progressInterval) clearInterval(progressInterval); - } - } -} diff --git a/src/runtime/error-classifier.ts b/src/runtime/error-classifier.ts new file mode 100644 index 0000000..ff13dea --- /dev/null +++ b/src/runtime/error-classifier.ts @@ -0,0 +1,76 @@ +/** + * Classifies errors from ClickHouse inserts (and the surrounding pipeline) + * into `permanent` (the data will never insert — retrying is pointless) and + * `transient` (network/load — retrying works). + * + * Unknown errors default to `transient`: misclassifying either way is + * recoverable (the chunk fails and is redone / replayed), but defaulting + * unknowns to retry means a new transient error class doesn't silently + * divert good data into the failed pile. + */ + +export type ErrorClass = 'permanent' | 'transient'; + +/** + * ClickHouse exception codes that indicate the payload itself is unacceptable. + * Source: ClickHouse ErrorCodes; verified empirically (e.g. 41 on bad DateTime). + */ +const PERMANENT_CH_CODES = new Set([ + '6', // CANNOT_PARSE_TEXT + '26', // CANNOT_PARSE_ESCAPE_SEQUENCE + '27', // CANNOT_PARSE_INPUT_ASSERTION_FAILED + '38', // CANNOT_PARSE_DATE + '41', // CANNOT_PARSE_DATETIME + '53', // TYPE_MISMATCH + '69', // ARGUMENT_OUT_OF_BOUND + '72', // CANNOT_PARSE_NUMBER + '117', // INCORRECT_DATA + '130', // CANNOT_READ_ARRAY_FROM_TEXT + '467', // CANNOT_PARSE_BOOL + '490', // CANNOT_PARSE_IPV4 + '491', // CANNOT_PARSE_IPV6 +]); + +/** + * ClickHouse capacity/limit codes — transient: they clear when the operator + * frees disk / load subsides. The retry→attempts→circuit-breaker chain turns + * a persistent disk-full into a paused engine awaiting the operator. + * 243 NOT_ENOUGH_SPACE, 202 TOO_MANY_SIMULTANEOUS_QUERIES, + * 209 SOCKET_TIMEOUT, 210 NETWORK_ERROR (already covered by default-transient, + * listed for documentation). + */ + +/** Node/undici-level network error codes — always transient. */ +const TRANSIENT_SYSTEM_CODES = new Set([ + 'ECONNRESET', 'ECONNREFUSED', 'EPIPE', 'ETIMEDOUT', 'EAI_AGAIN', + 'ENOTFOUND', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_SOCKET', +]); + +/** Client-side serialization failures that no retry can fix. */ +const PERMANENT_MESSAGE_PATTERNS = [ + /do not know how to serialize/i, // JSON.stringify on BigInt + /circular structure/i, +]; + +/** + * Classify an error thrown by a ClickHouse insert (or the code around it). + */ +export function classifyError(err: unknown): ErrorClass { + const e = err as { code?: unknown; message?: unknown } | null; + const code = e && e.code !== undefined ? String(e.code) : ''; + const message = e && typeof e.message === 'string' ? e.message : ''; + + if (PERMANENT_CH_CODES.has(code)) { + return 'permanent'; + } + if (TRANSIENT_SYSTEM_CODES.has(code)) { + return 'transient'; + } + for (const pattern of PERMANENT_MESSAGE_PATTERNS) { + if (pattern.test(message)) { + return 'permanent'; + } + } + // Unknown → transient (retry). See module doc for rationale. + return 'transient'; +} diff --git a/src/runtime/gc-controller.ts b/src/runtime/gc-controller.ts deleted file mode 100644 index cd695ab..0000000 --- a/src/runtime/gc-controller.ts +++ /dev/null @@ -1,310 +0,0 @@ -import type { Logger } from "pino"; -import { PerformanceObserver } from "node:perf_hooks"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export type GcMode = "after-batch" | "now" | "force"; - -export interface GcConfig { - enabled: boolean; - rssSoftLimitBytes: number; - rssHardLimitBytes: number; - heapUsedRatio: number; - everyNBatches: number; -} - -export interface GcEvent { - kind: number | null; - durationMs: number; - startTimeMs: number; - timestamp: string; -} - -export interface GcTelemetry { - gcAvailable: boolean; - gcState: "idle" | "pending" | "running"; - lastGcReason: string | null; - lastGcDurationMs: number; - heapUsedBefore: number; - heapUsedAfter: number; - rssBefore: number; - rssAfter: number; - gcCountTotal: number; - observedGcCount: number; - lastObservedGcDurationMs: number; - lastObservedGcKind: number | null; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/** - * Wraps `setImmediate` in a promise so we can `await` one full I/O cycle. - */ -function nextImmediate(): Promise { - return new Promise((resolve) => setImmediate(resolve)); -} - -// --------------------------------------------------------------------------- -// GcController -// --------------------------------------------------------------------------- - -/** - * Manual GC controller. - * - * Requires Node to be started with `--expose-gc` so that `global.gc` is - * available. When GC is unavailable the controller is a safe no-op; it - * records that fact in its telemetry so operators can tell from metrics. - */ -export class GcController { - private readonly config: GcConfig; - private readonly logger: Logger; - - private state: "idle" | "pending" | "running" = "idle"; - private pendingReason: string | null = null; - private gcCount = 0; - private lastReason: string | null = null; - private lastDurationMs = 0; - private lastHeapUsedBefore = 0; - private lastHeapUsedAfter = 0; - private lastRssBefore = 0; - private lastRssAfter = 0; - - /** PerformanceObserver tracking V8-initiated GC events. */ - private gcObserver: PerformanceObserver | null = null; - private lastObservedEvent: GcEvent | null = null; - private observedGcCount = 0; - - constructor(config: GcConfig, logger: Logger) { - this.config = config; - this.logger = logger.child({ component: "GcController" }); - - if (!this.isAvailable) { - this.logger.warn( - "global.gc is not available. Start Node with --expose-gc to enable manual GC.", - ); - } - } - - /** - * Start observing V8 GC events via PerformanceObserver. - * Call this after construction when the event loop is ready. - */ - start(): void { - if (this.gcObserver) return; - - this.gcObserver = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - const detail = entry as unknown as { detail?: { kind?: number }; kind?: number; duration: number; startTime: number }; - const kind: number | null = detail.detail?.kind ?? detail.kind ?? null; - - this.lastObservedEvent = { - kind, - durationMs: entry.duration, - startTimeMs: entry.startTime, - timestamp: new Date().toISOString(), - }; - this.observedGcCount++; - } - }); - - try { - this.gcObserver.observe({ entryTypes: ["gc"] }); - } catch { - this.logger.debug("PerformanceObserver for GC entryType not supported in this runtime"); - } - } - - // ----------------------------------------------------------------------- - // Accessors - // ----------------------------------------------------------------------- - - /** Check if GC is available (`global.gc` exists). */ - get isAvailable(): boolean { - return typeof global.gc === "function"; - } - - /** Check if GC has been marked as pending. */ - get isPending(): boolean { - return this.state === "pending"; - } - - // ----------------------------------------------------------------------- - // Telemetry - // ----------------------------------------------------------------------- - - /** Return a snapshot of the current GC telemetry. */ - getTelemetry(): GcTelemetry { - const lastEvent = this.lastObservedEvent; - return { - gcAvailable: this.isAvailable, - gcState: this.state, - lastGcReason: this.lastReason, - lastGcDurationMs: this.lastDurationMs, - heapUsedBefore: this.lastHeapUsedBefore, - heapUsedAfter: this.lastHeapUsedAfter, - rssBefore: this.lastRssBefore, - rssAfter: this.lastRssAfter, - gcCountTotal: this.gcCount, - observedGcCount: this.observedGcCount, - lastObservedGcDurationMs: lastEvent?.durationMs ?? 0, - lastObservedGcKind: lastEvent?.kind ?? null, - }; - } - - // ----------------------------------------------------------------------- - // Decision helpers - // ----------------------------------------------------------------------- - - /** - * Determine whether GC should run after the given batch sequence number. - * - * Checks three independent conditions (any one triggers a `true`): - * 1. `heapUsed / heapTotal >= heapUsedRatio` - * 2. `rss >= rssSoftLimitBytes` - * 3. `batchSeq % everyNBatches === 0` - */ - shouldRunAfterBatch(batchSeq: number): boolean { - if (!this.config.enabled) return false; - - const mem = process.memoryUsage(); - const heapRatio = mem.heapTotal > 0 ? mem.heapUsed / mem.heapTotal : 0; - - if (heapRatio >= this.config.heapUsedRatio) return true; - if (mem.rss >= this.config.rssSoftLimitBytes) return true; - if (this.config.everyNBatches > 0 && batchSeq % this.config.everyNBatches === 0) return true; - - return false; - } - - // ----------------------------------------------------------------------- - // Execution - // ----------------------------------------------------------------------- - - /** - * Mark GC as pending. The actual collection will happen when the current - * batch completes and the runner calls {@link runGc}. - */ - markPending(reason: string): void { - if (this.state === "idle") { - this.state = "pending"; - this.pendingReason = reason; - this.logger.debug({ reason }, "GC marked pending"); - } - } - - /** - * Execute garbage collection. - * - * @param mode - Controls preconditions: - * - `after-batch`: normal conditional GC (obeys thresholds). - * - `now`: only if currently idle (skip if already running). - * - `force`: bypass threshold checks, always collect. - * @param reason - Human-readable reason for log messages. - * @returns `true` if GC actually ran, `false` if skipped. - */ - async runGc(mode: GcMode, reason: string): Promise { - // Guard: GC not available - if (!this.isAvailable) { - this.logger.debug("Skipping GC: global.gc not available"); - return false; - } - - // Guard: not enabled (unless forced) - if (!this.config.enabled && mode !== "force") { - return false; - } - - // Guard: already running - if (this.state === "running") { - this.logger.debug("Skipping GC: already running"); - return false; - } - - // Mode-specific guards - if (mode === "now" && this.state !== "idle") { - this.logger.debug({ state: this.state }, "Skipping GC (mode=now): not idle"); - return false; - } - - // ------------------------------------------------------------------ - // Proceed with collection - // ------------------------------------------------------------------ - - this.state = "running"; - this.lastReason = reason; - - // 1. Await one microtask to let pending promises settle - await Promise.resolve(); - - // 2. Await one setImmediate to drain the I/O queue - await nextImmediate(); - - // 3. Record memory before - const memBefore = process.memoryUsage(); - this.lastHeapUsedBefore = memBefore.heapUsed; - this.lastRssBefore = memBefore.rss; - - // 4. Run GC - const startMs = performance.now(); - global.gc!(); - const elapsed = performance.now() - startMs; - this.lastDurationMs = Math.round(elapsed * 100) / 100; - - // 5. Record memory after - const memAfter = process.memoryUsage(); - this.lastHeapUsedAfter = memAfter.heapUsed; - this.lastRssAfter = memAfter.rss; - - // 6. Update counters & state - this.gcCount++; - this.state = "idle"; - this.pendingReason = null; - - const freedMb = ((memBefore.heapUsed - memAfter.heapUsed) / 1024 / 1024).toFixed(1); - - this.logger.info( - { - reason, - mode, - durationMs: this.lastDurationMs, - heapUsedBeforeMb: (memBefore.heapUsed / 1024 / 1024).toFixed(1), - heapUsedAfterMb: (memAfter.heapUsed / 1024 / 1024).toFixed(1), - freedMb, - rssMb: (memAfter.rss / 1024 / 1024).toFixed(1), - gcCountTotal: this.gcCount, - }, - "GC completed", - ); - - // Warn if RSS exceeds hard limit after collection - if (memAfter.rss >= this.config.rssHardLimitBytes) { - this.logger.warn( - { - rssBytes: memAfter.rss, - rssHardLimitBytes: this.config.rssHardLimitBytes, - }, - "RSS exceeds hard limit even after GC", - ); - } - - return true; - } - - // ----------------------------------------------------------------------- - // Cleanup - // ----------------------------------------------------------------------- - - /** Disconnect the PerformanceObserver and release resources. */ - dispose(): void { - try { - this.gcObserver?.disconnect(); - this.gcObserver = null; - } catch { - // observer may already be disconnected - } - } -} diff --git a/src/runtime/ledger-engine.ts b/src/runtime/ledger-engine.ts new file mode 100644 index 0000000..184da1d --- /dev/null +++ b/src/runtime/ledger-engine.ts @@ -0,0 +1,334 @@ +/** + * Bootstrap for the `ledger` engine (MIGRATION_ENGINE=ledger). + * + * Dependencies: MongoDB + ClickHouse. Deliberately NO Redis, no async batch + * writer, no per-batch manifest machinery — the chunk ledger (LedgerStore) + * is the only persistent state, and it is verified, not trusted (see + * ChunkOrchestrator). + */ + +import Fastify from 'fastify'; +import type { Logger } from 'pino'; +import { createClient as createClickHouseClient } from '@clickhouse/client'; +import type { Config } from '../config/schema.ts'; +import { MongoReader } from '../source/mongo-reader.ts'; +import { HashResolver } from '../transform/hash-resolver.ts'; +import { RetryPolicy } from './retry-policy.ts'; +import { LedgerStore } from '../state/ledger-store.ts'; +import { DlqStore } from '../state/dlq-store.ts'; +import { StagingManager } from '../target/staging-manager.ts'; +import { ClickHousePressure } from '../target/clickhouse-pressure.ts'; +import { ChunkOrchestrator } from './chunk-orchestrator.ts'; +import { wireExitOnComplete } from './exit-on-complete.ts'; +import { rebuildLedger, newRebuildProgress, type RebuildProgress } from './ledger-rebuild.ts'; + +export async function runLedgerEngine(config: Config, logger: Logger): Promise { + logger.info({ engine: 'ledger', runId: config.ledger.runId }, 'Starting ledger engine (no Redis)'); + + // Read preference 'auto' (the default): pick secondaryPreferred on replica + // sets — the source is frozen after cutover, so secondary reads are exact + // and the days-long scan stays off the primary. Explicit env wins. + if (config.source.readPreference === 'auto') { + const { MongoClient } = await import('mongodb'); + const probe = new MongoClient(config.source.uri); + try { + await probe.connect(); + const hello = await probe.db('admin').command({ hello: 1 }); + config.source.readPreference = hello.setName ? 'secondaryPreferred' : 'primary'; + config.source.readPreferenceAuto = true; + logger.info( + { readPreference: config.source.readPreference, replicaSet: hello.setName ?? null }, + 'Read preference auto-selected', + ); + } catch { + config.source.readPreference = 'primary'; + } finally { + await probe.close().catch(() => {}); + } + } + + const mongoReader = new MongoReader( + { + uri: config.source.uri, + database: config.source.db, + readPreference: config.source.readPreference, + readConcern: config.source.readConcern, + retryReads: config.source.retryReads, + appName: config.source.appName ?? config.service.name, + cursorBatchSize: config.source.cursorBatchSize, + maxTimeMs: config.source.maxTimeMs, + }, + logger, + ); + + const ledger = new LedgerStore(config.source.uri, config.state.manifestDb, logger); + const dlq = new DlqStore(config.source.uri, config.state.manifestDb, logger); + + const staging = new StagingManager( + { + url: config.target.url, + database: config.target.db, + table: config.target.table, + username: config.target.username, + password: config.target.password, + queryTimeoutMs: config.target.queryTimeoutMs, + }, + logger, + ); + + const retryPolicy = new RetryPolicy({ + maxRetries: config.target.maxRetries, + baseDelayMs: config.target.retryBaseDelayMs, + maxDelayMs: config.target.retryMaxDelayMs, + }); + + const hashResolver = new HashResolver( + { uri: config.source.uri, countlyDb: config.source.countlyDb }, + logger, + ); + + await mongoReader.connect(); + await ledger.connect(); + await dlq.connect(); + await staging.connect(); + await hashResolver.build(); + logger.info('Ledger engine: all services connected (MongoDB + ClickHouse only)'); + + // Backpressure sampler (TTL-cached inside the orchestrator — never per-batch) + const pressureClient = createClickHouseClient({ + url: config.target.url, + database: config.target.db, + username: config.target.username, + password: config.target.password, + request_timeout: config.target.queryTimeoutMs, + }); + const serverLimits = await ClickHousePressure.fetchServerLimits(pressureClient, logger); + config.backpressure.partsToThrowInsert = serverLimits.partsToThrowInsert; + config.backpressure.maxPartsInTotal = serverLimits.maxPartsInTotal; + const chPressure = new ClickHousePressure(pressureClient, config.backpressure, logger); + + const orchestrator = new ChunkOrchestrator({ + config, + logger, + mongoReader, + ledger, + dlq, + staging, + retryPolicy, + hashResolver, + chPressure, + }); + + // ── In-process dry-run runner (UI action) ───────────────────────────── + // Uses its OWN MongoReader/StagingManager so it can never disturb the main + // orchestrator's collection binding. Allowed only while the main run is + // not actively copying. + const dryState: { status: string; stats: Record | null; error: string | null } = + { status: 'not_run', stats: null, error: null }; + + async function startDryRun(): Promise<{ started: boolean; reason?: string }> { + if (dryState.status === 'running') return { started: false, reason: 'dry run already running' }; + if (orchestrator.getStatus() === 'running') return { started: false, reason: 'main migration is running — pause or wait for completion first' }; + dryState.status = 'running'; dryState.error = null; + + const dryConfig: Config = { ...config, ledger: { ...config.ledger, dryRun: true } }; + const dryReader = new MongoReader( + { + uri: config.source.uri, database: config.source.db, + readPreference: config.source.readPreference, readConcern: config.source.readConcern, + retryReads: config.source.retryReads, appName: `${config.service.name}-dry`, + cursorBatchSize: config.source.cursorBatchSize, maxTimeMs: config.source.maxTimeMs, + }, + logger, + ); + const dryStaging = new StagingManager( + { + url: config.target.url, database: config.target.db, table: config.target.table, + username: config.target.username, password: config.target.password, + queryTimeoutMs: config.target.queryTimeoutMs, + }, + logger, + ); + void (async () => { + try { + await dryReader.connect(); + await dryStaging.connect(); + const dryOrch = new ChunkOrchestrator({ + config: dryConfig, logger, mongoReader: dryReader, ledger, dlq, + staging: dryStaging, retryPolicy, hashResolver, + }); + await dryOrch.run(); + dryState.stats = dryOrch.getStats() as unknown as Record; + dryState.status = 'completed'; + } catch (err) { + dryState.status = 'failed'; + dryState.error = (err as Error).message; + } finally { + await dryReader.close().catch(() => {}); + await dryStaging.close().catch(() => {}); + } + })(); + return { started: true }; + } + + // ── Ledger rebuild (disaster recovery, UI action) ───────────────────── + // Regenerates mig_ranges from Mongo + ClickHouse counts when the ledger is + // lost. Guarded hard: single active pod, engine not copying, and an + // existing ledger is only replaced with force=true. + const rebuildState = newRebuildProgress(); + + async function startRebuild(force: boolean): Promise<{ started: boolean; reason?: string; existingChunks?: number }> { + if (rebuildState.status === 'running') return { started: false, reason: 'rebuild already running' }; + if (orchestrator.getStatus() === 'running') return { started: false, reason: 'main migration is running — a rebuild only makes sense when progress state is lost; stop/pause first' }; + if (dryState.status === 'running') return { started: false, reason: 'dry run in progress — wait for it to finish' }; + const pods = await ledger.podActivity(config.ledger.runId); + const others = pods.filter((row) => row.pod !== config.worker.podId && row.active > 0); + if (others.length > 0) return { started: false, reason: `other pods hold active chunks (${others.map((row) => row.pod).join(', ')}) — stop them first` }; + const existing = await ledger.countForRun(config.ledger.runId); + if (existing > 0 && !force) { + return { started: false, reason: `ledger already has ${existing} chunks for run "${config.ledger.runId}" — rebuilding replaces them; confirm with force`, existingChunks: existing }; + } + Object.assign(rebuildState, newRebuildProgress(), { status: 'running', startedAt: Date.now() }); + void rebuildLedger({ config, logger, ledger, dlq, hashResolver, progress: rebuildState }) + .then(() => { rebuildState.status = 'completed'; rebuildState.finishedAt = Date.now(); }) + .catch((err) => { + rebuildState.status = 'failed'; + rebuildState.error = (err as Error).message; + rebuildState.finishedAt = Date.now(); + logger.error({ err }, 'Ledger rebuild failed'); + }); + return { started: true }; + } + + // HTTP surface: health + stats + report + controls + branded dashboard (/viz) + const app = Fastify({ logger: false }); + app.get('/healthz', async () => { + const stats = orchestrator.getStats(); + return stats.fatalError + ? { status: 'error', engine: 'ledger', error: stats.fatalError } + : { status: 'ok', engine: 'ledger' }; + }); + app.get('/stats', async () => orchestrator.getStats()); + app.get('/report', async () => orchestrator.getReport()); + app.post('/control/pause', async () => { orchestrator.pause(); return { status: orchestrator.getStatus() }; }); + app.post('/control/resume', async () => { orchestrator.resume(); return { status: orchestrator.getStatus() }; }); + // Replay runs in the background: a mass DLQ (systematic failure on a + // 10B-doc run) can hold millions of entries — not one HTTP request's work. + const replayState: { status: string; result: Record | null; error: string | null } = + { status: 'not_run', result: null, error: null }; + app.post('/control/replay-dlq', async () => { + if (replayState.status === 'running') return { started: false, reason: 'replay already running' }; + replayState.status = 'running'; replayState.result = null; replayState.error = null; + void orchestrator.replayDlq() + .then((r) => { replayState.result = r as unknown as Record; replayState.status = 'completed'; }) + .catch((e) => { replayState.error = (e as Error).message; replayState.status = 'failed'; }); + return { started: true }; + }); + app.get('/api/replay', async () => ({ + status: replayState.status, progress: orchestrator.replayProgress, + result: replayState.result, error: replayState.error, + })); + app.post('/control/retry-failed', async () => orchestrator.retryFailed()); + app.post<{ Body: { ids?: string[] } }>('/control/waive-dlq', async (req) => ({ + waived: await dlq.waive(config.ledger.dryRun ? `${config.ledger.runId}-dry` : config.ledger.runId, req.body?.ids), + })); + app.post('/control/build-indexes', async () => orchestrator.startIndexBuilds()); + app.get('/api/index-progress', async () => orchestrator.indexBuildProgress()); + app.post('/control/dry-run', async () => startDryRun()); + app.post<{ Body: { force?: boolean } }>('/control/rebuild-ledger', async (req) => startRebuild(req.body?.force === true)); + app.get('/api/rebuild', async () => rebuildState); + + // ── Post-migration audits ────────────────────────────────────────────── + // Count-based chunk verification is the commit gate; these two answer what + // it cannot: (a) source audit — recount every window against the SOURCE + // (catches a self-consistent under-read); (b) content audit — sampled + // doc-per-doc field comparison (catches right-count-wrong-content). + const auditSourceState: RebuildProgress & { status: string } = newRebuildProgress() as never; + app.post('/control/audit-source', async () => { + if (auditSourceState.status === 'running') return { started: false, reason: 'source audit already running' }; + if (orchestrator.getStatus() === 'running') return { started: false, reason: 'main migration is running — audit after completion or while paused' }; + Object.assign(auditSourceState, newRebuildProgress(), { status: 'running', startedAt: Date.now() }); + void rebuildLedger({ config, logger, ledger, dlq, hashResolver, progress: auditSourceState, checkOnly: true }) + .then(() => { auditSourceState.status = 'completed'; auditSourceState.finishedAt = Date.now(); }) + .catch((e) => { auditSourceState.status = 'failed'; auditSourceState.error = (e as Error).message; }); + return { started: true }; + }); + app.get('/api/audit-source', async () => auditSourceState); + + const auditContentState: { status: string; result: Record | null; error: string | null } = + { status: 'not_run', result: null, error: null }; + app.post<{ Body: { samples?: number } }>('/control/audit-content', async (req) => { + if (auditContentState.status === 'running') return { started: false, reason: 'content audit already running' }; + if (orchestrator.getStatus() === 'running') return { started: false, reason: 'main migration is running — audit after completion or while paused' }; + const samples = Math.min(10_000, Math.max(50, req.body?.samples ?? 500)); + auditContentState.status = 'running'; auditContentState.result = null; auditContentState.error = null; + void orchestrator.contentAudit(samples) + .then((r) => { auditContentState.result = r as unknown as Record; auditContentState.status = 'completed'; }) + .catch((e) => { auditContentState.status = 'failed'; auditContentState.error = (e as Error).message; }); + return { started: true, samples }; + }); + app.get('/api/audit-content', async () => ({ ...auditContentState, progress: orchestrator.contentAuditProgress })); + app.get('/api/dryrun', async () => dryState); + app.get('/api/config', async () => ({ + knobs: [ + { env: 'LEDGER_CHUNK_DOCS_TARGET', value: config.ledger.chunkDocsTarget, def: 2_000_000, + hint: 'Docs per chunk — the unit of crash-redo and pod parallelism. Lower on unstable infra (cheaper redo); raise to shave per-chunk overhead.' }, + { env: 'LEDGER_MAX_CHUNK_DAYS', value: config.ledger.maxChunkDays, def: 7, + hint: 'Max time span per chunk — guards sizing against bad doc estimates.' }, + { env: 'MONGO_PAGE_SIZE', value: config.source.mongoPageSize, def: 10_000, + hint: 'Docs per read page / insert batch (held in memory whole). Lower to ≤1,000 for very large documents.' }, + { env: 'LEDGER_INSERT_INFLIGHT', value: config.ledger.insertInflight, def: 3, + hint: 'Concurrent inserts per chunk. Raise for high-latency ClickHouse; set 1 for a memory-tight one.' }, + { env: 'LEDGER_LEASE_SEC', value: config.ledger.leaseSec, def: 600, + hint: 'Chunk claim lease — how long before other pods reclaim a dead pod\u2019s chunk.' }, + { env: 'LEDGER_SOURCE_COUNT_CHECK', value: String(config.ledger.sourceCountCheck), def: 'true', + hint: 'Per-commit under-read guard: after each chunk promotes, recount its window in the SOURCE (tally-independent, ~1% overhead). Disable only if the source Mongo is severely IO-bound.' }, + { env: 'LEDGER_DLQ_PAUSE_THRESHOLD', value: config.ledger.dlqPauseThreshold, def: 1_000_000, + hint: 'Global guard the per-chunk breaker cannot provide: pause when total pending DLQ crosses this (evenly-spread failure never trips a per-chunk %). 0 disables.' }, + { env: 'LEDGER_BREAKER_PCT', value: config.ledger.breakerPct, def: 5, + hint: 'Circuit breaker: pause when more than this % of a chunk\u2019s docs fail.' }, + { env: 'MONGO_READ_PREFERENCE', value: config.source.readPreference + (config.source.readPreferenceAuto ? ' (auto)' : ''), def: 'auto', + hint: 'Auto-selected: secondaryPreferred on replica sets (offloads the primary; exact since the source is frozen), primary otherwise. Set explicitly only to override.' }, + ], + stateLocation: { + ledger: `${config.state.manifestDb}.mig_ranges (MongoDB)`, + dlq: `${config.state.manifestDb}.mig_dlq_docs (MongoDB)`, + note: 'Progress state is ~50-100 tiny documents with your MongoDB\u2019s durability. Recovery never trusts it blindly \u2014 chunks are count-verified. Changing a knob requires an engine restart (env vars).', + }, + })); + app.get('/api/pods', async () => ({ + pods: await ledger.podActivity(config.ledger.dryRun ? `${config.ledger.runId}-dry` : config.ledger.runId), + leaseSec: config.ledger.leaseSec, + })); + const { registerLedgerVizRoutes } = await import('../http/ledger-viz-route.ts'); + registerLedgerVizRoutes(app, { orchestrator, ledger, dlq, config }); + await app.listen({ port: config.service.port, host: config.service.host }); + logger.info({ port: config.service.port }, 'Ledger engine HTTP listening'); + + const runPromise = orchestrator.run(); + runPromise.catch((err) => { + // Keep the HTTP console alive: an operator with a typo'd MONGO_DB or a + // missing target table needs to SEE the error, not a dead process. + logger.fatal({ err }, 'ChunkOrchestrator crashed — console stays up so the error is visible at /'); + orchestrator.markFatal((err as Error).message); + }); + wireExitOnComplete(runPromise, config.service.exitOnComplete, logger); + + let shuttingDown = false; + async function shutdown(signal: string): Promise { + if (shuttingDown) return; + shuttingDown = true; + logger.info({ signal }, 'Ledger engine shutting down'); + orchestrator.stopAfterChunk(); + await app.close().catch(() => {}); + await mongoReader.close().catch(() => {}); + await staging.close().catch(() => {}); + await pressureClient.close().catch(() => {}); + await ledger.close().catch(() => {}); + await dlq.close().catch(() => {}); + await hashResolver.close().catch(() => {}); + process.exit(0); + } + process.on('SIGTERM', () => shutdown('SIGTERM')); + process.on('SIGINT', () => shutdown('SIGINT')); +} diff --git a/src/runtime/ledger-rebuild.ts b/src/runtime/ledger-rebuild.ts new file mode 100644 index 0000000..841b17f --- /dev/null +++ b/src/runtime/ledger-rebuild.ts @@ -0,0 +1,272 @@ +/** + * Ledger rebuild: regenerate `mig_ranges` from the data itself when the + * progress state in MongoDB is lost or corrupted. + * + * How it works — the source is frozen (post-cutover), so the chunk grid can + * be regenerated deterministically from Mongo's cd span, and each window's + * migration status is decided by comparing exact counts: + * + * Mongo docs in [lower_cd, upper_cd) vs live ClickHouse rows in the + * same window, scoped to this + * collection's (a, e) + * + * equal → done (window fully migrated) + * ClickHouse = 0 → pending (window not migrated yet) + * anything else → failed (partial — Retry failed chunks purges the + * window and redoes it cleanly) + * + * Live ingestion writing NEW data into the same table does not interfere: + * new rows carry cd >= cutover time, while every regenerated window ends at + * the frozen source's max cd — the windows cannot contain post-cutover rows. + * + * The one exception is the tool's own null-cd sweep: those rows carry + * ts-derived cd values that DO land inside regular windows. Their Mongo ids + * are known ({cd: null} in the frozen source), so the rebuild fetches their + * live cd values and subtracts them per window before comparing. + */ + +import type { Logger } from 'pino'; +import { MongoClient } from 'mongodb'; +import type { Config } from '../config/schema.ts'; +import type { HashResolver } from '../transform/hash-resolver.ts'; +import { LedgerStore, type ChunkDoc } from '../state/ledger-store.ts'; +import type { DlqStore } from '../state/dlq-store.ts'; +import { StagingManager } from '../target/staging-manager.ts'; +import { discoverCollections } from '../source/discover-collections.ts'; +import { computeChunkBounds } from './chunk-orchestrator.ts'; +import { chScopeOf } from '../transform/hash-resolver.ts'; +import { toEpochMillis, clampDateTime64 } from '../transform/validators.ts'; + +export interface RebuildCollectionSummary { + collection: string; + scoped: boolean; + chunks: number; + done: number; + pending: number; + failed: number; + mongoDocs: number; + liveRows: number; + nullCdDocs: number; + nullCdSwept: number; +} + +export interface RebuildProgress { + status: 'not_run' | 'running' | 'completed' | 'failed'; + phase: string; + collectionsDone: number; + collectionsTotal: number; + summary: RebuildCollectionSummary[]; + /** checkOnly audits: windows where source count != live count */ + mismatchedWindows: Array<{ collection: string; lowerCd: string; upperCd: string; source: number; live: number }>; + error: string | null; + startedAt: number | null; + finishedAt: number | null; +} + +export function newRebuildProgress(): RebuildProgress { + return { + status: 'not_run', phase: '', collectionsDone: 0, collectionsTotal: 0, + summary: [], mismatchedWindows: [], error: null, startedAt: null, finishedAt: null, + }; +} + +/** Cap on null-cd outlier ids held in memory per collection. */ +const MAX_NULLCD_IDS = 1_000_000; + +export async function rebuildLedger(opts: { + config: Config; + logger: Logger; + ledger: LedgerStore; + dlq: DlqStore; + hashResolver: HashResolver; + progress: RebuildProgress; + /** + * Audit mode: recount every window (source Mongo vs scoped live ClickHouse) + * and REPORT mismatches without touching the ledger. This is the defense + * against the one silent-loss class count-based chunk verification cannot + * see: a reader under-read whose tally is self-consistent — the SOURCE is + * the truth, not the tally. + */ + checkOnly?: boolean; +}): Promise { + const { config, ledger, dlq, hashResolver, progress, checkOnly = false } = opts; + const logger = opts.logger.child({ component: 'LedgerRebuild' }); + const runId = config.ledger.runId; + + // Own connections — never disturbs the main orchestrator's bindings. + const mongo = new MongoClient(config.source.uri); + const staging = new StagingManager( + { + url: config.target.url, database: config.target.db, table: config.target.table, + username: config.target.username, password: config.target.password, + queryTimeoutMs: config.target.queryTimeoutMs, + }, + logger, + ); + + try { + await mongo.connect(); + await staging.connect(); + const db = mongo.db(config.source.db); + + progress.phase = 'discovering collections'; + let collections = await discoverCollections(db, config.source.collectionPrefix, logger); + const skipEventNames = new Set(['[CLY]_apm_device', '[CLY]_apm_network']); + collections = collections.filter((name) => { + const defaults = hashResolver.resolveCollectionName(name, config.source.collectionPrefix); + return !(defaults && skipEventNames.has(defaults.e)); + }); + progress.collectionsTotal = collections.length; + + const now = new Date(); + const allDocs: ChunkDoc[] = []; + + for (const collection of collections) { + progress.phase = `analyzing ${collection}`; + const coll = db.collection(collection); + const defaults = hashResolver.resolveCollectionName(collection, config.source.collectionPrefix); + const scope = defaults ? chScopeOf(defaults) : null; + + // cd span of the frozen source (same probes the engine uses) + const [lowDoc] = await coll.find({ cd: { $type: 'date' } }).sort({ cd: 1 }).limit(1) + .project({ cd: 1 }).toArray(); + const [highDoc] = await coll.find({ cd: { $type: 'date' } }).sort({ cd: -1 }).limit(1) + .project({ cd: 1 }).toArray(); + + const summary: RebuildCollectionSummary = { + collection, scoped: !!scope, chunks: 0, done: 0, pending: 0, failed: 0, + mongoDocs: 0, liveRows: 0, nullCdDocs: 0, nullCdSwept: 0, + }; + + // Null-cd outliers: fetch ids + live cd values so sweep rows can be + // subtracted from the regular windows their ts-derived cd landed in. + const nullCdIds: string[] = []; + let derivedLo = Infinity, derivedHi = -Infinity; + const idCursor = coll.find({ cd: null }, { projection: { _id: 1, ts: 1 } }).batchSize(10_000); + for await (const doc of idCursor) { + nullCdIds.push(String(doc._id)); + const tsMs = toEpochMillis(doc.ts); + if (tsMs !== null && tsMs > 0) { + const d = clampDateTime64(tsMs); // the sweep's derived cd + if (d < derivedLo) derivedLo = d; + if (d > derivedHi) derivedHi = d; + } + if (nullCdIds.length >= MAX_NULLCD_IDS) { + throw new Error(`${collection}: more than ${MAX_NULLCD_IDS.toLocaleString('en-US')} null-cd documents — not outliers; rebuild does not support this shape`); + } + } + summary.nullCdDocs = nullCdIds.length; + const liveNullCd = nullCdIds.length > 0 + ? await staging.fetchLiveCdByIds(nullCdIds, derivedLo <= derivedHi ? { loMs: derivedLo, hiMs: derivedHi } : undefined) + : new Map(); + summary.nullCdSwept = liveNullCd.size; + const sweptCds = [...liveNullCd.values()].sort((a, b) => a - b); + + let bounds: Array<{ lowerCd: number; upperCd: number }> = []; + if (lowDoc && highDoc) { + const lowerCd = (lowDoc.cd as Date).getTime(); + const upperCd = (highDoc.cd as Date).getTime(); + const estimated = await coll.estimatedDocumentCount(); + bounds = computeChunkBounds(lowerCd, upperCd, estimated, config.ledger.chunkDocsTarget, config.ledger.maxChunkDays); + } + + let idx = 0; + for (const b of bounds) { + progress.phase = `counting ${collection} chunk ${idx + 1}/${bounds.length}`; + const mongoCount = await coll.countDocuments({ + cd: { $gte: new Date(b.lowerCd), $lt: new Date(b.upperCd) }, + }); + const liveRaw = await staging.countLiveInCdRange(b.lowerCd, b.upperCd, scope); + // Subtract this collection's sweep rows whose derived cd fell in-window + let lo = 0, hi = sweptCds.length; + while (lo < hi) { const m = (lo + hi) >> 1; if (sweptCds[m] < b.lowerCd) lo = m + 1; else hi = m; } + let sweptIn = 0; + for (let i = lo; i < sweptCds.length && sweptCds[i] < b.upperCd; i++) sweptIn++; + const live = liveRaw - sweptIn; + + // Docs in this window that are KNOWN unmigrated (pending/waived DLQ) + // legitimately explain source > live — without this, a window whose + // only shortfall is its own DLQ'd docs gets flagged/redone forever. + const unresolved = await dlq.countUnresolvedInWindow(runId, collection, b.lowerCd, b.upperCd); + + const status: ChunkDoc['status'] = + live + unresolved === mongoCount ? 'done' : live === 0 ? 'pending' : 'failed'; + // pending (live=0) is 'not migrated yet', not a disagreement + if (checkOnly && live + unresolved !== mongoCount && live !== 0 && progress.mismatchedWindows.length < 200) { + progress.mismatchedWindows.push({ + collection, lowerCd: new Date(b.lowerCd).toISOString(), upperCd: new Date(b.upperCd).toISOString(), + source: mongoCount, live, + }); + } + summary.mongoDocs += mongoCount; + summary.liveRows += live; + summary[status === 'done' ? 'done' : status === 'pending' ? 'pending' : 'failed']++; + + allDocs.push({ + _id: `${runId}:${collection}:${idx}`, + run_id: runId, collection, + scope_a: scope?.a ?? null, scope_e: scope?.e ?? null, scope_n: scope?.n ?? null, + idx, lower_cd: b.lowerCd, upper_cd: b.upperCd, + status, pod_id: null, lease_until: null, staging_table: null, + docs_read: status === 'done' ? mongoCount : 0, + docs_skipped: 0, + rows_expected: status === 'done' ? mongoCount : 0, + partitions: [], attached: [], + attach_method: null, attempts: 0, + last_error: status === 'failed' ? `rebuilt from data: live=${live} mongo=${mongoCount} — retry purges and redoes this window` : null, + transform_version: config.transform.version, + updated_at: now, + }); + idx++; + } + + // Sentinel sweep chunk for the null-cd outliers + if (nullCdIds.length > 0) { + const swept = liveNullCd.size; + const status: ChunkDoc['status'] = + swept === nullCdIds.length ? 'done' : swept === 0 ? 'pending' : 'failed'; + summary[status === 'done' ? 'done' : status === 'pending' ? 'pending' : 'failed']++; + allDocs.push({ + _id: `${runId}:${collection}:${idx}`, + run_id: runId, collection, + scope_a: scope?.a ?? null, scope_e: scope?.e ?? null, scope_n: scope?.n ?? null, + idx, lower_cd: -1, upper_cd: 0, + status, pod_id: null, lease_until: null, staging_table: null, + docs_read: status === 'done' ? nullCdIds.length : 0, + docs_skipped: 0, + rows_expected: status === 'done' ? nullCdIds.length : 0, + partitions: [], attached: [], + attach_method: null, attempts: 0, + last_error: status === 'failed' ? `rebuilt from data: swept=${swept} of ${nullCdIds.length} null-cd docs` : null, + transform_version: config.transform.version, + updated_at: now, + }); + idx++; + } + + summary.chunks = idx; + progress.summary.push(summary); + progress.collectionsDone++; + logger.info(summary, 'Collection analyzed'); + } + + if (checkOnly) { + progress.phase = 'done'; + logger.info( + { windows: allDocs.length, mismatches: progress.mismatchedWindows.length }, + 'Source audit complete — ledger untouched', + ); + } else { + progress.phase = 'writing ledger'; + await ledger.replaceAllForRun(runId, allDocs); + progress.phase = 'done'; + logger.info( + { chunks: allDocs.length, collections: collections.length }, + 'Ledger rebuilt from data — restart or resume the engine to continue the run', + ); + } + } finally { + await mongo.close().catch(() => {}); + await staging.close().catch(() => {}); + } +} diff --git a/src/runtime/process-metrics.ts b/src/runtime/process-metrics.ts deleted file mode 100644 index b8a7c89..0000000 --- a/src/runtime/process-metrics.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { monitorEventLoopDelay, type IntervalHistogram } from "node:perf_hooks"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface ProcessMetricsSnapshot { - rssBytes: number; - heapTotalBytes: number; - heapUsedBytes: number; - externalBytes: number; - arrayBuffersBytes: number; - eventLoopLagMs_p95_1m: number; - cpuUserSec: number; - cpuSystemSec: number; -} - -// --------------------------------------------------------------------------- -// ProcessMetricsCollector -// --------------------------------------------------------------------------- - -/** - * Collects process-level metrics: - * - * - Memory usage via `process.memoryUsage()` - * - CPU time via `process.cpuUsage()` - * - Event loop lag via `perf_hooks.monitorEventLoopDelay()` - * - * The event loop lag histogram is sampled at 20 ms resolution and the - * collector resets it every 60 seconds (the "1m" window). The p95 value - * returned in a snapshot is always from the most recently completed window - * so that callers see a stable, non-zero value rather than a histogram - * that was just reset. - */ -export class ProcessMetricsCollector { - private histogram: IntervalHistogram | null = null; - private lastP95Ns = 0; - private running = false; - private resetTimer: ReturnType | null = null; - - /** Resolution for the event loop delay histogram (20 ms). */ - private static readonly ELD_RESOLUTION_MS = 20; - /** Window duration before the histogram resets (60 s). */ - private static readonly ELD_WINDOW_MS = 60_000; - - constructor() { - // Histogram is created lazily in start() so the collector can be - // instantiated before the event loop is fully warmed up. - } - - // ----------------------------------------------------------------------- - // Lifecycle - // ----------------------------------------------------------------------- - - /** Start collecting event loop lag samples. */ - start(): void { - if (this.running) return; - - this.histogram = monitorEventLoopDelay({ - resolution: ProcessMetricsCollector.ELD_RESOLUTION_MS, - }); - this.histogram.enable(); - this.running = true; - - // Rotate the histogram every 60 s to keep a sliding 1 m window. - this.resetTimer = setInterval(() => { - if (this.histogram) { - // Capture p95 before resetting so snapshot() has a value. - this.lastP95Ns = this.histogram.percentile(95); - this.histogram.reset(); - } - }, ProcessMetricsCollector.ELD_WINDOW_MS); - - // Allow the process to exit even if the timer is still alive. - if (this.resetTimer && typeof this.resetTimer.unref === "function") { - this.resetTimer.unref(); - } - } - - /** Stop collecting event loop lag samples. */ - stop(): void { - if (!this.running) return; - - if (this.histogram) { - // Capture final p95 before disabling. - this.lastP95Ns = this.histogram.percentile(95); - this.histogram.disable(); - this.histogram = null; - } - - if (this.resetTimer !== null) { - clearInterval(this.resetTimer); - this.resetTimer = null; - } - - this.running = false; - } - - // ----------------------------------------------------------------------- - // Snapshot - // ----------------------------------------------------------------------- - - /** - * Return a point-in-time snapshot of process metrics. - * - * Memory and CPU values are instantaneous; the event loop lag p95 is - * from the most recently completed 60 s window (or the current window - * if no rotation has happened yet). - */ - snapshot(): ProcessMetricsSnapshot { - const mem = process.memoryUsage(); - const cpu = process.cpuUsage(); - - // p95 event loop lag: prefer the live histogram if it has samples, - // fall back to the last captured value from the most recent reset. - let p95Ns = this.lastP95Ns; - if (this.histogram) { - const liveP95 = this.histogram.percentile(95); - if (liveP95 > 0) { - p95Ns = liveP95; - } - } - - return { - rssBytes: mem.rss, - heapTotalBytes: mem.heapTotal, - heapUsedBytes: mem.heapUsed, - externalBytes: mem.external, - arrayBuffersBytes: mem.arrayBuffers ?? 0, - // Convert nanoseconds to milliseconds - eventLoopLagMs_p95_1m: p95Ns / 1e6, - // Convert microseconds to seconds - cpuUserSec: cpu.user / 1e6, - cpuSystemSec: cpu.system / 1e6, - }; - } -} diff --git a/src/runtime/range-coordinator.ts b/src/runtime/range-coordinator.ts deleted file mode 100644 index fb2b01a..0000000 --- a/src/runtime/range-coordinator.ts +++ /dev/null @@ -1,769 +0,0 @@ -import type { Redis } from "ioredis"; -import type { Logger } from "pino"; -import type { MongoReader } from "../source/mongo-reader.ts"; -import type { ManifestStore } from "../state/manifest-store.ts"; -import { RedisHotState } from "../state/redis-hot-state.ts"; -import type { ClickHouseWriter } from "../target/clickhouse-writer.ts"; -import type { ClickHousePressure, BackpressureConfig } from "../target/clickhouse-pressure.ts"; -import type { GcController } from "./gc-controller.ts"; -import type { RetryPolicy } from "./retry-policy.ts"; -import { BatchRunner } from "./batch-runner.ts"; -import type { SkipReason } from "../transform/skip-reasons.ts"; -import type { CollectionDefaults } from "../transform/hash-resolver.ts"; -import type { AsyncBatchWriter } from "../state/async-batch-writer.ts"; -import { serializeCursor, deserializeCursor } from "../types/cursor.ts"; -import { randomUUID } from "node:crypto"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface RangeEntry { - idx: number; - startCd: number; // epoch ms, inclusive - endCd: number; // epoch ms, inclusive - status: "pending" | "processing" | "done" | "failed"; - podId: string | null; - claimedAt: number | null; // epoch seconds -} - -export interface RangeCoordinatorConfig { - collectionName: string; - sourceNs: string; - targetTable: string; - transformVersion: string; - rangeCount: number; - rangeLeaseTtlSec: number; - batchRowsTarget: number; - mongoPageSize: number; - backpressure: BackpressureConfig; - useDedupToken: boolean; - database: string; - table: string; - snapshotInterval: number; - collectionDefaults?: CollectionDefaults; - podId: string; - redisKeyPrefix: string; -} - -export interface RangeCoordinatorDeps { - redis: Redis; - manifestStore: ManifestStore; - redisState: RedisHotState; - asyncBatchWriter?: AsyncBatchWriter; - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - chPressure: ClickHousePressure; - gcController: GcController; - retryPolicy: RetryPolicy; - logger: Logger; - config: RangeCoordinatorConfig; -} - -export interface RangeResult { - totalRanges: number; - completedRanges: number; - failedRanges: number; - totalDocsRead: number; - totalRowsInserted: number; - runId: string; -} - -// --------------------------------------------------------------------------- -// Lua: Claim next pending range (also reclaims stale ranges from dead pods) -// -// This runs atomically on the Redis server via ioredis .eval(). -// It is NOT child_process.exec — it is a Redis server-side Lua script. -// --------------------------------------------------------------------------- - -const CLAIM_RANGE_LUA = ` -local hash = KEYS[1] -local podId = ARGV[1] -local nowSec = tonumber(ARGV[2]) -local leaseTtlSec = tonumber(ARGV[3]) -local podKeyPrefix = ARGV[4] - -local fields = redis.call('HGETALL', hash) - --- First pass: reclaim stale ranges from dead pods -for i = 1, #fields, 2 do - local data = cjson.decode(fields[i+1]) - if data.status == 'processing' and data.claimedAt then - local elapsed = nowSec - tonumber(data.claimedAt) - if elapsed > leaseTtlSec then - local otherPodKey = podKeyPrefix .. tostring(data.podId) - local alive = redis.call('EXISTS', otherPodKey) - if alive == 0 then - data.status = 'pending' - data.podId = cjson.null - data.claimedAt = cjson.null - redis.call('HSET', hash, fields[i], cjson.encode(data)) - end - end - end -end - --- Second pass: claim first pending range -fields = redis.call('HGETALL', hash) -for i = 1, #fields, 2 do - local data = cjson.decode(fields[i+1]) - if data.status == 'pending' then - data.status = 'processing' - data.podId = podId - data.claimedAt = nowSec - redis.call('HSET', hash, fields[i], cjson.encode(data)) - return cjson.encode(data) - end -end -return nil -`; - -// --------------------------------------------------------------------------- -// Lua: Atomically mark a range as "done" or "failed". -// -// Only transitions if the range is still "processing" and owned by -// the calling pod. Prevents the race where CLAIM_RANGE_LUA resets a -// range to "pending" between a non-atomic HGET and HSET. -// --------------------------------------------------------------------------- - -const MARK_RANGE_TERMINAL_LUA = ` -local raw = redis.call('HGET', KEYS[1], ARGV[1]) -if not raw then return 0 end -local data = cjson.decode(raw) -if data.status ~= 'processing' then return 0 end -if tostring(data.podId) ~= ARGV[3] then return 0 end -data.status = ARGV[2] -redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(data)) -return 1 -`; - -// --------------------------------------------------------------------------- -// RangeCoordinator -// --------------------------------------------------------------------------- - -const BATCH_SEQ_SLOTS_PER_RANGE = 10_000; -const MAX_RANGE_RETRIES = 3; - -export class RangeCoordinator { - private readonly deps: RangeCoordinatorDeps; - private readonly logger: Logger; - private readonly rangesKey: string; - private readonly initKey: string; - private readonly runIdKey: string; - private readonly metaKey: string; - private stopping = false; - activeBatchRunner: BatchRunner | null = null; - private readonly rangeRetryCounts = new Map(); - - /** Accumulated docs read across all completed ranges (updated live). */ - totalDocsRead = 0; - /** Accumulated rows inserted across all completed ranges (updated live). */ - totalRowsInserted = 0; - /** Accumulated docs skipped across all completed ranges. */ - totalDocsSkipped = 0; - /** Accumulated skip reasons across all completed ranges. */ - skipsByReason: Record = {} as Record; - /** Timestamp when range processing started. */ - startTime = 0; - - constructor(deps: RangeCoordinatorDeps) { - this.deps = deps; - this.logger = deps.logger.child({ component: "RangeCoordinator", collection: deps.config.collectionName }); - const prefix = deps.config.redisKeyPrefix; - const coll = deps.config.collectionName; - this.rangesKey = `${prefix}:ranges:${coll}`; - this.initKey = `${prefix}:ranges:${coll}:init`; - this.runIdKey = `${prefix}:ranges:${coll}:runId`; - this.metaKey = `${prefix}:ranges:${coll}:meta`; - } - - async run(): Promise { - const { manifestStore, config } = this.deps; - - // 1. Initialize ranges (first pod only, via SETNX) - const runId = await this.initRanges(); - - this.logger.info({ runId, rangeCount: config.rangeCount }, "Starting range-parallel processing"); - - this.totalDocsRead = 0; - this.totalRowsInserted = 0; - this.totalDocsSkipped = 0; - this.skipsByReason = {} as Record; - this.startTime = Date.now(); - let completedRanges = 0; - let failedRanges = 0; - - // 2. Claim and process ranges (with self-healing retry for failed ranges) - for (let retryRound = 0; retryRound <= MAX_RANGE_RETRIES; retryRound++) { - if (this.stopping) break; - - if (retryRound > 0) { - this.logger.info({ retryRound }, "Starting retry round for failed ranges"); - } - - // Inner claim loop: claim and process until no pending ranges remain - while (!this.stopping) { - const range = await this.claimNextRange(); - - if (range) { - this.logger.info( - { rangeIdx: range.idx, startCd: new Date(range.startCd).toISOString(), endCd: new Date(range.endCd).toISOString(), retryRound }, - "Claimed range", - ); - - try { - const result = await this.processRange(range, runId); - this.totalDocsRead += result.docsRead; - this.totalRowsInserted += result.rowsInserted; - this.totalDocsSkipped += result.docsSkipped; - for (const [reason, count] of Object.entries(result.skipsByReason) as Array<[SkipReason, number]>) { - this.skipsByReason[reason] = (this.skipsByReason[reason] ?? 0) + count; - } - await this.markRangeDone(range.idx); - completedRanges++; - this.logger.info({ rangeIdx: range.idx, docsRead: result.docsRead, rowsInserted: result.rowsInserted }, "Range completed"); - } catch (err) { - const error = err instanceof Error ? err.message : String(err); - await this.markRangeFailed(range.idx); - failedRanges++; - this.logger.error({ rangeIdx: range.idx, error, retryRound }, "Range failed"); - } - continue; - } - - // No pending range — check if any are still processing by other pods - const status = await this.getRangeStatus(); - if (status.processing > 0) { - this.logger.info({ processing: status.processing, done: status.done }, "Waiting for other pods to finish ranges"); - await new Promise(r => setTimeout(r, 10_000)); - continue; - } - - // All ranges terminal (done or failed) - break; - } - - // Check if any failed ranges can be retried - if (this.stopping) break; - if (retryRound >= MAX_RANGE_RETRIES) break; - - const resetCount = await this.resetFailedRanges(); - if (resetCount === 0) break; // no failed ranges or all exhausted retries - - this.logger.info( - { resetCount, retryRound: retryRound + 1, maxRetries: MAX_RANGE_RETRIES }, - "Re-queued failed ranges for retry", - ); - } - - // 2.5. Null-cd sweep after all cd-ranges complete - const preNullCdStatus = await this.getRangeStatus(); - if (preNullCdStatus.pending === 0 - && preNullCdStatus.processing === 0 - && preNullCdStatus.failed === 0 - && !this.stopping) { - - const sweepKey = `${this.rangesKey}:null_cd_sweep`; - const sweepDoneKey = `${sweepKey}:done`; - - let sweepHandled = false; - while (!this.stopping && !sweepHandled) { - const alreadyDone = await this.deps.redis.get(sweepDoneKey); - if (alreadyDone) { sweepHandled = true; break; } - - const acquired = await this.deps.redis.set( - sweepKey, config.podId, "EX", 3600, "NX", - ); - - if (acquired) { - sweepHandled = true; - try { - const bounds = await this.deps.mongoReader.getNullCdBounds(); - if (bounds) { - await this.deps.manifestStore.updateRunPhase( - runId, "null_cd", bounds.upper, - ); - const sweepResult = await this.runNullCdSweep(runId, bounds.upper); - this.totalDocsRead += sweepResult.docsRead; - this.totalRowsInserted += sweepResult.rowsInserted; - this.totalDocsSkipped += sweepResult.docsSkipped; - } - await this.deps.redis.set(sweepDoneKey, "1", "EX", 86400); - } catch (err) { - await this.deps.redis.del(sweepKey); - throw err; - } - } else { - const lockHolder = await this.deps.redis.get(sweepKey); - if (lockHolder) { - const podKeyPrefix = `${config.redisKeyPrefix}:pod:`; - const holderAlive = await this.deps.redis.exists( - `${podKeyPrefix}${lockHolder}`, - ); - if (!holderAlive) { - await this.deps.redis.del(sweepKey); - this.logger.info( - { stalePod: lockHolder }, - "Reclaimed stale null-cd sweep lock from dead pod", - ); - continue; - } - } - await new Promise(r => setTimeout(r, 5_000)); - } - } - } - - // 3. Mark run complete if all ranges are terminal (only one pod finalizes via SETNX) - const finalStatus = await this.getRangeStatus(); - if (finalStatus.pending === 0 && finalStatus.processing === 0) { - const finalizeKey = `${this.rangesKey}:finalized`; - const acquired = await this.deps.redis.set(finalizeKey, config.podId, "EX", 60, "NX"); - if (acquired) { - const runStatus = finalStatus.failed > 0 ? "failed" as const : "completed" as const; - await manifestStore.updateRunStatus(runId, runStatus); - this.logger.info({ runId, runStatus, failedRanges: finalStatus.failed }, "Run finalized by this pod"); - } - } - - return { - totalRanges: config.rangeCount, - completedRanges, - failedRanges, - totalDocsRead: this.totalDocsRead, - totalRowsInserted: this.totalRowsInserted, - runId, - }; - } - - stop(): void { - this.stopping = true; - this.activeBatchRunner?.stopAfterBatch(); - } - - pause(): void { - this.activeBatchRunner?.pause(); - } - - resume(): void { - this.activeBatchRunner?.resume(); - } - - async getRangeStatus(): Promise<{ pending: number; processing: number; done: number; failed: number }> { - const all = await this.deps.redis.hgetall(this.rangesKey); - let pending = 0, processing = 0, done = 0, failed = 0; - for (const val of Object.values(all)) { - const entry = JSON.parse(val) as RangeEntry; - if (entry.status === "pending") pending++; - else if (entry.status === "processing") processing++; - else if (entry.status === "done") done++; - else if (entry.status === "failed") failed++; - } - return { pending, processing, done, failed }; - } - - // ----------------------------------------------------------------------- - // Private - // ----------------------------------------------------------------------- - - /** - * Reset eligible failed ranges back to "pending" for retry. - * Tracks per-range retry counts to cap retries at MAX_RANGE_RETRIES. - * Returns the number of ranges reset. - */ - private async resetFailedRanges(): Promise { - const all = await this.deps.redis.hgetall(this.rangesKey); - let resetCount = 0; - - for (const [key, val] of Object.entries(all)) { - const entry = JSON.parse(val) as RangeEntry; - if (entry.status !== "failed") continue; - - const retries = this.rangeRetryCounts.get(entry.idx) ?? 0; - if (retries >= MAX_RANGE_RETRIES) continue; - - entry.status = "pending"; - entry.podId = null as any; - entry.claimedAt = null as any; - await this.deps.redis.hset(this.rangesKey, key, JSON.stringify(entry)); - this.rangeRetryCounts.set(entry.idx, retries + 1); - resetCount++; - - this.logger.info( - { rangeIdx: entry.idx, retryCount: retries + 1, maxRetries: MAX_RANGE_RETRIES }, - "Reset failed range for retry", - ); - } - - return resetCount; - } - - private async initRanges(): Promise { - const { redis, mongoReader, manifestStore, config } = this.deps; - - // Check if ranges already initialized by another pod - const existingRunId = await redis.get(this.runIdKey); - if (existingRunId) { - this.logger.info({ runId: existingRunId }, "Ranges already initialized by another pod"); - return existingRunId; - } - - // Try to become the coordinator via SETNX - const acquired = await redis.set(this.initKey, config.podId, "EX", 60, "NX"); - if (!acquired) { - // Check if the lock holder is still alive; reclaim if dead - const lockHolder = await redis.get(this.initKey); - if (lockHolder) { - const podKeyPrefix = `${config.redisKeyPrefix}:pod:`; - const holderAlive = await redis.exists(`${podKeyPrefix}${lockHolder}`); - if (!holderAlive) { - await redis.del(this.initKey); - this.logger.info( - { stalePod: lockHolder }, - "Reclaimed stale range init lock from dead pod", - ); - // Retry acquisition after reclaim - return this.initRanges(); - } - } - - this.logger.info("Another pod is initializing ranges, waiting..."); - for (let i = 0; i < 30; i++) { - await new Promise(r => setTimeout(r, 2_000)); - const rid = await redis.get(this.runIdKey); - if (rid) return rid; - - // Re-check pod liveness periodically to avoid waiting for a dead pod - const currentHolder = await redis.get(this.initKey); - if (currentHolder) { - const podKeyPrefix = `${config.redisKeyPrefix}:pod:`; - const stillAlive = await redis.exists(`${podKeyPrefix}${currentHolder}`); - if (!stillAlive) { - await redis.del(this.initKey); - this.logger.info( - { stalePod: currentHolder }, - "Reclaimed stale range init lock from dead pod during wait", - ); - return this.initRanges(); - } - } else { - // initKey expired (TTL), retry - return this.initRanges(); - } - } - throw new Error("Timed out waiting for range initialization"); - } - - // We are the coordinator — wrapped in try/catch to release initKey on failure - try { - const lowerBound = await mongoReader.getLowerBound(); - const upperBound = await mongoReader.getUpperBound(); - if (!lowerBound || !upperBound) { - const hasNullCd = await mongoReader.hasNullCdDocuments(); - if (!hasNullCd) { - throw new Error("Collection is empty, cannot initialize ranges"); - } - const runId = randomUUID(); - const now = new Date().toISOString(); - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: config.sourceNs, - target_table: config.targetTable, - upper_bound_cursor: serializeCursor({ cd: 0, id: "\uffff".repeat(24) }), - transform_version: config.transformVersion, - created_at: now, - }); - await redis.set(this.runIdKey, runId); - await redis.del(this.initKey); - this.logger.info({ runId }, "All-null collection — skipping cd-ranges, will sweep via null-cd phase"); - return runId; - } - - const minCd = lowerBound.cd; - const maxCd = upperBound.cd; - const rangeCount = config.rangeCount; - const spanMs = maxCd - minCd; - const stepMs = Math.max(1, Math.ceil(spanMs / rangeCount)); - - // Create shared run in ManifestStore - const runId = randomUUID(); - const now = new Date().toISOString(); - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: config.sourceNs, - target_table: config.targetTable, - upper_bound_cursor: serializeCursor(upperBound), - transform_version: config.transformVersion, - created_at: now, - }); - - // Create all ranges in Redis (atomic pipeline) - const pipeline = redis.multi(); - for (let i = 0; i < rangeCount; i++) { - const startCd = minCd + (i * stepMs); - const endCd = i === rangeCount - 1 ? maxCd : minCd + ((i + 1) * stepMs); - const entry: RangeEntry = { - idx: i, - startCd, - endCd, - status: "pending", - podId: null, - claimedAt: null, - }; - pipeline.hset(this.rangesKey, String(i), JSON.stringify(entry)); - } - pipeline.set(this.runIdKey, runId); - pipeline.set(this.metaKey, JSON.stringify({ minCd, maxCd, rangeCount, createdAt: now })); - pipeline.del(this.initKey); - await pipeline.exec(); - - this.logger.info( - { runId, minCd: new Date(minCd).toISOString(), maxCd: new Date(maxCd).toISOString(), rangeCount, stepMs }, - "Ranges initialized", - ); - return runId; - } catch (err) { - // Release initKey so other pods (or retries) aren't blocked - await redis.del(this.initKey).catch(() => {}); - throw err; - } - } - - private async claimNextRange(): Promise { - const { redis, config } = this.deps; - const nowSec = String(Math.floor(Date.now() / 1000)); - const podKeyPrefix = `${config.redisKeyPrefix}:pod:`; - - // ioredis .eval() runs a Lua script atomically on the Redis server - const result = await redis.eval( - CLAIM_RANGE_LUA, - 1, - this.rangesKey, - config.podId, - nowSec, - String(config.rangeLeaseTtlSec), - podKeyPrefix, - ) as string | null; - - if (!result) return null; - return JSON.parse(result) as RangeEntry; - } - - /** - * Atomically mark a range as "done" or "failed" via Lua script. - * Only transitions if still "processing" and owned by this pod. - * Uses ioredis .eval() to run a Lua script on the Redis server (NOT JS eval). - */ - private async markRangeTerminal(idx: number, status: "done" | "failed"): Promise { - // ioredis .eval() sends a Lua script to Redis for atomic server-side execution - await this.deps.redis.eval( - MARK_RANGE_TERMINAL_LUA, - 1, - this.rangesKey, - String(idx), - status, - this.deps.config.podId, - ); - } - - private async markRangeDone(idx: number): Promise { - await this.markRangeTerminal(idx, "done"); - } - - private async markRangeFailed(idx: number): Promise { - await this.markRangeTerminal(idx, "failed"); - } - - private async processRange(range: RangeEntry, runId: string): Promise<{ docsRead: number; rowsInserted: number; docsSkipped: number; skipsByReason: Record }> { - const { manifestStore, asyncBatchWriter, mongoReader, chWriter, chPressure, gcController, retryPolicy, config } = this.deps; - - // Cursor bounds: [startCd, endCd) for non-final ranges, [startCd, maxCd] for final - const startCursorStr = serializeCursor({ cd: range.startCd, id: "" }); - const isFinalRange = range.idx === config.rangeCount - 1; - const upperBoundStr = isFinalRange - ? serializeCursor({ cd: range.endCd, id: "\uffff".repeat(24) }) - : serializeCursor({ cd: range.endCd, id: "" }); - - const batchSeqOffset = range.idx * BATCH_SEQ_SLOTS_PER_RANGE; - const rangeStartedAt = Date.now(); - - // Per-range RedisHotState (shares connection, isolated key prefix) - const rangeRedisState = RedisHotState.fromExistingConnection( - this.deps.redis, - `${config.redisKeyPrefix}:${config.collectionName}:range${range.idx}`, - ); - - // Write initial range live stats - await this.deps.redisState.setRangeLiveStats(config.collectionName, range.idx, { - idx: range.idx, - status: "processing", - podId: config.podId, - docsRead: 0, - rowsInserted: 0, - batchesDone: 0, - docsPerSecond: 0, - startedAt: rangeStartedAt, - }).catch(() => {}); - - const batchRunner = new BatchRunner({ - manifestStore, - redisState: rangeRedisState, - globalRedisState: this.deps.redisState, - asyncBatchWriter, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger: this.deps.logger, - config: { - runId, - transformVersion: config.transformVersion, - sourceNs: config.sourceNs, - targetTable: config.targetTable, - upperBoundId: upperBoundStr, - batchRowsTarget: config.batchRowsTarget, - mongoPageSize: config.mongoPageSize, - backpressure: config.backpressure, - useDedupToken: config.useDedupToken, - database: config.database, - table: config.table, - snapshotInterval: config.snapshotInterval, - collectionDefaults: config.collectionDefaults, - batchSeqOffset, - collectionName: config.collectionName, - podId: config.podId, - rangeIdx: range.idx, - batchSeqMax: batchSeqOffset + BATCH_SEQ_SLOTS_PER_RANGE, - }, - }); - - this.activeBatchRunner = batchRunner; - try { - await batchRunner.run(startCursorStr); - } finally { - this.activeBatchRunner = null; - } - - const stats = batchRunner.getStats(); - - // If the BatchRunner ended in a failed state (batch failed after all retries), - // propagate as an error so the range is marked failed and can be retried. - if (stats.status === "failed") { - throw new Error( - `Range ${range.idx} BatchRunner ended with status "failed" ` + - `(${stats.batchesFailed} batch(es) failed after retries)`, - ); - } - - // Layer 3: Completion guard — detect silent data skips - const rangeNewBatches = stats.batchSeq - batchSeqOffset; - if (rangeNewBatches === 0 && stats.totalDocsRead === 0) { - // Range produced zero work — probe to verify it's genuinely empty - const probe = await mongoReader.readPage( - deserializeCursor(startCursorStr), - deserializeCursor(upperBoundStr), - 1, - ); - if (probe.docs.length > 0) { - throw new Error( - `Range ${range.idx} [${new Date(range.startCd).toISOString()} → ` + - `${new Date(range.endCd).toISOString()}] completed with 0 docs but has data — ` + - `possible cursor bleed or resume bug`, - ); - } - this.deps.logger.info({ rangeIdx: range.idx }, "Range is genuinely empty, skipping"); - } - - // Update range live stats on completion - const elapsedSec = (Date.now() - rangeStartedAt) / 1000; - await this.deps.redisState.setRangeLiveStats(config.collectionName, range.idx, { - idx: range.idx, - status: "done", - podId: config.podId, - docsRead: stats.totalDocsRead, - rowsInserted: stats.totalRowsInserted, - batchesDone: (stats.batchSeq - batchSeqOffset) - stats.batchesFailed, - docsPerSecond: elapsedSec > 0 ? stats.totalDocsRead / elapsedSec : 0, - startedAt: rangeStartedAt, - }).catch(() => {}); - - return { - docsRead: stats.totalDocsRead, - rowsInserted: stats.totalRowsInserted, - docsSkipped: stats.totalDocsSkipped, - skipsByReason: stats.skipsByReason ?? {}, - }; - } - - private async runNullCdSweep( - runId: string, - nullCdUpperBound: string, - ): Promise<{ docsRead: number; rowsInserted: number; docsSkipped: number }> { - const { config } = this.deps; - const upperBoundStr = serializeCursor({ cd: 0, id: nullCdUpperBound }); - const batchSeqOffset = config.rangeCount * BATCH_SEQ_SLOTS_PER_RANGE; - - const sweepRedisState = RedisHotState.fromExistingConnection( - this.deps.redis, - `${config.redisKeyPrefix}:${config.collectionName}:null_cd_sweep`, - ); - - const batchRunner = new BatchRunner({ - manifestStore: this.deps.manifestStore, - redisState: sweepRedisState, - globalRedisState: this.deps.redisState, - asyncBatchWriter: this.deps.asyncBatchWriter, - mongoReader: this.deps.mongoReader, - chWriter: this.deps.chWriter, - chPressure: this.deps.chPressure, - gcController: this.deps.gcController, - retryPolicy: this.deps.retryPolicy, - logger: this.deps.logger, - config: { - runId, - nullCdMode: true, - nullCdUpperBound, - upperBoundId: upperBoundStr, - batchSeqOffset, - batchSeqMax: batchSeqOffset + BATCH_SEQ_SLOTS_PER_RANGE, - transformVersion: config.transformVersion, - sourceNs: config.sourceNs, - targetTable: config.targetTable, - batchRowsTarget: config.batchRowsTarget, - mongoPageSize: config.mongoPageSize, - backpressure: config.backpressure, - useDedupToken: config.useDedupToken, - database: config.database, - table: config.table, - snapshotInterval: config.snapshotInterval, - collectionDefaults: config.collectionDefaults, - collectionName: config.collectionName, - podId: config.podId, - rangeIdx: config.rangeCount, - }, - }); - - this.activeBatchRunner = batchRunner; - try { - await batchRunner.run(); - } finally { - this.activeBatchRunner = null; - } - - const stats = batchRunner.getStats(); - if (stats.status === "failed") { - throw new Error( - `Null-cd sweep failed (${stats.batchesFailed} batch(es) failed after retries)`, - ); - } - - return { - docsRead: stats.totalDocsRead, - rowsInserted: stats.totalRowsInserted, - docsSkipped: stats.totalDocsSkipped, - }; - } -} diff --git a/src/runtime/resolve-run.ts b/src/runtime/resolve-run.ts deleted file mode 100644 index 48b86fd..0000000 --- a/src/runtime/resolve-run.ts +++ /dev/null @@ -1,159 +0,0 @@ -import type { Logger } from 'pino'; -import type { ManifestStore } from '../state/manifest-store.ts'; -import type { RedisHotState } from '../state/redis-hot-state.ts'; -import type { MongoReader } from '../source/mongo-reader.ts'; -import { serializeCursor } from '../types/cursor.ts'; -import { randomUUID } from 'node:crypto'; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface ResolvedRun { - runId: string; - upperBoundId: string; - /** True if the source collection was empty — caller should skip migration. */ - isEmpty?: boolean; -} - -export interface ResolveRunOpts { - rerunMode: 'resume' | 'clone-run' | 'new-run'; - manifestStore: ManifestStore; - redisState: RedisHotState; - mongoReader: MongoReader; - sourceNs: string; - targetTable: string; - transformVersion: string; - logger: Logger; -} - -// --------------------------------------------------------------------------- -// resolveRun -// --------------------------------------------------------------------------- - -export async function resolveRun(opts: ResolveRunOpts): Promise { - const { rerunMode, manifestStore, redisState, mongoReader, sourceNs, targetTable, transformVersion, logger } = opts; - - async function createNewRun(rid: string, ubId: string): Promise { - const now = new Date().toISOString(); - await manifestStore.createRun({ - run_id: rid, - status: 'active', - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: ubId, - transform_version: transformVersion, - created_at: now, - updated_at: now, - }); - await redisState.setActiveRun(rid); - await redisState.setState(rid, { - runId: rid, - status: 'active', - sourceNs, - targetTable, - upperBoundCursor: ubId, - lastCommittedCursor: null, - transformVersion, - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - } - - if (rerunMode === 'resume') { - // Priority 1: active run for THIS collection (crash recovery) - const activeRun = await manifestStore.getActiveRun(sourceNs, targetTable); - if (activeRun) { - logger.info({ runId: activeRun.run_id }, 'Resuming active run'); - return { runId: activeRun.run_id, upperBoundId: activeRun.upper_bound_cursor }; - } - - // Priority 2: most recent paused/stopped run (operator restart) - const resumableRun = await manifestStore.getResumableRun(sourceNs, targetTable, transformVersion); - if (resumableRun) { - await manifestStore.updateRunStatus(resumableRun.run_id, 'active'); - await redisState.setActiveRun(resumableRun.run_id); - - await manifestStore.insertEvent({ - run_id: resumableRun.run_id, - event_type: 'run_resumed', - message: `Run resumed from ${resumableRun.status} state`, - metadata: { prior_status: resumableRun.status, last_cursor: resumableRun.last_committed_cursor }, - created_at: new Date().toISOString(), - }); - - logger.info( - { runId: resumableRun.run_id, priorStatus: resumableRun.status }, - 'Resuming stopped/paused run', - ); - return { runId: resumableRun.run_id, upperBoundId: resumableRun.upper_bound_cursor }; - } - - // Priority 3: no resumable run, create new - const runId = randomUUID(); - const upperBound = await mongoReader.getUpperBound(); - if (!upperBound) { - const hasNullCd = await mongoReader.hasNullCdDocuments(); - if (!hasNullCd) { - return { runId, upperBoundId: '', isEmpty: true }; - } - const upperBoundId = serializeCursor({ cd: 0, id: "\uffff".repeat(24) }); - await createNewRun(runId, upperBoundId); - logger.info({ runId, sourceNs, targetTable }, 'All-null collection — created run for null-cd sweep'); - return { runId, upperBoundId }; - } - const upperBoundId = serializeCursor(upperBound); - await createNewRun(runId, upperBoundId); - logger.info({ runId, upperBoundId, sourceNs, targetTable }, 'Created new migration run'); - return { runId, upperBoundId }; - } - - if (rerunMode === 'new-run') { - const runId = randomUUID(); - const upperBound = await mongoReader.getUpperBound(); - if (!upperBound) { - const hasNullCd = await mongoReader.hasNullCdDocuments(); - if (!hasNullCd) { - return { runId, upperBoundId: '', isEmpty: true }; - } - const upperBoundId = serializeCursor({ cd: 0, id: "\uffff".repeat(24) }); - const activeRun = await manifestStore.getActiveRun(sourceNs, targetTable); - if (activeRun) { - const deleted = await manifestStore.deleteRunData(activeRun.run_id); - logger.info({ oldRunId: activeRun.run_id, deletedRecords: deleted }, 'Cleaned old run data for fresh start'); - await manifestStore.updateRunStatus(activeRun.run_id, 'completed'); - } - await createNewRun(runId, upperBoundId); - logger.info({ runId, sourceNs, targetTable }, 'All-null collection — created run for null-cd sweep (new-run mode)'); - return { runId, upperBoundId }; - } - const upperBoundId = serializeCursor(upperBound); - const activeRun = await manifestStore.getActiveRun(sourceNs, targetTable); - if (activeRun) { - // Clean old run data before starting fresh - const deleted = await manifestStore.deleteRunData(activeRun.run_id); - logger.info({ oldRunId: activeRun.run_id, deletedRecords: deleted }, 'Cleaned old run data for fresh start'); - await manifestStore.updateRunStatus(activeRun.run_id, 'completed'); - } - await createNewRun(runId, upperBoundId); - logger.info({ runId, upperBoundId, sourceNs, targetTable }, 'Created new migration run (new-run mode)'); - return { runId, upperBoundId }; - } - - if (rerunMode === 'clone-run') { - const activeRun = await manifestStore.getActiveRun(sourceNs, targetTable); - if (!activeRun) throw new Error('No existing run to clone from'); - const runId = randomUUID(); - const upperBoundId = activeRun.upper_bound_cursor; - // Clean old run data before starting fresh - const deleted = await manifestStore.deleteRunData(activeRun.run_id); - logger.info({ oldRunId: activeRun.run_id, deletedRecords: deleted }, 'Cleaned old run data for fresh start'); - await manifestStore.updateRunStatus(activeRun.run_id, 'completed'); - await createNewRun(runId, upperBoundId); - logger.info({ runId, upperBoundId, sourceNs, targetTable }, 'Created new migration run (clone-run mode)'); - return { runId, upperBoundId }; - } - - throw new Error(`Unknown rerun mode: ${rerunMode satisfies never}`); -} diff --git a/src/runtime/retry-policy.ts b/src/runtime/retry-policy.ts index e6184ca..313b938 100644 --- a/src/runtime/retry-policy.ts +++ b/src/runtime/retry-policy.ts @@ -70,6 +70,7 @@ export class RetryPolicy { label: string, logger: Logger, onError?: (attempt: number, error: Error) => Promise, + classifier?: (err: unknown) => 'permanent' | 'transient', ): Promise { let lastError: Error | null = null; @@ -83,6 +84,16 @@ export class RetryPolicy { await onError(attempt, lastError).catch(() => {}); } + // Permanent errors (bad data) can never succeed on retry — fail fast + // instead of burning the full backoff schedule per doomed batch. + if (classifier && classifier(err) === 'permanent') { + logger.warn( + { label, error: lastError.message }, + 'Permanent error — failing immediately without retries', + ); + throw lastError; + } + if (this.shouldRetry(attempt + 1)) { const delay = this.getDelay(attempt); logger.warn( diff --git a/src/source/mongo-reader.ts b/src/source/mongo-reader.ts index 0618dc6..0bd9496 100644 --- a/src/source/mongo-reader.ts +++ b/src/source/mongo-reader.ts @@ -10,7 +10,6 @@ export interface MongoReaderConfig { readConcern: string; retryReads: boolean; appName: string; - batchRowsTarget: number; cursorBatchSize: number; maxTimeMs: number; } @@ -268,58 +267,60 @@ export class MongoReader { }; } - async readPage(lastCursor: Cursor | null, upperBound: Cursor, limit?: number): Promise { + /** + * Stream documents between two cursors using ONE long-lived MongoDB cursor + * (no fresh find() per page — measured 25-40% faster than paged reads, and + * no per-page boundary re-reads). Yields pages of `pageSize` docs. + * + * NOTE: `start` is INCLUSIVE (min bound). When resuming from a cursor that + * points at an already-processed doc, the caller must skip the first doc if + * it equals the start cursor. On cursor death, reopen from the last yielded + * cursor — the generator itself does not retry. + */ + async *readStream( + start: Cursor | null, + upperBound: Cursor, + pageSize: number, + ): AsyncGenerator { this.ensureConnected(); - const { cursorBatchSize, maxTimeMs } = this.config; - const pageLimit = limit ?? this.config.batchRowsTarget; - - const ucd = new Date(upperBound.cd); - const startMs = performance.now(); let query = this.collection! .find({ cd: { $ne: null } }) .sort({ cd: 1, _id: 1 }) .hint({ cd: 1, _id: 1 }) - .max({ cd: ucd, _id: upperBound.id }) - .limit(pageLimit) + .max({ cd: new Date(upperBound.cd), _id: upperBound.id }) .batchSize(cursorBatchSize) .project(PROJECTION) .maxTimeMS(maxTimeMs); - if (lastCursor !== null) { - const lcd = new Date(lastCursor.cd); - query = query.min({ cd: lcd, _id: lastCursor.id }); + if (start !== null) { + query = query.min({ cd: new Date(start.cd), _id: start.id }); } - const docs = await query.toArray(); - - const fetchMs = Math.round(performance.now() - startMs); - - const lastDoc = docs[docs.length - 1]; - const lastCursorResult: Cursor | null = docs.length > 0 - ? { cd: cdToEpoch(lastDoc.cd), id: String(lastDoc._id) } - : null; - - // Guard: min() is inclusive, so the first doc may equal lastCursor. - // If it's the only doc returned, lastCursorResult === lastCursor → infinite loop. - // Signal "done" to the caller instead. - if (lastCursor !== null && lastCursorResult !== null - && lastCursorResult.cd === lastCursor.cd - && lastCursorResult.id === lastCursor.id) { - return { docs: [], lastCursor: null, fetchMs: Math.round(performance.now() - startMs) }; + let page: SourceDocument[] = []; + let pageStart = performance.now(); + for await (const doc of query) { + page.push(doc as SourceDocument); + if (page.length >= pageSize) { + const last = page[page.length - 1]; + yield { + docs: page, + lastCursor: { cd: cdToEpoch(last.cd), id: String(last._id) }, + fetchMs: Math.round(performance.now() - pageStart), + }; + page = []; + pageStart = performance.now(); + } + } + if (page.length > 0) { + const last = page[page.length - 1]; + yield { + docs: page, + lastCursor: { cd: cdToEpoch(last.cd), id: String(last._id) }, + fetchMs: Math.round(performance.now() - pageStart), + }; } - - this.logger.debug( - { docsRead: docs.length, lastCursor: lastCursorResult, fetchMs }, - "Page read complete", - ); - - return { - docs: docs as SourceDocument[], - lastCursor: lastCursorResult, - fetchMs, - }; } isConnected(): boolean { diff --git a/src/state/async-batch-writer.ts b/src/state/async-batch-writer.ts deleted file mode 100644 index 29c70fa..0000000 --- a/src/state/async-batch-writer.ts +++ /dev/null @@ -1,185 +0,0 @@ -import type { Logger } from "pino"; -import type { ManifestStore, Batch, BatchStatus } from "./manifest-store.ts"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -interface QueuedBatchWrite { - batch: Batch; - upperInclusiveCursor: string; - queuedAt: number; -} - -export interface AsyncBatchWriterConfig { - flushIntervalMs: number; - flushBatchSize: number; - maxQueueDepth?: number; -} - -// --------------------------------------------------------------------------- -// AsyncBatchWriter -// --------------------------------------------------------------------------- - -/** - * Async write queue that makes Redis the hot-path commit point. - * - * On batch completion: - * 1. Write cursor + bitmap to Redis (sync — this is the commit point) - * 2. Queue MongoDB batch record for async bulk flush - * - * Background loop flushes queued records to MongoDB periodically. - */ -export class AsyncBatchWriter { - private queue: QueuedBatchWrite[] = []; - private flushTimer: ReturnType | null = null; - private flushing = false; - private stopped = false; - private readonly manifestStore: ManifestStore; - private readonly config: AsyncBatchWriterConfig; - private readonly logger: Logger; - - constructor( - manifestStore: ManifestStore, - config: AsyncBatchWriterConfig, - logger: Logger, - ) { - this.manifestStore = manifestStore; - this.config = config; - this.logger = logger.child({ component: "AsyncBatchWriter" }); - } - - /** - * Queue a completed batch for async MongoDB flush. - * - * Caller (BatchRunner) is responsible for writing cursor + bitmap to - * the correct per-collection/per-range RedisHotState BEFORE calling this. - * This method only handles the MongoDB write queue. - */ - async queueBatch( - batch: Batch, - upperInclusiveCursor: string, - ): Promise { - // Queue MongoDB write (bounded) - const maxDepth = this.config.maxQueueDepth ?? 1000; - if (this.queue.length >= maxDepth) { - this.logger.error( - { queueDepth: this.queue.length, maxQueueDepth: maxDepth }, - "Async write queue at max depth — MongoDB may be unreachable. Forcing flush.", - ); - await this.flush(); - } - const doc: Batch = { - ...batch, - status: "done" as BatchStatus, - finished_at: new Date().toISOString(), - error_history: batch.error_history ?? [], - digest_match: batch.digest_match ?? null, - }; - this.queue.push({ batch: doc, upperInclusiveCursor, queuedAt: Date.now() }); - - // 3. Flush if queue is large enough - if (this.queue.length >= this.config.flushBatchSize) { - this.triggerFlush(); - } - } - - /** Start the periodic flush timer. */ - startPeriodicFlush(): void { - if (this.flushTimer) return; - this.flushTimer = setInterval(() => { - this.triggerFlush(); - }, this.config.flushIntervalMs); - } - - /** Flush all queued writes and stop the timer. */ - async drainAndStop(): Promise { - this.stopped = true; - if (this.flushTimer) { - clearInterval(this.flushTimer); - this.flushTimer = null; - } - if (this.queue.length > 0) { - this.logger.info({ pending: this.queue.length }, "Draining async write queue on shutdown"); - } - try { - await this.flush(); - } catch (err) { - this.logger.error( - { error: err instanceof Error ? err.message : String(err), lostRecords: this.queue.length }, - "Failed to drain async write queue on shutdown — batch records may be missing from MongoDB", - ); - } - } - - /** Number of writes waiting to be flushed. */ - getPendingCount(): number { - return this.queue.length; - } - - // ----------------------------------------------------------------------- - // Private - // ----------------------------------------------------------------------- - - private triggerFlush(): void { - if (this.flushing || this.queue.length === 0) return; - this.flush().catch(err => { - this.logger.warn({ error: err instanceof Error ? err.message : String(err) }, "Async flush failed, will retry"); - }); - } - - private async flush(): Promise { - if (this.flushing || this.queue.length === 0) return; - this.flushing = true; - - // Drain the current queue - const items = this.queue.splice(0); - const batchDocs = items.map(i => i.batch); - - // Find the cursor with the highest batch_seq (monotonically increasing) - let latestCursor = items[0].upperInclusiveCursor; - let latestRunId = items[0].batch.run_id; - let maxBatchSeq = items[0].batch.batch_seq; - for (const item of items) { - if (item.batch.batch_seq > maxBatchSeq) { - maxBatchSeq = item.batch.batch_seq; - latestCursor = item.upperInclusiveCursor; - latestRunId = item.batch.run_id; - } - } - - try { - // Bulk upsert batch records (handles duplicates via update) - await this.manifestStore.bulkInsertBatches(batchDocs); - - // Advance cursor to latest position - await this.manifestStore.advanceCursor(latestRunId, latestCursor); - - this.logger.debug( - { flushed: batchDocs.length, latestBatchSeq: batchDocs[batchDocs.length - 1].batch_seq }, - "Async flush completed", - ); - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - // Re-queue failed items (capped to prevent unbounded growth) - const maxDepth = this.config.maxQueueDepth ?? 1000; - const totalAfterRequeue = this.queue.length + items.length; - if (totalAfterRequeue <= maxDepth) { - this.queue.unshift(...items); - } else { - const dropped = items.length - (maxDepth - this.queue.length); - this.queue.unshift(...items.slice(0, Math.max(0, maxDepth - this.queue.length))); - this.logger.error( - { dropped, queueDepth: this.queue.length }, - "Dropped batch records exceeding max queue depth — MongoDB writes lost", - ); - } - this.logger.warn( - { error: errMsg, count: items.length, queueDepth: this.queue.length }, - "Async flush to MongoDB failed, re-queued", - ); - } finally { - this.flushing = false; - } - } -} diff --git a/src/state/collection-lock.ts b/src/state/collection-lock.ts deleted file mode 100644 index c35eb04..0000000 --- a/src/state/collection-lock.ts +++ /dev/null @@ -1,393 +0,0 @@ -import type { Redis } from "ioredis"; -import type { Logger } from "pino"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export type AcquireResult = "acquired" | "stolen" | "locked"; - -export interface LockInfo { - collectionName: string; - podId: string; - acquiredAt: string; - ttlSec: number; -} - -export interface CollectionLockConfig { - lockTtlSec: number; // default 300 (5 minutes) - renewIntervalMs: number; // default 60_000 (1 minute) - podHeartbeatMs: number; // default 30_000 - podDeadAfterSec: number; // default 180 (3 minutes) - keyPrefix: string; // default "mig" -} - -// --------------------------------------------------------------------------- -// Lua scripts (executed atomically on the Redis server via EVAL) -// --------------------------------------------------------------------------- - -/** - * Atomic lock acquisition with pod-liveness check. - * - * KEYS[1] = lock key (mig:lock:{collection}) - * ARGV[1] = podId (caller) - * ARGV[2] = lockTtlSec - * ARGV[3] = JSON payload {podId, acquiredAt} - * ARGV[4] = pod key prefix (mig:pod:) - * - * Returns: 1 = acquired, 2 = stolen from dead pod, 0 = locked by alive pod - */ -const ACQUIRE_LUA = ` -local current = redis.call('GET', KEYS[1]) -if not current then - redis.call('SET', KEYS[1], ARGV[3], 'EX', tonumber(ARGV[2])) - return 1 -end - -local data = cjson.decode(current) -if data.podId == ARGV[1] then - redis.call('SET', KEYS[1], ARGV[3], 'EX', tonumber(ARGV[2])) - return 1 -end - -local otherPodKey = ARGV[4] .. data.podId -local otherPod = redis.call('GET', otherPodKey) -if otherPod then - return 0 -end - -redis.call('SET', KEYS[1], ARGV[3], 'EX', tonumber(ARGV[2])) -return 2 -`; - -/** - * Renew lock TTL if owned by this pod. - * - * KEYS[1] = lock key - * ARGV[1] = podId - * ARGV[2] = lockTtlSec - * - * Returns: 1 = renewed, 0 = lock lost - */ -const RENEW_LUA = ` -local current = redis.call('GET', KEYS[1]) -if not current then - return 0 -end -local data = cjson.decode(current) -if data.podId ~= ARGV[1] then - return 0 -end -redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) -return 1 -`; - -/** - * Release lock if owned by this pod. - * - * KEYS[1] = lock key - * ARGV[1] = podId - * - * Returns: 1 = released, 0 = not owned - */ -const RELEASE_LUA = ` -local current = redis.call('GET', KEYS[1]) -if not current then - return 1 -end -local data = cjson.decode(current) -if data.podId ~= ARGV[1] then - return 0 -end -redis.call('DEL', KEYS[1]) -return 1 -`; - -// --------------------------------------------------------------------------- -// CollectionLock -// --------------------------------------------------------------------------- - -export class CollectionLock { - private readonly redis: Redis; - private readonly podId: string; - private readonly config: CollectionLockConfig; - private readonly logger: Logger; - - private readonly heldLocks = new Set(); - private lockRenewTimer: ReturnType | null = null; - private podHeartbeatTimer: ReturnType | null = null; - private consecutiveHeartbeatFailures = 0; - private _heartbeatHealthy = true; - onLockLost: ((collectionName: string) => void) | null = null; - - get heartbeatHealthy(): boolean { return this._heartbeatHealthy; } - - constructor(redis: Redis, podId: string, config: CollectionLockConfig, logger: Logger) { - this.redis = redis; - this.podId = podId; - this.config = config; - this.logger = logger.child({ component: "CollectionLock", podId }); - } - - // ----------------------------------------------------------------------- - // Lock operations - // ----------------------------------------------------------------------- - - async tryAcquire(collectionName: string): Promise { - const lockKey = this.lockKey(collectionName); - const payload = JSON.stringify({ podId: this.podId, acquiredAt: new Date().toISOString() }); - const podKeyPrefix = `${this.config.keyPrefix}:pod:`; - - // ioredis .eval() runs a Lua script atomically on the Redis server - const result = await this.redis.eval( - ACQUIRE_LUA, - 1, - lockKey, - this.podId, - String(this.config.lockTtlSec), - payload, - podKeyPrefix, - ) as number; - - if (result === 1) { - this.heldLocks.add(collectionName); - this.logger.debug({ collection: collectionName }, "Lock acquired"); - return "acquired"; - } - if (result === 2) { - this.heldLocks.add(collectionName); - this.logger.info({ collection: collectionName }, "Lock stolen from dead pod"); - return "stolen"; - } - return "locked"; - } - - async release(collectionName: string): Promise { - const lockKey = this.lockKey(collectionName); - - await this.redis.eval( - RELEASE_LUA, - 1, - lockKey, - this.podId, - ); - - this.heldLocks.delete(collectionName); - this.logger.debug({ collection: collectionName }, "Lock released"); - } - - async releaseAll(): Promise { - const collections = [...this.heldLocks]; - await Promise.allSettled( - collections.map(name => this.release(name)), - ); - } - - // ----------------------------------------------------------------------- - // Heartbeat - // ----------------------------------------------------------------------- - - startHeartbeat(): void { - // Renew held locks - this.lockRenewTimer = setInterval(() => { - this.renewAll().catch(err => { - this.logger.warn({ error: err instanceof Error ? err.message : String(err) }, "Lock renewal failed"); - }); - }, this.config.renewIntervalMs); - - // Pod liveness heartbeat (with failure tracking) - this.podHeartbeatTimer = setInterval(() => { - this.updatePodLiveness() - .then(() => { - this.consecutiveHeartbeatFailures = 0; - this._heartbeatHealthy = true; - }) - .catch(err => { - this.consecutiveHeartbeatFailures++; - const msg = err instanceof Error ? err.message : String(err); - if (this.consecutiveHeartbeatFailures >= 3 && this._heartbeatHealthy) { - this._heartbeatHealthy = false; - this.logger.error( - { consecutive: this.consecutiveHeartbeatFailures, error: msg }, - "Pod heartbeat failed 3+ consecutive times — lock acquisition paused until recovery", - ); - } else { - this.logger.warn( - { error: msg, consecutive: this.consecutiveHeartbeatFailures }, - "Pod heartbeat failed", - ); - } - }); - }, this.config.podHeartbeatMs); - - // Initial pod liveness write - this.updatePodLiveness().catch(() => {}); - } - - stopHeartbeat(): void { - if (this.lockRenewTimer) { - clearInterval(this.lockRenewTimer); - this.lockRenewTimer = null; - } - if (this.podHeartbeatTimer) { - clearInterval(this.podHeartbeatTimer); - this.podHeartbeatTimer = null; - } - } - - // ----------------------------------------------------------------------- - // Query - // ----------------------------------------------------------------------- - - async listAllLocks(): Promise { - const pattern = `${this.config.keyPrefix}:lock:*`; - const keys: string[] = []; - const stream = this.redis.scanStream({ match: pattern, count: 100 }); - for await (const batch of stream) { - keys.push(...(batch as string[])); - } - if (keys.length === 0) return []; - - // Fetch values and TTLs in parallel via pipeline - const pipeline = this.redis.pipeline(); - for (const key of keys) { - pipeline.get(key); - pipeline.ttl(key); - } - const results = await pipeline.exec(); - - const prefix = `${this.config.keyPrefix}:lock:`; - const locks: LockInfo[] = []; - - for (let i = 0; i < keys.length; i++) { - const val = results?.[i * 2]?.[1] as string | null; - const ttl = results?.[i * 2 + 1]?.[1] as number ?? -1; - if (!val) continue; - try { - const data = JSON.parse(val) as { podId: string; acquiredAt: string }; - locks.push({ - collectionName: keys[i].slice(prefix.length), - podId: data.podId, - acquiredAt: data.acquiredAt, - ttlSec: ttl, - }); - } catch { - // skip malformed entries - } - } - return locks; - } - - getHeldLocks(): string[] { - return [...this.heldLocks]; - } - - // ----------------------------------------------------------------------- - // Admin operations (lock management) - // ----------------------------------------------------------------------- - - /** Force-release a lock regardless of owner. Admin override. */ - async forceRelease(collectionName: string): Promise { - const lockKey = this.lockKey(collectionName); - await this.redis.del(lockKey); - this.heldLocks.delete(collectionName); - this.logger.info({ collection: collectionName }, "Lock force-released (admin)"); - } - - /** Delete a pod's heartbeat key, marking it as dead for lock stealing. */ - async deletePodKey(podId: string): Promise { - const podKey = `${this.config.keyPrefix}:pod:${podId}`; - await this.redis.del(podKey); - this.logger.info({ targetPod: podId }, "Pod key deleted (admin)"); - } - - /** List all pod heartbeat keys. */ - async listAllPodKeys(): Promise> { - const pattern = `${this.config.keyPrefix}:pod:*`; - const keys: string[] = []; - const stream = this.redis.scanStream({ match: pattern, count: 100 }); - for await (const batch of stream) { - keys.push(...(batch as string[])); - } - if (keys.length === 0) return []; - - const values = await this.redis.mget(...keys); - const results: Array<{ podId: string; lastHeartbeat: string; collectionsActive: string[] }> = []; - - for (const val of values) { - if (!val) continue; - try { - results.push(JSON.parse(val) as { podId: string; lastHeartbeat: string; collectionsActive: string[] }); - } catch { - // skip - } - } - return results; - } - - /** Release all locks held by a specific (dead) pod. Returns released collection names. */ - async releaseLocksForPod(podId: string): Promise { - const allLocks = await this.listAllLocks(); - const toRelease = allLocks.filter(l => l.podId === podId); - const released: string[] = []; - - for (const lock of toRelease) { - const lockKey = this.lockKey(lock.collectionName); - await this.redis.del(lockKey); - released.push(lock.collectionName); - } - - this.logger.info({ targetPod: podId, releasedCount: released.length, collections: released }, "Released locks for dead pod"); - return released; - } - - // ----------------------------------------------------------------------- - // Private helpers - // ----------------------------------------------------------------------- - - private lockKey(collectionName: string): string { - return `${this.config.keyPrefix}:lock:${collectionName}`; - } - - private async renewAll(): Promise { - const collections = [...this.heldLocks]; - if (collections.length === 0) return; - - const results = await Promise.allSettled( - collections.map(async name => { - const lockKey = this.lockKey(name); - const result = await this.redis.eval( - RENEW_LUA, - 1, - lockKey, - this.podId, - String(this.config.lockTtlSec), - ) as number; - if (result === 0) { - this.heldLocks.delete(name); - this.logger.warn({ collection: name }, "Lock lost during renewal — another pod may have taken it"); - try { this.onLockLost?.(name); } catch { /* callback must not throw */ } - } - }), - ); - - const failures = results.filter(r => r.status === "rejected"); - if (failures.length > 0) { - this.logger.warn({ failedRenewals: failures.length }, "Some lock renewals failed"); - } - } - - private async updatePodLiveness(): Promise { - const podKey = `${this.config.keyPrefix}:pod:${this.podId}`; - await this.redis.set( - podKey, - JSON.stringify({ - podId: this.podId, - lastHeartbeat: new Date().toISOString(), - collectionsActive: [...this.heldLocks], - }), - "EX", - this.config.podDeadAfterSec, - ); - } -} diff --git a/src/state/coverage.ts b/src/state/coverage.ts deleted file mode 100644 index ac6e2dd..0000000 --- a/src/state/coverage.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { type Cursor, compareCursors, deserializeCursor } from '../types/cursor.ts'; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface CoverageInterval { - lowerExclusive: Cursor; - upperInclusive: Cursor; -} - -/** - * Minimal batch shape required for coverage computation. - * Compatible with the full Batch type from manifest-store. - */ -export interface CompletedBatch { - lower_exclusive_cursor: string; // serialized Cursor JSON - upper_inclusive_cursor: string; // serialized Cursor JSON - status: string; -} - -// --------------------------------------------------------------------------- -// Coverage functions -// --------------------------------------------------------------------------- - -/** - * Build sorted coverage intervals from completed (status === "done") batches. - * - * The intervals are sorted by their lower-exclusive cursor using compound - * cursor comparison (cd first, then _id as tiebreaker). - */ -export function buildCoverageFromBatches( - batches: CompletedBatch[], -): CoverageInterval[] { - const done = batches.filter((b) => b.status === "done"); - - const ZERO_CURSOR: Cursor = { cd: 0, id: '' }; - - const intervals: CoverageInterval[] = done.map((b) => ({ - lowerExclusive: b.lower_exclusive_cursor - ? deserializeCursor(b.lower_exclusive_cursor) - : ZERO_CURSOR, - upperInclusive: deserializeCursor(b.upper_inclusive_cursor), - })); - - intervals.sort((a, b) => - compareCursors(a.lowerExclusive, b.lowerExclusive), - ); - - return intervals; -} - -/** - * Merge adjacent or overlapping intervals into the smallest set of - * contiguous intervals. - * - * Two intervals are considered adjacent when the upper bound of one equals - * the lower bound of the next (i.e., they share a boundary cursor). - * - * Intervals MUST be sorted by `lowerExclusive` before calling this - * function (use `buildCoverageFromBatches` which already sorts). - */ -export function compactIntervals( - intervals: CoverageInterval[], -): CoverageInterval[] { - if (intervals.length === 0) return []; - - // Assumes input is already sorted by lowerExclusive (buildCoverageFromBatches sorts). - const merged: CoverageInterval[] = [{ ...intervals[0] }]; - - for (let i = 1; i < intervals.length; i++) { - const current = intervals[i]; - const last = merged[merged.length - 1]; - - // Adjacent: last.upper === current.lower (boundary cursors match) - // Overlapping: last.upper >= current.lower - if (compareCursors(last.upperInclusive, current.lowerExclusive) >= 0) { - // Extend upper bound if the current interval reaches further - if (compareCursors(current.upperInclusive, last.upperInclusive) > 0) { - last.upperInclusive = current.upperInclusive; - } - } else { - merged.push({ ...current }); - } - } - - return merged; -} - diff --git a/src/state/dlq-store.ts b/src/state/dlq-store.ts new file mode 100644 index 0000000..dfe2bc8 --- /dev/null +++ b/src/state/dlq-store.ts @@ -0,0 +1,185 @@ +/** + * DlqStore — the dead-letter queue for the ledger engine. + * + * One MongoDB document per source document that could not be migrated, + * carrying the FULL RAW source doc — that is what makes the pile replayable + * later (after a transform fix) without ever re-reading the customer's + * source collection. + */ + +import { MongoClient, type Collection } from 'mongodb'; +import { toEpochMillis, clampDateTime64 } from '../transform/validators.ts'; +import type { Logger } from 'pino'; + +export type DlqReason = 'insert_rejected' | 'transform_error' | 'skipped'; +export type DlqStatus = 'pending' | 'resolved' | 'waived'; + +export interface DlqDoc { + _id: string; // `${runId}:${sourceId}` + run_id: string; + collection: string; + chunk_id: string; + source_id: string; + raw_doc: Record; + reason: DlqReason; + error: string; + transform_version: string; // version that failed + /** + * The doc's cd (or ts-derived fallback) in epoch ms, when parseable — + * lets audits attribute unmigrated docs to their cd window, so a window + * whose shortfall is exactly its pending/waived DLQ docs is not flagged + * as a disagreement. Null when the doc has no usable cd/ts (those can't + * land in any window anyway). + */ + cd_ms: number | null; + status: DlqStatus; + resolved_by_version: string | null; + created_at: Date; + updated_at: Date; +} + +export class DlqStore { + private client: MongoClient; + private coll: Collection | null = null; + private readonly logger: Logger; + private readonly dbName: string; + + constructor(uri: string, dbName: string, logger: Logger) { + this.client = new MongoClient(uri); + this.dbName = dbName; + this.logger = logger.child({ component: 'DlqStore' }); + } + + async connect(): Promise { + await this.client.connect(); + this.coll = this.client.db(this.dbName).collection('mig_dlq_docs'); + await this.coll.createIndex({ run_id: 1, status: 1 }); + await this.coll.createIndex({ run_id: 1, collection: 1, cd_ms: 1 }); + this.logger.info({ db: this.dbName }, 'DlqStore connected'); + } + + async close(): Promise { + await this.client.close(); + } + + private c(): Collection { + if (!this.coll) throw new Error('DlqStore not connected'); + return this.coll; + } + + async add(entries: Array & { cd_ms?: number | null }>): Promise { + if (entries.length === 0) return; + const now = new Date(); + const deriveCdMs = (raw: Record): number | null => { + const cd = toEpochMillis(raw.cd); + if (cd !== null && cd > 0) return clampDateTime64(cd); + const ts = toEpochMillis(raw.ts); + return ts !== null && ts > 0 ? clampDateTime64(ts) : null; + }; + const docs: DlqDoc[] = entries.map((e) => ({ + cd_ms: e.cd_ms !== undefined ? e.cd_ms : deriveCdMs(e.raw_doc), + ...e, + _id: `${e.run_id}:${e.source_id}`, + status: 'pending', + resolved_by_version: null, + created_at: now, + updated_at: now, + })); + try { + await this.c().insertMany(docs, { ordered: false }); + } catch (err: unknown) { + if ((err as { code?: number }).code !== 11000) throw err; // re-DLQ of same doc is fine + } + } + + /** Keyset page for full-drain loops: pending entries with _id > after. */ + async listPendingAfter(runId: string, afterId: string | null, limit = 500): Promise { + const filter: Record = { run_id: runId, status: 'pending' }; + if (afterId) filter._id = { $gt: afterId }; + return this.c().find(filter).sort({ _id: 1 }).limit(limit).toArray(); + } + + async listPending(runId: string, limit = 10_000, skip = 0): Promise { + // Stable order so pagination pages don't shuffle between polls + return this.c().find({ run_id: runId, status: 'pending' }).sort({ _id: 1 }).skip(skip).limit(limit).toArray(); + } + + /** Disk cost of the DLQ (raw docs are stored whole) + manifest-DB headroom. */ + async storageStats(): Promise<{ dlqBytes: number; dlqDocs: number; diskFreePct: number | null }> { + const db = this.client.db(this.dbName); + let dlqBytes = 0, dlqDocs = 0, diskFreePct: number | null = null; + try { + const cs = await db.command({ collStats: 'mig_dlq_docs' }); + dlqBytes = cs.storageSize ?? 0; + dlqDocs = cs.count ?? 0; + } catch { /* collection may not exist yet */ } + try { + const ds = await db.stats(); + if (ds.fsTotalSize) diskFreePct = Math.round(((ds.fsTotalSize - ds.fsUsedSize) / ds.fsTotalSize) * 100); + } catch { /* no permission — omit */ } + return { dlqBytes, dlqDocs, diskFreePct }; + } + + /** + * Known-unmigrated docs (pending/waived — anything not resolved) whose cd + * falls in a window. Audits subtract these: their absence from the live + * table is accounted for, not a disagreement. Entries written before the + * cd_ms field (or with unparseable cd/ts) can't be attributed and count 0. + */ + async countUnresolvedInWindow(runId: string, collection: string, lowerCdMs: number, upperCdMs: number): Promise { + return this.c().countDocuments({ + run_id: runId, collection, + cd_ms: { $gte: lowerCdMs, $lt: upperCdMs }, + status: { $ne: 'resolved' }, + }); + } + + async countByStatus(runId: string): Promise> { + const rows = await this.c() + .aggregate<{ _id: string; n: number }>([ + { $match: { run_id: runId } }, + { $group: { _id: '$status', n: { $sum: 1 } } }, + ]) + .toArray(); + return Object.fromEntries(rows.map((r) => [r._id, r.n])); + } + + async topErrors(runId: string, limit = 10): Promise> { + const rows = await this.c() + .aggregate<{ _id: string; n: number }>([ + { $match: { run_id: runId, status: 'pending' } }, + { $group: { _id: '$error', n: { $sum: 1 } } }, + { $sort: { n: -1 } }, + { $limit: limit }, + ]) + .toArray(); + return rows.map((r) => ({ error: r._id, n: r.n })); + } + + async markResolved(ids: string[], version: string): Promise { + if (ids.length === 0) return; + await this.c().updateMany( + { _id: { $in: ids } }, + { $set: { status: 'resolved', resolved_by_version: version, updated_at: new Date() } }, + ); + } + + /** Update error on a still-failing pending entry (replay attempt failed again). */ + async recordRetryError(id: string, error: string): Promise { + await this.c().updateOne({ _id: id }, { $set: { error, updated_at: new Date() } }); + } + + /** + * Waive pending entries: the explicit operator decision that these docs + * will NOT be migrated. The raw docs stay in the DLQ permanently as the + * record of what was excluded — waived is terminal, but reversible (an + * operator can flip back to pending and replay after a later fix). + * With no ids given, waives everything currently pending for the run. + */ + async waive(runId: string, ids?: string[]): Promise { + const filter: Record = { run_id: runId, status: 'pending' }; + if (ids && ids.length > 0) filter._id = { $in: ids }; + const res = await this.c().updateMany(filter, { $set: { status: 'waived', updated_at: new Date() } }); + return res.modifiedCount; + } +} diff --git a/src/state/global-progress.ts b/src/state/global-progress.ts deleted file mode 100644 index dbecfce..0000000 --- a/src/state/global-progress.ts +++ /dev/null @@ -1,141 +0,0 @@ -import type { Redis } from "ioredis"; -import type { Logger } from "pino"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface CollectionProgress { - collectionName: string; - podId: string; - status: "processing" | "completed" | "failed" | "skipped"; - runId: string; - docsRead: number; - rowsInserted: number; - estimatedTotal: number; - batchSeq: number; - startedAt: string; - updatedAt: string; - error?: string; - isRangeParallel?: boolean; - rangeCount?: number; - throughput?: number; -} - -export interface PodInfo { - podId: string; - lastHeartbeat: string; - collectionsActive: string[]; -} - -export interface GlobalCommands { - pause: boolean; - stop: boolean; -} - -// --------------------------------------------------------------------------- -// GlobalProgress -// --------------------------------------------------------------------------- - -export class GlobalProgress { - private readonly redis: Redis; - private readonly podId: string; - private readonly keyPrefix: string; - private readonly logger: Logger; - - constructor(redis: Redis, podId: string, keyPrefix: string, logger: Logger) { - this.redis = redis; - this.podId = podId; - this.keyPrefix = keyPrefix; - this.logger = logger.child({ component: "GlobalProgress", podId }); - } - - // ----------------------------------------------------------------------- - // Per-collection progress - // ----------------------------------------------------------------------- - - async updateCollectionProgress(progress: CollectionProgress): Promise { - const key = `${this.keyPrefix}:progress:${progress.collectionName}:${progress.podId}`; - await this.redis.set( - key, - JSON.stringify(progress), - "EX", - 1800, // 30 minute TTL, renewed on each update - ); - } - - async getCollectionProgress(collectionName: string): Promise { - const key = `${this.keyPrefix}:progress:${collectionName}:${this.podId}`; - const raw = await this.redis.get(key); - if (!raw) return null; - return JSON.parse(raw) as CollectionProgress; - } - - async getAllCollectionProgress(): Promise { - const pattern = `${this.keyPrefix}:progress:*`; - const keys: string[] = []; - const stream = this.redis.scanStream({ match: pattern, count: 100 }); - for await (const batch of stream) { - keys.push(...(batch as string[])); - } - if (keys.length === 0) return []; - - const values = await this.redis.mget(...keys); - const results: CollectionProgress[] = []; - - for (const val of values) { - if (!val) continue; - try { - results.push(JSON.parse(val) as CollectionProgress); - } catch { - // skip malformed entries - } - } - return results; - } - - // ----------------------------------------------------------------------- - // Pod registry - // ----------------------------------------------------------------------- - - async getAllPods(): Promise { - const pattern = `${this.keyPrefix}:pod:*`; - const keys: string[] = []; - const stream = this.redis.scanStream({ match: pattern, count: 100 }); - for await (const batch of stream) { - keys.push(...(batch as string[])); - } - if (keys.length === 0) return []; - - const values = await this.redis.mget(...keys); - const pods: PodInfo[] = []; - - for (const val of values) { - if (!val) continue; - try { - pods.push(JSON.parse(val) as PodInfo); - } catch { - // skip malformed entries - } - } - return pods; - } - - // ----------------------------------------------------------------------- - // Global commands - // ----------------------------------------------------------------------- - - async setGlobalCommand(command: keyof GlobalCommands, value: boolean): Promise { - const key = `${this.keyPrefix}:cmd:global`; - await this.redis.hset(key, command, value ? "true" : "false"); - } - - async getGlobalCommands(): Promise { - const key = `${this.keyPrefix}:cmd:global`; - const raw = await this.redis.hgetall(key); - return { - pause: raw.pause === "true", - stop: raw.stop === "true", - }; - } -} diff --git a/src/state/ledger-store.ts b/src/state/ledger-store.ts new file mode 100644 index 0000000..c3386ed --- /dev/null +++ b/src/state/ledger-store.ts @@ -0,0 +1,395 @@ +/** + * LedgerStore — the chunk checklist for the `ledger` engine. + * + * One MongoDB document per chunk of work. This is the ONLY progress state the + * ledger engine keeps (no Redis): transitions happen a few times per chunk + * (~30-60 min of work), and recovery never trusts the ledger blindly — it + * verifies chunks against actual staging-table counts (see ChunkOrchestrator). + * + * Chunk lifecycle: + * pending → in_progress → written → attaching → done + * ↘ failed (operator-visible, redo via reset) + * + * Claiming is an atomic findOneAndUpdate with a lease; a pod that dies simply + * lets its lease expire and another pod reclaims the chunk (drop staging, redo). + */ + +import { MongoClient, type Collection } from 'mongodb'; +import type { Logger } from 'pino'; + +export type ChunkStatus = 'pending' | 'in_progress' | 'written' | 'attaching' | 'done' | 'failed' | 'superseded'; + +export interface ChunkDoc { + _id: string; // `${runId}:${collection}:${idx}` + run_id: string; + collection: string; + // Collection identity in ClickHouse terms. Hashed collections map 1:1 to an + // (app, event) pair — every cd-window query against the LIVE table must be + // scoped by these, because collections overlap in wall-clock time and the + // live table holds them all. Null for unresolvable/base collections. + scope_a: string | null; + scope_e: string | null; + scope_n: string | null; // set for custom events (e='[CLY]_custom') + idx: number; + lower_cd: number; // inclusive, epoch ms + upper_cd: number; // exclusive, epoch ms + status: ChunkStatus; + pod_id: string | null; + lease_until: Date | null; + staging_table: string | null; + docs_read: number; + docs_skipped: number; + rows_expected: number; + partitions: string[]; // partition ids discovered in staging at attach time + attached: string[]; // partition ids confirmed attached to the live table + attach_method: 'attach' | 'insert_select' | null; + attempts: number; + last_error: string | null; + transform_version: string; + updated_at: Date; +} + +export class LedgerStore { + private client: MongoClient; + private coll: Collection | null = null; + private readonly logger: Logger; + private readonly dbName: string; + private readonly collectionName: string; + + constructor(uri: string, dbName: string, logger: Logger, collectionName = 'mig_ranges') { + this.client = new MongoClient(uri); + this.dbName = dbName; + this.collectionName = collectionName; + this.logger = logger.child({ component: 'LedgerStore' }); + } + + async connect(): Promise { + await this.client.connect(); + this.coll = this.client.db(this.dbName).collection(this.collectionName); + await this.coll.createIndex({ run_id: 1, collection: 1, status: 1, idx: -1 }); + this.logger.info({ db: this.dbName, collection: this.collectionName }, 'LedgerStore connected'); + } + + async close(): Promise { + await this.client.close(); + } + + private c(): Collection { + if (!this.coll) throw new Error('LedgerStore not connected'); + return this.coll; + } + + /** + * Aggregated run summary — the UI's primary data source, so the dashboard + * stays O(collections), not O(chunks) (a 10TB run can have tens of + * thousands of chunks; shipping them all every 2s does not scale). + */ + async summarize(runId: string): Promise<{ + total: number; + byStatus: Record; + docsDone: number; + perCollection: Array<{ collection: string; byStatus: Record; docsDone: number; doneDocsRead: number; nonDoneRowsExpected: number }>; + }> { + const rows = await this.c().aggregate<{ + _id: { c: string; s: string }; n: number; docsDone: number; docsRead: number; nonDoneExpected: number; + }>([ + { $match: { run_id: runId } }, + { $group: { + _id: { c: '$collection', s: '$status' }, + n: { $sum: 1 }, + docsDone: { $sum: { $cond: [{ $eq: ['$status', 'done'] }, '$rows_expected', 0] } }, + docsRead: { $sum: { $cond: [{ $eq: ['$status', 'done'] }, '$docs_read', 0] } }, + nonDoneExpected: { $sum: { $cond: [{ $in: ['$status', ['pending', 'in_progress', 'written', 'attaching', 'failed']] }, '$rows_expected', 0] } }, + } }, + ]).toArray(); + const perColl = new Map; docsDone: number; doneDocsRead: number; nonDoneRowsExpected: number }>(); + const byStatus: Record = {}; + let total = 0, docsDone = 0; + for (const r of rows) { + const e = perColl.get(r._id.c) ?? { collection: r._id.c, byStatus: {}, docsDone: 0, doneDocsRead: 0, nonDoneRowsExpected: 0 }; + e.byStatus[r._id.s] = (e.byStatus[r._id.s] ?? 0) + r.n; + e.docsDone += r.docsDone; + e.doneDocsRead += r.docsRead; + e.nonDoneRowsExpected += r.nonDoneExpected; + perColl.set(r._id.c, e); + byStatus[r._id.s] = (byStatus[r._id.s] ?? 0) + r.n; + total += r.n; + docsDone += r.docsDone; + } + return { total, byStatus, docsDone, perCollection: [...perColl.values()].sort((a, b) => a.collection.localeCompare(b.collection)) }; + } + + /** Non-terminal + failed chunk details, capped — the interesting ones on huge runs. */ + async listActive(runId: string, limit = 500): Promise { + return this.c() + .find({ run_id: runId, status: { $in: ['pending', 'in_progress', 'written', 'attaching', 'failed'] } }) + .sort({ collection: 1, idx: 1 }) + .limit(limit) + .toArray() as never; + } + + /** Rebuild support: replace this run's entire ledger with regenerated chunks. */ + async replaceAllForRun(runId: string, docs: ChunkDoc[]): Promise { + await this.c().deleteMany({ run_id: runId }); + if (docs.length > 0) await this.c().insertMany(docs, { ordered: false }); + return docs.length; + } + + async countForRun(runId: string): Promise { + return this.c().countDocuments({ run_id: runId }); + } + + /** + * Idempotently create the chunk list for a collection. If any chunks + * already exist for (runId, collection) this is a no-op — resume keeps + * whatever bounds were originally cut. + */ + async initChunks( + runId: string, + collection: string, + bounds: Array<{ lowerCd: number; upperCd: number }>, + transformVersion: string, + scope?: { a: string; e: string; n?: string } | null, + ): Promise { + const existing = await this.c().countDocuments({ run_id: runId, collection }, { limit: 1 }); + if (existing > 0) return 0; + + const now = new Date(); + const docs: ChunkDoc[] = bounds.map((b, idx) => ({ + _id: `${runId}:${collection}:${idx}`, + run_id: runId, + collection, + scope_a: scope?.a ?? null, + scope_e: scope?.e ?? null, + scope_n: scope?.n ?? null, + idx, + lower_cd: b.lowerCd, + upper_cd: b.upperCd, + status: 'pending', + pod_id: null, + lease_until: null, + staging_table: null, + docs_read: 0, + docs_skipped: 0, + rows_expected: 0, + partitions: [], + attached: [], + attach_method: null, + attempts: 0, + last_error: null, + transform_version: transformVersion, + updated_at: now, + })); + + try { + await this.c().insertMany(docs, { ordered: false }); + } catch (err: unknown) { + // Duplicate keys mean another pod initialized concurrently — fine. + if ((err as { code?: number }).code !== 11000) throw err; + } + return docs.length; + } + + /** + * Atomically claim the next pending chunk, newest data first (highest idx). + */ + async claimNext( + runId: string, + collection: string, + podId: string, + leaseSec: number, + excludeSentinel = false, + ): Promise { + const filter: Record = { run_id: runId, collection, status: 'pending' }; + // The null-cd sweep (sentinel bounds lower_cd=-1) must run strictly AFTER + // all regular chunks: its rows carry cd derived from ts, which lands + // inside regular chunks' cd windows and would poison their + // verify-then-attach checks. + if (excludeSentinel) filter.lower_cd = { $gte: 0 }; + return this.c().findOneAndUpdate( + filter, + { + $set: { + status: 'in_progress', + pod_id: podId, + lease_until: new Date(Date.now() + leaseSec * 1000), + updated_at: new Date(), + }, + $inc: { attempts: 1 }, + }, + { sort: { idx: -1 }, returnDocument: 'after' }, + ); + } + + /** Extend the lease of a chunk this pod is working on. */ + async heartbeat(chunkId: string, podId: string, leaseSec: number): Promise { + await this.c().updateOne( + { _id: chunkId, pod_id: podId, status: { $in: ['in_progress', 'written', 'attaching'] } }, + { $set: { lease_until: new Date(Date.now() + leaseSec * 1000), updated_at: new Date() } }, + ); + } + + /** + * Guarded state transition. Returns the updated doc or null when the guard + * failed (someone else moved the chunk — treat as lost claim). + */ + async transition( + chunkId: string, + from: ChunkStatus | ChunkStatus[], + to: ChunkStatus, + patch: Partial = {}, + ): Promise { + const fromArr = Array.isArray(from) ? from : [from]; + return this.c().findOneAndUpdate( + { _id: chunkId, status: { $in: fromArr } }, + { $set: { ...patch, status: to, updated_at: new Date() } }, + { returnDocument: 'after' }, + ); + } + + /** Append one attached partition id (crash-safe attach progress). */ + async recordAttached(chunkId: string, partitionId: string): Promise { + await this.c().updateOne( + { _id: chunkId }, + { $addToSet: { attached: partitionId }, $set: { updated_at: new Date() } }, + ); + } + + /** + * Chunks needing recovery: leases expired mid-work, or non-terminal states + * left behind by a crashed pod (when includeAll, e.g. single-pod startup). + */ + async findRecoverable(runId: string, collection: string, includeAll: boolean): Promise { + const nonTerminal: ChunkStatus[] = ['in_progress', 'written', 'attaching']; + const filter = includeAll + ? { run_id: runId, collection, status: { $in: nonTerminal } } + : { run_id: runId, collection, status: { $in: nonTerminal }, lease_until: { $lt: new Date() } }; + return this.c().find(filter).toArray(); + } + + async listByStatus(runId: string, collection: string, status: ChunkStatus): Promise { + return this.c().find({ run_id: runId, collection, status }).toArray(); + } + + /** Status → count map for progress reporting. */ + async statusCounts(runId: string, collection?: string): Promise> { + const match: Record = { run_id: runId }; + if (collection) match.collection = collection; + const rows = await this.c() + .aggregate<{ _id: string; n: number }>([ + { $match: match }, + { $group: { _id: '$status', n: { $sum: 1 } } }, + ]) + .toArray(); + return Object.fromEntries(rows.map((r) => [r._id, r.n])); + } + + /** + * Poison-pill quarantine: replace a chunk that keeps crashing the process + * with `parts` fresh sub-chunks over its cd span. Repeated splitting + * converges on a tiny window around the poison document. The original + * chunk becomes `superseded` (terminal). + */ + async splitChunk(chunk: ChunkDoc, parts: number): Promise { + const maxDoc = await this.c() + .find({ run_id: chunk.run_id, collection: chunk.collection }) + .sort({ idx: -1 }).limit(1).project({ idx: 1 }).toArray(); + const baseIdx = (maxDoc[0]?.idx ?? 0) + 1; + + const span = chunk.upper_cd - chunk.lower_cd; + const now = new Date(); + const subs: ChunkDoc[] = []; + for (let i = 0; i < parts; i++) { + const lo = chunk.lower_cd + Math.floor((span * i) / parts); + const hi = i === parts - 1 ? chunk.upper_cd : chunk.lower_cd + Math.floor((span * (i + 1)) / parts); + if (hi <= lo) continue; + subs.push({ + _id: `${chunk.run_id}:${chunk.collection}:${baseIdx + i}`, + run_id: chunk.run_id, + collection: chunk.collection, + scope_a: chunk.scope_a ?? null, + scope_e: chunk.scope_e ?? null, + scope_n: chunk.scope_n ?? null, + idx: baseIdx + i, + lower_cd: lo, + upper_cd: hi, + status: 'pending', + pod_id: null, + lease_until: null, + staging_table: null, + docs_read: 0, + docs_skipped: 0, + rows_expected: 0, + partitions: [], + attached: [], + attach_method: null, + attempts: 0, + last_error: null, + transform_version: chunk.transform_version, + updated_at: now, + }); + } + await this.c().insertMany(subs, { ordered: false }); + await this.transition(chunk._id, ['in_progress', 'failed'], 'superseded', { + last_error: `split into ${subs.length} sub-chunks (idx ${baseIdx}..${baseIdx + subs.length - 1}) after repeated crashes`, + }); + return subs.length; + } + + /** All chunks of a run (dashboard feed) — trimmed projection, idx order. */ + async listAll(runId: string): Promise>> { + return this.c() + .find( + { run_id: runId }, + { projection: { collection: 1, scope_a: 1, scope_e: 1, scope_n: 1, idx: 1, status: 1, lower_cd: 1, upper_cd: 1, + docs_read: 1, docs_skipped: 1, rows_expected: 1, attempts: 1, last_error: 1, pod_id: 1, updated_at: 1 } }, + ) + .sort({ collection: 1, idx: 1 }) + .toArray() as never; + } + + /** Non-terminal REGULAR (non-sentinel) chunks — gates the null-cd sweep. */ + async countRegularNonTerminal(runId: string, collection: string): Promise { + return this.c().countDocuments({ + run_id: runId, + collection, + lower_cd: { $gte: 0 }, + status: { $in: ['pending', 'in_progress', 'written', 'attaching'] }, + }); + } + + /** The null-cd sentinel chunk of a collection, if any. */ + async getSentinel(runId: string, collection: string): Promise { + return this.c().findOne({ run_id: runId, collection, lower_cd: -1, upper_cd: 0 }); + } + + /** Per-pod activity summary (Pods panel): who did what, who is alive. */ + async podActivity(runId: string): Promise> { + const rows = await this.c() + .aggregate<{ _id: string; done: number; active: number; lastSeen: Date }>([ + { $match: { run_id: runId, pod_id: { $ne: null } } }, + { $group: { + _id: '$pod_id', + done: { $sum: { $cond: [{ $eq: ['$status', 'done'] }, 1, 0] } }, + active: { $sum: { $cond: [{ $in: ['$status', ['in_progress', 'written', 'attaching']] }, 1, 0] } }, + lastSeen: { $max: '$updated_at' }, + } }, + { $sort: { done: -1 } }, + ]) + .toArray(); + return rows.map((r) => ({ pod: r._id, done: r.done, active: r.active, lastSeen: r.lastSeen ?? null })); + } + + /** Sum of expected rows for done chunks — used by full re-verification. */ + async expectedRows(runId: string, collection: string): Promise { + const rows = await this.c() + .aggregate<{ total: number }>([ + { $match: { run_id: runId, collection, status: 'done' } }, + { $group: { _id: null, total: { $sum: '$rows_expected' } } }, + ]) + .toArray(); + return rows[0]?.total ?? 0; + } +} diff --git a/src/state/manifest-store.ts b/src/state/manifest-store.ts deleted file mode 100644 index 5907ecc..0000000 --- a/src/state/manifest-store.ts +++ /dev/null @@ -1,610 +0,0 @@ -import { MongoClient, type Collection, type Db } from "mongodb"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export type RunStatus = "active" | "completed" | "failed" | "paused" | "stopped"; - -export interface RunSummary { - finished_at: string; - duration_ms: number; - total_docs_read: number; - total_rows_inserted: number; - total_docs_skipped: number; - avg_docs_per_second: number; - avg_rows_per_second: number; - total_batches: number; - batches_done: number; - batches_failed: number; - batches_skipped_empty: number; - skip_reasons: Record; - total_errors: number; - failed_batch_seqs: number[]; - digest_mismatches: number; - estimated_duplicate_rows: number; - coverage_pct: number; -} - -export type BatchStatus = - | "prepared" - | "inflight" - | "done" - | "failed" - | "skipped_empty"; - -export interface Run { - run_id: string; - status: RunStatus; - source_ns: string; - target_table: string; - upper_bound_cursor: string; - last_committed_cursor: string | null; - transform_version: string; - created_at: string; - updated_at: string; - summary: RunSummary | null; - phase?: "cursor" | "null_cd"; - null_cd_upper_bound?: string; -} - -export interface Batch { - run_id: string; - batch_seq: number; - lower_exclusive_cursor: string; - upper_inclusive_cursor: string; - source_docs_read: number; - docs_skipped: number; - rows_to_insert: number; - payload_digest: string; - insert_dedup_token: string; - query_id: string; - status: BatchStatus; - retry_count: number; - last_error: string | null; - started_at: string | null; - finished_at: string | null; - phase?: "cursor" | "null_cd"; - error_history?: CompactError[]; - digest_match?: boolean | null; -} - -export interface SkipSample { - run_id: string; - batch_seq: number; - doc_id: string; - reason: string; - captured_at: string; -} - -export interface CompactError { - attempt: number; - error: string; - timestamp: string; -} - -export interface EventRecord { - run_id: string; - event_type: string; - message: string; - metadata: Record | null; - created_at: string; -} - -export interface BatchSeqRange { - min: number; // inclusive - max: number; // exclusive -} - -export interface GetBatchesOptions { - status?: BatchStatus; - limit?: number; - batchSeqRange?: BatchSeqRange; -} - -// --------------------------------------------------------------------------- -// ManifestStore (MongoDB) -// --------------------------------------------------------------------------- - -interface Collections { - runs: Collection; - batches: Collection; - skipSamples: Collection; - events: Collection; -} - -export class ManifestStore { - private client: MongoClient; - private readonly dbName: string; - private collections: Collections | null = null; - private _lastWriteLatencyMs = 0; - - constructor(uri: string, dbName: string) { - this.dbName = dbName; - this.client = new MongoClient(uri, { - writeConcern: { w: "majority", journal: true }, - }); - } - - async connect(): Promise { - await this.client.connect(); - const db = this.client.db(this.dbName); - - this.collections = { - runs: db.collection("mig_runs"), - batches: db.collection("mig_batches"), - skipSamples: db.collection("mig_skip_samples"), - events: db.collection("mig_events"), - }; - - // Ensure indexes - await this.collections.runs.createIndex({ run_id: 1 }, { unique: true }); - await this.collections.runs.createIndex({ status: 1 }); - await this.collections.batches.createIndex({ run_id: 1, batch_seq: 1 }, { unique: true }); - await this.collections.batches.createIndex({ run_id: 1, status: 1 }); - await this.collections.skipSamples.createIndex({ run_id: 1, batch_seq: 1 }); - await this.collections.events.createIndex({ run_id: 1, created_at: 1 }); - } - - private ensureConnected(): Collections { - if (!this.collections) { - throw new Error("ManifestStore is not connected. Call connect() first."); - } - return this.collections; - } - - // ----------------------------------------------------------------------- - // Runs - // ----------------------------------------------------------------------- - - async createRun(run: Omit & { - last_committed_cursor?: string | null; - updated_at?: string; - }): Promise { - const { runs } = this.ensureConnected(); - const now = new Date().toISOString(); - await runs.insertOne({ - ...run, - last_committed_cursor: run.last_committed_cursor ?? null, - updated_at: run.updated_at ?? now, - summary: null, - } as Run); - } - - async getRun(runId: string): Promise { - const { runs } = this.ensureConnected(); - const doc = await runs.findOne({ run_id: runId }, { projection: { _id: 0 } }); - return doc ?? undefined; - } - - async listRuns(opts: { - status?: RunStatus; - limit?: number; - offset?: number; - } = {}): Promise<{ runs: Run[]; total: number }> { - const { runs } = this.ensureConnected(); - const filter: { status?: RunStatus } = {}; - if (opts.status !== undefined) { - filter.status = opts.status; - } - const limit = opts.limit ?? 20; - const offset = opts.offset ?? 0; - - const [docs, total] = await Promise.all([ - runs.find(filter, { projection: { _id: 0 } }) - .sort({ updated_at: -1 }) - .skip(offset) - .limit(limit) - .toArray() as Promise, - runs.countDocuments(filter), - ]); - - return { runs: docs, total }; - } - - async getActiveRun(sourceNs?: string, targetTable?: string): Promise { - const { runs } = this.ensureConnected(); - const filter: Record = { status: "active" }; - if (sourceNs) filter.source_ns = sourceNs; - if (targetTable) filter.target_table = targetTable; - const doc = await runs.findOne(filter, { projection: { _id: 0 } }); - return doc ?? undefined; - } - - async getResumableRun( - sourceNs: string, - targetTable: string, - transformVersion: string, - ): Promise { - const { runs } = this.ensureConnected(); - const doc = await runs.findOne( - { - status: { $in: ["paused", "stopped"] as RunStatus[] }, - source_ns: sourceNs, - target_table: targetTable, - transform_version: transformVersion, - }, - { sort: { updated_at: -1 }, projection: { _id: 0 } }, - ); - return doc ?? undefined; - } - - async updateRunLastCommittedCursor(runId: string, lastCommittedCursor: string): Promise { - const { runs } = this.ensureConnected(); - const now = new Date().toISOString(); - const start = performance.now(); - await runs.updateOne( - { run_id: runId }, - { $set: { last_committed_cursor: lastCommittedCursor, updated_at: now } }, - ); - this._lastWriteLatencyMs = Math.round(performance.now() - start); - } - - async updateRunStatus(runId: string, status: RunStatus): Promise { - const { runs } = this.ensureConnected(); - const now = new Date().toISOString(); - await runs.updateOne( - { run_id: runId }, - { $set: { status, updated_at: now } }, - ); - } - - async writeSummary(runId: string, status: RunStatus, summary: RunSummary): Promise { - const { runs } = this.ensureConnected(); - const now = new Date().toISOString(); - await runs.updateOne( - { run_id: runId }, - { $set: { status, summary, updated_at: now } }, - ); - } - - async updateRunPhase( - runId: string, - phase: "cursor" | "null_cd", - nullCdUpperBound?: string, - ): Promise { - const { runs, events } = this.ensureConnected(); - const now = new Date().toISOString(); - const $set: Record = { - phase, - last_committed_cursor: null, - updated_at: now, - }; - if (nullCdUpperBound !== undefined) { - $set.null_cd_upper_bound = nullCdUpperBound; - } - await runs.updateOne( - { run_id: runId }, - { $set }, - ); - await events.insertOne({ - run_id: runId, - event_type: "null_cd_sweep_started", - message: `Run transitioned to ${phase} phase`, - metadata: { phase, null_cd_upper_bound: nullCdUpperBound ?? null }, - created_at: now, - }); - } - - // ----------------------------------------------------------------------- - // Batches - // ----------------------------------------------------------------------- - - async insertBatch(batch: Omit & { - error_history?: CompactError[]; - digest_match?: boolean | null; - }): Promise { - const { batches } = this.ensureConnected(); - await batches.insertOne({ - ...batch, - error_history: batch.error_history ?? [], - digest_match: batch.digest_match ?? null, - } as Batch); - } - - async updateBatchStatus( - runId: string, - batchSeq: number, - status: BatchStatus, - error?: string, - ): Promise { - const { batches } = this.ensureConnected(); - const now = new Date().toISOString(); - const $set: Record = { status }; - - // Only set finished_at for terminal statuses - if (status === "done" || status === "failed" || status === "skipped_empty") { - $set.finished_at = now; - } - - if (error !== undefined) { - $set.last_error = error; - } - - await batches.updateOne( - { run_id: runId, batch_seq: batchSeq }, - error !== undefined - ? { $set, $inc: { retry_count: 1 } } - : { $set }, - ); - } - - /** - * Mark a batch as done and advance the run's last_committed_cursor. - * Two-step write (not transactional) — the resume path's `getLastDoneBatch` - * fallback compensates if the process crashes between the two writes. - */ - async completeBatch( - runId: string, - batchSeq: number, - lastCommittedCursor: string, - ): Promise { - const { batches, runs } = this.ensureConnected(); - const now = new Date().toISOString(); - const start = performance.now(); - - await batches.updateOne( - { run_id: runId, batch_seq: batchSeq }, - { $set: { status: "done" as BatchStatus, finished_at: now } }, - ); - - await runs.updateOne( - { run_id: runId }, - { $set: { last_committed_cursor: lastCommittedCursor, updated_at: now } }, - ); - - this._lastWriteLatencyMs = Math.round(performance.now() - start); - } - - /** - * Insert a completed batch and advance the run cursor in one go. - * Skips the intermediate "prepared"/"inflight" states — the batch record - * is written only after successful ClickHouse insertion. - */ - async insertCompletedBatch( - batch: Omit, - lastCommittedCursor: string, - ): Promise { - const { batches, runs } = this.ensureConnected(); - const now = new Date().toISOString(); - const start = performance.now(); - - const doc = { - ...batch, - status: "done" as BatchStatus, - finished_at: now, - error_history: [], - digest_match: null, - }; - - await batches.insertOne(doc as Batch); - - await runs.updateOne( - { run_id: batch.run_id }, - { $set: { last_committed_cursor: lastCommittedCursor, updated_at: now } }, - ); - - this._lastWriteLatencyMs = Math.round(performance.now() - start); - } - - /** - * Bulk insert completed batch records (used by async batch writer). - */ - async bulkInsertBatches(batchDocs: Batch[]): Promise { - if (batchDocs.length === 0) return; - const { batches } = this.ensureConnected(); - const start = performance.now(); - // Use upsert to handle duplicates — if (run_id, batch_seq) exists, update it - const ops = batchDocs.map(doc => ({ - updateOne: { - filter: { run_id: doc.run_id, batch_seq: doc.batch_seq }, - update: { $set: doc }, - upsert: true, - }, - })); - await batches.bulkWrite(ops, { ordered: false }); - this._lastWriteLatencyMs = Math.round(performance.now() - start); - } - - /** - * Advance the run cursor to the latest position (used by async batch writer). - */ - async advanceCursor(runId: string, cursor: string): Promise { - const { runs } = this.ensureConnected(); - const now = new Date().toISOString(); - await runs.updateOne( - { run_id: runId }, - { $set: { last_committed_cursor: cursor, updated_at: now } }, - ); - } - - async updateBatchDigestMatch( - runId: string, - batchSeq: number, - digestMatch: boolean, - ): Promise { - const { batches } = this.ensureConnected(); - await batches.updateOne( - { run_id: runId, batch_seq: batchSeq }, - { $set: { digest_match: digestMatch } }, - ); - } - - async pushBatchError( - runId: string, - batchSeq: number, - error: CompactError, - ): Promise { - const { batches } = this.ensureConnected(); - await batches.updateOne( - { run_id: runId, batch_seq: batchSeq }, - { - $push: { error_history: { $each: [error], $slice: -50 } as any }, - $set: { last_error: error.error }, - }, - ); - } - - async getLastBatch(runId: string, batchSeqRange?: BatchSeqRange): Promise { - const { batches } = this.ensureConnected(); - const filter: Record = { run_id: runId }; - if (batchSeqRange) { - filter.batch_seq = { $gte: batchSeqRange.min, $lt: batchSeqRange.max }; - } - const doc = await batches.findOne( - filter, - { sort: { batch_seq: -1 }, projection: { _id: 0 } }, - ); - return doc ?? null; - } - - async getLastDoneBatch(runId: string, batchSeqRange?: BatchSeqRange): Promise { - const { batches } = this.ensureConnected(); - const filter: Record = { run_id: runId, status: "done" }; - if (batchSeqRange) { - filter.batch_seq = { $gte: batchSeqRange.min, $lt: batchSeqRange.max }; - } - const doc = await batches.findOne( - filter, - { sort: { batch_seq: -1 }, projection: { _id: 0 } }, - ); - return doc ?? null; - } - - async existsCompletedRun(sourceNs: string, targetTable: string): Promise { - const { runs } = this.ensureConnected(); - const count = await runs.countDocuments( - { status: "completed", source_ns: sourceNs, target_table: targetTable }, - { limit: 1 }, - ); - return count > 0; - } - - /** Get the most recent completed run for a given sourceNs/targetTable. */ - async getCompletedRun(sourceNs: string, targetTable: string): Promise { - const { runs } = this.ensureConnected(); - const doc = await runs.findOne( - { status: "completed", source_ns: sourceNs, target_table: targetTable }, - { sort: { created_at: -1 }, projection: { _id: 0 } }, - ); - return (doc as Run | null) ?? null; - } - - /** Sum source_docs_read and rows_to_insert from all done batches for a run (optionally scoped to a batchSeq range). */ - async sumCompletedBatchStats(runId: string, batchSeqRange?: BatchSeqRange): Promise<{ docsRead: number; rowsInserted: number }> { - const { batches } = this.ensureConnected(); - const match: Record = { run_id: runId, status: "done" }; - if (batchSeqRange) { - match.batch_seq = { $gte: batchSeqRange.min, $lt: batchSeqRange.max }; - } - const pipeline = [ - { $match: match }, - { $group: { _id: null, docsRead: { $sum: "$source_docs_read" }, rowsInserted: { $sum: "$rows_to_insert" } } }, - ]; - const result = await batches.aggregate(pipeline).toArray(); - if (result.length === 0) return { docsRead: 0, rowsInserted: 0 }; - return { docsRead: result[0].docsRead ?? 0, rowsInserted: result[0].rowsInserted ?? 0 }; - } - - async getBatches(runId: string, opts: GetBatchesOptions = {}): Promise { - const { batches } = this.ensureConnected(); - const filter: Record = { run_id: runId }; - if (opts.status !== undefined) { - filter.status = opts.status; - } - if (opts.batchSeqRange) { - filter.batch_seq = { $gte: opts.batchSeqRange.min, $lt: opts.batchSeqRange.max }; - } - let cursor = batches.find(filter, { projection: { _id: 0 } }).sort({ batch_seq: 1 }); - if (opts.limit !== undefined) { - cursor = cursor.limit(opts.limit); - } - return cursor.toArray() as Promise; - } - - async getFailedBatches(runId: string): Promise { - const { batches } = this.ensureConnected(); - return batches.find( - { run_id: runId, status: "failed" }, - { projection: { _id: 0 } }, - ).sort({ batch_seq: 1 }).toArray() as Promise; - } - - // ----------------------------------------------------------------------- - // Skip samples - // ----------------------------------------------------------------------- - - async insertSkipSample(sample: SkipSample): Promise { - const { skipSamples } = this.ensureConnected(); - await skipSamples.insertOne({ ...sample }); - } - - async insertSkipSamples(samples: SkipSample[]): Promise { - if (samples.length === 0) return; - const { skipSamples } = this.ensureConnected(); - await skipSamples.insertMany(samples); - } - - /** - * Delete all batch records and skip samples for a given run. - * Used when starting a fresh (non-resume) run to prevent stale data. - */ - async deleteRunData(runId: string): Promise { - const { batches, skipSamples, events } = this.ensureConnected(); - const [bResult, sResult, eResult] = await Promise.all([ - batches.deleteMany({ run_id: runId }), - skipSamples.deleteMany({ run_id: runId }), - events.deleteMany({ run_id: runId }), - ]); - return (bResult.deletedCount ?? 0) + (sResult.deletedCount ?? 0) + (eResult.deletedCount ?? 0); - } - - // ----------------------------------------------------------------------- - // Events - // ----------------------------------------------------------------------- - - async insertEvent(event: EventRecord): Promise { - const { events } = this.ensureConnected(); - await events.insertOne({ ...event }); - } - - async countEvents(runId: string, eventType?: string): Promise { - const { events } = this.ensureConnected(); - const filter: { run_id: string; event_type?: string } = { run_id: runId }; - if (eventType !== undefined) { - filter.event_type = eventType; - } - return events.countDocuments(filter); - } - - // ----------------------------------------------------------------------- - // Telemetry - // ----------------------------------------------------------------------- - - getWriteLatency(): number { - return this._lastWriteLatencyMs; - } - - // ----------------------------------------------------------------------- - // Health - // ----------------------------------------------------------------------- - - async isWritable(): Promise { - try { - const db = this.client.db(this.dbName); - await db.command({ ping: 1 }); - return true; - } catch { - return false; - } - } - - // ----------------------------------------------------------------------- - // Lifecycle - // ----------------------------------------------------------------------- - - async close(): Promise { - await this.client.close(); - this.collections = null; - } -} diff --git a/src/state/redis-hot-state.ts b/src/state/redis-hot-state.ts deleted file mode 100644 index 3cc8afc..0000000 --- a/src/state/redis-hot-state.ts +++ /dev/null @@ -1,597 +0,0 @@ -import { Redis } from "ioredis"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface RunState { - runId: string; - status: string; - sourceNs: string; - targetTable: string; - upperBoundCursor: string; // serialized Cursor JSON - lastCommittedCursor: string | null; // serialized Cursor JSON - transformVersion: string; - totalBatches: number; - completedBatches: number; - startedAt: string; -} - -export interface RunStats { - docsRead: number; - docsSkipped: number; - rowsInserted: number; - batchesDone: number; - batchesFailed: number; - batchesInflight: number; - elapsedMs: number; - docsPerSecond: number; - lastBatchSeq: number; - lastBatchFinishedAt: string | null; -} - -export interface CommandFlags { - pause?: boolean; - abort?: boolean; - resume?: boolean; - gc?: boolean; - stopAfterBatch?: boolean; - skipBatch?: number | null; -} - -export interface RecentError { - batchSeq: number; - error: string; - timestamp: string; - retryCount: number; -} - -export interface TimelineSnapshot { - timestamp: string; - batch_seq: number; - docs_read: number; - rows_inserted: number; - docs_skipped: number; - docs_per_second: number; - rows_per_second: number; - skip_reasons: Record; - digest_mismatches: number; - estimated_duplicate_rows: number; - batches_failed: number; - heap_used_mb: number; - rss_mb: number; -} - -export interface VerboseError { - attempt: number; - error: string; - stack: string | null; - timestamp: string; - context: Record; -} - -export type BatchPhase = "READING" | "TRANSFORMING" | "WRITING" | "COMMITTING"; - -export interface LiveBatchData { - collection: string; - podId: string; - batchSeq: number; - phase: BatchPhase; - docsRead: number; - rowsToInsert: number; - startedAt: number; - rangeIdx?: number; -} - -export interface RangeLiveStats { - idx: number; - status: string; - podId: string; - docsRead: number; - rowsInserted: number; - batchesDone: number; - docsPerSecond: number; - startedAt: number; -} - -// --------------------------------------------------------------------------- -// Redis key helpers -// --------------------------------------------------------------------------- - -function k(prefix: string, ...parts: string[]): string { - return [prefix, ...parts].join(":"); -} - -// --------------------------------------------------------------------------- -// RedisHotState -// --------------------------------------------------------------------------- - -const RECENT_ERRORS_CAP = 100; - -export class RedisHotState { - private readonly redis: Redis; - private readonly prefix: string; - private readonly ownsConnection: boolean; - - private _lastStateWriteMs: number = 0; - private _lastError: string | null = null; - - constructor(redisUrl: string, prefix: string) { - this.redis = new Redis(redisUrl, { - maxRetriesPerRequest: 3, - enableReadyCheck: true, - lazyConnect: true, - connectTimeout: 10_000, - commandTimeout: 5_000, - retryStrategy(times: number) { - return Math.min(times * 200, 3_000); - }, - }); - this.prefix = prefix; - this.ownsConnection = true; - } - - /** - * Create a RedisHotState that reuses an existing Redis connection - * with a different key prefix. Useful for per-collection state isolation. - * The returned instance does NOT own the connection and will not close it. - */ - static fromExistingConnection(redis: Redis, prefix: string): RedisHotState { - const instance = Object.create(RedisHotState.prototype) as RedisHotState; - Object.defineProperty(instance, 'redis', { value: redis, writable: false }); - Object.defineProperty(instance, 'prefix', { value: prefix, writable: false }); - Object.defineProperty(instance, 'ownsConnection', { value: false, writable: false }); - Object.defineProperty(instance, '_lastStateWriteMs', { value: 0, writable: true }); - Object.defineProperty(instance, '_lastError', { value: null, writable: true }); - return instance; - } - - /** Expose the underlying Redis client for creating derived instances. */ - getRedisClient(): Redis { - return this.redis; - } - - async connect(): Promise { - await this.redis.connect(); - } - - // ----------------------------------------------------------------------- - // Active run - // ----------------------------------------------------------------------- - - async setActiveRun(runId: string): Promise { - await this.redis.set(k(this.prefix, "active_run"), runId); - } - - async getActiveRun(): Promise { - return this.redis.get(k(this.prefix, "active_run")); - } - - // ----------------------------------------------------------------------- - // Run state (JSON blob) - // ----------------------------------------------------------------------- - - async setState(runId: string, state: RunState): Promise { - try { - const start = performance.now(); - const key = k(this.prefix, "run", runId, "state"); - await this.redis.set(key, JSON.stringify(state)); - this._lastStateWriteMs = Math.round(performance.now() - start); - } catch (err) { - this._lastError = err instanceof Error ? err.message : String(err); - throw err; - } - } - - async getState(runId: string): Promise { - const raw = await this.redis.get( - k(this.prefix, "run", runId, "state"), - ); - if (!raw) return null; - return JSON.parse(raw) as RunState; - } - - // ----------------------------------------------------------------------- - // Batch completion bitmap (SETBIT / GETBIT) - // ----------------------------------------------------------------------- - - async markBatchDone(runId: string, batchSeq: number): Promise { - await this.redis.setbit( - k(this.prefix, "run", runId, "done_bitmap"), - batchSeq, - 1, - ); - } - - /** - * Atomically commit a batch: set cursor + mark bitmap in a single MULTI/EXEC. - * Eliminates the crash window between separate SET and SETBIT calls. - */ - async commitBatch(runId: string, cursor: string, batchSeq: number): Promise { - const pipeline = this.redis.multi(); - pipeline.set(k(this.prefix, "run", runId, "cursor"), cursor); - pipeline.setbit(k(this.prefix, "run", runId, "done_bitmap"), batchSeq, 1); - await pipeline.exec(); - } - - async getBitmapCount(runId: string): Promise { - return this.redis.bitcount( - k(this.prefix, "run", runId, "done_bitmap"), - ); - } - - // ----------------------------------------------------------------------- - // Stats (JSON) - // ----------------------------------------------------------------------- - - async updateStats(runId: string, stats: RunStats): Promise { - const key = k(this.prefix, "run", runId, "stats", "latest"); - await this.redis.set(key, JSON.stringify(stats)); - } - - async getStats(runId: string): Promise { - const raw = await this.redis.get( - k(this.prefix, "run", runId, "stats", "latest"), - ); - if (!raw) return null; - return JSON.parse(raw) as RunStats; - } - - // ----------------------------------------------------------------------- - // Command flags (JSON) - // ----------------------------------------------------------------------- - - async setCommand( - runId: string, - command: keyof CommandFlags, - value: boolean | number | null, - ): Promise { - const key = k(this.prefix, "run", runId, "commands"); - await this.redis.hset(key, command, JSON.stringify(value)); - } - - async getCommands(runId: string): Promise { - const raw = await this.redis.hgetall( - k(this.prefix, "run", runId, "commands"), - ); - if (!raw || Object.keys(raw).length === 0) return {}; - const result: CommandFlags = {}; - const KNOWN_COMMANDS = new Set(['pause', 'abort', 'resume', 'gc', 'stopAfterBatch', 'skipBatch']); - for (const [key, val] of Object.entries(raw)) { - if (KNOWN_COMMANDS.has(key)) { - (result as Record)[key] = JSON.parse(val as string); - } - } - return result; - } - - // ----------------------------------------------------------------------- - // Recent errors (capped list) - // ----------------------------------------------------------------------- - - async pushError(runId: string, error: RecentError): Promise { - const key = k(this.prefix, "run", runId, "recent_errors"); - const pipeline = this.redis.multi(); - pipeline.lpush(key, JSON.stringify(error)); - pipeline.ltrim(key, 0, RECENT_ERRORS_CAP - 1); - await pipeline.exec(); - } - - async getRecentErrors(runId: string): Promise { - const raw = await this.redis.lrange( - k(this.prefix, "run", runId, "recent_errors"), - 0, - -1, - ); - return raw.map((entry: string) => JSON.parse(entry) as RecentError); - } - - // ----------------------------------------------------------------------- - // Timeline - // ----------------------------------------------------------------------- - - async pushTimelineSnapshot(runId: string, snapshot: TimelineSnapshot): Promise { - const key = k(this.prefix, "run", runId, "timeline"); - const pipeline = this.redis.multi(); - pipeline.rpush(key, JSON.stringify(snapshot)); - pipeline.ltrim(key, -1000, -1); - await pipeline.exec(); - } - - async getTimeline(runId: string): Promise { - const raw = await this.redis.lrange( - k(this.prefix, "run", runId, "timeline"), - 0, - -1, - ); - return raw.map((entry: string) => JSON.parse(entry) as TimelineSnapshot); - } - - // ----------------------------------------------------------------------- - // Throughput sliding window (5-min window for accurate real-time throughput) - // ----------------------------------------------------------------------- - - async pushThroughputSample(runId: string, sample: { ts: number; docsRead: number }): Promise { - const key = k(this.prefix, "run", runId, "throughput_window"); - const pipeline = this.redis.multi(); - pipeline.lpush(key, JSON.stringify(sample)); - pipeline.ltrim(key, 0, 59); - pipeline.expire(key, 600); - await pipeline.exec(); - } - - async getThroughputWindow(runId: string): Promise> { - const key = k(this.prefix, "run", runId, "throughput_window"); - const raw = await this.redis.lrange(key, 0, -1); - return raw.map(r => JSON.parse(r) as { ts: number; docsRead: number }); - } - - // ----------------------------------------------------------------------- - // Verbose errors - // ----------------------------------------------------------------------- - - async pushVerboseError( - runId: string, - batchSeq: number, - error: VerboseError, - ): Promise { - const key = k(this.prefix, "run", runId, "batch", String(batchSeq), "errors"); - const pipeline = this.redis.multi(); - pipeline.rpush(key, JSON.stringify(error)); - pipeline.ltrim(key, -20, -1); - await pipeline.exec(); - } - - async getVerboseErrors( - runId: string, - batchSeq: number, - ): Promise { - const raw = await this.redis.lrange( - k(this.prefix, "run", runId, "batch", String(batchSeq), "errors"), - 0, - -1, - ); - return raw.map((entry: string) => JSON.parse(entry) as VerboseError); - } - - // ----------------------------------------------------------------------- - // Last committed cursor (hot-path authority for async writes) - // ----------------------------------------------------------------------- - - async setLastCommittedCursor(runId: string, cursor: string): Promise { - await this.redis.set(k(this.prefix, "run", runId, "cursor"), cursor); - } - - async getLastCommittedCursor(runId: string): Promise { - return this.redis.get(k(this.prefix, "run", runId, "cursor")); - } - - // ----------------------------------------------------------------------- - // Live batch tracking (phase visibility for dashboard) - // ----------------------------------------------------------------------- - - async setLiveBatch(collection: string, data: LiveBatchData): Promise { - const key = k(this.prefix, "liveBatch", collection); - await this.redis.set(key, JSON.stringify(data), "EX", 30); - } - - async getLiveBatch(collection: string): Promise { - const raw = await this.redis.get(k(this.prefix, "liveBatch", collection)); - if (!raw) return null; - return JSON.parse(raw) as LiveBatchData; - } - - async clearLiveBatch(collection: string): Promise { - await this.redis.del(k(this.prefix, "liveBatch", collection)); - } - - async getAllLiveBatches(): Promise { - const keys = await this.scanKeys(k(this.prefix, "liveBatch", "*")); - if (keys.length === 0) return []; - const pipeline = this.redis.pipeline(); - for (const key of keys) pipeline.get(key); - const results = await pipeline.exec(); - const batches: LiveBatchData[] = []; - if (results) { - for (const [err, val] of results) { - if (!err && val && typeof val === "string") { - batches.push(JSON.parse(val) as LiveBatchData); - } - } - } - return batches; - } - - // ----------------------------------------------------------------------- - // Per-range live stats (range-parallel dashboard visibility) - // ----------------------------------------------------------------------- - - async setRangeLiveStats(collection: string, rangeIdx: number, stats: RangeLiveStats): Promise { - const key = k(this.prefix, "rangeLive", collection, String(rangeIdx)); - await this.redis.set(key, JSON.stringify(stats), "EX", 60); - } - - async getRangeLiveStats(collection: string): Promise { - const keys = await this.scanKeys(k(this.prefix, "rangeLive", collection, "*")); - if (keys.length === 0) return []; - const pipeline = this.redis.pipeline(); - for (const key of keys) pipeline.get(key); - const results = await pipeline.exec(); - const stats: RangeLiveStats[] = []; - if (results) { - for (const [err, val] of results) { - if (!err && val && typeof val === "string") { - stats.push(JSON.parse(val) as RangeLiveStats); - } - } - } - return stats; - } - - async clearRangeLiveStats(collection: string, rangeIdx: number): Promise { - await this.redis.del(k(this.prefix, "rangeLive", collection, String(rangeIdx))); - } - - // ----------------------------------------------------------------------- - // Persistent collection estimates (no TTL) - // ----------------------------------------------------------------------- - - /** Bulk-write estimated doc counts for all collections (MSET, single round-trip). */ - async setCollectionEstimates(estimates: Map): Promise { - if (estimates.size === 0) return; - const args: string[] = []; - for (const [collection, count] of estimates) { - args.push(k(this.prefix, "est", collection), String(count)); - } - await this.redis.mset(...args); - } - - /** Read all persisted collection estimates (SCAN + MGET). */ - async getAllCollectionEstimates(): Promise> { - const pattern = k(this.prefix, "est", "*"); - const keys = await this.scanKeys(pattern); - if (keys.length === 0) return new Map(); - - const values = await this.redis.mget(...keys); - const prefix = k(this.prefix, "est") + ":"; - const result = new Map(); - for (let i = 0; i < keys.length; i++) { - const val = values[i]; - if (val !== null) { - const collection = keys[i].slice(prefix.length); - result.set(collection, Number(val)); - } - } - return result; - } - - // ----------------------------------------------------------------------- - // Persistent collection completion aggregates (no TTL) - // ----------------------------------------------------------------------- - - /** Write completion aggregate for one collection per pod (SET, no TTL). */ - async setCollectionCompleted( - collection: string, - podId: string, - data: { docsRead: number; rowsInserted: number; runId: string; completedAt: string }, - ): Promise { - const key = k(this.prefix, "completed", collection, podId); - await this.redis.set(key, JSON.stringify(data)); - } - - /** Read all completion aggregates (SCAN + MGET), summing per-pod entries per collection. */ - async getAllCollectionCompleted(): Promise> { - const pattern = k(this.prefix, "completed", "*"); - const keys = await this.scanKeys(pattern); - if (keys.length === 0) return new Map(); - - const values = await this.redis.mget(...keys); - const prefix = k(this.prefix, "completed") + ":"; - const result = new Map(); - for (let i = 0; i < keys.length; i++) { - const val = values[i]; - if (val !== null) { - try { - const parsed = JSON.parse(val) as { docsRead: number; rowsInserted: number; runId: string; completedAt: string }; - // Key format: {prefix}:completed:{collection}:{podId} - // Extract collection name by stripping prefix and last :podId segment - const suffix = keys[i].slice(prefix.length); - const lastColon = suffix.lastIndexOf(":"); - const collection = lastColon > 0 ? suffix.slice(0, lastColon) : suffix; - - const existing = result.get(collection); - if (existing) { - existing.docsRead += parsed.docsRead; - existing.rowsInserted += parsed.rowsInserted; - if (parsed.completedAt > existing.completedAt) { - existing.completedAt = parsed.completedAt; - } - } else { - result.set(collection, { ...parsed }); - } - } catch { - // skip malformed - } - } - } - return result; - } - - // ----------------------------------------------------------------------- - // Key management - // ----------------------------------------------------------------------- - - async scanKeys(pattern: string): Promise { - const keys: string[] = []; - const stream = this.redis.scanStream({ match: pattern, count: 100 }); - for await (const batch of stream) { - keys.push(...(batch as string[])); - } - return keys; - } - - async cleanupRun(runId: string): Promise { - const runKeys = [ - k(this.prefix, "run", runId, "state"), - k(this.prefix, "run", runId, "done_bitmap"), - k(this.prefix, "run", runId, "stats", "latest"), - k(this.prefix, "run", runId, "commands"), - k(this.prefix, "run", runId, "recent_errors"), - k(this.prefix, "run", runId, "timeline"), - k(this.prefix, "run", runId, "cursor"), - ]; - - const batchErrorKeys = await this.scanKeys( - k(this.prefix, "run", runId, "batch", "*", "errors"), - ); - // liveBatch:* and rangeLive:* keys have TTLs (30s/60s) and self-expire - const allKeys = [...runKeys, ...batchErrorKeys]; - - const activeRunKey = k(this.prefix, "active_run"); - const luaScript = ` - if redis.call('GET', KEYS[1]) == ARGV[1] then - redis.call('UNLINK', KEYS[1]) - return 1 - end - return 0 - `; - await this.redis.call("EVAL", luaScript, "1", activeRunKey, runId); - - if (allKeys.length === 0) return 0; - return this.redis.unlink(...allKeys); - } - - // ----------------------------------------------------------------------- - // Health - // ----------------------------------------------------------------------- - - async isHealthy(): Promise { - try { - const pong = await this.redis.ping(); - return pong === "PONG"; - } catch { - return false; - } - } - - // ----------------------------------------------------------------------- - // Metrics - // ----------------------------------------------------------------------- - - getMetrics(): { lastStateWriteMs: number; lastError: string | null } { - return { - lastStateWriteMs: this._lastStateWriteMs, - lastError: this._lastError, - }; - } - - // ----------------------------------------------------------------------- - // Lifecycle - // ----------------------------------------------------------------------- - - async close(): Promise { - if (this.ownsConnection) { - await this.redis.quit(); - } - } -} diff --git a/src/target/clickhouse-writer.ts b/src/target/clickhouse-writer.ts deleted file mode 100644 index 81f2c57..0000000 --- a/src/target/clickhouse-writer.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { createClient, type ClickHouseClient } from '@clickhouse/client'; -import type { Logger } from 'pino'; -import type { OutputRow } from '../transform/normalize.ts'; - -export interface ClickHouseWriterConfig { - url: string; - database: string; - table: string; - username: string; - password: string; - queryTimeoutMs: number; - useDedupToken: boolean; -} - -export interface InsertBatchParams { - runId: string; - batchSeq: number; - rows: OutputRow[]; -} - -export interface InsertResult { - insertMs: number; - rowsInserted: number; -} - -export class ClickHouseWriter { - private readonly config: ClickHouseWriterConfig; - private readonly logger: Logger; - private client: ClickHouseClient | null = null; - private _connected: boolean = false; - - constructor(config: ClickHouseWriterConfig, logger: Logger) { - this.config = config; - this.logger = logger.child({ component: 'clickhouse-writer' }); - } - - async connect(): Promise { - this.logger.info({ url: this.config.url, database: this.config.database }, 'connecting to ClickHouse'); - - this.client = createClient({ - url: this.config.url, - database: this.config.database, - username: this.config.username, - password: this.config.password, - compression: { - request: true, - }, - clickhouse_settings: { - date_time_input_format: 'best_effort', - optimize_on_insert: 0, - async_insert: 1, - wait_for_async_insert: 0, - }, - request_timeout: this.config.queryTimeoutMs, - }); - - await this.client.ping(); - this._connected = true; - - this.logger.info('connected to ClickHouse'); - } - - async insertBatch(params: InsertBatchParams): Promise { - if (!this.client) { - throw new Error('ClickHouseWriter is not connected. Call connect() first.'); - } - - const { runId, batchSeq, rows } = params; - const queryId = `mig__${runId}__${batchSeq}`; - const dedupToken = `mig:${runId}:${batchSeq}`; - - this.logger.debug( - { queryId, rowCount: rows.length }, - 'inserting batch', - ); - - const start = performance.now(); - - const clickhouseSettings: Record = {}; - if (this.config.useDedupToken) { - clickhouseSettings.insert_deduplication_token = dedupToken; - } - - await this.client.insert({ - table: this.config.table, - values: rows, - format: 'JSONEachRow', - clickhouse_settings: clickhouseSettings, - query_id: queryId, - }); - - const insertMs = performance.now() - start; - - this.logger.debug( - { queryId, insertMs: Math.round(insertMs), rowsInserted: rows.length }, - 'batch inserted', - ); - - return { - insertMs, - rowsInserted: rows.length, - }; - } - - isConnected(): boolean { - return this._connected; - } - - async close(): Promise { - if (this.client) { - this.logger.info('closing ClickHouse connection'); - await this.client.close(); - this.client = null; - this._connected = false; - } - } -} diff --git a/src/target/staging-manager.ts b/src/target/staging-manager.ts new file mode 100644 index 0000000..372face --- /dev/null +++ b/src/target/staging-manager.ts @@ -0,0 +1,468 @@ +/** + * StagingManager — ClickHouse side of the `ledger` engine. + * + * Owns per-chunk staging tables: create (clone of the live table), insert + * (SYNCHRONOUS — errors surface to the caller, unlike the classic engine's + * fire-and-forget async inserts), count (exact, metadata-served), promote + * (verify-then-ATTACH per partition, with INSERT SELECT fallback), drop. + * + * Also runs the startup dedup canary: insert_deduplication_token only works + * when the target engine supports a dedup window (Replicated* by default; + * plain MergeTree needs non_replicated_deduplication_window > 0). The canary + * measures this instead of assuming. + */ + +import { createClient, type ClickHouseClient } from '@clickhouse/client'; +import type { Logger } from 'pino'; +import type { OutputRow } from '../transform/normalize.ts'; + +export interface StagingManagerConfig { + url: string; + database: string; + table: string; // live target table + username: string; + password: string; + queryTimeoutMs: number; +} + +export class StagingManager { + private client: ClickHouseClient | null = null; + private readonly logger: Logger; + private readonly config: StagingManagerConfig; + public dedupWorks: boolean | null = null; + + constructor(config: StagingManagerConfig, logger: Logger) { + this.config = config; + this.logger = logger.child({ component: 'StagingManager' }); + } + + async connect(): Promise { + this.client = createClient({ + url: this.config.url, + database: this.config.database, + username: this.config.username, + password: this.config.password, + compression: { request: true }, + // Synchronous inserts: an acked insert is parsed, validated, and visible. + clickhouse_settings: { + date_time_input_format: 'best_effort', + optimize_on_insert: 0, + }, + request_timeout: this.config.queryTimeoutMs, + }); + await this.client.ping(); + this.logger.info('StagingManager connected (sync inserts)'); + } + + async close(): Promise { + if (this.client) { + await this.client.close(); + this.client = null; + } + } + + private ch(): ClickHouseClient { + if (!this.client) throw new Error('StagingManager not connected'); + return this.client; + } + + private fq(table: string): string { + return `\`${this.config.database}\`.\`${table}\``; + } + + // ------------------------------------------------------------------------- + // Dedup canary + // ------------------------------------------------------------------------- + + /** + * Empirically verify whether insert_deduplication_token is honored on a + * clone of the live table. Logs loudly either way; the result is exposed + * so operators can see it in /stats. + */ + async runDedupCanary(): Promise { + // Per-process name: concurrent pods each probe their own canary table + // (a shared name races on create/drop and false-flags dedup as inert). + const canary = `${this.config.table}_mig_canary_${process.pid}`; + const row = [{ _id: 'canary', a: 'canary', e: 'canary', n: 'canary', uid: 'canary', did: '', + ts: '2000-01-01 00:00:00.000', up: {}, sg: {}, c: 1, s: 0, dur: 0, cd: '2000-01-01 00:00:00.000' }]; + try { + await this.ch().command({ query: `DROP TABLE IF EXISTS ${this.fq(canary)}` }); + await this.ch().command({ query: `CREATE TABLE ${this.fq(canary)} AS ${this.fq(this.config.table)}` }); + await this.applyDedupWindow(canary); + for (let i = 0; i < 2; i++) { + await this.ch().insert({ + table: canary, + values: row, + format: 'JSONEachRow', + clickhouse_settings: { insert_deduplication_token: 'mig-canary' }, + }); + } + const n = await this.countRows(canary); + this.dedupWorks = n === 1; + } catch (err) { + this.logger.warn({ err: (err as Error).message }, 'Dedup canary failed to run — assuming dedup inert'); + this.dedupWorks = false; + } finally { + await this.ch().command({ query: `DROP TABLE IF EXISTS ${this.fq(canary)}` }).catch(() => {}); + } + if (this.dedupWorks) { + this.logger.info('Dedup canary: insert_deduplication_token WORKS on this target'); + } else { + this.logger.warn( + 'Dedup canary: dedup token is INERT on this target — within-chunk duplicate protection ' + + 'relies on chunk redo semantics only (safe, but ambiguous insert retries may duplicate ' + + 'rows inside a chunk until it is verified)', + ); + } + return this.dedupWorks; + } + + /** Best-effort: give a staging table a dedup window so tokens work on plain MergeTree. */ + private async applyDedupWindow(table: string): Promise { + try { + await this.ch().command({ + query: `ALTER TABLE ${this.fq(table)} MODIFY SETTING non_replicated_deduplication_window = 100`, + }); + } catch { + // Replicated / Shared engines reject or don't need this — token works there natively. + } + } + + // ------------------------------------------------------------------------- + // Dry-run target (Null engine: full parse/type validation, nothing stored) + // ------------------------------------------------------------------------- + + get dryRunTable(): string { + return `${this.config.table}_dryrun`; + } + + async createDryRunTable(): Promise { + await this.ch().command({ query: `DROP TABLE IF EXISTS ${this.fq(this.dryRunTable)}` }); + await this.ch().command({ + query: `CREATE TABLE ${this.fq(this.dryRunTable)} AS ${this.fq(this.config.table)} ENGINE = Null`, + }); + this.logger.info({ table: this.dryRunTable }, 'Dry-run Null-engine table created'); + } + + /** Direct insert into the live table (DLQ replay path). */ + async insertIntoLive(rows: OutputRow[], dedupToken: string): Promise { + await this.ch().insert({ + table: this.config.table, + values: rows, + format: 'JSONEachRow', + clickhouse_settings: { insert_deduplication_token: dedupToken }, + }); + } + + // ------------------------------------------------------------------------- + // Staging table lifecycle + // ------------------------------------------------------------------------- + + async createStaging(stagingTable: string): Promise { + await this.ch().command({ query: `DROP TABLE IF EXISTS ${this.fq(stagingTable)}` }); + await this.ch().command({ + query: `CREATE TABLE ${this.fq(stagingTable)} AS ${this.fq(this.config.table)}`, + }); + await this.applyDedupWindow(stagingTable); + } + + async dropStaging(stagingTable: string): Promise { + await this.ch().command({ query: `DROP TABLE IF EXISTS ${this.fq(stagingTable)}` }); + } + + async insertBatch( + stagingTable: string, + rows: OutputRow[], + dedupToken: string, + queryId: string, + ): Promise { + await this.ch().insert({ + table: stagingTable, + values: rows, + format: 'JSONEachRow', + clickhouse_settings: { insert_deduplication_token: dedupToken }, + query_id: queryId, + }); + } + + /** Exact row count (metadata-served on MergeTree — instant). */ + async countRows(table: string): Promise { + const res = await this.ch().query({ + query: `SELECT count() AS c FROM ${this.fq(table)}`, + format: 'JSONEachRow', + }); + const rows = await res.json<{ c: string }>(); + return Number(rows[0]?.c ?? 0); + } + + // ------------------------------------------------------------------------- + // Promotion (verify-then-attach) + // ------------------------------------------------------------------------- + + /** Active partition ids of a staging table. */ + async listPartitions(stagingTable: string): Promise { + const res = await this.ch().query({ + query: `SELECT DISTINCT partition_id FROM system.parts + WHERE database = {db:String} AND table = {table:String} AND active + ORDER BY partition_id`, + query_params: { db: this.config.database, table: stagingTable }, + format: 'JSONEachRow', + }); + const rows = await res.json<{ partition_id: string }>(); + return rows.map((r) => r.partition_id); + } + + /** + * Attach-recovery check: are any of this staging partition's rows already + * live? Matches (_id, cd) PAIRS — exact provenance with no schema changes: + * an id alone is ambiguous (a cross-cutover SDK retry lands the same _id + * in the same ts-month partition), but the retry copy's cd is stamped at + * post-cutover insert time and can never equal the staged row's historical + * cd. Also precise across sibling collections sharing the partition. + */ + async countLiveByStagedIds(stagingTable: string, partitionId: string): Promise { + const res = await this.ch().query({ + query: `SELECT count() AS c FROM ${this.fq(this.config.table)} + WHERE _partition_id = {pid:String} + AND (_id, cd) IN (SELECT _id, cd FROM ${this.fq(stagingTable)} WHERE _partition_id = {pid:String} LIMIT 100)`, + query_params: { pid: partitionId }, + format: 'JSONEachRow', + }); + const rows = await res.json<{ c: string }>(); + return Number(rows[0]?.c ?? 0); + } + + /** + * Attach one partition of a staging table into the live table. + * Parts-level (no rewrite). Throws on failure — caller decides fallback. + */ + async attachPartition(stagingTable: string, partitionId: string): Promise { + await this.ch().command({ + query: `ALTER TABLE ${this.fq(this.config.table)} ATTACH PARTITION ID '${partitionId}' FROM ${this.fq(stagingTable)}`, + }); + } + + /** + * Fallback promotion when ATTACH is unavailable (e.g. engine/settings + * mismatch in some environments): copy rows. Slower but always works. + */ + async insertSelect(stagingTable: string): Promise { + await this.ch().command({ + query: `INSERT INTO ${this.fq(this.config.table)} SELECT * FROM ${this.fq(stagingTable)}`, + }); + } + + /** + * Purge the live table's rows in a chunk's cd window (lightweight DELETE). + * Used when retrying a chunk that was already (partially) promoted — redo + * must start from a clean window or verify-then-attach would skip it. + */ + private scopeSql(scope?: { a: string; e: string; n?: string } | null): string { + if (!scope) return ''; + return 'AND a = {sa:String} AND e = {se:String}' + (scope.n !== undefined ? ' AND n = {sn:String}' : ''); + } + + private scopeParams(scope?: { a: string; e: string; n?: string } | null): Record { + if (!scope) return {}; + return { sa: scope.a, se: scope.e, ...(scope.n !== undefined ? { sn: scope.n } : {}) }; + } + + async deleteLiveCdRange(lowerCdMs: number, upperCdMs: number, scope?: { a: string; e: string; n?: string } | null): Promise { + await this.ch().command({ + query: `DELETE FROM ${this.fq(this.config.table)} + WHERE cd >= fromUnixTimestamp64Milli({lo:Int64}) + AND cd < fromUnixTimestamp64Milli({hi:Int64}) + ${this.scopeSql(scope)}`, + query_params: { lo: lowerCdMs, hi: upperCdMs, ...this.scopeParams(scope) }, + }); + } + + /** ClickHouse disk headroom (preflight). */ + async diskSpace(): Promise<{ freeBytes: number; totalBytes: number } | null> { + try { + const res = await this.ch().query({ + query: `SELECT sum(free_space) AS f, sum(total_space) AS t FROM system.disks`, + format: 'JSONEachRow', + }); + const rows = await res.json<{ f: string; t: string }>(); + return { freeBytes: Number(rows[0]?.f ?? 0), totalBytes: Number(rows[0]?.t ?? 0) }; + } catch { return null; } + } + + + /** Rows live ingestion wrote recently (preflight: is new ingestion flowing?). */ + async countRecentLive(minutes: number): Promise { + const res = await this.ch().query({ + query: `SELECT count() AS c FROM ${this.fq(this.config.table)} + WHERE cd >= now64(3) - INTERVAL {m:UInt32} MINUTE`, + query_params: { m: minutes }, + format: 'JSONEachRow', + }); + const rows = await res.json<{ c: string }>(); + return Number(rows[0]?.c ?? 0); + } + + /** + * Full rows for the sampled content audit, keyed by _id. Timestamps come + * back in the same 'YYYY-MM-DD hh:mm:ss.SSS' text form the transform + * emits, so scalar comparison is direct string/number equality. + */ + async fetchRowsByIds(ids: string[], cdBounds?: { loMs: number; hiMs: number }): Promise>> { + const bound = cdBounds + ? 'AND cd >= fromUnixTimestamp64Milli({blo:Int64}) AND cd <= fromUnixTimestamp64Milli({bhi:Int64})' + : ''; + const out = new Map>(); + for (let i = 0; i < ids.length; i += 5_000) { + const page = ids.slice(i, i + 5_000); + const res = await this.ch().query({ + query: `SELECT _id, a, e, n, uid, uid_canon, did, lsid, + toString(ts) AS ts_txt, toString(cd) AS cd_txt, + c, s, dur, up, sg, custom, cmp + FROM ${this.fq(this.config.table)} WHERE _id IN {ids:Array(String)} ${bound}`, + query_params: { ids: page, ...(cdBounds ? { blo: cdBounds.loMs, bhi: cdBounds.hiMs } : {}) }, + format: 'JSONEachRow', + }); + for (const r of await res.json>()) out.set(String(r._id), r); + } + return out; + } + + /** ClickHouse server wall-clock (preflight clock-skew check). */ + async serverNowMs(): Promise { + const res = await this.ch().query({ + query: `SELECT toUnixTimestamp64Milli(now64(3)) AS n`, + format: 'JSONEachRow', + }); + const rows = await res.json<{ n: string }>(); + return Number(rows[0]?.n ?? 0); + } + + /** + * Exact duplicate statistics, memory-bounded for billion-row tables: + * scans partition by partition (legitimate duplicate copies always share + * their ts month — same document ⇒ same ts ⇒ same partition — so + * per-partition GROUP BY is exact for every duplicate class we act on, + * while a global uniqExact/GROUP BY over billions of ids is not safe). + */ + async duplicateStats(boundaryMs: number, sampleLimit = 20): Promise<{ + rows: number; duplicates: number; + sample: Array<{ _id: string; copies: number; migratedCopies: number; min_cd_ms: number; max_cd_ms: number }>; + }> { + const parts = await this.ch().query({ + query: `SELECT partition_id AS partition, sum(rows) AS r FROM system.parts + WHERE database = {db:String} AND table = {t:String} AND active + GROUP BY partition_id ORDER BY partition_id`, + query_params: { db: this.config.database, t: this.config.table }, + format: 'JSONEachRow', + }); + const partitions = await parts.json<{ partition: string; r: string }>(); + let rows = 0, duplicates = 0; + const sample: Array<{ _id: string; copies: number; migratedCopies: number; min_cd_ms: number; max_cd_ms: number }> = []; + for (const p of partitions) { + rows += Number(p.r); + const res = await this.ch().query({ + query: `SELECT _id, count() AS c, countIf(cd < fromUnixTimestamp64Milli({b:Int64})) AS mc, + toUnixTimestamp64Milli(min(cd)) AS lo, toUnixTimestamp64Milli(max(cd)) AS hi, + sum(c - 1) OVER () AS excess + FROM (SELECT _id, cd FROM ${this.fq(this.config.table)} WHERE _partition_id = {p:String}) + GROUP BY _id HAVING c > 1 + ORDER BY mc DESC, c DESC LIMIT {lim:UInt32}`, + query_params: { b: boundaryMs, p: p.partition, lim: sampleLimit }, + format: 'JSONEachRow', + clickhouse_settings: { max_bytes_before_external_group_by: '4000000000' }, + }); + const groups = await res.json<{ _id: string; c: string; mc: string; lo: string; hi: string; excess: string }>(); + if (groups.length > 0) duplicates += Number(groups[0].excess); + for (const g of groups) { + if (sample.length >= sampleLimit) break; + sample.push({ _id: g._id, copies: Number(g.c), migratedCopies: Number(g.mc), min_cd_ms: Number(g.lo), max_cd_ms: Number(g.hi) }); + } + } + return { rows, duplicates, sample }; + } + + + /** Does the live target table exist / how many rows does it hold? */ + async targetTableInfo(): Promise<{ exists: boolean; rows: number }> { + try { + const rows = await this.countRows(this.config.table); + return { exists: true, rows }; + } catch { + return { exists: false, rows: 0 }; + } + } + + /** + * Precise purge by (_id, cd) pairs — used where no cd window exists (the + * null-cd sweep) or the collection is unresolvable. Pair matching means a + * live cross-cutover retry copy (same _id, post-cutover cd) is untouchable. + */ + async deleteLiveByPairs(pairs: Array<{ id: string; cdMs: number }>): Promise { + if (pairs.length === 0) return; + // Two parallel arrays zipped server-side — the HTTP interface cannot + // parse a JS array-of-arrays as Array(Tuple(...)). The cd min/max bound + // lets the mutation prune to the pairs' partitions instead of scanning + // the whole table. + const lo = Math.min(...pairs.map((p) => p.cdMs)); + const hi = Math.max(...pairs.map((p) => p.cdMs)); + await this.ch().command({ + query: `DELETE FROM ${this.fq(this.config.table)} + WHERE cd >= fromUnixTimestamp64Milli({blo:Int64}) AND cd <= fromUnixTimestamp64Milli({bhi:Int64}) + AND (_id, toUnixTimestamp64Milli(cd)) IN ( + SELECT arrayJoin(arrayZip({ids:Array(String)}, {cds:Array(Int64)})) + )`, + query_params: { ids: pairs.map((p) => p.id), cds: pairs.map((p) => p.cdMs), blo: lo, bhi: hi }, + }); + } + + /** Staging tables left behind by crashes (crash between done and drop). */ + async listStagingTables(prefix: string): Promise { + const res = await this.ch().query({ + query: `SELECT name FROM system.tables WHERE database = {db:String} AND name LIKE {p:String}`, + query_params: { db: this.config.database, p: `${prefix}%` }, + format: 'JSONEachRow', + }); + const rows = await res.json<{ name: string }>(); + return rows.map((r) => r.name); + } + + /** Grouped verification: rows in the live table within given cd bounds. */ + async countLiveInCdRange(lowerCdMs: number, upperCdMs: number, scope?: { a: string; e: string; n?: string } | null): Promise { + const res = await this.ch().query({ + query: `SELECT count() AS c FROM ${this.fq(this.config.table)} + WHERE cd >= fromUnixTimestamp64Milli({lo:Int64}) + AND cd < fromUnixTimestamp64Milli({hi:Int64}) + ${this.scopeSql(scope)}`, + query_params: { lo: lowerCdMs, hi: upperCdMs, ...this.scopeParams(scope) }, + format: 'JSONEachRow', + }); + const rows = await res.json<{ c: string }>(); + return Number(rows[0]?.c ?? 0); + } + + /** + * Which of these ids exist in the live table, and at what cd? Paged IN + * queries; used by ledger rebuild to attribute null-cd sweep rows (their + * cd is ts-derived and lands inside regular chunks' windows). + */ + async fetchLiveCdByIds(ids: string[], cdBounds?: { loMs: number; hiMs: number }): Promise> { + // _id is not in the ORDER BY — without cd bounds this is a full-column + // scan on a 10B-row table. Callers know their rows' cd values; pass them. + const bound = cdBounds + ? 'AND cd >= fromUnixTimestamp64Milli({blo:Int64}) AND cd <= fromUnixTimestamp64Milli({bhi:Int64})' + : ''; + const out = new Map(); + for (let i = 0; i < ids.length; i += 10_000) { + const page = ids.slice(i, i + 10_000); + const res = await this.ch().query({ + query: `SELECT _id, toUnixTimestamp64Milli(cd) AS cd_ms FROM ${this.fq(this.config.table)} + WHERE _id IN {ids:Array(String)} ${bound}`, + query_params: { ids: page, ...(cdBounds ? { blo: cdBounds.loMs, bhi: cdBounds.hiMs } : {}) }, + format: 'JSONEachRow', + }); + const rows = await res.json<{ _id: string; cd_ms: string }>(); + for (const r of rows) out.set(r._id, Number(r.cd_ms)); + } + return out; + } +} diff --git a/src/transform/coercions.ts b/src/transform/coercions.ts new file mode 100644 index 0000000..990201a --- /dev/null +++ b/src/transform/coercions.ts @@ -0,0 +1,50 @@ +/** + * CoercionCounter — per-(rule, key) accounting of every value the transform + * had to alter, with samples. Feeds the dry-run / final report so "what did + * we change?" is always answerable. + * + * The coercion RULES themselves live in validators.ts (clampUInt32, + * clampDateTime64, sanitizeJsonValue) — they are part of the shared + * normalization spec enforced by tests/differential/ and must match + * countly-platform's live ingestion exactly. This module only counts. + */ + +export interface CoercionSample { + key: string; + original: string; + coerced: string; +} + +export class CoercionCounter { + private counts = new Map(); + private samples = new Map(); + private static readonly MAX_SAMPLED_KEYS = 200; + // Distinct (rule, field) keys are bounded so pathological data (millions of + // distinct field names) cannot grow this map without limit; the overflow + // bucket keeps the TOTAL exact either way. + private static readonly MAX_DISTINCT_KEYS = 10_000; + private static readonly OVERFLOW_KEY = 'other:distinct-key-cap-reached'; + + record(rule: string, key: string, original: unknown, coerced: unknown): void { + let k = `${rule}:${key}`; + if (!this.counts.has(k) && this.counts.size >= CoercionCounter.MAX_DISTINCT_KEYS) { + k = CoercionCounter.OVERFLOW_KEY; + } + this.counts.set(k, (this.counts.get(k) ?? 0) + 1); + if (!this.samples.has(k) && this.samples.size < CoercionCounter.MAX_SAMPLED_KEYS) { + this.samples.set(k, { key, original: String(original).slice(0, 100), coerced: String(coerced).slice(0, 100) }); + } + } + + getTotal(): number { + let t = 0; + for (const n of this.counts.values()) t += n; + return t; + } + + getReport(): Array<{ rule_key: string; count: number; sample: CoercionSample | null }> { + return [...this.counts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([ruleKey, count]) => ({ rule_key: ruleKey, count, sample: this.samples.get(ruleKey) ?? null })); + } +} diff --git a/src/transform/hash-resolver.ts b/src/transform/hash-resolver.ts index 0354d9d..50870c5 100644 --- a/src/transform/hash-resolver.ts +++ b/src/transform/hash-resolver.ts @@ -16,6 +16,22 @@ import type { Logger } from "pino"; // Public types // ───────────────────────────────────────────────────────────────────────────── +/** + * ClickHouse row identity of a collection's rows. The transform maps custom + * events to e='[CLY]_custom' with the original name in `n` — so a custom + * event's rows are identified by (a, e, n), an internal [CLY]_ event's by + * (a, e) (its n is data-dependent: view name, widget id, …). + * Caveat: a source doc carrying its own non-blank `n` keeps it (dedup + * identity with live ingestion); legacy drill documents never have one. + */ +export interface ChScope { a: string; e: string; n?: string } + +export function chScopeOf(defaults: CollectionDefaults): ChScope { + return defaults.e.startsWith('[CLY]_') + ? { a: defaults.a, e: defaults.e } + : { a: defaults.a, e: '[CLY]_custom', n: defaults.e }; +} + /** Default `a` (appId) and `e` (eventName) values derived from a collection hash. */ export interface CollectionDefaults { a: string; diff --git a/src/transform/normalize.ts b/src/transform/normalize.ts index bbb8e8a..01a5a24 100644 --- a/src/transform/normalize.ts +++ b/src/transform/normalize.ts @@ -4,6 +4,26 @@ * Transforms raw MongoDB event documents into ClickHouse-ready rows, * applying field validation, event-name derivation, and timestamp * normalization. + * + * This module implements the drill-event normalization spec. The spec was + * defined together with a matching rewrite of countly-platform's + * api/utils/eventTransformer.ts (branch claude/jovial-shannon-b3dd29) and the + * goldens in tests/differential/ were generated from that code — but this + * tool does NOT depend on that branch merging. It writes ClickHouse directly; + * platform code never runs on the migration path. The differential harness + * pins THIS repo's behavior against the frozen goldens. + * + * What consistency actually requires (and why it holds against platform + * main unmerged): row semantics that span history + live queries — custom + * events as e='[CLY]_custom' with the name in n (confirmed live behavior), + * uid_canon left to the identity machinery (both sides), cd = historical + * time for migrated rows vs receive-time for live rows. Everything else in + * this spec (NaN/Decimal128/Long stringification, ts heuristics, clamps, + * skip rules) concerns BSON-only shapes that live SDK ingestion can never + * receive through JSON — divergence there is unobservable. + * + * If the platform PR merges with behavior changes, regenerate goldens there + * and re-sync tests/differential/. */ import { SkipReason, SkipCounter } from './skip-reasons.ts'; @@ -12,24 +32,20 @@ import { asString, toEpochMillis, toDouble, + clampUInt32, + clampDateTime64, + isPlainObject, + sanitizeJsonValue, formatTimestamp, firstNonBlank, } from './validators.ts'; import type { CollectionDefaults } from './hash-resolver.ts'; +import type { CoercionCounter } from './coercions.ts'; // ──────────────────────────────────────────────────────────────────────────── // Constants // ──────────────────────────────────────────────────────────────────────────── -/** - * The complete set of fields that may appear in a transformed output row. - * Anything not in this set is stripped before the row is emitted. - */ -const KNOWN_FIELDS = new Set([ - 'a', 'e', 'n', 'uid', 'uid_canon', 'did', 'lsid', - '_id', 'ts', 'up', 'custom', 'cmp', 'sg', 'c', 's', 'dur', 'lu', 'cd', -]); - const CLY_PREFIX = '[CLY]_'; // ──────────────────────────────────────────────────────────────────────────── @@ -52,8 +68,8 @@ export interface SourceDocument { ts?: unknown; up?: unknown; custom?: unknown; - cmp?: string; - sg?: Record; + cmp?: unknown; + sg?: unknown; c?: unknown; s?: unknown; dur?: unknown; @@ -73,9 +89,9 @@ export interface OutputRow { did: string; lsid?: string; ts: string; - up?: unknown; - custom?: unknown; - cmp?: string; + up?: Record; + custom?: Record; + cmp?: Record; sg?: Record; c: number; s: number; @@ -100,9 +116,13 @@ export interface TransformResult { * * Returns `{ row, skipReason }` where exactly one of the two is non-null. */ -export function transformDocument(doc: SourceDocument, defaults?: CollectionDefaults): TransformResult { +export function transformDocument( + doc: SourceDocument, + defaults?: CollectionDefaults, + coercions?: CoercionCounter, +): TransformResult { try { - return doTransform(doc, defaults); + return doTransform(doc, defaults, coercions); } catch { return { row: null, skipReason: SkipReason.TRANSFORM_ERROR }; } @@ -124,13 +144,14 @@ export function transformBatch( docs: SourceDocument[], skipCounter: SkipCounter, defaults?: CollectionDefaults, + coercions?: CoercionCounter, ): { rows: OutputRow[]; skippedSamples: Array<{ _id: string; reason: SkipReason }> } { const rows: OutputRow[] = []; const skippedSamples: Array<{ _id: string; reason: SkipReason }> = []; const MAX_SKIP_SAMPLES = 10; for (const doc of docs) { - const { row, skipReason } = transformDocument(doc, defaults); + const { row, skipReason } = transformDocument(doc, defaults, coercions); if (row !== null) { rows.push(row); @@ -150,7 +171,11 @@ export function transformBatch( // Internal helpers // ──────────────────────────────────────────────────────────────────────────── -function doTransform(doc: SourceDocument, defaults?: CollectionDefaults): TransformResult { +function doTransform( + doc: SourceDocument, + defaults?: CollectionDefaults, + coercions?: CoercionCounter, +): TransformResult { // ── Skip if already migrated ────────────────────────────────────────── if (doc.migrated === true) { return { row: null, skipReason: SkipReason.ALREADY_MARKED_MIGRATED }; @@ -185,80 +210,87 @@ function doTransform(doc: SourceDocument, defaults?: CollectionDefaults): Transf return { row: null, skipReason: SkipReason.INVALID_TS }; } - // ── Build output row ────────────────────────────────────────────────── - // We build via a mutable bag and cast at the end since all required - // OutputRow fields are guaranteed to be set by the code below. - const row: Record = {}; - - // Copy all known fields from the source document - for (const key of KNOWN_FIELDS) { - if (key in doc) { - row[key] = (doc as Record)[key]; - } - } - - // Overwrite validated / required fields - row['a'] = a; - row['e'] = e; - row['uid'] = uid; - row['_id'] = _id; - - // ── Defaults ────────────────────────────────────────────────────────── - const did = asString(doc.did); - row['did'] = did ?? ''; - - row['s'] = toDouble(doc.s, 0.0); - row['dur'] = toDouble(doc.dur, 0.0); - row['c'] = Math.max(0, Math.floor(toDouble(doc.c, 0))); - // ── Event name derivation ───────────────────────────────────────────── + // An existing non-blank doc.n always wins so migrated rows match the rows + // live ingestion produced for the same document (dedup identity). Legacy + // documents have no `n`, so for them the sg-derived name applies as before. let eventName = e; - let n: string | null = null; + let n: string | null = asString(doc.n); if (e.startsWith(CLY_PREFIX)) { - const sg = (doc.sg ?? {}) as Record; - - switch (e) { - case '[CLY]_view': - n = asString(sg['name']); - break; - case '[CLY]_action': - n = firstNonBlank(asString(sg['name']), asString(sg['view'])); - break; - case '[CLY]_nps': - case '[CLY]_survey': - case '[CLY]_star_rating': - n = asString(sg['widget_id']); - break; - case '[CLY]_crash': - n = asString(sg['group']); - break; - default: - // Keep existing n from the document, if any - n = asString(doc.n) ?? null; - break; + if (n === null) { + const sg = (isPlainObject(doc.sg) ? doc.sg : {}) as Record; + + switch (e) { + case '[CLY]_view': + n = asString(sg['name']); + break; + case '[CLY]_action': + n = firstNonBlank(asString(sg['name']), asString(sg['view'])); + break; + case '[CLY]_nps': + case '[CLY]_survey': + case '[CLY]_star_rating': + n = asString(sg['widget_id']); + break; + case '[CLY]_crash': + n = asString(sg['group']); + break; + default: + n = null; + break; + } } } else { // Custom event: n = original event name, e becomes [CLY]_custom - n = eventName; + if (n === null) { + n = eventName; + } eventName = '[CLY]_custom'; - row['e'] = eventName; } // Final fallback: if n is still blank, use e if (isBlank(n)) { n = eventName; } - row['n'] = n; + + // ── Build output row ────────────────────────────────────────────────── + const row: OutputRow = { + _id, + a, + e: eventName, + n: n as string, + uid, + did: asString(doc.did) ?? '', + ts: '', + c: clampUInt32(doc.c), // counted below when clamping changed the value + s: toDouble(doc.s, 0.0), + dur: toDouble(doc.dur, 0.0), + cd: '', + }; + + { + const cRaw = Math.floor(toDouble(doc.c, 0)); + if (row.c !== cRaw) coercions?.record('clamp_uint32', 'c', cRaw, row.c); + } + + const uidCanon = asString(doc.uid_canon); + if (uidCanon !== null) { + row.uid_canon = uidCanon; + } + const lsid = asString(doc.lsid); + if (lsid !== null) { + row.lsid = lsid; + } // ── Timestamp normalisation ─────────────────────────────────────────── - row['ts'] = formatTimestamp(tsMillis); + // Countly-owned timestamps clamp to the DateTime64(3) column range. + if (clampDateTime64(tsMillis) !== tsMillis) coercions?.record('clamp_datetime', 'ts', tsMillis, clampDateTime64(tsMillis)); + row.ts = formatTimestamp(clampDateTime64(tsMillis)); const luMillis = toEpochMillis(doc.lu); - if (luMillis !== null) { - row['lu'] = formatTimestamp(luMillis); - } else { - delete row['lu']; + if (luMillis !== null && luMillis > 0) { + row.lu = formatTimestamp(clampDateTime64(luMillis)); } // `cd` must always be emitted. The ClickHouse column is declared @@ -274,7 +306,24 @@ function doTransform(doc: SourceDocument, defaults?: CollectionDefaults): Transf // keeps them in a plausible term and, unlike now64(3), is deterministic, so // re-running or resuming a migration is idempotent. const cdMillis = toEpochMillis(doc.cd); - row['cd'] = formatTimestamp(cdMillis !== null && cdMillis > 0 ? cdMillis : tsMillis); + row.cd = formatTimestamp(clampDateTime64(cdMillis !== null && cdMillis > 0 ? cdMillis : tsMillis)); + + // ── JSON columns ────────────────────────────────────────────────────── + // Only plain objects are insertable into the JSON columns; customer-owned + // values that don't fit JSON numeric representation are stringified + // losslessly (NaN/±Infinity, BSON Decimal128/Long, bigint). + if (isPlainObject(doc.up)) { + row.up = sanitizeJsonValue(doc.up, (k, o, c) => coercions?.record(k, 'up', o, c)) as Record; + } + if (isPlainObject(doc.custom)) { + row.custom = sanitizeJsonValue(doc.custom, (k, o, c) => coercions?.record(k, 'custom', o, c)) as Record; + } + if (isPlainObject(doc.cmp)) { + row.cmp = sanitizeJsonValue(doc.cmp, (k, o, c) => coercions?.record(k, 'cmp', o, c)) as Record; + } + if (isPlainObject(doc.sg)) { + row.sg = sanitizeJsonValue(doc.sg, (k, o, c) => coercions?.record(k, 'sg', o, c)) as Record; + } - return { row: row as unknown as OutputRow, skipReason: null }; + return { row, skipReason: null }; } diff --git a/src/transform/validators.ts b/src/transform/validators.ts index f06e614..6651d29 100644 --- a/src/transform/validators.ts +++ b/src/transform/validators.ts @@ -1,7 +1,19 @@ /** * Field validation and conversion utilities for the migration transform layer. + * + * These helpers implement the shared drill-event normalization spec that + * countly-platform's live ingestion transformer + * (api/utils/eventTransformer.ts) also implements. Both sides must produce + * IDENTICAL countly_drill.drill_events rows for the same input document; the + * differential harness in tests/differential/ enforces this in CI. */ +const UINT32_MAX = 4294967295; + +/** DateTime64(3) representable range: 1900-01-01 00:00:00.000 .. 2299-12-31 23:59:59.999 UTC. */ +export const DATETIME64_MIN_MS = Date.UTC(1900, 0, 1, 0, 0, 0, 0); +export const DATETIME64_MAX_MS = Date.UTC(2299, 11, 31, 23, 59, 59, 999); + /** * Returns true if the value is null, undefined, or an empty/whitespace-only string. */ @@ -73,6 +85,14 @@ export function toEpochMillis(ts: unknown): number | null { return null; } +/** + * Clamps epoch milliseconds to the DateTime64(3) column range + * (Countly-owned timestamps clamp to column ranges by policy). + */ +export function clampDateTime64(epochMs: number): number { + return Math.min(Math.max(epochMs, DATETIME64_MIN_MS), DATETIME64_MAX_MS); +} + /** * Parses a value as a double (floating-point number), returning the default if * the value is null, undefined, or not parseable. @@ -88,6 +108,79 @@ export function toDouble(val: unknown, defaultVal: number): number { return (isNaN(parsed) || !isFinite(parsed)) ? defaultVal : parsed; } +/** + * Clamps a count value to the UInt32 column range: [0, 4294967295], integer. + */ +export function clampUInt32(val: unknown): number { + const num = Math.floor(toDouble(val, 0)); + return Math.min(Math.max(num, 0), UINT32_MAX); +} + +/** + * Returns true for plain (non-array, non-Date) objects usable as JSON column payloads. + */ +export function isPlainObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date); +} + +/** + * Deep-sanitizes customer-owned JSON column values (up/custom/cmp/sg). + * Values JSON cannot carry numerically are stringified losslessly instead of + * degrading to null: NaN/±Infinity become "NaN"/"Infinity"/"-Infinity", + * BSON Decimal128/Long become their decimal string, bigint becomes a string. + * `undefined` values are dropped from objects and become null inside arrays, + * matching JSON serialization. + */ +export type OnCoerce = (kind: string, original: unknown, coerced: unknown) => void; + +export function sanitizeJsonValue(value: unknown, onCoerce?: OnCoerce): unknown { + if (typeof value === 'number') { + if (isNaN(value)) { + onCoerce?.('stringify_nonfinite', value, 'NaN'); + return 'NaN'; + } + if (value === Infinity) { + onCoerce?.('stringify_nonfinite', value, 'Infinity'); + return 'Infinity'; + } + if (value === -Infinity) { + onCoerce?.('stringify_nonfinite', value, '-Infinity'); + return '-Infinity'; + } + return value; + } + if (typeof value === 'bigint') { + onCoerce?.('stringify_bigint', value, value.toString()); + return value.toString(); + } + if (value === null || typeof value !== 'object') { + return value; + } + if (value instanceof Date) { + return value; + } + const bsonType = (value as Record)['_bsontype']; + if (bsonType === 'Decimal128' || bsonType === 'Long') { + const coerced = String(value); + onCoerce?.('stringify_bson_' + String(bsonType).toLowerCase(), value, coerced); + return coerced; + } + if (Array.isArray(value)) { + return value.map((item) => { + const sanitized = sanitizeJsonValue(item, onCoerce); + return sanitized === undefined ? null : sanitized; + }); + } + const out: Record = {}; + for (const key of Object.keys(value as Record)) { + const sanitized = sanitizeJsonValue((value as Record)[key], onCoerce); + if (sanitized !== undefined) { + out[key] = sanitized; + } + } + return out; +} + /** * Formats an epoch-millisecond timestamp as 'yyyy-MM-dd HH:mm:ss.SSS' in UTC. */ diff --git a/tests/differential/README.md b/tests/differential/README.md new file mode 100644 index 0000000..25d6668 --- /dev/null +++ b/tests/differential/README.md @@ -0,0 +1,43 @@ +# Ingestion-matching transformer differential harness (migration side) + +Drill migration overhaul item **D4** ([#6](https://github.com/Countly/migration/issues/6)). + +`differential.test.ts` asserts that this repo's transform +(`src/transform/normalize.ts`) reproduces, for every document in the shared fixture +corpus, exactly the canonical `countly_drill.drill_events` row of the agreed +normalization spec. + +**Status of the platform side:** the spec was co-developed with a rewrite of +countly-platform's `api/utils/eventTransformer.ts` (branch +`claude/jovial-shannon-b3dd29`), and these goldens were generated from that code. +That platform PR is currently UNMERGED, and this tool does not require it: the +migrator writes ClickHouse directly and never runs platform code. The goldens +therefore act as this repo's frozen spec. The platform PR remains desirable for +platform-side replay paths (anything feeding old drill docs through +KafkaEventSink on main today re-stamps `cd` to insert time and skips +sanitization) — but that is hardening for the platform, not a dependency of +the migration. The cd re-stamping specifically is fixed by the surgical +[countly-platform#1105](https://github.com/Countly/countly-platform/pull/1105), +split out of the branch because it also affects LIVE rows: without it, +Kafka offset replay and connector redelivery re-date live events too. + +**Vendored files — do not edit here:** `corpus.json`, `goldens.json`, `decode.mjs`, +`canonicalize.mjs` are synced byte-identical from countly-platform +`test/unit/fixtures/drill-transform-differential/`. The goldens are generated from the +platform's live ingestion code (`generate-goldens.mjs` there) — the platform is the +source of truth. The goldens embed sha256 hashes of the corpus and of +decode+canonicalize, so a partial sync fails the first test. + +When this suite fails: + +- **after a change in this repo** — the migration transform drifted from live + ingestion; fix `src/transform/`, never the vendored goldens. +- **after syncing fresh fixtures** — countly-platform changed normalization behavior; + align `src/transform/` to match (the golden diff in the platform PR describes the + change). + +The full shared normalization spec and the pre-alignment divergence report live next +to the generator in the countly-platform fixture directory. + +Runs in CI via `.github/workflows/ci.yml` (`npx vitest run tests/differential`) — +pure unit tests, no MongoDB/ClickHouse/Redis needed. diff --git a/tests/differential/canonicalize.mjs b/tests/differential/canonicalize.mjs new file mode 100644 index 0000000..fb8ccf6 --- /dev/null +++ b/tests/differential/canonicalize.mjs @@ -0,0 +1,236 @@ +/** + * Canonical ClickHouse row model for the ingestion-matching transformer + * differential harness. + * + * Both pipelines write the same table but over different wire formats: + * - countly-platform: transformToKafkaEventFormat() -> JSON.stringify -> + * Kafka -> ClickHouse Kafka Connect sink (numeric epoch-ms timestamps) + * - Countly/migration: doTransform() -> @clickhouse/client JSONEachRow + * ('yyyy-MM-dd HH:mm:ss.SSS' timestamp strings) + * + * This module maps either wire payload to the effective countly_drill.drill_events + * row so the two can be deep-equal compared. Values that a given wire format + * could not insert cleanly (type mismatch for the target column, out-of-range + * numerics, null into a non-nullable column) are represented as explicit + * "__UNINSERTABLE__(...)" sentinel strings instead of being smoothed over — + * an aligned implementation must never produce them, so any sentinel in a + * golden or a diff is itself a reportable divergence. + * + * Assumption (deployment invariant, asserted by the migration's + * datetime-handling integration test): the ClickHouse server timezone is UTC, + * so the migration's naive timestamp strings and the connector's epoch-ms + * numbers denote the same instant. + * + * This file is part of the cross-repo differential contract and must stay + * byte-identical between: + * countly-platform: test/unit/fixtures/drill-transform-differential/canonicalize.mjs + * Countly/migration: tests/differential/canonicalize.mjs + */ + +/** Columns of countly_drill.drill_events (plugins/clickhouse/api/sql/01-drill_events.sql). */ +export const SCHEMA_COLUMNS = [ + 'a', 'e', 'n', 'uid', 'uid_canon', 'did', 'lsid', '_id', + 'ts', 'up', 'custom', 'cmp', 'sg', 'c', 's', 'dur', 'lu', 'cd', +]; + +const UINT32_MAX = 4294967295; +/** DateTime64(3) representable range: 1900-01-01 00:00:00.000 .. 2299-12-31 23:59:59.999 UTC. */ +export const DATETIME64_MIN_MS = Date.UTC(1900, 0, 1, 0, 0, 0, 0); +export const DATETIME64_MAX_MS = Date.UTC(2299, 11, 31, 23, 59, 59, 999); + +const TS_STRING_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/; + +/** + * Format an epoch-ms timestamp as 'yyyy-MM-dd HH:mm:ss.SSS' in UTC + * (identical to the migration's validators.formatTimestamp). + * @param {number} epochMs - epoch milliseconds + * @returns {string} formatted timestamp + */ +export function formatTimestamp(epochMs) { + const d = new Date(epochMs); + const yyyy = d.getUTCFullYear().toString().padStart(4, '0'); + const MM = (d.getUTCMonth() + 1).toString().padStart(2, '0'); + const dd = d.getUTCDate().toString().padStart(2, '0'); + const HH = d.getUTCHours().toString().padStart(2, '0'); + const mm = d.getUTCMinutes().toString().padStart(2, '0'); + const ss = d.getUTCSeconds().toString().padStart(2, '0'); + const SSS = d.getUTCMilliseconds().toString().padStart(3, '0'); + return `${yyyy}-${MM}-${dd} ${HH}:${mm}:${ss}.${SSS}`; +} + +/** + * Deep-sort object keys so JSON-column comparisons are order-insensitive. + * @param {*} value - any JSON value + * @returns {*} value with all object keys sorted + */ +export function deepSortKeys(value) { + if (Array.isArray(value)) { + return value.map(deepSortKeys); + } + if (value !== null && typeof value === 'object') { + const out = {}; + for (const key of Object.keys(value).sort()) { + out[key] = deepSortKeys(value[key]); + } + return out; + } + return value; +} + +/** + * Build an uninsertable-value sentinel. + * @param {string} column - column name + * @param {string} why - short reason + * @param {*} value - offending value (post JSON round-trip) + * @returns {string} sentinel string + */ +function uninsertable(column, why, value) { + return `__UNINSERTABLE__(${column}: ${why}: ${JSON.stringify(value)})`; +} + +/** + * Canonicalize one String-typed column value. + * @param {object} payload - JSON-round-tripped wire payload + * @param {string} column - column name + * @param {{nullable?: boolean}} [opts] - column options + * @returns {*} canonical value + */ +function stringColumn(payload, column, opts = {}) { + const value = payload[column]; + if (value === undefined || value === null) { + return opts.nullable ? null : (column in payload ? uninsertable(column, 'null into non-nullable String', value) : ''); + } + if (typeof value !== 'string') { + return uninsertable(column, 'non-string into String column', value); + } + return value; +} + +/** + * Canonicalize a DateTime64(3) column from a wire value that may be an + * epoch-ms number (Kafka path) or a pre-formatted string (JSONEachRow path). + * @param {*} value - wire value + * @param {string} column - column name + * @returns {*} canonical 'yyyy-MM-dd HH:mm:ss.SSS' string or sentinel + */ +function dateTimeColumn(value, column) { + if (typeof value === 'number') { + if (!Number.isFinite(value) || !Number.isInteger(value)) { + return uninsertable(column, 'non-integer epoch value', value); + } + if (value < DATETIME64_MIN_MS || value > DATETIME64_MAX_MS) { + return uninsertable(column, 'epoch-ms outside DateTime64(3) range', value); + } + return formatTimestamp(value); + } + if (typeof value === 'string') { + if (!TS_STRING_RE.test(value)) { + return uninsertable(column, 'unparseable DateTime64 string', value); + } + const ms = Date.parse(value.replace(' ', 'T') + 'Z'); + if (Number.isNaN(ms) || ms < DATETIME64_MIN_MS || ms > DATETIME64_MAX_MS) { + return uninsertable(column, 'DateTime64 string outside range', value); + } + return formatTimestamp(ms); + } + return uninsertable(column, 'unsupported DateTime64 wire type', value); +} + +/** + * Canonicalize a JSON-typed column (up/custom/cmp/sg). + * @param {object} payload - JSON-round-tripped wire payload + * @param {string} column - column name + * @param {{nullable?: boolean}} [opts] - column options + * @returns {*} canonical value + */ +function jsonColumn(payload, column, opts = {}) { + const value = payload[column]; + if (value === undefined || value === null) { + // Missing/null -> Nullable(JSON) NULL, non-nullable JSON default {}. + return opts.nullable ? null : {}; + } + if (typeof value !== 'object' || Array.isArray(value)) { + return uninsertable(column, 'non-object into JSON column', value); + } + return deepSortKeys(value); +} + +/** + * Canonicalize the c (UInt32) column. + * @param {object} payload - JSON-round-tripped wire payload + * @returns {*} canonical value + */ +function uint32Column(payload) { + const value = payload.c; + if (value === undefined) { + return 0; // column type default + } + if (typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > UINT32_MAX) { + return uninsertable('c', 'not a UInt32', value); + } + return value; +} + +/** + * Canonicalize a Float64 column (s/dur). + * @param {object} payload - JSON-round-tripped wire payload + * @param {string} column - column name + * @returns {*} canonical value + */ +function float64Column(payload, column) { + const value = payload[column]; + if (value === undefined) { + return 0; + } + if (typeof value !== 'number') { + return uninsertable(column, 'not a Float64', value); + } + return value; +} + +/** + * Canonicalize a transformed wire payload (either pipeline) into the effective + * drill_events row. Pass the transform's direct output; the JSON round-trip + * both wire formats perform (JSON.stringify for Kafka, JSONEachRow + * serialization for the migration) is applied here, so toJSON() conversions, + * NaN->null, and undefined-dropping match the real wire behavior. + * + * @param {object|null} output - transform output (null/undefined = skipped) + * @returns {{skip: true}|{row: object}} canonical result + */ +export function canonicalizeOutput(output) { + if (output === null || output === undefined) { + return { skip: true }; + } + const payload = JSON.parse(JSON.stringify(output)); + const row = {}; + row.a = stringColumn(payload, 'a'); + row.e = stringColumn(payload, 'e'); + row.n = stringColumn(payload, 'n'); + row.uid = stringColumn(payload, 'uid'); + row.uid_canon = stringColumn(payload, 'uid_canon', { nullable: true }); + row.did = stringColumn(payload, 'did'); + row.lsid = stringColumn(payload, 'lsid', { nullable: true }); + row._id = stringColumn(payload, '_id'); + + if (payload.ts === undefined || payload.ts === null) { + row.ts = uninsertable('ts', 'missing required timestamp', payload.ts); + } + else { + row.ts = dateTimeColumn(payload.ts, 'ts'); + } + row.lu = (payload.lu === undefined || payload.lu === null) ? null : dateTimeColumn(payload.lu, 'lu'); + // cd has DEFAULT now64(3): omitting it yields a nondeterministic + // ingestion-time stamp, which can never match a deterministic transform. + row.cd = (payload.cd === undefined) ? '__NONDETERMINISTIC__(cd: now64(3) column default)' : dateTimeColumn(payload.cd, 'cd'); + + row.up = jsonColumn(payload, 'up'); + row.custom = jsonColumn(payload, 'custom', { nullable: true }); + row.cmp = jsonColumn(payload, 'cmp', { nullable: true }); + row.sg = jsonColumn(payload, 'sg'); + + row.c = uint32Column(payload); + row.s = float64Column(payload, 's'); + row.dur = float64Column(payload, 'dur'); + return { row }; +} diff --git a/tests/differential/corpus.json b/tests/differential/corpus.json new file mode 100644 index 0000000..1da3327 --- /dev/null +++ b/tests/differential/corpus.json @@ -0,0 +1,1133 @@ +{ + "description": "Input corpus for the ingestion-matching transformer differential harness (drill migration overhaul item D4, Countly/migration#6). Each entry's doc is a MongoDB drill event document as the Node driver (default options) would deliver it. Values plain JSON cannot express use {\"$$t\": ...} wrappers decoded by decode.mjs. This file must stay byte-identical between countly-platform test/unit/fixtures/drill-transform-differential/ and Countly/migration tests/differential/.", + "baseline": "ts=1704067200000 (2024-01-01 00:00:00.000 UTC), cd=1704067205000", + "entries": [ + { + "id": "custom-event-full", + "group": "known-events", + "notes": "New-format custom event exactly as processToDrill writes it, all fields populated.", + "doc": { + "_id": "req1hash_u1_1704067200000_0", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "_uid": "6543ab", + "did": "device-001", + "lsid": "lsid-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "lu": { "$$t": "date", "ms": 1703980800000 }, + "c": 2, + "s": 19.99, + "dur": 3.5, + "up": { "cc": "US", "p": "iOS", "av": "2.1.0", "hour": 14, "dow": 1 }, + "custom": { "tier": "premium", "beta": "true" }, + "cmp": { "c": "Organic" }, + "sg": { "item": "sku-123", "qty": 2, "price": 9.995, "gift": false } + } + }, + { + "id": "view-visit", + "group": "known-events", + "notes": "New-format [CLY]_view with visit=1; processToDrill sets n to the event key, not sg.name.", + "doc": { + "_id": "5f1c_u1_view1", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_view", + "n": "[CLY]_view", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "s": 0, + "dur": 0, + "up": { "p": "Android" }, + "sg": { "name": "Home", "visit": 1, "start": 1, "segment": "Android" } + } + }, + { + "id": "view-update", + "group": "known-events", + "notes": "[CLY]_view_update carries duration; not in the derivation switch on either side.", + "doc": { + "_id": "5f1c_u1_view1_up", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_view_update", + "n": "[CLY]_view_update", + "uid": "u1", + "did": "device-001", + "ts": 1704067201000, + "cd": { "$$t": "date", "ms": 1704067206000 }, + "c": 1, + "s": 0, + "dur": 12.5, + "up": { "p": "Android" }, + "sg": { "name": "Home", "exit": 1, "bounce": 1 } + } + }, + { + "id": "action-event", + "group": "known-events", + "notes": "New-format [CLY]_action with n already set by ingestion.", + "doc": { + "_id": "req2_u1_1704067202000_0", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_action", + "n": "[CLY]_action", + "uid": "u1", + "did": "device-001", + "ts": 1704067202000, + "cd": { "$$t": "date", "ms": 1704067207000 }, + "c": 1, + "s": 0, + "dur": 0, + "up": { "p": "web" }, + "sg": { "type": "click", "x": 100, "y": 250, "view": "/home", "width": 1920, "height": 1080 } + } + }, + { + "id": "nps-event", + "group": "known-events", + "notes": "New-format [CLY]_nps.", + "doc": { + "_id": "req3_u1_1704067203000_0", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_nps", + "n": "[CLY]_nps", + "uid": "u1", + "did": "device-001", + "ts": 1704067203000, + "cd": { "$$t": "date", "ms": 1704067208000 }, + "c": 1, + "s": 0, + "dur": 0, + "up": {}, + "sg": { "widget_id": "64f0aa11bb22cc33dd44ee55", "rating": 9, "comment": "great app" } + } + }, + { + "id": "survey-event", + "group": "known-events", + "notes": "New-format [CLY]_survey.", + "doc": { + "_id": "req4_u1_1704067204000_0", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_survey", + "n": "[CLY]_survey", + "uid": "u1", + "did": "device-001", + "ts": 1704067204000, + "cd": { "$$t": "date", "ms": 1704067209000 }, + "c": 1, + "s": 0, + "dur": 0, + "up": {}, + "sg": { "widget_id": "64f0aa11bb22cc33dd44ee66", "answers": { "q1": "yes", "q2": "no" } } + } + }, + { + "id": "star-rating-event", + "group": "known-events", + "notes": "New-format [CLY]_star_rating.", + "doc": { + "_id": "req5_u1_1704067205000_0", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_star_rating", + "n": "[CLY]_star_rating", + "uid": "u1", + "did": "device-001", + "ts": 1704067205000, + "cd": { "$$t": "date", "ms": 1704067210000 }, + "c": 1, + "s": 0, + "dur": 0, + "up": {}, + "sg": { "widget_id": "64f0aa11bb22cc33dd44ee77", "rating": 5, "platform": "iOS", "app_version": "2.1.0" } + } + }, + { + "id": "crash-event", + "group": "known-events", + "notes": "New-format [CLY]_crash.", + "doc": { + "_id": "req6_u1_1704067206000_0", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_crash", + "n": "[CLY]_crash", + "uid": "u1", + "did": "device-001", + "ts": 1704067206000, + "cd": { "$$t": "date", "ms": 1704067211000 }, + "c": 1, + "s": 0, + "dur": 0, + "up": { "p": "Android" }, + "sg": { "group": "8f7a6b5c4d3e2f1a", "nonfatal": false, "os": "Android 14" } + } + }, + { + "id": "session-event", + "group": "known-events", + "notes": "New-format [CLY]_session; _id is the request id.", + "doc": { + "_id": "req7requestid", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_session", + "n": "[CLY]_session", + "uid": "u1", + "did": "device-001", + "lsid": "lsid-002", + "ts": 1704067207000, + "cd": { "$$t": "date", "ms": 1704067212000 }, + "c": 1, + "s": 0, + "dur": 0, + "up": { "cc": "DE", "p": "iOS" } + } + }, + { + "id": "consent-event-extra-fields", + "group": "known-events", + "notes": "Non-schema doc fields (after/cvid/pvid/id/peid/_uid/up_extra) must not leak into the row.", + "doc": { + "_id": "req8_u1_1704067208000_0", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_consent", + "n": "[CLY]_consent", + "uid": "u1", + "_uid": "6543ab", + "did": "device-001", + "ts": 1704067208000, + "cd": { "$$t": "date", "ms": 1704067213000 }, + "c": 1, + "s": 0, + "dur": 0, + "up": {}, + "sg": { "change": "{\"sessions\":true}" }, + "after": { "sessions": true }, + "cvid": "view-1", + "pvid": "view-0", + "id": "evt-1", + "peid": "parent-1", + "up_extra": { "seg": "beta" } + } + }, + { + "id": "legacy-custom-event", + "group": "legacy-events", + "notes": "Legacy doc: raw event name in e, no n. Must become e=[CLY]_custom, n=.", + "doc": { + "_id": "656565656565656565656565", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "s": 5.5, + "up": { "cc": "US" }, + "sg": { "item": "sku-9" } + } + }, + { + "id": "legacy-view-derive-n", + "group": "legacy-events", + "notes": "Legacy [CLY]_view without n: n derived from sg.name.", + "doc": { + "_id": "656565656565656565656566", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_view", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "name": "Main Menu", "visit": 1 } + } + }, + { + "id": "legacy-action-view-fallback", + "group": "legacy-events", + "notes": "Legacy [CLY]_action with blank sg.name: n falls back to sg.view.", + "doc": { + "_id": "656565656565656565656567", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_action", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "name": " ", "view": "/cart", "type": "click" } + } + }, + { + "id": "legacy-nps-widget-id", + "group": "legacy-events", + "notes": "Legacy [CLY]_nps without n: n derived from sg.widget_id.", + "doc": { + "_id": "656565656565656565656568", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_nps", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "widget_id": "64f0aa11bb22cc33dd44ee88", "rating": 3 } + } + }, + { + "id": "legacy-star-rating-no-sg", + "group": "legacy-events", + "notes": "Legacy [CLY]_star_rating with no sg at all: n falls back to e.", + "doc": { + "_id": "656565656565656565656569", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_star_rating", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "legacy-crash-group", + "group": "legacy-events", + "notes": "Legacy [CLY]_crash without n: n derived from sg.group.", + "doc": { + "_id": "65656565656565656565656a", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_crash", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "group": "deadbeefcafe0123", "nonfatal": true } + } + }, + { + "id": "legacy-session-no-n", + "group": "legacy-events", + "notes": "Legacy [CLY]_session without n: default branch, n falls back to e.", + "doc": { + "_id": "65656565656565656565656b", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_session", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "s": 0, + "dur": 42 + } + }, + { + "id": "legacy-unknown-internal-event", + "group": "legacy-events", + "notes": "Internal event not in the derivation switch ([CLY]_push_action): n falls back to e.", + "doc": { + "_id": "65656565656565656565656c", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_push_action", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "i": "message-1", "b": 0 } + } + }, + { + "id": "n-preservation-view", + "group": "legacy-events", + "notes": "Dedup-parity rule: when the doc already has non-blank n, it wins over sg-derived n. A [CLY]_view doc with n=[CLY]_view and sg.name=Other must keep n=[CLY]_view, matching the live-ingested row.", + "doc": { + "_id": "65656565656565656565656d", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_view", + "n": "[CLY]_view", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "name": "Other", "visit": 1 } + } + }, + { + "id": "missing-a", + "group": "required-fields", + "notes": "No a and no collection defaults: skip.", + "doc": { + "_id": "65656565656565656565656e", + "e": "purchase", + "uid": "u1", + "ts": 1704067200000 + } + }, + { + "id": "blank-a", + "group": "required-fields", + "notes": "Whitespace-only a is blank: skip.", + "doc": { + "_id": "65656565656565656565656f", + "a": " ", + "e": "purchase", + "uid": "u1", + "ts": 1704067200000 + } + }, + { + "id": "numeric-a", + "group": "required-fields", + "notes": "Numeric a must be stringified for the LowCardinality(String) column.", + "doc": { + "_id": "656565656565656565656570", + "a": 12345, + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "missing-e", + "group": "required-fields", + "notes": "No e and no collection defaults: skip.", + "doc": { + "_id": "656565656565656565656571", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "uid": "u1", + "ts": 1704067200000 + } + }, + { + "id": "blank-e", + "group": "required-fields", + "notes": "Empty-string e is blank: skip.", + "doc": { + "_id": "656565656565656565656572", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "", + "uid": "u1", + "ts": 1704067200000 + } + }, + { + "id": "missing-uid", + "group": "required-fields", + "notes": "uid is part of row identity and the sharding key: skip.", + "doc": { + "_id": "656565656565656565656573", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "did": "device-001", + "ts": 1704067200000 + } + }, + { + "id": "numeric-uid", + "group": "required-fields", + "notes": "Numeric uid must be stringified.", + "doc": { + "_id": "656565656565656565656574", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": 42, + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "missing-id", + "group": "required-fields", + "notes": "No _id: skip.", + "doc": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "ts": 1704067200000 + } + }, + { + "id": "migrated-flag-set", + "group": "required-fields", + "notes": "Doc already marked migrated: skip (idempotent re-runs).", + "doc": { + "_id": "656565656565656565656575", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "ts": 1704067200000, + "migrated": true + } + }, + { + "id": "ts-missing", + "group": "timestamps", + "notes": "No ts: skip.", + "doc": { + "_id": "656565656565656565656576", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1" + } + }, + { + "id": "ts-zero", + "group": "timestamps", + "notes": "ts=0: skip.", + "doc": { + "_id": "656565656565656565656577", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "ts": 0 + } + }, + { + "id": "ts-small-integer", + "group": "timestamps", + "notes": "ts below the seconds heuristic floor (9.5e8): skip.", + "doc": { + "_id": "656565656565656565656578", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "ts": 12345 + } + }, + { + "id": "ts-seconds-precision", + "group": "timestamps", + "notes": "Seconds-precision ts (>=9.5e8, <9.5e11) is multiplied by 1000.", + "doc": { + "_id": "656565656565656565656579", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "ts-seconds-fractional", + "group": "timestamps", + "notes": "Fractional seconds ts keeps sub-second precision after the *1000 conversion.", + "doc": { + "_id": "65656565656565656565657a", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200.5, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "ts-millis-fractional", + "group": "timestamps", + "notes": "Fractional-millisecond ts is floored.", + "doc": { + "_id": "65656565656565656565657b", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000.9, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "ts-numeric-string", + "group": "timestamps", + "notes": "Numeric-string ts is parsed with Number() and then unit-normalized.", + "doc": { + "_id": "65656565656565656565657c", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": "1704067200000", + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "ts-iso-string", + "group": "timestamps", + "notes": "ISO-8601 string ts is NOT accepted (Number() parse only, no Date.parse ambiguity): skip.", + "doc": { + "_id": "65656565656565656565657d", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "ts": "2024-01-01T00:00:00.000Z" + } + }, + { + "id": "ts-date-object", + "group": "timestamps", + "notes": "BSON date (JS Date) ts uses getTime().", + "doc": { + "_id": "65656565656565656565657e", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": { "$$t": "date", "ms": 1704067200123 }, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "ts-extended-json-number", + "group": "timestamps", + "notes": "Raw Mongo Extended JSON {$date: } object (from EJSON-dumped data) recurses on the value.", + "doc": { + "_id": "65656565656565656565657f", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": { "$date": 1704067200000 }, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "ts-extended-json-nested-long", + "group": "timestamps", + "notes": "Canonical Extended JSON {$date: {$numberLong: ...}} is NOT resolvable (inner object is not a timestamp): skip on both sides, by agreement.", + "doc": { + "_id": "656565656565656565656580", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "ts": { "$date": { "$numberLong": "1704067200000" } } + } + }, + { + "id": "ts-below-seconds-floor", + "group": "timestamps", + "notes": "ts just below 9.5e8: skip.", + "doc": { + "_id": "656565656565656565656581", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "ts": 949999999 + } + }, + { + "id": "ts-beyond-datetime64-range", + "group": "timestamps", + "notes": "Countly-owned timestamp clamps to the DateTime64(3) column range (policy). Driver-promoted Int64 1e17 ms lands far beyond 2299 and clamps to 2299-12-31 23:59:59.999.", + "doc": { + "_id": "656565656565656565656582", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": { "$$t": "driverLong", "v": "100000000000000000" }, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "lu-variants-valid", + "group": "timestamps", + "notes": "lu as seconds-precision number is unit-normalized like ts.", + "doc": { + "_id": "656565656565656565656583", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "lu": 1703980800, + "c": 1 + } + }, + { + "id": "lu-garbage-string", + "group": "timestamps", + "notes": "Unparseable lu is dropped (row still emitted, lu=NULL).", + "doc": { + "_id": "656565656565656565656584", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "lu": "not-a-date", + "c": 1 + } + }, + { + "id": "lu-epoch-negative-date", + "group": "timestamps", + "notes": "A pre-1970 Date in lu (negative epoch) is dropped: only lu > 0 is kept.", + "doc": { + "_id": "656565656565656565656585", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "lu": { "$$t": "date", "ms": -5000 }, + "c": 1 + } + }, + { + "id": "cd-missing-falls-back-to-ts", + "group": "timestamps", + "notes": "Docs predating cd: cd must be emitted deterministically as ts, never left to the now64(3) column default.", + "doc": { + "_id": "656565656565656565656586", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "c": 1 + } + }, + { + "id": "cd-epoch-zero-date", + "group": "timestamps", + "notes": "cd at epoch 0 is not a valid creation stamp: falls back to ts.", + "doc": { + "_id": "656565656565656565656587", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 0 }, + "c": 1 + } + }, + { + "id": "cd-numeric-millis", + "group": "timestamps", + "notes": "Numeric cd (legacy oddity) is normalized like any Countly timestamp.", + "doc": { + "_id": "656565656565656565656588", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": 1704067205123, + "c": 1 + } + }, + { + "id": "cd-beyond-datetime64-range", + "group": "timestamps", + "notes": "cd in year 2500 clamps to the DateTime64(3) maximum.", + "doc": { + "_id": "656565656565656565656589", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 16725225600000 }, + "c": 1 + } + }, + { + "id": "c-float", + "group": "countly-numerics", + "notes": "Fractional count floors to an integer for UInt32.", + "doc": { + "_id": "65656565656565656565658a", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 2.9 + } + }, + { + "id": "c-negative", + "group": "countly-numerics", + "notes": "Negative count clamps to 0 (UInt32 lower bound).", + "doc": { + "_id": "65656565656565656565658b", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": -5 + } + }, + { + "id": "c-beyond-uint32", + "group": "countly-numerics", + "notes": "Count beyond 2^32-1 clamps to the UInt32 maximum (policy: Countly-owned fields clamp to column ranges).", + "doc": { + "_id": "65656565656565656565658c", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1000000000000 + } + }, + { + "id": "c-numeric-string", + "group": "countly-numerics", + "notes": "Numeric-string count parses.", + "doc": { + "_id": "65656565656565656565658d", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": "7" + } + }, + { + "id": "c-garbage-string", + "group": "countly-numerics", + "notes": "Unparseable count becomes 0.", + "doc": { + "_id": "65656565656565656565658e", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": "abc" + } + }, + { + "id": "c-nan", + "group": "countly-numerics", + "notes": "NaN count becomes 0.", + "doc": { + "_id": "65656565656565656565658f", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": { "$$t": "nan" } + } + }, + { + "id": "s-dur-numeric-strings", + "group": "countly-numerics", + "notes": "Numeric-string sum/duration parse to Float64.", + "doc": { + "_id": "656565656565656565656590", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "s": "99.99", + "dur": "30.5" + } + }, + { + "id": "s-nan-dur-infinity", + "group": "countly-numerics", + "notes": "Non-finite sum/duration become 0 (Float64 columns are non-nullable; JSON cannot carry NaN/Infinity).", + "doc": { + "_id": "656565656565656565656591", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "s": { "$$t": "nan" }, + "dur": { "$$t": "inf" } + } + }, + { + "id": "s-negative-dur-negative", + "group": "countly-numerics", + "notes": "Negative sum/duration are within Float64 range and pass through unchanged.", + "doc": { + "_id": "656565656565656565656592", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "s": -2.5, + "dur": -1 + } + }, + { + "id": "c-s-dur-missing", + "group": "countly-numerics", + "notes": "Missing count/sum/duration default to 0/0/0 in the row (explicit on the wire; SDK-level count defaulting to 1 happens at ingestion, before the doc exists).", + "doc": { + "_id": "656565656565656565656593", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 } + } + }, + { + "id": "sg-mixed-types", + "group": "customer-json", + "notes": "Mixed value types per key including nested structures, unicode and explicit null pass through unchanged.", + "doc": { + "_id": "656565656565656565656594", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { + "str": "text", + "int": 42, + "float": 3.14159, + "bool": true, + "nul": null, + "arr": ["a", 1, false], + "nested": { "k1": "v1", "k2": { "k3": [1, 2, 3] } }, + "unicode": "žąčęėįšųū 中文 🎉" + } + } + }, + { + "id": "sg-nan-infinity", + "group": "customer-json", + "notes": "Policy: customer-owned values that do not fit JSON numeric representation are stringified losslessly (NaN -> \"NaN\", Infinity -> \"Infinity\", -Infinity -> \"-Infinity\"), never silently nulled.", + "doc": { + "_id": "656565656565656565656595", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "bad1": { "$$t": "nan" }, "bad2": { "$$t": "inf" }, "bad3": { "$$t": "-inf" }, "ok": 1 }, + "custom": { "score": { "$$t": "nan" } } + } + }, + { + "id": "sg-decimal128", + "group": "customer-json", + "notes": "BSON Decimal128 (not promoted by the driver) is stringified losslessly to its decimal representation, not dumped as {$numberDecimal:...}.", + "doc": { + "_id": "656565656565656565656596", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "price": { "$$t": "decimal128", "v": "19.99999999999999999999999999999999" } } + } + }, + { + "id": "sg-int64-driver-promoted", + "group": "customer-json", + "notes": "BSON Int64 beyond 2^53 read with default driver options arrives as an already-lossy double; both sides emit that same double. Residual precision loss is a documented driver-level limitation (fixing it requires useBigInt64 at read time).", + "doc": { + "_id": "656565656565656565656597", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "big": { "$$t": "driverLong", "v": "9223372036854775807" }, "safe": 9007199254740991 } + } + }, + { + "id": "sg-date-value", + "group": "customer-json", + "notes": "Date values inside customer JSON serialize to ISO-8601 strings (JSON.stringify behavior, identical on both wires).", + "doc": { + "_id": "656565656565656565656598", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "when": { "$$t": "date", "ms": 1703980800500 } } + } + }, + { + "id": "sg-undefined-value", + "group": "customer-json", + "notes": "undefined values inside customer JSON are dropped (key disappears).", + "doc": { + "_id": "656565656565656565656599", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "gone": { "$$t": "undef" }, "kept": "v" } + } + }, + { + "id": "sg-deep-nesting", + "group": "customer-json", + "notes": "Six levels of nesting pass through unchanged.", + "doc": { + "_id": "65656565656565656565659a", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "l1": { "l2": { "l3": { "l4": { "l5": { "l6": "deep", "arr": [{ "x": 1 }] } } } } } } + } + }, + { + "id": "sg-oversized-string", + "group": "customer-json", + "notes": "A 32KiB string value passes through untruncated on both sides.", + "doc": { + "_id": "65656565656565656565659b", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "sg": { "blob": { "$$t": "bigstring", "c": "x", "n": 32768 } } + } + }, + { + "id": "json-fields-wrong-types", + "group": "customer-json", + "notes": "Non-object up/custom/cmp/sg (strings, arrays, numbers) cannot land in JSON columns: treated as absent (up/sg -> {}, custom/cmp -> NULL).", + "doc": { + "_id": "65656565656565656565659c", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1, + "up": "not-an-object", + "custom": 42, + "cmp": [1, 2, 3], + "sg": "also-not-an-object" + } + }, + { + "id": "json-fields-all-absent", + "group": "customer-json", + "notes": "No up/custom/cmp/sg at all: up={}, sg={}, custom=NULL, cmp=NULL.", + "doc": { + "_id": "65656565656565656565659d", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "lsid-blank", + "group": "identity-fields", + "notes": "Blank lsid is treated as absent (NULL), matching the live path.", + "doc": { + "_id": "65656565656565656565659e", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "did": "device-001", + "lsid": "", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "uid-canon-preserved", + "group": "identity-fields", + "notes": "uid_canon must survive the transform on both sides.", + "doc": { + "_id": "65656565656565656565659f", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "uid_canon": "canonical-u1", + "did": "device-001", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + }, + { + "id": "did-missing-defaults-empty", + "group": "identity-fields", + "notes": "Missing did defaults to '' (non-nullable String column).", + "doc": { + "_id": "6565656565656565656565a0", + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "purchase", + "uid": "u1", + "ts": 1704067200000, + "cd": { "$$t": "date", "ms": 1704067205000 }, + "c": 1 + } + } + ] +} diff --git a/tests/differential/decode.mjs b/tests/differential/decode.mjs new file mode 100644 index 0000000..992e839 --- /dev/null +++ b/tests/differential/decode.mjs @@ -0,0 +1,72 @@ +/** + * Corpus decoder for the ingestion-matching transformer differential harness. + * + * The fixture corpus (corpus.json) is plain JSON, but the input documents it + * describes contain values plain JSON cannot express (NaN, Infinity, Date + * instances, BSON Decimal128, driver-promoted Int64 doubles, huge strings). + * Those are encoded as `{"$$t": ...}` wrapper objects and decoded here into + * the exact JavaScript values the MongoDB Node driver (default options, + * promoteLongs=true) would hand to either transform. + * + * This file is part of the cross-repo differential contract and must stay + * byte-identical between: + * countly-platform: test/unit/fixtures/drill-transform-differential/decode.mjs + * Countly/migration: tests/differential/decode.mjs + */ + +import { Decimal128, Long } from 'mongodb'; + +/** + * Recursively decode a corpus-encoded value into the runtime value the + * transforms receive. + * @param {*} value - encoded corpus value + * @returns {*} decoded runtime value + */ +export function decodeValue(value) { + if (Array.isArray(value)) { + return value.map(decodeValue); + } + if (value === null || typeof value !== 'object') { + return value; + } + if (typeof value.$$t === 'string') { + switch (value.$$t) { + case 'nan': + return NaN; + case 'inf': + return Infinity; + case '-inf': + return -Infinity; + case 'undef': + return undefined; + case 'date': + // A real Date instance, as the driver returns for BSON dates. + return new Date(value.ms); + case 'decimal128': + // Decimal128 is NOT promoted by the driver; transforms see the BSON object. + return Decimal128.fromString(value.v); + case 'driverLong': + // BSON Int64 read with default driver options (promoteLongs=true): + // the transform sees a (possibly precision-lossy) JS double. + return Long.fromString(value.v).toNumber(); + case 'bigstring': + return value.c.repeat(value.n); + default: + throw new Error(`Unknown corpus wrapper type: ${value.$$t}`); + } + } + const out = {}; + for (const key of Object.keys(value)) { + out[key] = decodeValue(value[key]); + } + return out; +} + +/** + * Decode one corpus entry's `doc` into a runtime input document. + * @param {{id: string, doc: object}} entry - corpus entry + * @returns {object} decoded document + */ +export function decodeDoc(entry) { + return decodeValue(entry.doc); +} diff --git a/tests/differential/differential.test.ts b/tests/differential/differential.test.ts new file mode 100644 index 0000000..c48ee24 --- /dev/null +++ b/tests/differential/differential.test.ts @@ -0,0 +1,92 @@ +/** + * Ingestion-matching transformer differential harness — migration side. + * (Drill migration overhaul item D4, https://github.com/Countly/migration/issues/6) + * + * Asserts that this repo's transform (src/transform/normalize.ts) reproduces, + * for every document in the shared fixture corpus, EXACTLY the canonical + * ClickHouse row that countly-platform's live ingestion normalization + * produces (checked-in goldens generated by the platform repo). + * + * The fixture set (corpus.json, goldens.json, decode.mjs, canonicalize.mjs) + * is vendored byte-identical from countly-platform + * test/unit/fixtures/drill-transform-differential/. The goldens' embedded + * hashes tie the three artifacts together, so a partial sync fails loudly. + * + * If this test fails: + * - after a change in THIS repo: the migration transform drifted from live + * ingestion — fix the transform, do not touch the goldens here. + * - after syncing new fixtures from countly-platform: the platform changed + * normalization behavior — align src/transform/ to match. + * + * Pure unit test: no MongoDB/ClickHouse/Redis needed. + */ + +import { describe, it, expect } from 'vitest'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +// eslint-disable-next-line import/no-unresolved +import { decodeDoc } from './decode.mjs'; +// eslint-disable-next-line import/no-unresolved +import { canonicalizeOutput } from './canonicalize.mjs'; +import { transformDocument, transformBatch, type SourceDocument } from '../../src/transform/normalize.ts'; +import { SkipCounter } from '../../src/transform/skip-reasons.ts'; + +const here = path.dirname(fileURLToPath(import.meta.url)); + +const corpus = JSON.parse(readFileSync(path.join(here, 'corpus.json'), 'utf8')) as { + entries: Array<{ id: string; group: string; notes: string; doc: Record }>; +}; +const goldens = JSON.parse(readFileSync(path.join(here, 'goldens.json'), 'utf8')) as { + corpusSha256: string; + contractSha256: string; + entries: Record }>; +}; + +function fileSha256(fileName: string): string { + return createHash('sha256').update(readFileSync(path.join(here, fileName))).digest('hex'); +} + +describe('differential: migration transform vs countly-platform live ingestion', () => { + it('vendored fixture set is in sync (corpus matches the goldens generation)', () => { + expect(fileSha256('corpus.json'), 'corpus.json does not match the corpus the goldens were generated from — re-sync ALL fixture files from countly-platform test/unit/fixtures/drill-transform-differential/').toBe(goldens.corpusSha256); + const contractSha = createHash('sha256') + .update(readFileSync(path.join(here, 'decode.mjs'))) + .update(readFileSync(path.join(here, 'canonicalize.mjs'))) + .digest('hex'); + expect(contractSha, 'decode.mjs/canonicalize.mjs do not match the contract the goldens were generated with — re-sync ALL fixture files from countly-platform').toBe(goldens.contractSha256); + }); + + it('every corpus fixture has a golden and vice versa', () => { + const corpusIds = corpus.entries.map((entry) => entry.id).sort(); + expect(Object.keys(goldens.entries).sort()).toEqual(corpusIds); + }); + + for (const entry of corpus.entries) { + it(`reproduces the live-ingestion row for "${entry.id}" (${entry.group})`, () => { + const golden = goldens.entries[entry.id]; + const result = transformDocument(decodeDoc(entry) as SourceDocument); + const canonical = canonicalizeOutput(result.row); + expect(canonical, `${entry.notes}\nGoldens are generated by countly-platform live ingestion normalization; fix src/transform/, never the vendored goldens.`).toEqual(golden); + if ('skip' in golden && golden.skip === true) { + expect(result.skipReason, 'skipped documents must carry a skip reason').not.toBeNull(); + } + const serialized = JSON.stringify(canonical); + expect(serialized).not.toContain('__UNINSERTABLE__'); + expect(serialized).not.toContain('__NONDETERMINISTIC__'); + }); + } + + it('transformBatch matches per-document transformDocument over the whole corpus', () => { + const docs = corpus.entries.map((entry) => decodeDoc(entry) as SourceDocument); + const counter = new SkipCounter(); + const { rows } = transformBatch(docs, counter); + const expectedRows = corpus.entries + .map((entry) => transformDocument(decodeDoc(entry) as SourceDocument).row) + .filter((row) => row !== null); + expect(rows).toEqual(expectedRows); + expect(rows.length + counter.getTotal()).toBe(corpus.entries.length); + }); +}); diff --git a/tests/differential/goldens.json b/tests/differential/goldens.json new file mode 100644 index 0000000..3a4faa6 --- /dev/null +++ b/tests/differential/goldens.json @@ -0,0 +1,1468 @@ +{ + "description": "Expected countly_drill.drill_events rows (canonical form, see canonicalize.mjs) produced by countly-platform live ingestion normalization for each corpus.json input document. Generated by generate-goldens.mjs — do not edit by hand. Countly/migration CI asserts its transform reproduces these exactly.", + "generator": "test/unit/fixtures/drill-transform-differential/generate-goldens.mjs", + "sourceOfTruth": "api/utils/eventTransformer.ts (transformToKafkaEventFormat)", + "corpusSha256": "db0f0ea6f03287a4da3863a654751cebf5ff0037840473c87d714362d90ea134", + "contractSha256": "2464ff30319b8185e373329ef6856454c1dbb1a6db4a3a50680a128cceab7134", + "entries": { + "custom-event-full": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": "lsid-001", + "_id": "req1hash_u1_1704067200000_0", + "ts": "2024-01-01 00:00:00.000", + "lu": "2023-12-31 00:00:00.000", + "cd": "2024-01-01 00:00:05.000", + "up": { + "av": "2.1.0", + "cc": "US", + "dow": 1, + "hour": 14, + "p": "iOS" + }, + "custom": { + "beta": "true", + "tier": "premium" + }, + "cmp": { + "c": "Organic" + }, + "sg": { + "gift": false, + "item": "sku-123", + "price": 9.995, + "qty": 2 + }, + "c": 2, + "s": 19.99, + "dur": 3.5 + } + }, + "view-visit": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_view", + "n": "[CLY]_view", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "5f1c_u1_view1", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": { + "p": "Android" + }, + "custom": null, + "cmp": null, + "sg": { + "name": "Home", + "segment": "Android", + "start": 1, + "visit": 1 + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "view-update": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_view_update", + "n": "[CLY]_view_update", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "5f1c_u1_view1_up", + "ts": "2024-01-01 00:00:01.000", + "lu": null, + "cd": "2024-01-01 00:00:06.000", + "up": { + "p": "Android" + }, + "custom": null, + "cmp": null, + "sg": { + "bounce": 1, + "exit": 1, + "name": "Home" + }, + "c": 1, + "s": 0, + "dur": 12.5 + } + }, + "action-event": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_action", + "n": "[CLY]_action", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "req2_u1_1704067202000_0", + "ts": "2024-01-01 00:00:02.000", + "lu": null, + "cd": "2024-01-01 00:00:07.000", + "up": { + "p": "web" + }, + "custom": null, + "cmp": null, + "sg": { + "height": 1080, + "type": "click", + "view": "/home", + "width": 1920, + "x": 100, + "y": 250 + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "nps-event": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_nps", + "n": "[CLY]_nps", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "req3_u1_1704067203000_0", + "ts": "2024-01-01 00:00:03.000", + "lu": null, + "cd": "2024-01-01 00:00:08.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "comment": "great app", + "rating": 9, + "widget_id": "64f0aa11bb22cc33dd44ee55" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "survey-event": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_survey", + "n": "[CLY]_survey", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "req4_u1_1704067204000_0", + "ts": "2024-01-01 00:00:04.000", + "lu": null, + "cd": "2024-01-01 00:00:09.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "answers": { + "q1": "yes", + "q2": "no" + }, + "widget_id": "64f0aa11bb22cc33dd44ee66" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "star-rating-event": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_star_rating", + "n": "[CLY]_star_rating", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "req5_u1_1704067205000_0", + "ts": "2024-01-01 00:00:05.000", + "lu": null, + "cd": "2024-01-01 00:00:10.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "app_version": "2.1.0", + "platform": "iOS", + "rating": 5, + "widget_id": "64f0aa11bb22cc33dd44ee77" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "crash-event": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_crash", + "n": "[CLY]_crash", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "req6_u1_1704067206000_0", + "ts": "2024-01-01 00:00:06.000", + "lu": null, + "cd": "2024-01-01 00:00:11.000", + "up": { + "p": "Android" + }, + "custom": null, + "cmp": null, + "sg": { + "group": "8f7a6b5c4d3e2f1a", + "nonfatal": false, + "os": "Android 14" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "session-event": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_session", + "n": "[CLY]_session", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": "lsid-002", + "_id": "req7requestid", + "ts": "2024-01-01 00:00:07.000", + "lu": null, + "cd": "2024-01-01 00:00:12.000", + "up": { + "cc": "DE", + "p": "iOS" + }, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "consent-event-extra-fields": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_consent", + "n": "[CLY]_consent", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "req8_u1_1704067208000_0", + "ts": "2024-01-01 00:00:08.000", + "lu": null, + "cd": "2024-01-01 00:00:13.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "change": "{\"sessions\":true}" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "legacy-custom-event": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656565", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": { + "cc": "US" + }, + "custom": null, + "cmp": null, + "sg": { + "item": "sku-9" + }, + "c": 1, + "s": 5.5, + "dur": 0 + } + }, + "legacy-view-derive-n": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_view", + "n": "Main Menu", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656566", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "name": "Main Menu", + "visit": 1 + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "legacy-action-view-fallback": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_action", + "n": "/cart", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656567", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "name": " ", + "type": "click", + "view": "/cart" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "legacy-nps-widget-id": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_nps", + "n": "64f0aa11bb22cc33dd44ee88", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656568", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "rating": 3, + "widget_id": "64f0aa11bb22cc33dd44ee88" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "legacy-star-rating-no-sg": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_star_rating", + "n": "[CLY]_star_rating", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656569", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "legacy-crash-group": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_crash", + "n": "deadbeefcafe0123", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565656a", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "group": "deadbeefcafe0123", + "nonfatal": true + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "legacy-session-no-n": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_session", + "n": "[CLY]_session", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565656b", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 42 + } + }, + "legacy-unknown-internal-event": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_push_action", + "n": "[CLY]_push_action", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565656c", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "b": 0, + "i": "message-1" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "n-preservation-view": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_view", + "n": "[CLY]_view", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565656d", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "name": "Other", + "visit": 1 + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "missing-a": { + "skip": true + }, + "blank-a": { + "skip": true + }, + "numeric-a": { + "row": { + "a": "12345", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656570", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "missing-e": { + "skip": true + }, + "blank-e": { + "skip": true + }, + "missing-uid": { + "skip": true + }, + "numeric-uid": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "42", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656574", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "missing-id": { + "skip": true + }, + "migrated-flag-set": { + "skip": true + }, + "ts-missing": { + "skip": true + }, + "ts-zero": { + "skip": true + }, + "ts-small-integer": { + "skip": true + }, + "ts-seconds-precision": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656579", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "ts-seconds-fractional": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565657a", + "ts": "2024-01-01 00:00:00.500", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "ts-millis-fractional": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565657b", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "ts-numeric-string": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565657c", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "ts-iso-string": { + "skip": true + }, + "ts-date-object": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565657e", + "ts": "2024-01-01 00:00:00.123", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "ts-extended-json-number": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565657f", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "ts-extended-json-nested-long": { + "skip": true + }, + "ts-below-seconds-floor": { + "skip": true + }, + "ts-beyond-datetime64-range": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656582", + "ts": "2299-12-31 23:59:59.999", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "lu-variants-valid": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656583", + "ts": "2024-01-01 00:00:00.000", + "lu": "2023-12-31 00:00:00.000", + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "lu-garbage-string": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656584", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "lu-epoch-negative-date": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656585", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "cd-missing-falls-back-to-ts": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656586", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:00.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "cd-epoch-zero-date": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656587", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:00.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "cd-numeric-millis": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656588", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.123", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "cd-beyond-datetime64-range": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656589", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2299-12-31 23:59:59.999", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "c-float": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565658a", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 2, + "s": 0, + "dur": 0 + } + }, + "c-negative": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565658b", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 0, + "s": 0, + "dur": 0 + } + }, + "c-beyond-uint32": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565658c", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 4294967295, + "s": 0, + "dur": 0 + } + }, + "c-numeric-string": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565658d", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 7, + "s": 0, + "dur": 0 + } + }, + "c-garbage-string": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565658e", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 0, + "s": 0, + "dur": 0 + } + }, + "c-nan": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565658f", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 0, + "s": 0, + "dur": 0 + } + }, + "s-dur-numeric-strings": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656590", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 99.99, + "dur": 30.5 + } + }, + "s-nan-dur-infinity": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656591", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "s-negative-dur-negative": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656592", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": -2.5, + "dur": -1 + } + }, + "c-s-dur-missing": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656593", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 0, + "s": 0, + "dur": 0 + } + }, + "sg-mixed-types": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656594", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "arr": [ + "a", + 1, + false + ], + "bool": true, + "float": 3.14159, + "int": 42, + "nested": { + "k1": "v1", + "k2": { + "k3": [ + 1, + 2, + 3 + ] + } + }, + "nul": null, + "str": "text", + "unicode": "žąčęėįšųū 中文 🎉" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "sg-nan-infinity": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656595", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": { + "score": "NaN" + }, + "cmp": null, + "sg": { + "bad1": "NaN", + "bad2": "Infinity", + "bad3": "-Infinity", + "ok": 1 + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "sg-decimal128": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656596", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "price": "19.99999999999999999999999999999999" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "sg-int64-driver-promoted": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656597", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "big": 9223372036854776000, + "safe": 9007199254740991 + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "sg-date-value": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656598", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "when": "2023-12-31T00:00:00.500Z" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "sg-undefined-value": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "656565656565656565656599", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "kept": "v" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "sg-deep-nesting": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565659a", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "l1": { + "l2": { + "l3": { + "l4": { + "l5": { + "arr": [ + { + "x": 1 + } + ], + "l6": "deep" + } + } + } + } + } + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "sg-oversized-string": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565659b", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": { + "blob": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "json-fields-wrong-types": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565659c", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "json-fields-all-absent": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565659d", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "lsid-blank": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565659e", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "uid-canon-preserved": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": "canonical-u1", + "did": "device-001", + "lsid": null, + "_id": "65656565656565656565659f", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + }, + "did-missing-defaults-empty": { + "row": { + "a": "5f1c8e2a9b3d4c001a2b3c4d", + "e": "[CLY]_custom", + "n": "purchase", + "uid": "u1", + "uid_canon": null, + "did": "", + "lsid": null, + "_id": "6565656565656565656565a0", + "ts": "2024-01-01 00:00:00.000", + "lu": null, + "cd": "2024-01-01 00:00:05.000", + "up": {}, + "custom": null, + "cmp": null, + "sg": {}, + "c": 1, + "s": 0, + "dur": 0 + } + } + } +} diff --git a/tests/helpers/seed-mongo.ts b/tests/helpers/seed-mongo.ts deleted file mode 100644 index 2a53bde..0000000 --- a/tests/helpers/seed-mongo.ts +++ /dev/null @@ -1,290 +0,0 @@ -/** - * MongoDB test data seeding utilities. - * - * Creates drill_events collections with realistic documents that match - * the production schema and exercise all code paths in the transform layer. - */ -import { ObjectId, type Db } from "mongodb"; -import { getMongoDb, TEST_COLLECTION_PREFIX } from "./setup.ts"; -import crypto from "node:crypto"; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface SeedOptions { - /** Number of documents to insert. */ - count: number; - /** App ID for all docs. Defaults to a fixed test app ID. */ - appId?: string; - /** Event name. Defaults to "test_event". */ - eventName?: string; - /** Start date for cd/ts spread. Defaults to 2024-01-01. */ - startDate?: Date; - /** End date for cd/ts spread. Defaults to 2025-01-01. */ - endDate?: Date; - /** Fraction of docs to mark as migrated (0–1). Default 0. */ - migratedFraction?: number; - /** Fraction of docs missing uid (will be skipped). Default 0. */ - missingUidFraction?: number; - /** Whether to create the {cd:1, _id:1} index. Default true. */ - createIndex?: boolean; - /** Fraction of docs with cd set to null (0–1). Default 0. */ - nullCdFraction?: number; - /** Fraction of docs with invalid ts (ts=0, will be skipped). Default 0. */ - invalidTsFraction?: number; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -const DEFAULT_APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; - -/** Compute the drill_events collection name from appId + eventName. */ -export function collectionName(eventName: string, appId: string): string { - const hash = crypto.createHash("sha1").update(eventName + appId).digest("hex"); - return `${TEST_COLLECTION_PREFIX}${hash}`; -} - -/** Generate a realistic test document. */ -function makeDoc(opts: { - appId: string; - eventName: string; - ts: number; - cd: Date; - idx: number; - migrated?: boolean; - missingUid?: boolean; -}): Record { - const doc: Record = { - _id: new ObjectId().toHexString(), - a: opts.appId, - e: opts.eventName, - uid: opts.missingUid ? undefined : `user-${(opts.idx % 1000).toString().padStart(4, "0")}`, - did: `device-${(opts.idx % 500).toString().padStart(4, "0")}`, - ts: opts.ts, - cd: opts.cd, - c: Math.floor(Math.random() * 5) + 1, - s: Math.random() * 100, - dur: Math.random() * 60, - n: opts.eventName, - }; - - if (opts.migrated) { - doc.migrated = true; - } - if (opts.missingUid) { - delete doc.uid; - } - - // Add some segment data for view events - if (opts.eventName.includes("view")) { - doc.sg = { name: `Page ${opts.idx % 20}` }; - } - - return doc; -} - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -/** - * Seed a single drill_events collection with test documents. - * Returns the collection name and the count of non-skippable docs. - */ -export async function seedCollection(opts: SeedOptions): Promise<{ - collName: string; - totalDocs: number; - expectedRows: number; // docs that should make it to ClickHouse -}> { - const db = await getMongoDb(); - const appId = opts.appId ?? DEFAULT_APP_ID; - const eventName = opts.eventName ?? "test_event"; - const start = (opts.startDate ?? new Date("2024-01-01")).getTime(); - const end = (opts.endDate ?? new Date("2025-01-01")).getTime(); - const migratedFrac = opts.migratedFraction ?? 0; - const missingUidFrac = opts.missingUidFraction ?? 0; - - const collName = collectionName(eventName, appId); - const coll = db.collection(collName); - - // Drop if exists - await coll.drop().catch(() => {}); - - // Generate docs - const docs: Record[] = []; - let expectedRows = 0; - - for (let i = 0; i < opts.count; i++) { - const fraction = i / Math.max(1, opts.count - 1); - const ts = start + Math.floor(fraction * (end - start)); - const cd = new Date(ts); - - const migrated = Math.random() < migratedFrac; - const missingUid = !migrated && Math.random() < missingUidFrac; - const nullCd = !migrated && !missingUid && Math.random() < (opts.nullCdFraction ?? 0); - const invalidTs = !migrated && !missingUid && !nullCd && Math.random() < (opts.invalidTsFraction ?? 0); - - docs.push(makeDoc({ - appId, - eventName, - ts: invalidTs ? 0 : ts, - cd: nullCd ? null as any : cd, - idx: i, - migrated, - missingUid, - })); - - if (!migrated && !missingUid && !invalidTs) { - expectedRows++; - } - } - - // Bulk insert in chunks of 5000 - const CHUNK = 5000; - for (let i = 0; i < docs.length; i += CHUNK) { - await coll.insertMany(docs.slice(i, i + CHUNK)); - } - - // Create the required compound index - if (opts.createIndex !== false) { - await coll.createIndex({ cd: 1, _id: 1 }); - } - - return { collName, totalDocs: opts.count, expectedRows }; -} - -/** - * Seed multiple collections with varying sizes. - * Returns metadata for each collection. - */ -export async function seedMultipleCollections(configs: Array<{ - eventName: string; - count: number; - appId?: string; - startDate?: Date; - endDate?: Date; - migratedFraction?: number; - missingUidFraction?: number; -}>): Promise> { - const results: Array<{ - collName: string; - eventName: string; - totalDocs: number; - expectedRows: number; - }> = []; - - for (const cfg of configs) { - const r = await seedCollection({ - count: cfg.count, - eventName: cfg.eventName, - appId: cfg.appId, - startDate: cfg.startDate, - endDate: cfg.endDate, - migratedFraction: cfg.migratedFraction, - missingUidFraction: cfg.missingUidFraction, - }); - results.push({ ...r, eventName: cfg.eventName }); - } - - return results; -} - -/** - * Seed documents with specific cd timestamps for boundary testing. - * Each entry in `timestamps` creates one doc at that exact time. - */ -export async function seedAtTimestamps( - eventName: string, - appId: string, - timestamps: Date[], -): Promise<{ collName: string; totalDocs: number }> { - const db = await getMongoDb(); - const collName = collectionName(eventName, appId); - const coll = db.collection(collName); - - await coll.drop().catch(() => {}); - - const docs = timestamps.map((ts, i) => makeDoc({ - appId, - eventName, - ts: ts.getTime(), - cd: ts, - idx: i, - })); - - if (docs.length > 0) { - await coll.insertMany(docs); - } - - await coll.createIndex({ cd: 1, _id: 1 }); - - return { collName, totalDocs: docs.length }; -} - -/** - * Register app/event hash in the countly database for HashResolver. - * This makes the HashResolver able to resolve the collection name - * back to { a: appId, e: eventName }. - */ -export async function registerEventHash( - appId: string, - eventName: string, - db?: Db, -): Promise { - const mongoDb = db ?? await getMongoDb(); - // The hash resolver reads from the countly db's events collection - // For tests, we'll use a simpler approach: the collection name is deterministic - // from SHA1(eventName + appId), so the resolver should work if it has the app/event data. - // In practice, tests pass collectionDefaults directly to the transform layer. -} - -/** - * Seed a collection where ALL documents have cd: null. - */ -export async function seedNullCdCollection(opts: { - count: number; - appId?: string; - eventName?: string; -}): Promise<{ collName: string; totalDocs: number; expectedRows: number }> { - const db = await getMongoDb(); - const appId = opts.appId ?? DEFAULT_APP_ID; - const eventName = opts.eventName ?? "test_null_cd_event"; - const collName = collectionName(eventName, appId); - const coll = db.collection(collName); - - await coll.drop().catch(() => {}); - - const docs: Record[] = []; - for (let i = 0; i < opts.count; i++) { - docs.push({ - _id: new ObjectId().toHexString(), - a: appId, - e: eventName, - uid: `user-${(i % 1000).toString().padStart(4, "0")}`, - did: `device-${(i % 500).toString().padStart(4, "0")}`, - ts: Date.now() - (opts.count - i) * 1000, - cd: null, - c: 1, - s: 0, - dur: 0, - n: eventName, - }); - } - - const CHUNK = 5000; - for (let i = 0; i < docs.length; i += CHUNK) { - await coll.insertMany(docs.slice(i, i + CHUNK)); - } - - await coll.createIndex({ cd: 1, _id: 1 }); - - return { collName, totalDocs: opts.count, expectedRows: opts.count }; -} diff --git a/tests/helpers/setup.ts b/tests/helpers/setup.ts deleted file mode 100644 index e000c76..0000000 --- a/tests/helpers/setup.ts +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Global test setup: connections to MongoDB, ClickHouse, Redis. - * Shared across all integration tests. - */ -import { MongoClient, type Db } from "mongodb"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; -import Redis from "ioredis"; - -// --------------------------------------------------------------------------- -// Test constants -// --------------------------------------------------------------------------- -export const TEST_MONGO_URI = "mongodb://localhost:27017/?directConnection=true"; -export const TEST_MONGO_DB = "test_mig_integration"; -export const TEST_MANIFEST_DB = "test_mig_integration"; -export const TEST_CH_URL = "http://localhost:8123"; -export const TEST_CH_DB = "test_mig_integration"; -export const TEST_CH_TABLE = "drill_events"; -export const TEST_REDIS_URL = "redis://localhost:6379"; -export const TEST_REDIS_PREFIX = "test_mig"; -export const TEST_COLLECTION_PREFIX = "drill_events"; - -// --------------------------------------------------------------------------- -// Singleton connections (reused across tests in a file) -// --------------------------------------------------------------------------- -let mongoClient: MongoClient | null = null; -let chClient: ClickHouseClient | null = null; -let redisClient: Redis | null = null; - -export async function getMongoClient(): Promise { - if (!mongoClient) { - mongoClient = new MongoClient(TEST_MONGO_URI); - await mongoClient.connect(); - } - return mongoClient; -} - -export async function getMongoDb(): Promise { - const client = await getMongoClient(); - return client.db(TEST_MONGO_DB); -} - -export async function getClickHouseClient(): Promise { - if (!chClient) { - chClient = createClient({ - url: TEST_CH_URL, - username: "default", - password: "", - database: TEST_CH_DB, - clickhouse_settings: { - date_time_input_format: "best_effort", - }, - }); - } - return chClient; -} - -export async function getRedis(): Promise { - if (!redisClient) { - redisClient = new Redis(TEST_REDIS_URL); - } - return redisClient; -} - -// --------------------------------------------------------------------------- -// Setup / Teardown -// --------------------------------------------------------------------------- - -/** Create the ClickHouse test database and drill_events table. */ -export async function setupClickHouse(): Promise { - // Create DB using a temporary client connected to 'default' (the DB may not exist yet) - const adminCh = createClient({ - url: TEST_CH_URL, - username: "default", - password: "", - database: "default", - }); - await adminCh.command({ - query: `CREATE DATABASE IF NOT EXISTS ${TEST_CH_DB}`, - }); - await adminCh.close(); - - const ch = await getClickHouseClient(); - - // Create table matching production schema - await ch.command({ - query: ` - CREATE TABLE IF NOT EXISTS ${TEST_CH_DB}.${TEST_CH_TABLE} - ( - a LowCardinality(String), - e LowCardinality(String), - n String, - uid String, - uid_canon Nullable(String), - did String, - lsid Nullable(String), - _id String, - ts DateTime64(3), - up String DEFAULT '{}', - custom Nullable(String), - cmp Nullable(String), - sg String DEFAULT '{}', - c UInt32, - s Float64, - dur Float64, - lu Nullable(DateTime64(3)) CODEC(Delta, LZ4), - cd DateTime64(3) DEFAULT now64(3) CODEC(Delta, LZ4) - ) - ENGINE = MergeTree - PARTITION BY toYYYYMM(ts, 'UTC') - ORDER BY (a, e, n, ts) - SETTINGS index_granularity = 8192 - `, - }); -} - -/** Drop the ClickHouse test table (fresh start). */ -export async function teardownClickHouse(): Promise { - try { - const ch = await getClickHouseClient(); - await ch.command({ query: `DROP TABLE IF EXISTS ${TEST_CH_DB}.${TEST_CH_TABLE}` }); - } catch { - // DB may not exist yet — ignore - } -} - -/** Drop the MongoDB test database. */ -export async function teardownMongo(): Promise { - const db = await getMongoDb(); - await db.dropDatabase(); -} - -/** Flush all Redis keys with the test prefix. */ -export async function teardownRedis(): Promise { - const redis = await getRedis(); - const keys = await redis.keys(`${TEST_REDIS_PREFIX}*`); - if (keys.length > 0) { - await redis.del(...keys); - } -} - -/** Full cleanup: all three stores. */ -export async function cleanAll(): Promise { - await Promise.all([ - teardownMongo(), - teardownClickHouse(), - teardownRedis(), - ]); -} - -/** Close all connections. Call in afterAll(). */ -export async function closeAll(): Promise { - if (mongoClient) { - await mongoClient.close(); - mongoClient = null; - } - if (chClient) { - await chClient.close(); - chClient = null; - } - if (redisClient) { - await redisClient.quit(); - redisClient = null; - } -} - -/** Flush ClickHouse async insert queue to ensure all data is visible. */ -export async function flushClickHouse(): Promise { - const ch = await getClickHouseClient(); - await ch.command({ query: "SYSTEM FLUSH ASYNC INSERT QUEUE" }).catch(() => {}); -} - -/** Query ClickHouse row count for the test table. */ -export async function chRowCount(where?: string): Promise { - await flushClickHouse(); - const ch = await getClickHouseClient(); - const q = where - ? `SELECT count() AS cnt FROM ${TEST_CH_TABLE} WHERE ${where}` - : `SELECT count() AS cnt FROM ${TEST_CH_TABLE}`; - const result = await ch.query({ query: q, format: "JSONEachRow" }); - const rows = await result.json<{ cnt: string }[]>(); - return Number(rows[0]?.cnt ?? 0); -} - -/** Query ClickHouse for specific rows. */ -export async function chQuery>(query: string): Promise { - await flushClickHouse(); - const ch = await getClickHouseClient(); - const result = await ch.query({ query, format: "JSONEachRow" }); - return result.json(); -} diff --git a/tests/integration/basic-migration.test.ts b/tests/integration/basic-migration.test.ts deleted file mode 100644 index 0f3b1ec..0000000 --- a/tests/integration/basic-migration.test.ts +++ /dev/null @@ -1,441 +0,0 @@ -/** - * Integration test: basic MongoDB-to-ClickHouse migration flow. - * - * Seeds test documents into MongoDB, runs the BatchRunner programmatically, - * and verifies ClickHouse row counts match expectations. - */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter } from "../../src/target/clickhouse-writer.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { BatchRunner, type BatchRunnerDeps } from "../../src/runtime/batch-runner.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - getMongoDb, - getRedis, - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, - TEST_COLLECTION_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection, collectionName } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const EVENT_NAME = "test_event"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Helpers to construct migration components per test -// --------------------------------------------------------------------------- - -let chClientForPressure: ClickHouseClient | null = null; - -function getChClientForPressure(): ClickHouseClient { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return chClientForPressure; -} - -async function buildDeps(overrides: { - collName: string; - appId?: string; - eventName?: string; -}): Promise<{ - deps: BatchRunnerDeps; - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; - runId: string; - upperBoundId: string; -}> { - const appId = overrides.appId ?? APP_ID; - const eventName = overrides.eventName ?? EVENT_NAME; - const collName = overrides.collName; - - // MongoReader - const mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "integration-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - await mongoReader.switchCollection(collName); - - // ClickHouseWriter - const chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: true, - }, - logger, - ); - await chWriter.connect(); - - // ManifestStore - const manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - // RedisHotState - const redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - // ClickHousePressure (disabled for tests) - const chPressure = new ClickHousePressure( - getChClientForPressure(), - BACKPRESSURE_OFF, - logger, - ); - - // GcController (no-op since --expose-gc is not set in test) - const gcController = new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); - - // RetryPolicy - const retryPolicy = new RetryPolicy({ - maxRetries: 3, - baseDelayMs: 100, - maxDelayMs: 1000, - }); - - // Determine upper bound - const upperBound = await mongoReader.getUpperBound(); - const upperBoundId = upperBound ? serializeCursor(upperBound) : ""; - - // Create run in manifest - const runId = randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await redisState.setActiveRun(runId); - await redisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - - const deps: BatchRunnerDeps = { - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: true, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: appId, e: eventName }, - }, - }; - - return { - deps, - mongoReader, - chWriter, - manifestStore, - redisState, - gcController, - runId, - upperBoundId, - }; -} - -async function cleanupDeps(parts: { - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; -}): Promise { - await parts.mongoReader.close().catch(() => {}); - await parts.chWriter.close().catch(() => {}); - await parts.manifestStore.close().catch(() => {}); - await parts.redisState.close().catch(() => {}); - parts.gcController.dispose(); -} - -// --------------------------------------------------------------------------- -// Test suite -// --------------------------------------------------------------------------- - -describe("basic-migration", () => { - beforeAll(async () => { - // Clean slate: drop everything and recreate the CH table - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - afterAll(async () => { - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await closeAll(); - }); - - // ----------------------------------------------------------------------- - // Test 1: full migration of 5000 docs - // ----------------------------------------------------------------------- - - it("should migrate all documents from MongoDB to ClickHouse", async () => { - // Seed - const { collName, expectedRows } = await seedCollection({ - count: 5000, - appId: APP_ID, - eventName: EVENT_NAME, - }); - expect(expectedRows).toBe(5000); - - // Build components - const parts = await buildDeps({ collName }); - - try { - // Run migration - const runner = new BatchRunner(parts.deps); - await runner.run(); - - // Allow a moment for ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify ClickHouse count (min() inclusivity may cause 1-2 extra duplicates per page) - const count = await chRowCount(); - expect(count).toBeGreaterThanOrEqual(5000); - expect(count).toBeLessThanOrEqual(5020); // small tolerance for min() inclusivity - } finally { - await cleanupDeps(parts); - } - }); - - // ----------------------------------------------------------------------- - // Test 2: skip migrated documents (10%) - // ----------------------------------------------------------------------- - - it("should skip documents marked as migrated", async () => { - // Clean CH table for this test - await teardownClickHouse(); - await setupClickHouse(); - - // Seed with 10% migrated - const { collName, expectedRows, totalDocs } = await seedCollection({ - count: 5000, - appId: APP_ID, - eventName: "migrated_test", - migratedFraction: 0.1, - }); - - // expectedRows should be approximately 4500 (5000 - ~10% migrated) - expect(totalDocs).toBe(5000); - expect(expectedRows).toBeLessThan(5000); - expect(expectedRows).toBeGreaterThan(4000); - - // Build components - const parts = await buildDeps({ - collName, - eventName: "migrated_test", - }); - - try { - const runner = new BatchRunner(parts.deps); - await runner.run(); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - const count = await chRowCount(); - // min() inclusivity may cause small duplicate count; expectedRows is minimum - expect(count).toBeGreaterThanOrEqual(expectedRows); - expect(count).toBeLessThanOrEqual(expectedRows + 20); - } finally { - await cleanupDeps(parts); - } - }); - - // ----------------------------------------------------------------------- - // Test 3: skip documents missing uid (5%) - // ----------------------------------------------------------------------- - - it("should skip documents missing uid", async () => { - // Clean CH table for this test - await teardownClickHouse(); - await setupClickHouse(); - - // Seed with 5% missing uid - const { collName, expectedRows, totalDocs } = await seedCollection({ - count: 5000, - appId: APP_ID, - eventName: "missing_uid_test", - missingUidFraction: 0.05, - }); - - expect(totalDocs).toBe(5000); - expect(expectedRows).toBeLessThan(5000); - expect(expectedRows).toBeGreaterThan(4500); - - // Build components - const parts = await buildDeps({ - collName, - eventName: "missing_uid_test", - }); - - try { - const runner = new BatchRunner(parts.deps); - await runner.run(); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - const count = await chRowCount(); - expect(count).toBeGreaterThanOrEqual(expectedRows); - expect(count).toBeLessThanOrEqual(expectedRows + 20); - } finally { - await cleanupDeps(parts); - } - }); - - // ----------------------------------------------------------------------- - // Test 4: empty collection - // ----------------------------------------------------------------------- - - it("should handle empty collection gracefully", async () => { - // Clean CH table for this test - await teardownClickHouse(); - await setupClickHouse(); - - // Seed with 0 docs - const { collName, expectedRows } = await seedCollection({ - count: 0, - appId: APP_ID, - eventName: "empty_test", - }); - expect(expectedRows).toBe(0); - - // For an empty collection, getUpperBound() returns null. - // Build MongoReader manually and check. - const mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "integration-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - await mongoReader.switchCollection(collName); - - try { - const upperBound = await mongoReader.getUpperBound(); - expect(upperBound).toBeNull(); - - // With a null upper bound the migration has nothing to do. - // Verify no rows in ClickHouse. - const count = await chRowCount(); - expect(count).toBe(0); - } finally { - await mongoReader.close(); - } - }); -}); diff --git a/tests/integration/completion-guard.test.ts b/tests/integration/completion-guard.test.ts deleted file mode 100644 index 5682c21..0000000 --- a/tests/integration/completion-guard.test.ts +++ /dev/null @@ -1,424 +0,0 @@ -/** - * Integration test: Layer 3 completion guard in RangeCoordinator's processRange(). - * - * The completion guard detects "falsely empty" ranges -- ranges where the - * BatchRunner returns 0 docs but the actual time window contains data - * (indicating a cursor bleed or resume bug). It also verifies that - * genuinely empty ranges (no documents in the time window) pass through - * without error. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import pino from "pino"; -import { createClient } from "@clickhouse/client"; - -import { - RangeCoordinator, - type RangeCoordinatorConfig, - type RangeCoordinatorDeps, -} from "../../src/runtime/range-coordinator.ts"; -import { MongoReader, type MongoReaderConfig } from "../../src/source/mongo-reader.ts"; -import { - ClickHouseWriter, - type ClickHouseWriterConfig, -} from "../../src/target/clickhouse-writer.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedAtTimestamps, collectionName } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "silent" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 100_000, - partitionPctHigh: 0.8, - partitionPctLow: 0.6, - totalPctHigh: 0.8, - totalPctLow: 0.6, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Shared resources -// --------------------------------------------------------------------------- - -let manifestStore: ManifestStore; -let redisState: RedisHotState; -let mongoReader: MongoReader; -let chWriter: ClickHouseWriter; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function createChPressure(): ClickHousePressure { - const client = createClient({ - url: TEST_CH_URL, - username: "default", - password: "", - database: TEST_CH_DB, - }); - return new ClickHousePressure(client, BACKPRESSURE_OFF, logger); -} - -function createGcController(): GcController { - return new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.85, - everyNBatches: 999_999, - }, - logger, - ); -} - -function createRetryPolicy(): RetryPolicy { - return new RetryPolicy({ - maxRetries: 2, - baseDelayMs: 50, - maxDelayMs: 200, - }); -} - -function buildRangeCoordinatorDeps( - collName: string, - eventName: string, - overrides?: { rangeCount?: number }, -): RangeCoordinatorDeps { - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - const config: RangeCoordinatorConfig = { - collectionName: collName, - sourceNs, - targetTable, - transformVersion: "v1", - rangeCount: overrides?.rangeCount ?? 4, - rangeLeaseTtlSec: 300, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 100, - collectionDefaults: { a: APP_ID, e: eventName }, - podId: "test-pod-guard", - redisKeyPrefix: TEST_REDIS_PREFIX, - }; - - return { - redis: redisState.getRedisClient(), - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure: createChPressure(), - gcController: createGcController(), - retryPolicy: createRetryPolicy(), - logger, - config, - }; -} - -// --------------------------------------------------------------------------- -// Suite -// --------------------------------------------------------------------------- - -describe("completion-guard", () => { - beforeAll(async () => { - // Connect shared resources - manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const mongoReaderConfig: MongoReaderConfig = { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "completion-guard-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }; - mongoReader = new MongoReader(mongoReaderConfig, logger); - await mongoReader.connect(); - - const chWriterConfig: ClickHouseWriterConfig = { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }; - chWriter = new ClickHouseWriter(chWriterConfig, logger); - await chWriter.connect(); - }); - - afterAll(async () => { - await mongoReader.close().catch(() => {}); - await chWriter.close().catch(() => {}); - await manifestStore.close().catch(() => {}); - await redisState.close().catch(() => {}); - await closeAll(); - }); - - beforeEach(async () => { - // Clean slate for each test - await teardownMongo(); - await teardownClickHouse(); - await setupClickHouse(); - await teardownRedis(); - - // Re-create manifest store indexes (teardownMongo drops the DB) - const freshManifest = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await freshManifest.connect(); - await freshManifest.close(); - }); - - // ------------------------------------------------------------------------- - // Test 1: empty range detected as genuinely empty - // ------------------------------------------------------------------------- - - it("empty range detected as genuinely empty", async () => { - const eventName = "guard_empty_range"; - const collName = collectionName(eventName, APP_ID); - - // Seed docs ONLY in the first half of a broad date range, leaving a gap. - // Date range: 2024-01-01 to 2024-12-31 - // Docs only in Jan-Mar, leaving Apr-Dec empty. - // With rangeCount=4, ranges will roughly be: - // range 0: Jan-Mar (has data) - // range 1: Apr-Jun (empty) - // range 2: Jul-Sep (empty) - // range 3: Oct-Dec (empty -- but endCd is maxCd so it is the final range) - // - // We place docs at the very start and very end of the full range to ensure - // the coordinator can determine min/max cd bounds, then cluster data only - // in the first quarter. - - const timestamps: Date[] = []; - - // One doc at the very start (bookend for range calculation) - timestamps.push(new Date("2024-01-01T00:00:00Z")); - - // 200 docs clustered in Jan-Feb - for (let i = 0; i < 200; i++) { - const d = new Date("2024-01-02T00:00:00Z"); - d.setHours(d.getHours() + i * 2); // spread over ~400 hours (~17 days) - timestamps.push(d); - } - - // One doc at the very end (bookend) - timestamps.push(new Date("2024-12-31T23:59:59Z")); - - const { collName: seededColl, totalDocs } = await seedAtTimestamps( - eventName, - APP_ID, - timestamps, - ); - - expect(seededColl).toBe(collName); - expect(totalDocs).toBe(timestamps.length); - - await mongoReader.switchCollection(collName); - - const deps = buildRangeCoordinatorDeps(collName, eventName, { rangeCount: 4 }); - const coordinator = new RangeCoordinator(deps); - const result = await coordinator.run(); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // The migration should complete without error -- the empty ranges should - // pass the completion guard (genuinely empty, probe returns 0 docs). - const run = await manifestStore.getRun(result.runId); - expect(run).toBeDefined(); - // The run should be "completed" because all ranges finished (some empty, some with data) - expect(run!.status).toBe("completed"); - expect(result.failedRanges).toBe(0); - - // ClickHouse should contain the non-empty range data - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThan(0); - expect(totalRows).toBe(result.totalRowsInserted); - - // Verify the total docs processed match what was seeded - // (all 202 docs should have been read, though some ranges read 0) - expect(result.totalDocsRead).toBe(totalDocs); - }); - - // ------------------------------------------------------------------------- - // Test 2: completion guard catches falsely empty range - // ------------------------------------------------------------------------- - - it("completion guard catches falsely empty range", async () => { - const eventName = "guard_false_empty"; - const collName = collectionName(eventName, APP_ID); - - // Seed docs uniformly across the full date range to ensure every range - // has data. With rangeCount=4, each range should have ~75 docs. - const timestamps: Date[] = []; - const start = new Date("2024-01-01T00:00:00Z").getTime(); - const end = new Date("2024-12-31T23:59:59Z").getTime(); - const step = (end - start) / 299; // 300 docs evenly spread - - for (let i = 0; i < 300; i++) { - timestamps.push(new Date(start + Math.floor(i * step))); - } - - await seedAtTimestamps(eventName, APP_ID, timestamps); - await mongoReader.switchCollection(collName); - - // Create a Proxy around the MongoReader that makes readPage return empty - // results for a specific range's time window. This simulates a cursor bug - // where a range appears empty even though it has data. The completion - // guard's probe call goes through the *real* readPage, exposing the lie. - // - // Strategy: intercept the BatchRunner's processing by wrapping mongoReader - // such that the *batch* readPage calls (which use the internal cursor) - // return empty when the startCd is in range 1's window. However, the - // guard's probe call uses different cursor arguments, so we need to be - // more surgical. - // - // Simpler approach: wrap the CH writer to silently drop all writes for - // range 1's batch_seq slots AND make the mongoReader return 0 docs for - // range 1. This triggers the guard because the range has data but the - // BatchRunner sees 0 docs. - // - // Even simpler: We can directly test the guard by using a Proxy on - // mongoReader.readPage. For calls during the batch loop where the cursor - // falls within range 1's time window, return empty results. The guard's - // subsequent probe will also go through readPage, but we let the probe - // through (returning real data), which will trigger the guard error. - - const blockRange1 = true; - - // Determine approximate range 1 boundaries - // With 4 ranges over [start, end], range 1 covers roughly [start + 1/4*span, start + 2/4*span) - const span = end - start; - const range1StartApprox = start + Math.floor(span / 4); - const range1EndApprox = start + Math.floor((2 * span) / 4); - - const proxiedReader = new Proxy(mongoReader, { - get(target, prop, receiver) { - if (prop === "readPage") { - return async (...args: Parameters) => { - const [lastCursor, upperBound, limit] = args; - - // Let probe calls through (limit=1 is the guard's probe signature) - if (limit === 1) { - return target.readPage(lastCursor, upperBound, limit); - } - - // Check if this readPage is for range 1's time window - if (blockRange1 && lastCursor) { - const cursorCd = lastCursor.cd; - if (cursorCd >= range1StartApprox && cursorCd < range1EndApprox) { - // Return empty page -- simulating a cursor bug - return { docs: [], lastCursor: null, fetchMs: 0 }; - } - } - - return target.readPage(lastCursor, upperBound, limit); - }; - } - return Reflect.get(target, prop, receiver); - }, - }); - - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - const config: RangeCoordinatorConfig = { - collectionName: collName, - sourceNs, - targetTable, - transformVersion: "v1", - rangeCount: 4, - rangeLeaseTtlSec: 300, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 100, - collectionDefaults: { a: APP_ID, e: eventName }, - podId: "test-pod-guard-false", - redisKeyPrefix: TEST_REDIS_PREFIX, - }; - - const deps: RangeCoordinatorDeps = { - redis: redisState.getRedisClient(), - manifestStore, - redisState, - mongoReader: proxiedReader as MongoReader, - chWriter, - chPressure: createChPressure(), - gcController: createGcController(), - retryPolicy: createRetryPolicy(), - logger: pino({ level: "silent" }), - config, - }; - - const coordinator = new RangeCoordinator(deps); - const result = await coordinator.run(); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // The range that was artificially emptied should have been caught by the - // completion guard and marked as failed. - // With MAX_RANGE_RETRIES=3, the coordinator will retry range 1 multiple times, - // and each time the guard fires because our proxy still returns empty. - // After exhausting retries, the run should be marked as "failed". - const run = await manifestStore.getRun(result.runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("failed"); - expect(result.failedRanges).toBeGreaterThan(0); - - // Ranges 0, 2, and 3 should have their data in ClickHouse - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThan(0); - - // The number of completed ranges should be less than 4 - // (at least range 1 failed) - expect(result.completedRanges).toBeLessThan(4); - expect(result.completedRanges).toBeGreaterThanOrEqual(2); - }); -}); diff --git a/tests/integration/crash-recovery.test.ts b/tests/integration/crash-recovery.test.ts deleted file mode 100644 index 00e6c46..0000000 --- a/tests/integration/crash-recovery.test.ts +++ /dev/null @@ -1,592 +0,0 @@ -/** - * Integration test: crash recovery scenarios for BatchRunner. - * - * Validates that resumeFromInterruption() correctly handles: - * 1. Inflight batches left after a crash (CH write done, manifest not marked done) - * 2. Redis cursor loss with manifest fallback - * 3. Counter recovery from manifest aggregate when Redis stats are lost - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter } from "../../src/target/clickhouse-writer.ts"; -import { ManifestStore, type Batch, type BatchStatus } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { BatchRunner, type BatchRunnerDeps } from "../../src/runtime/batch-runner.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - chQuery, - getRedis, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const EVENT_NAME = "crash_recovery_event"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -let chClientForPressure: ClickHouseClient | null = null; - -function getChClientForPressure(): ClickHouseClient { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return chClientForPressure; -} - -interface BuildDepsResult { - deps: BatchRunnerDeps; - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; - runId: string; - upperBoundId: string; - sourceNs: string; - targetTable: string; -} - -/** - * Build a full set of BatchRunner dependencies for a given collection. - * Optionally reuse an existing runId and upperBoundId (for resume scenarios). - */ -async function buildDeps(opts: { - collName: string; - appId?: string; - eventName?: string; - batchRowsTarget?: number; - existingRunId?: string; - existingUpperBoundId?: string; - skipRunCreation?: boolean; -}): Promise { - const appId = opts.appId ?? APP_ID; - const eventName = opts.eventName ?? EVENT_NAME; - const collName = opts.collName; - const batchRowsTarget = opts.batchRowsTarget ?? 500; - - // MongoReader - const mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "crash-recovery-test", - batchRowsTarget, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - await mongoReader.switchCollection(collName); - - // ClickHouseWriter - const chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: true, - }, - logger, - ); - await chWriter.connect(); - - // ManifestStore - const manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - // RedisHotState - const redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - // ClickHousePressure (disabled) - const chPressure = new ClickHousePressure( - getChClientForPressure(), - BACKPRESSURE_OFF, - logger, - ); - - // GcController (no-op) - const gcController = new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); - - // RetryPolicy - const retryPolicy = new RetryPolicy({ - maxRetries: 3, - baseDelayMs: 100, - maxDelayMs: 1000, - }); - - // Determine upper bound (or reuse existing) - let upperBoundId: string; - if (opts.existingUpperBoundId) { - upperBoundId = opts.existingUpperBoundId; - } else { - const upperBound = await mongoReader.getUpperBound(); - upperBoundId = upperBound ? serializeCursor(upperBound) : ""; - } - - const runId = opts.existingRunId ?? randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - if (!opts.skipRunCreation) { - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await redisState.setActiveRun(runId); - await redisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - } - - const deps: BatchRunnerDeps = { - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget, - mongoPageSize: batchRowsTarget, - backpressure: BACKPRESSURE_OFF, - useDedupToken: true, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: appId, e: eventName }, - }, - }; - - return { - deps, - mongoReader, - chWriter, - manifestStore, - redisState, - gcController, - runId, - upperBoundId, - sourceNs, - targetTable, - }; -} - -async function cleanupDeps(parts: { - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; -}): Promise { - await parts.mongoReader.close().catch(() => {}); - await parts.chWriter.close().catch(() => {}); - await parts.manifestStore.close().catch(() => {}); - await parts.redisState.close().catch(() => {}); - parts.gcController.dispose(); -} - -// --------------------------------------------------------------------------- -// Test suite -// --------------------------------------------------------------------------- - -describe("crash-recovery", () => { - beforeAll(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - beforeEach(async () => { - // Clean CH table + Redis between tests to avoid cross-contamination - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - afterAll(async () => { - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await closeAll(); - }); - - // ----------------------------------------------------------------------- - // Test 1: recovers inflight batch after crash - // ----------------------------------------------------------------------- - - it("recovers inflight batch after crash", async () => { - // Seed 2000 docs - const { collName, expectedRows } = await seedCollection({ - count: 2000, - appId: APP_ID, - eventName: "crash_inflight", - }); - expect(expectedRows).toBe(2000); - - // --- Phase 1: Run first batch, then simulate crash --- - const parts1 = await buildDeps({ - collName, - eventName: "crash_inflight", - batchRowsTarget: 500, - }); - - const runner1 = new BatchRunner(parts1.deps); - - // Use a wrapping CH writer that stops after 1 successful insert - let insertCount = 0; - const origInsertBatch = parts1.chWriter.insertBatch.bind(parts1.chWriter); - parts1.chWriter.insertBatch = async (params) => { - const result = await origInsertBatch(params); - insertCount++; - if (insertCount >= 1) { - // Stop after first batch completes - runner1.stopAfterBatch(); - } - return result; - }; - - await runner1.run(); - await new Promise((r) => setTimeout(r, 1500)); - - // Confirm some rows were written - const rowsAfterPhase1 = await chRowCount(); - expect(rowsAfterPhase1).toBeGreaterThan(0); - - // Get the last done batch to create a simulated inflight batch after it - const lastDone = await parts1.manifestStore.getLastDoneBatch(parts1.runId); - expect(lastDone).not.toBeNull(); - - // Now simulate an "inflight" batch: create a batch record in manifest - // as if the process crashed after CH write but before marking done. - // Read the next page to build a realistic batch record. - const lastCursorStr = lastDone!.upper_inclusive_cursor; - const nextBatchSeq = lastDone!.batch_seq + 1; - - // Read the next page from Mongo to get realistic cursor values - const { deserializeCursor } = await import("../../src/types/cursor.ts"); - const { transformBatch } = await import("../../src/transform/normalize.ts"); - const { SkipCounter } = await import("../../src/transform/skip-reasons.ts"); - - const lastCursor = deserializeCursor(lastCursorStr); - const upperBound = deserializeCursor(parts1.upperBoundId); - const page = await parts1.deps.mongoReader.readPage(lastCursor, upperBound, 500); - - let inflightUpperCursor: string; - if (page.docs.length > 0 && page.lastCursor) { - inflightUpperCursor = serializeCursor(page.lastCursor); - - const skipCounter = new SkipCounter(); - const { rows } = transformBatch(page.docs, skipCounter, { a: APP_ID, e: "crash_inflight" }); - - // Insert as inflight batch in manifest - await parts1.manifestStore.insertBatch({ - run_id: parts1.runId, - batch_seq: nextBatchSeq, - lower_exclusive_cursor: lastCursorStr, - upper_inclusive_cursor: inflightUpperCursor, - source_docs_read: page.docs.length, - docs_skipped: page.docs.length - rows.length, - rows_to_insert: rows.length, - payload_digest: String(rows.length), - insert_dedup_token: `mig:${parts1.runId}:${nextBatchSeq}`, - query_id: `mig__${parts1.runId}__${nextBatchSeq}`, - status: "inflight" as BatchStatus, - retry_count: 0, - last_error: null, - started_at: new Date().toISOString(), - finished_at: null, - }); - } - - await cleanupDeps(parts1); - - // --- Phase 2: Create NEW BatchRunner and resume --- - const parts2 = await buildDeps({ - collName, - eventName: "crash_inflight", - batchRowsTarget: 500, - existingRunId: parts1.runId, - existingUpperBoundId: parts1.upperBoundId, - skipRunCreation: true, - }); - - // Re-activate the run for the new runner - await parts2.manifestStore.updateRunStatus(parts1.runId, "active"); - - const runner2 = new BatchRunner(parts2.deps); - await runner2.run(); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify: total CH rows = all expected docs (no gaps) - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThanOrEqual(expectedRows); - expect(totalRows).toBeLessThanOrEqual(expectedRows + 20); - - // Verify: inflight batch was recovered (check events for batch_recovered) - const recoveryEvents = await parts2.manifestStore.countEvents(parts1.runId, "batch_recovered"); - expect(recoveryEvents).toBeGreaterThanOrEqual(1); - - await cleanupDeps(parts2); - }); - - // ----------------------------------------------------------------------- - // Test 2: recovers from Redis cursor loss (manifest fallback) - // ----------------------------------------------------------------------- - - it("recovers from Redis cursor loss (manifest fallback)", async () => { - // Seed 3000 docs - const { collName, expectedRows } = await seedCollection({ - count: 3000, - appId: APP_ID, - eventName: "redis_loss", - }); - expect(expectedRows).toBe(3000); - - // --- Phase 1: Run a few batches --- - const parts1 = await buildDeps({ - collName, - eventName: "redis_loss", - batchRowsTarget: 500, - }); - - const runner1 = new BatchRunner(parts1.deps); - - // Stop after 2 batches (~1000 docs) - let batchCount = 0; - const origInsert1 = parts1.chWriter.insertBatch.bind(parts1.chWriter); - parts1.chWriter.insertBatch = async (params) => { - const result = await origInsert1(params); - batchCount++; - if (batchCount >= 2) { - runner1.stopAfterBatch(); - } - return result; - }; - - await runner1.run(); - await new Promise((r) => setTimeout(r, 1500)); - - const rowsAfterPhase1 = await chRowCount(); - expect(rowsAfterPhase1).toBeGreaterThan(0); - expect(rowsAfterPhase1).toBeLessThan(expectedRows); - - // Verify manifest has committed cursor - const runDoc = await parts1.manifestStore.getRun(parts1.runId); - expect(runDoc).toBeDefined(); - expect(runDoc!.last_committed_cursor).not.toBeNull(); - - const savedRunId = parts1.runId; - const savedUpperBoundId = parts1.upperBoundId; - await cleanupDeps(parts1); - - // --- Delete all Redis keys (simulate Redis flush) --- - await teardownRedis(); - - // --- Phase 2: Create NEW BatchRunner and resume --- - const parts2 = await buildDeps({ - collName, - eventName: "redis_loss", - batchRowsTarget: 500, - existingRunId: savedRunId, - existingUpperBoundId: savedUpperBoundId, - skipRunCreation: true, - }); - - // Re-activate the run - await parts2.manifestStore.updateRunStatus(savedRunId, "active"); - - const runner2 = new BatchRunner(parts2.deps); - await runner2.run(); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify: all docs end up in ClickHouse - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThanOrEqual(expectedRows); - expect(totalRows).toBeLessThanOrEqual(expectedRows + 20); - - await cleanupDeps(parts2); - }); - - // ----------------------------------------------------------------------- - // Test 3: counter recovery from manifest aggregate - // ----------------------------------------------------------------------- - - it("counter recovery from manifest aggregate", async () => { - // Seed 2000 docs - const { collName, expectedRows } = await seedCollection({ - count: 2000, - appId: APP_ID, - eventName: "counter_recovery", - }); - expect(expectedRows).toBe(2000); - - // --- Phase 1: Run some batches to create done batch records --- - const parts1 = await buildDeps({ - collName, - eventName: "counter_recovery", - batchRowsTarget: 500, - }); - - const runner1 = new BatchRunner(parts1.deps); - - // Stop after 2 batches - let batchCount = 0; - const origInsert = parts1.chWriter.insertBatch.bind(parts1.chWriter); - parts1.chWriter.insertBatch = async (params) => { - const result = await origInsert(params); - batchCount++; - if (batchCount >= 2) { - runner1.stopAfterBatch(); - } - return result; - }; - - await runner1.run(); - await new Promise((r) => setTimeout(r, 1500)); - - // Check that we have some done batches in the manifest - const doneBatches = await parts1.manifestStore.getBatches(parts1.runId, { status: "done" }); - expect(doneBatches.length).toBeGreaterThanOrEqual(2); - - // Sum up what the manifest says we should have - const manifestAggregate = await parts1.manifestStore.sumCompletedBatchStats(parts1.runId); - expect(manifestAggregate.docsRead).toBeGreaterThan(0); - - const savedRunId = parts1.runId; - const savedUpperBoundId = parts1.upperBoundId; - await cleanupDeps(parts1); - - // --- Delete all Redis keys (simulate Redis stats loss) --- - await teardownRedis(); - - // --- Phase 2: Create new BatchRunner and verify counter recovery --- - const parts2 = await buildDeps({ - collName, - eventName: "counter_recovery", - batchRowsTarget: 500, - existingRunId: savedRunId, - existingUpperBoundId: savedUpperBoundId, - skipRunCreation: true, - }); - - // Re-activate the run - await parts2.manifestStore.updateRunStatus(savedRunId, "active"); - - const runner2 = new BatchRunner(parts2.deps); - await runner2.run(); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify: the BatchRunner recovered counters from manifest aggregate - const stats = runner2.getStats(); - - // totalDocsRead should be the full count (recovered portion + newly read) - expect(stats.totalDocsRead).toBeGreaterThanOrEqual(expectedRows); - expect(stats.totalDocsRead).toBeLessThanOrEqual(expectedRows + 20); - - // totalDocsRead should not be less than the manifest aggregate - // (it should start from the aggregate, not 0) - expect(stats.totalDocsRead).toBeGreaterThanOrEqual(manifestAggregate.docsRead); - - // All rows should be in ClickHouse - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThanOrEqual(expectedRows); - expect(totalRows).toBeLessThanOrEqual(expectedRows + 20); - - await cleanupDeps(parts2); - }); -}); diff --git a/tests/integration/cursor-isolation.test.ts b/tests/integration/cursor-isolation.test.ts deleted file mode 100644 index 27d5497..0000000 --- a/tests/integration/cursor-isolation.test.ts +++ /dev/null @@ -1,596 +0,0 @@ -/** - * Integration test: cursor isolation in range-parallel mode. - * - * Verifies the fix for the critical bug where resumeFromInterruption() - * would read the globally last-done batch across ALL ranges instead of - * scoping its manifest query to the current range's batch_seq slot. - * - * In range-parallel mode: - * - All ranges share one runId - * - Each range gets its own batch_seq slot: - * range 0 = [0, 10000), range 1 = [10000, 20000), etc. - * - Manifest queries MUST be scoped with batchSeqRange: { min, max } - */ - -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import pino from "pino"; - -import { - getMongoDb, - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, - TEST_MANIFEST_DB, -} from "../helpers/setup.ts"; -import { seedCollection, collectionName } from "../helpers/seed-mongo.ts"; - -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { MongoReader, type MongoReaderConfig } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter, type ClickHouseWriterConfig } from "../../src/target/clickhouse-writer.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { BatchRunner, type BatchRunnerConfig, type BatchRunnerDeps } from "../../src/runtime/batch-runner.ts"; -import { serializeCursor, deserializeCursor, type Cursor } from "../../src/types/cursor.ts"; -import { createClient } from "@clickhouse/client"; - -// --------------------------------------------------------------------------- -// Shared logger (silent for tests) -// --------------------------------------------------------------------------- -const logger = pino({ level: "silent" }); - -// --------------------------------------------------------------------------- -// Test-scoped constants -// --------------------------------------------------------------------------- -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const EVENT_NAME = "cursor_isolation_event"; -const COLL_NAME = collectionName(EVENT_NAME, APP_ID); -const SOURCE_NS = `${TEST_MONGO_DB}.${COLL_NAME}`; -const RUN_ID = "cursor-isolation-test-run"; - -// Date ranges: 6 months, one per range -const RANGE_DATES = [ - { start: new Date("2024-01-01"), end: new Date("2024-03-01") }, - { start: new Date("2024-03-01"), end: new Date("2024-05-01") }, - { start: new Date("2024-05-01"), end: new Date("2024-07-01") }, - { start: new Date("2024-07-01"), end: new Date("2024-09-01") }, - { start: new Date("2024-09-01"), end: new Date("2024-11-01") }, - { start: new Date("2024-11-01"), end: new Date("2025-01-01") }, -]; - -// Backpressure config (disabled for tests) -const BACKPRESSURE: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 100_000, - partitionPctHigh: 0.8, - partitionPctLow: 0.6, - totalPctHigh: 0.8, - totalPctLow: 0.6, - pollIntervalMs: 1000, - maxPauseEpisodeMs: 60_000, -}; - -// --------------------------------------------------------------------------- -// Shared resources -// --------------------------------------------------------------------------- -let manifestStore: ManifestStore; -let redisState: RedisHotState; -let mongoReader: MongoReader; -let chWriter: ClickHouseWriter; - -async function createChPressure(): Promise { - const client = createClient({ - url: TEST_CH_URL, - username: "default", - password: "", - database: TEST_CH_DB, - }); - return new ClickHousePressure(client, BACKPRESSURE, logger); -} - -function createGcController(): GcController { - return new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.85, - everyNBatches: 50, - }, - logger, - ); -} - -function createRetryPolicy(): RetryPolicy { - return new RetryPolicy({ - maxRetries: 2, - baseDelayMs: 100, - maxDelayMs: 500, - }); -} - -// --------------------------------------------------------------------------- -// Suite -// --------------------------------------------------------------------------- -describe("cursor-isolation (range-parallel)", () => { - beforeAll(async () => { - // Setup external stores - await setupClickHouse(); - - manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MANIFEST_DB); - await manifestStore.connect(); - - redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const mongoReaderConfig: MongoReaderConfig = { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "cursor-isolation-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }; - mongoReader = new MongoReader(mongoReaderConfig, logger); - await mongoReader.connect(); - - const chWriterConfig: ClickHouseWriterConfig = { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }; - chWriter = new ClickHouseWriter(chWriterConfig, logger); - await chWriter.connect(); - }); - - afterAll(async () => { - await mongoReader.close(); - await chWriter.close(); - await manifestStore.close(); - await redisState.close(); - await closeAll(); - }); - - beforeEach(async () => { - // Clean slate for each test - await teardownMongo(); - await teardownClickHouse(); - await setupClickHouse(); - await teardownRedis(); - - // Re-create manifest store indexes (teardownMongo drops the DB) - const freshManifest = new ManifestStore(TEST_MONGO_URI, TEST_MANIFEST_DB); - await freshManifest.connect(); - await freshManifest.close(); - }); - - // ------------------------------------------------------------------------- - // Test 1: range 3 starts from its own cursor after ranges 0-2 complete - // ------------------------------------------------------------------------- - it("range 3 starts from its own startCd after ranges 0-2 complete", async () => { - // Seed 6000 docs spanning the full date range - const { collName } = await seedCollection({ - count: 6000, - appId: APP_ID, - eventName: EVENT_NAME, - startDate: RANGE_DATES[0].start, - endDate: RANGE_DATES[5].end, - }); - - // Compute cursor boundaries for each range by querying MongoDB - const db = await getMongoDb(); - const coll = db.collection(collName); - - // Get docs sorted by cd to determine range boundaries - const allDocs = await coll - .find({}) - .sort({ cd: 1, _id: 1 }) - .project({ cd: 1, _id: 1 }) - .toArray(); - - // Split into 6 ranges - const docsPerRange = Math.ceil(allDocs.length / 6); - const rangeBoundaries: { startCursor: Cursor; endCursor: Cursor }[] = []; - for (let r = 0; r < 6; r++) { - const startIdx = r * docsPerRange; - const endIdx = Math.min((r + 1) * docsPerRange - 1, allDocs.length - 1); - const startDoc = allDocs[startIdx]; - const endDoc = allDocs[endIdx]; - rangeBoundaries.push({ - startCursor: { - cd: new Date(startDoc.cd as Date).getTime(), - id: String(startDoc._id), - }, - endCursor: { - cd: new Date(endDoc.cd as Date).getTime(), - id: String(endDoc._id), - }, - }); - } - - // Create a run in the manifest - const runId = `${RUN_ID}-test1-${Date.now()}`; - const upperBound = rangeBoundaries[5].endCursor; - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: SOURCE_NS, - target_table: TEST_CH_TABLE, - upper_bound_cursor: serializeCursor(upperBound), - transform_version: "v1", - created_at: new Date().toISOString(), - }); - - // Insert fake "done" batches for ranges 0-2 - // Each range uses batch_seq slots: range 0=[0,10000), range 1=[10000,20000), range 2=[20000,30000) - for (let rangeIdx = 0; rangeIdx < 3; rangeIdx++) { - const offset = rangeIdx * 10000; - const rangeStart = rangeBoundaries[rangeIdx].startCursor; - const rangeEnd = rangeBoundaries[rangeIdx].endCursor; - - // Insert 3 done batches per range (simulating completed work) - for (let i = 0; i < 3; i++) { - const fraction = (i + 1) / 3; - const batchCd = rangeStart.cd + fraction * (rangeEnd.cd - rangeStart.cd); - const lowerCd = rangeStart.cd + (i / 3) * (rangeEnd.cd - rangeStart.cd); - - await manifestStore.insertCompletedBatch( - { - run_id: runId, - batch_seq: offset + i, - lower_exclusive_cursor: serializeCursor({ - cd: lowerCd, - id: `fake-lower-${rangeIdx}-${i}`, - }), - upper_inclusive_cursor: serializeCursor({ - cd: batchCd, - id: `fake-upper-${rangeIdx}-${i}`, - }), - source_docs_read: 100, - docs_skipped: 0, - rows_to_insert: 100, - payload_digest: "100", - insert_dedup_token: `dedup:${runId}:${offset + i}`, - query_id: `query:${runId}:${offset + i}`, - status: "done", - retry_count: 0, - last_error: null, - started_at: new Date().toISOString(), - finished_at: new Date().toISOString(), - }, - serializeCursor({ cd: batchCd, id: `fake-upper-${rangeIdx}-${i}` }), - ); - } - } - - // Now set up a BatchRunner for range 3 - const range3Start = rangeBoundaries[3].startCursor; - const range3End = rangeBoundaries[3].endCursor; - - await mongoReader.switchCollection(collName); - - // Create a per-range RedisHotState (reuses connection but different prefix) - const range3Redis = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:range3`, - ); - - const chPressure = await createChPressure(); - - const config: BatchRunnerConfig = { - runId, - transformVersion: "v1", - sourceNs: SOURCE_NS, - targetTable: TEST_CH_TABLE, - upperBoundId: serializeCursor(range3End), - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 100, - collectionDefaults: { a: APP_ID, e: EVENT_NAME }, - batchSeqOffset: 30000, - batchSeqMax: 40000, - rangeIdx: 3, - collectionName: collName, - podId: "test-pod", - }; - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: range3Redis, - mongoReader, - chWriter, - chPressure, - gcController: createGcController(), - retryPolicy: createRetryPolicy(), - logger, - config, - }; - - const runner = new BatchRunner(deps); - - // Run with startCursor for range 3 - await runner.run(serializeCursor(range3Start)); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // Verify the runner completed - expect(runner.getStatus()).toBe("completed"); - - // Verify ClickHouse has rows with cd values in range 3's window - // Range 3 timestamps should be around 2024-07-01 to 2024-09-01 - const range3StartMs = range3Start.cd; - const range3EndMs = range3End.cd; - - // Query ClickHouse for rows inserted - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThan(0); - - // Verify all inserted rows have cd within range 3's time window - const range3StartDate = new Date(range3StartMs).toISOString().replace("T", " ").replace("Z", ""); - const range3EndDate = new Date(range3EndMs + 1000).toISOString().replace("T", " ").replace("Z", ""); - - const rowsInRange = await chRowCount( - `ts >= '${range3StartDate}' AND ts <= '${range3EndDate}'`, - ); - - // All rows should be within range 3's window (the runner should not have - // started from range 2's final cursor) - expect(rowsInRange).toBe(totalRows); - - // Verify the runner's stats show it processed docs - const stats = runner.getStats(); - expect(stats.totalDocsRead).toBeGreaterThan(0); - expect(stats.totalRowsInserted).toBeGreaterThan(0); - }); - - // ------------------------------------------------------------------------- - // Test 2: bounds guard discards out-of-range cursor - // ------------------------------------------------------------------------- - it("bounds guard discards out-of-range cursor", async () => { - const { collName } = await seedCollection({ - count: 2000, - appId: APP_ID, - eventName: EVENT_NAME, - startDate: new Date("2024-01-01"), - endDate: new Date("2024-12-31"), - }); - - const db = await getMongoDb(); - const coll = db.collection(collName); - - const allDocs = await coll - .find({}) - .sort({ cd: 1, _id: 1 }) - .project({ cd: 1, _id: 1 }) - .toArray(); - - // Range 1 covers the second quarter of docs - const docsPerRange = Math.ceil(allDocs.length / 4); - const range1StartIdx = docsPerRange; - const range1EndIdx = Math.min(2 * docsPerRange - 1, allDocs.length - 1); - const range1Start: Cursor = { - cd: new Date(allDocs[range1StartIdx].cd as Date).getTime(), - id: String(allDocs[range1StartIdx]._id), - }; - const range1End: Cursor = { - cd: new Date(allDocs[range1EndIdx].cd as Date).getTime(), - id: String(allDocs[range1EndIdx]._id), - }; - - const runId = `${RUN_ID}-test2-${Date.now()}`; - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: SOURCE_NS, - target_table: TEST_CH_TABLE, - upper_bound_cursor: serializeCursor(range1End), - transform_version: "v1", - created_at: new Date().toISOString(), - }); - - // Set a Redis cursor that is WAY past range 1's endCd (e.g., year 2025) - const outOfRangeCursor = serializeCursor({ - cd: new Date("2025-06-01").getTime(), - id: "zzz-out-of-range", - }); - - const range1Redis = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:range1`, - ); - await range1Redis.setLastCommittedCursor(runId, outOfRangeCursor); - - await mongoReader.switchCollection(collName); - - const chPressure = await createChPressure(); - - const config: BatchRunnerConfig = { - runId, - transformVersion: "v1", - sourceNs: SOURCE_NS, - targetTable: TEST_CH_TABLE, - upperBoundId: serializeCursor(range1End), - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 100, - collectionDefaults: { a: APP_ID, e: EVENT_NAME }, - batchSeqOffset: 10000, - batchSeqMax: 20000, - rangeIdx: 1, - collectionName: collName, - podId: "test-pod", - }; - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: range1Redis, - mongoReader, - chWriter, - chPressure, - gcController: createGcController(), - retryPolicy: createRetryPolicy(), - logger, - config, - }; - - const runner = new BatchRunner(deps); - await runner.run(serializeCursor(range1Start)); - - expect(runner.getStatus()).toBe("completed"); - - // The runner should have processed data starting from range1Start - // (the out-of-range cursor was discarded by the bounds guard) - const stats = runner.getStats(); - expect(stats.totalDocsRead).toBeGreaterThan(0); - expect(stats.totalRowsInserted).toBeGreaterThan(0); - - // Verify all rows are within range 1's time window - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThan(0); - - const range1StartDate = new Date(range1Start.cd) - .toISOString().replace("T", " ").replace("Z", ""); - const range1EndDate = new Date(range1End.cd + 1000) - .toISOString().replace("T", " ").replace("Z", ""); - - const rowsInRange = await chRowCount( - `ts >= '${range1StartDate}' AND ts <= '${range1EndDate}'`, - ); - expect(rowsInRange).toBe(totalRows); - }); - - // ------------------------------------------------------------------------- - // Test 3: scoped getLastDoneBatch returns only this range's batches - // ------------------------------------------------------------------------- - it("scoped getLastDoneBatch returns only this range's batches", async () => { - const runId = `${RUN_ID}-test3-${Date.now()}`; - - // Create a fresh ManifestStore for this test to avoid stale state - const store = new ManifestStore(TEST_MONGO_URI, TEST_MANIFEST_DB); - await store.connect(); - - try { - await store.createRun({ - run_id: runId, - status: "active", - source_ns: SOURCE_NS, - target_table: TEST_CH_TABLE, - upper_bound_cursor: serializeCursor({ cd: Date.now(), id: "upper" }), - transform_version: "v1", - created_at: new Date().toISOString(), - }); - - const baseCd = new Date("2024-06-01").getTime(); - - // Insert done batches for range 0: batch_seq 0..10 - for (let i = 0; i <= 10; i++) { - const cursorCd = baseCd + i * 1000; - await store.insertCompletedBatch( - { - run_id: runId, - batch_seq: i, - lower_exclusive_cursor: serializeCursor({ cd: cursorCd - 500, id: `r0-lower-${i}` }), - upper_inclusive_cursor: serializeCursor({ cd: cursorCd, id: `r0-upper-${i}` }), - source_docs_read: 50, - docs_skipped: 0, - rows_to_insert: 50, - payload_digest: "50", - insert_dedup_token: `dedup:${runId}:${i}`, - query_id: `query:${runId}:${i}`, - status: "done", - retry_count: 0, - last_error: null, - started_at: new Date().toISOString(), - finished_at: new Date().toISOString(), - }, - serializeCursor({ cd: cursorCd, id: `r0-upper-${i}` }), - ); - } - - // Insert done batches for range 1: batch_seq 10000..10010 - for (let i = 0; i <= 10; i++) { - const seq = 10000 + i; - const cursorCd = baseCd + 100_000 + i * 1000; - await store.insertCompletedBatch( - { - run_id: runId, - batch_seq: seq, - lower_exclusive_cursor: serializeCursor({ cd: cursorCd - 500, id: `r1-lower-${i}` }), - upper_inclusive_cursor: serializeCursor({ cd: cursorCd, id: `r1-upper-${i}` }), - source_docs_read: 50, - docs_skipped: 0, - rows_to_insert: 50, - payload_digest: "50", - insert_dedup_token: `dedup:${runId}:${seq}`, - query_id: `query:${runId}:${seq}`, - status: "done", - retry_count: 0, - last_error: null, - started_at: new Date().toISOString(), - finished_at: new Date().toISOString(), - }, - serializeCursor({ cd: cursorCd, id: `r1-upper-${i}` }), - ); - } - - // ----- Assertions ----- - - // Unscoped: should return the globally last batch (seq 10010) - const globalLast = await store.getLastDoneBatch(runId); - expect(globalLast).not.toBeNull(); - expect(globalLast!.batch_seq).toBe(10010); - - // Scoped to range 0 [0, 10000): should return batch 10 - const range0Last = await store.getLastDoneBatch(runId, { min: 0, max: 10000 }); - expect(range0Last).not.toBeNull(); - expect(range0Last!.batch_seq).toBe(10); - - // Verify range 0's cursor is from range 0's data - const range0Cursor = deserializeCursor(range0Last!.upper_inclusive_cursor); - expect(range0Cursor.id).toBe("r0-upper-10"); - - // Scoped to range 1 [10000, 20000): should return batch 10010 - const range1Last = await store.getLastDoneBatch(runId, { min: 10000, max: 20000 }); - expect(range1Last).not.toBeNull(); - expect(range1Last!.batch_seq).toBe(10010); - - // Verify range 1's cursor is from range 1's data - const range1Cursor = deserializeCursor(range1Last!.upper_inclusive_cursor); - expect(range1Cursor.id).toBe("r1-upper-10"); - - // Scoped to range 2 [20000, 30000): should return null (no batches) - const range2Last = await store.getLastDoneBatch(runId, { min: 20000, max: 30000 }); - expect(range2Last).toBeNull(); - } finally { - await store.close(); - } - }); -}); diff --git a/tests/integration/datetime-handling.test.ts b/tests/integration/datetime-handling.test.ts deleted file mode 100644 index f32e65b..0000000 --- a/tests/integration/datetime-handling.test.ts +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Integration tests for cdToEpoch() and DateTime64(3) handling. - * - * Validates that the cursor utility correctly normalises MongoDB cd field - * values into epoch milliseconds, and that millisecond precision is preserved - * end-to-end through the ClickHouse DateTime64(3) column. - */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { cdToEpoch } from "../../src/types/cursor.ts"; -import { formatTimestamp } from "../../src/transform/validators.ts"; -import { - getClickHouseClient, - setupClickHouse, - teardownClickHouse, - closeAll, - TEST_CH_DB, - TEST_CH_TABLE, -} from "../helpers/setup.ts"; - -// --------------------------------------------------------------------------- -// Lifecycle -// --------------------------------------------------------------------------- - -beforeAll(async () => { - await setupClickHouse(); -}); - -afterAll(async () => { - await teardownClickHouse(); - await closeAll(); -}); - -// --------------------------------------------------------------------------- -// cdToEpoch unit-level tests -// --------------------------------------------------------------------------- - -describe("cdToEpoch", () => { - it("converts Date objects to epoch milliseconds", () => { - const d = new Date("2024-03-27T10:00:00.123Z"); - expect(cdToEpoch(d)).toBe(d.getTime()); - }); - - it("returns the same value for epoch milliseconds (number >= 9.5e11)", () => { - const ms = 1711525200123; // 2024-03-27T09:00:00.123Z - expect(cdToEpoch(ms)).toBe(ms); - }); - - it("returns the same value for a large epoch millis without fractional part", () => { - const ms = 1700000000000; - expect(cdToEpoch(ms)).toBe(1700000000000); - }); - - it("multiplies epoch seconds (number >= 9.5e8 and < 9.5e11) by 1000", () => { - const sec = 1711525200; // 2024-03-27T09:00:00Z - expect(cdToEpoch(sec)).toBe(1711525200000); - }); - - it("multiplies a borderline epoch-seconds value (just above 9.5e8) by 1000", () => { - const sec = 950000001; - expect(cdToEpoch(sec)).toBe(950000001000); - }); - - it("parses string numbers and applies the same numeric rules", () => { - // String that looks like epoch milliseconds - expect(cdToEpoch("1711525200123")).toBe(1711525200123); - - // String that looks like epoch seconds - expect(cdToEpoch("1711525200")).toBe(1711525200000); - }); - - it("returns 0 for null", () => { - expect(cdToEpoch(null)).toBe(0); - }); - - it("returns 0 for undefined", () => { - expect(cdToEpoch(undefined)).toBe(0); - }); - - it("returns 0 for NaN", () => { - expect(cdToEpoch(NaN)).toBe(0); - }); - - it("returns 0 for Infinity", () => { - expect(cdToEpoch(Infinity)).toBe(0); - }); - - it("returns 0 for -Infinity", () => { - expect(cdToEpoch(-Infinity)).toBe(0); - }); - - it("returns 0 for non-numeric strings", () => { - expect(cdToEpoch("not-a-number")).toBe(0); - expect(cdToEpoch("")).toBe(0); - }); - - it("returns 0 for numbers below the epoch-seconds threshold (< 9.5e8)", () => { - expect(cdToEpoch(100)).toBe(0); - expect(cdToEpoch(0)).toBe(0); - expect(cdToEpoch(-1)).toBe(0); - }); - - it("returns 0 for an invalid Date object", () => { - expect(cdToEpoch(new Date("invalid"))).toBe(0); - }); - - it("floors fractional millisecond values", () => { - expect(cdToEpoch(1711525200123.999)).toBe(1711525200123); - }); -}); - -// --------------------------------------------------------------------------- -// DateTime64(3) precision end-to-end -// --------------------------------------------------------------------------- - -describe("DateTime64(3) precision", () => { - const PRECISION_ID = "__dt64_precision_test__"; - const TS_WITH_MS = 1711525200123; // has .123 milliseconds - - beforeAll(async () => { - const ch = await getClickHouseClient(); - - // Clean up any previous test row - await ch.command({ - query: `ALTER TABLE ${TEST_CH_DB}.${TEST_CH_TABLE} DELETE WHERE _id = '${PRECISION_ID}'`, - }); - - // Wait for the mutation to apply (lightweight table, should be fast) - await new Promise((r) => setTimeout(r, 1000)); - - // Insert a row with a known millisecond-precise timestamp - const tsFormatted = formatTimestamp(TS_WITH_MS); // '2024-03-27 09:00:00.123' - await ch.insert({ - table: `${TEST_CH_DB}.${TEST_CH_TABLE}`, - values: [ - { - _id: PRECISION_ID, - a: "test_app", - e: "[CLY]_custom", - n: "precision_test", - uid: "uid_precision", - did: "did_precision", - ts: tsFormatted, - c: 1, - s: 0, - dur: 0, - }, - ], - format: "JSONEachRow", - }); - }); - - it("preserves .123 millisecond precision in ClickHouse ts column", async () => { - const ch = await getClickHouseClient(); - const result = await ch.query({ - query: ` - SELECT - toUnixTimestamp64Milli(ts) AS ts_ms, - formatDateTime(ts, '%Y-%m-%d %H:%i:%S', 'UTC') AS ts_sec, - toString(ts) AS ts_full - FROM ${TEST_CH_DB}.${TEST_CH_TABLE} - WHERE _id = '${PRECISION_ID}' - `, - format: "JSONEachRow", - }); - - const rows = await result.json< - { ts_ms: string; ts_sec: string; ts_full: string }[] - >(); - expect(rows).toHaveLength(1); - - const row = rows[0]; - // The epoch millis should match exactly - expect(Number(row.ts_ms)).toBe(TS_WITH_MS); - - // The formatted value should include .123 - expect(row.ts_full).toContain(".123"); - }); - - it("formatTimestamp produces the expected DateTime64(3) string", () => { - const formatted = formatTimestamp(TS_WITH_MS); - // Verify format is yyyy-MM-dd HH:mm:ss.SSS and preserves .123 ms - expect(formatted).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/); - expect(formatted).toContain(".123"); - }); - - it("formatTimestamp preserves .000 for exact-second timestamps", () => { - const exactSecond = 1711525200000; - const formatted = formatTimestamp(exactSecond); - expect(formatted).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.000$/); - }); -}); diff --git a/tests/integration/exit-on-complete.test.ts b/tests/integration/exit-on-complete.test.ts deleted file mode 100644 index 098c5c0..0000000 --- a/tests/integration/exit-on-complete.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Unit tests for wireExitOnComplete: verifies the orchestrator-completion → - * SIGTERM wiring used in main.ts. The helper is pure (no infra), so this is a - * lightweight test even though it lives under tests/integration/ for - * collocation with the rest of the suite. - */ -import { describe, it, expect, vi } from 'vitest'; -import pino from 'pino'; -import { wireExitOnComplete } from '../../src/runtime/exit-on-complete.ts'; - -const silentLogger = pino({ level: 'silent' }); - -// Two ticks: one for the runPromise's .then() callback, one for the test continuation. -// Deterministic, unlike a setTimeout-based flush. -const flushMicrotasks = async () => { - await Promise.resolve(); - await Promise.resolve(); -}; - -describe('wireExitOnComplete', () => { - it('does not call kill when disabled, even if the run promise resolves', async () => { - const kill = vi.fn(); - wireExitOnComplete(Promise.resolve(), false, silentLogger, kill, 12345); - await flushMicrotasks(); - expect(kill).not.toHaveBeenCalled(); - }); - - it('sends SIGTERM and logs when enabled and the run promise resolves', async () => { - const kill = vi.fn(); - const logSpy = vi.spyOn(silentLogger, 'info'); - wireExitOnComplete(Promise.resolve(), true, silentLogger, kill, 12345); - await flushMicrotasks(); - expect(kill).toHaveBeenCalledTimes(1); - expect(kill).toHaveBeenCalledWith(12345, 'SIGTERM'); - expect(logSpy).toHaveBeenCalledWith( - 'Orchestrator completed; exit-on-complete set, signaling shutdown', - ); - logSpy.mockRestore(); - }); - - it('does not call kill if enabled but the run promise rejects', async () => { - const kill = vi.fn(); - const rejected = Promise.reject(new Error('orchestrator crashed')); - rejected.catch(() => {}); - wireExitOnComplete(rejected, true, silentLogger, kill, 12345); - await flushMicrotasks(); - expect(kill).not.toHaveBeenCalled(); - }); -}); diff --git a/tests/integration/index-management.test.ts b/tests/integration/index-management.test.ts deleted file mode 100644 index 548f207..0000000 --- a/tests/integration/index-management.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Integration tests for MongoDB index management used by the migration. - * - * Verifies hasRequiredIndex(), startIndexCreation(), and that readPage - * successfully uses the { cd: 1, _id: 1 } compound index hint. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import { ObjectId } from "mongodb"; -import pino from "pino"; - -import { MongoReader, type MongoReaderConfig } from "../../src/source/mongo-reader.ts"; -import { - getMongoDb, - teardownMongo, - closeAll, - TEST_MONGO_URI, - TEST_MONGO_DB, -} from "../helpers/setup.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "silent" }); - -const MONGO_READER_CONFIG: MongoReaderConfig = { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "integration-test-index", - batchRowsTarget: 100, - cursorBatchSize: 100, - maxTimeMs: 30_000, -}; - -const TEST_COLL = "drill_events_index_test"; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -async function seedSmallCollection(withIndex: boolean): Promise { - const db = await getMongoDb(); - const coll = db.collection(TEST_COLL); - - await coll.drop().catch(() => {}); - - const baseTime = new Date("2024-06-01T00:00:00Z"); - const docs = []; - for (let i = 0; i < 20; i++) { - const cd = new Date(baseTime.getTime() + i * 60_000); - docs.push({ - _id: new ObjectId().toHexString(), - a: "test_app", - e: "test_event", - uid: `user-${i}`, - did: `device-${i}`, - ts: cd.getTime(), - cd, - c: 1, - s: 0, - dur: 0, - n: "test_event", - }); - } - - await coll.insertMany(docs); - - if (withIndex) { - await coll.createIndex({ cd: 1, _id: 1 }); - } -} - -// --------------------------------------------------------------------------- -// Lifecycle -// --------------------------------------------------------------------------- - -let reader: MongoReader; - -beforeAll(async () => { - await teardownMongo(); -}); - -afterAll(async () => { - if (reader) { - await reader.close().catch(() => {}); - } - await teardownMongo(); - await closeAll(); -}); - -beforeEach(async () => { - // Close previous reader if open - if (reader) { - await reader.close().catch(() => {}); - } - reader = new MongoReader(MONGO_READER_CONFIG, logger); - await reader.connect(); -}); - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -describe("index-management", () => { - it("hasRequiredIndex returns false when index missing", async () => { - await seedSmallCollection(false); - - const has = await reader.hasRequiredIndex(TEST_COLL); - expect(has).toBe(false); - }); - - it("hasRequiredIndex returns true when index exists", async () => { - await seedSmallCollection(true); - - const has = await reader.hasRequiredIndex(TEST_COLL); - expect(has).toBe(true); - }); - - it("startIndexCreation creates compound index", async () => { - await seedSmallCollection(false); - - // Verify index does not exist yet - const before = await reader.hasRequiredIndex(TEST_COLL); - expect(before).toBe(false); - - // Create the index - await reader.startIndexCreation(TEST_COLL); - - // Verify it now exists - const after = await reader.hasRequiredIndex(TEST_COLL); - expect(after).toBe(true); - }); - - it("readPage uses the compound index hint", async () => { - await seedSmallCollection(true); - - await reader.switchCollection(TEST_COLL); - - // Get the upper bound to use as the scan limit - const upperBound = await reader.getUpperBound(); - expect(upperBound).not.toBeNull(); - - // readPage should not throw — it uses .hint({ cd: 1, _id: 1 }) internally, - // and if the index does not exist, MongoDB returns an error. - const page = await reader.readPage(null, upperBound!); - - expect(page.docs.length).toBeGreaterThan(0); - expect(page.docs.length).toBeLessThanOrEqual(20); - expect(page.fetchMs).toBeGreaterThanOrEqual(0); - }); -}); diff --git a/tests/integration/ledger-engine.test.ts b/tests/integration/ledger-engine.test.ts new file mode 100644 index 0000000..6060f02 --- /dev/null +++ b/tests/integration/ledger-engine.test.ts @@ -0,0 +1,310 @@ +/** + * Ledger engine tests: classifier (pure), coercions (pure), LedgerStore + * claim/lease/transition semantics, and an end-to-end chunk pipeline run + * against real MongoDB + ClickHouse — asserting exact counts, DLQ capture + * with raw docs, and coercion accounting. No Redis anywhere. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pino from 'pino'; +import { MongoClient } from 'mongodb'; +import { createClient, type ClickHouseClient } from '@clickhouse/client'; + +import { classifyError } from '../../src/runtime/error-classifier.ts'; +import { CoercionCounter } from '../../src/transform/coercions.ts'; +import { sanitizeJsonValue } from '../../src/transform/validators.ts'; +import { transformDocument } from '../../src/transform/normalize.ts'; +import { LedgerStore } from '../../src/state/ledger-store.ts'; +import { DlqStore } from '../../src/state/dlq-store.ts'; +import { StagingManager } from '../../src/target/staging-manager.ts'; +import { MongoReader } from '../../src/source/mongo-reader.ts'; +import { RetryPolicy } from '../../src/runtime/retry-policy.ts'; +import { HashResolver } from '../../src/transform/hash-resolver.ts'; +import { ChunkOrchestrator } from '../../src/runtime/chunk-orchestrator.ts'; +import { loadConfig } from '../../src/config/loader.ts'; + +const MONGO_URI = 'mongodb://localhost:27017/?directConnection=true'; +const CH_URL = process.env.TEST_CLICKHOUSE_URL ?? 'http://localhost:8123'; +const CH_PASSWORD = process.env.TEST_CLICKHOUSE_PASSWORD ?? ''; +const DB = 'test_mig_ledger'; +const logger = pino({ level: 'silent' }); + +// --------------------------------------------------------------------------- +// Pure units +// --------------------------------------------------------------------------- + +describe('error-classifier', () => { + it('classifies ClickHouse data-error codes as permanent', () => { + for (const code of ['41', '53', '72', '117', '6']) { + expect(classifyError({ code, message: 'x' })).toBe('permanent'); + } + }); + it('classifies network errors as transient', () => { + expect(classifyError({ code: 'ECONNRESET', message: 'socket hang up' })).toBe('transient'); + expect(classifyError({ code: 'ETIMEDOUT', message: '' })).toBe('transient'); + }); + it('classifies BigInt serialization as permanent', () => { + expect(classifyError(new TypeError('Do not know how to serialize a BigInt'))).toBe('permanent'); + }); + it('defaults unknown errors to transient', () => { + expect(classifyError(new Error('some novel failure'))).toBe('transient'); + expect(classifyError({ code: '999', message: 'unknown CH code' })).toBe('transient'); + }); +}); + +describe('coercions (shared spec: only non-JSON-carriable values change)', () => { + it('stringifies NaN/Infinity/bigint losslessly; finite large doubles STAY numbers (matches live ingestion)', () => { + const out = sanitizeJsonValue({ ok: 42, big: 9.2e25, nan: NaN, inf: Infinity, huge: 10n ** 30n, str: 'hello' }) as Record; + expect(out.nan).toBe('NaN'); + expect(out.inf).toBe('Infinity'); + expect(out.huge).toBe('1000000000000000000000000000000'); + expect(out.big).toBe(9.2e25); // finite double: live keeps it numeric — so do we + expect(out.ok).toBe(42); + }); + it('counts coercions per key through the transform', () => { + const counter = new CoercionCounter(); + const { row } = transformDocument( + { _id: 'x', a: 'app', e: 'ev', uid: 'u1', ts: 1750000000000, sg: { weird: NaN } }, + undefined, + counter, + ); + expect((row?.sg as Record).weird).toBe('NaN'); + expect(counter.getTotal()).toBe(1); + expect(counter.getReport()[0].rule_key).toBe('stringify_nonfinite:sg'); + }); + it('caps distinct coercion keys; the overflow bucket keeps totals exact', () => { + const counter = new CoercionCounter(); + for (let i = 0; i < 10_050; i++) counter.record('stringify_nonfinite', `field_${i}`, NaN, 'NaN'); + expect(counter.getTotal()).toBe(10_050); + const report = counter.getReport(); + expect(report.length).toBe(10_001); // 10k distinct + 1 overflow bucket + expect(report.some((r) => r.rule_key === 'other:distinct-key-cap-reached' && r.count === 50)).toBe(true); + }); + + it('clamps the Countly-owned counter c to UInt32', () => { + const counter = new CoercionCounter(); + const { row } = transformDocument( + { _id: 'x', a: 'app', e: 'ev', uid: 'u1', ts: 1750000000000, c: 99_999_999_999 }, + undefined, + counter, + ); + expect(row?.c).toBe(4_294_967_295); + expect(counter.getTotal()).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// LedgerStore semantics (real MongoDB) +// --------------------------------------------------------------------------- + +describe('LedgerStore', () => { + let ledger: LedgerStore; + + beforeAll(async () => { + ledger = new LedgerStore(MONGO_URI, DB, logger); + await ledger.connect(); + }); + afterAll(async () => { + const mc = new MongoClient(MONGO_URI); + await mc.connect(); + await mc.db(DB).dropDatabase(); + await mc.close(); + await ledger.close(); + }); + + it('initChunks is idempotent and claims are newest-first with a lease', async () => { + const bounds = [0, 1, 2].map((i) => ({ lowerCd: i * 1000, upperCd: (i + 1) * 1000 })); + expect(await ledger.initChunks('r1', 'coll', bounds, 'v1')).toBe(3); + expect(await ledger.initChunks('r1', 'coll', bounds, 'v1')).toBe(0); // no-op + + const first = await ledger.claimNext('r1', 'coll', 'podA', 60); + expect(first?.idx).toBe(2); // newest first + expect(first?.status).toBe('in_progress'); + expect(first?.lease_until!.getTime()).toBeGreaterThan(Date.now()); + + const second = await ledger.claimNext('r1', 'coll', 'podB', 60); + expect(second?.idx).toBe(1); // podA's claim is not re-claimable + }); + + it('guarded transitions reject wrong from-state', async () => { + const moved = await ledger.transition('r1:coll:2', 'pending', 'done'); + expect(moved).toBeNull(); // it's in_progress, not pending + const ok = await ledger.transition('r1:coll:2', 'in_progress', 'written', { rows_expected: 10 }); + expect(ok?.status).toBe('written'); + }); + + it('findRecoverable honors lease expiry for multi-pod reclaim', async () => { + // Nothing expired yet + expect((await ledger.findRecoverable('r1', 'coll', false)).length).toBe(0); + // includeAll (single-pod startup) sees all non-terminal chunks + const all = await ledger.findRecoverable('r1', 'coll', true); + expect(all.length).toBe(2); // idx2 written + idx1 in_progress + }); +}); + +// --------------------------------------------------------------------------- +// End-to-end chunk pipeline (real MongoDB + ClickHouse, no Redis) +// --------------------------------------------------------------------------- + +describe('ledger engine end-to-end', () => { + const CLEAN_DOCS = 2_000; + let ch: ClickHouseClient; + let mc: MongoClient; + let orchestrator: ChunkOrchestrator; + let dlq: DlqStore; + const closers: Array<() => Promise> = []; + + beforeAll(async () => { + mc = new MongoClient(MONGO_URI); + await mc.connect(); + await mc.db(DB).dropDatabase(); + + ch = createClient({ url: CH_URL, password: CH_PASSWORD }); + await ch.command({ query: `CREATE DATABASE IF NOT EXISTS ${DB}` }); + await ch.command({ query: `DROP TABLE IF EXISTS ${DB}.drill_events` }); + await ch.command({ + query: `CREATE TABLE ${DB}.drill_events ( + \`a\` LowCardinality(String), \`e\` LowCardinality(String), \`n\` String, + \`uid\` String, \`uid_canon\` Nullable(String), \`did\` String, \`lsid\` Nullable(String), + \`_id\` String, \`ts\` DateTime64(3), \`up\` JSON(max_dynamic_paths = 32), + \`custom\` Nullable(JSON(max_dynamic_paths = 0)), \`cmp\` Nullable(JSON(max_dynamic_paths = 0)), + \`sg\` JSON(max_dynamic_paths = 0), \`c\` UInt32, \`s\` Float64, \`dur\` Float64, + \`lu\` Nullable(DateTime64(3)), \`cd\` DateTime64(3) DEFAULT now64(3)) + ENGINE = MergeTree PARTITION BY toYYYYMM(ts, 'UTC') ORDER BY (a, e, n, ts)`, + }); + + // Seed: clean docs + 3 transform-poisoned docs (bad ts) + 1 coercion doc + const coll = mc.db(DB).collection('drill_events'); + const base = Date.UTC(2026, 0, 1); + const docs: Record[] = []; + for (let i = 0; i < CLEAN_DOCS; i++) { + const ts = base + i * 60_000; + docs.push({ + _id: `doc_${i}`, a: 'app1', e: i % 3 === 0 ? '[CLY]_view' : 'my_event', + uid: String(i % 50), did: `d${i}`, ts, cd: new Date(ts), + up: { p: 'iOS' }, sg: i % 3 === 0 ? { name: '/home' } : { price: 9.99 }, c: 1, + }); + } + for (let i = 0; i < 3; i++) { + docs.push({ _id: `poison_${i}`, a: 'app1', e: 'bad', uid: 'u', ts: 'not-a-ts', cd: new Date(base + i) }); + } + docs.push({ + _id: 'coerce_me', a: 'app1', e: 'big_int_event', uid: 'u9', did: 'd9', + ts: base + 1, cd: new Date(base + 1), sg: { order_id: 9.2e25, weird: Number.POSITIVE_INFINITY }, c: 1, + }); + // Docs with no cd value — must be picked up by the null-cd sweep chunk + docs.push({ _id: 'nocd_1', a: 'app1', e: 'legacy_event', uid: 'u1', did: 'd', ts: base - 86_400_000 }); + docs.push({ _id: 'nocd_2', a: 'app1', e: 'legacy_event', uid: 'u2', did: 'd', ts: base - 86_400_000, cd: null }); + // REGRESSION: null-cd doc whose derived cd (from ts) lands INSIDE the + // regular chunks' windows. If the sweep ran before regular chunks, its + // row would poison their verify-then-attach check → silently missing + // data. The sweep must be ordered strictly last. + docs.push({ _id: 'nocd_overlap', a: 'app1', e: 'legacy_event', uid: 'u3', did: 'd', ts: base + 120_000 }); + await coll.insertMany(docs as never[]); + await coll.createIndex({ cd: 1, _id: 1 }); + + // Engine wiring (mirrors ledger-engine.ts, minus HTTP) + process.env.SERVICE_NAME = 'ledger-e2e'; + process.env.MONGO_URI = MONGO_URI; + process.env.MONGO_DB = DB; + process.env.MANIFEST_DB = DB; + process.env.CLICKHOUSE_URL = CH_URL; + process.env.CLICKHOUSE_PASSWORD = CH_PASSWORD; + process.env.CLICKHOUSE_DB = DB; + process.env.LEDGER_RUN_ID = 'e2e-1'; + process.env.LEDGER_CHUNK_DOCS_TARGET = '500'; + process.env.LEDGER_MONITOR_INTERVAL_MS = '0'; + process.env.BACKPRESSURE_ENABLED = 'false'; + const config = loadConfig(); + + const mongoReader = new MongoReader({ + uri: MONGO_URI, database: DB, readPreference: 'primary', readConcern: 'local', + retryReads: true, appName: 'e2e', cursorBatchSize: 500, maxTimeMs: 60_000, + }, logger); + const ledger = new LedgerStore(MONGO_URI, DB, logger); + dlq = new DlqStore(MONGO_URI, DB, logger); + const staging = new StagingManager({ + url: CH_URL, database: DB, table: 'drill_events', username: 'default', password: CH_PASSWORD, queryTimeoutMs: 60_000, + }, logger); + const retryPolicy = new RetryPolicy({ maxRetries: 2, baseDelayMs: 50, maxDelayMs: 200 }); + const hashResolver = new HashResolver({ uri: MONGO_URI, countlyDb: `${DB}_countly` }, logger); + + await mongoReader.connect(); + await ledger.connect(); + await dlq.connect(); + await staging.connect(); + await hashResolver.build(); + closers.push(() => mongoReader.close(), () => ledger.close(), () => dlq.close(), () => staging.close(), () => hashResolver.close()); + + orchestrator = new ChunkOrchestrator({ + config, logger, mongoReader, ledger, dlq, staging, retryPolicy, hashResolver, + }); + }, 60_000); + + afterAll(async () => { + for (const close of closers) await close().catch(() => {}); + await ch.command({ query: `DROP DATABASE IF EXISTS ${DB}` }).catch(() => {}); + await ch.close(); + await mc.db(DB).dropDatabase().catch(() => {}); + await mc.close(); + }); + + it('migrates exactly: clean docs land once, poisoned docs go to DLQ with raw docs, coercions counted', async () => { + await orchestrator.run(); + + const res = await ch.query({ + query: `SELECT count() AS t, uniqExact(_id) AS u FROM ${DB}.drill_events`, + format: 'JSONEachRow', + }); + const [row] = await res.json<{ t: string; u: string }>(); + // clean docs + coercion doc + 3 null-cd docs land; the 3 poisoned do not + expect(Number(row.t)).toBe(CLEAN_DOCS + 4); + expect(Number(row.u)).toBe(CLEAN_DOCS + 4); // zero duplicates + + // Null-cd docs arrived via the sweep chunk + const nocd = await ch.query({ + query: `SELECT count() AS c FROM ${DB}.drill_events WHERE _id LIKE 'nocd_%'`, + format: 'JSONEachRow', + }); + expect(Number((await nocd.json<{ c: string }>())[0].c)).toBe(3); + + // DLQ carries the poisoned docs WITH their raw source docs + const pending = await dlq.listPending('e2e-1'); + expect(pending.length).toBe(3); + expect(pending.every((p) => p.reason === 'skipped' && p.error === 'skip:invalid_ts')).toBe(true); + expect(pending.every((p) => typeof p.raw_doc === 'object' && p.raw_doc.ts === 'not-a-ts')).toBe(true); + + // Spec behavior: finite large double stays numeric; Infinity stringified + const coerced = await ch.query({ + query: `SELECT sg.order_id AS v, sg.weird AS w FROM ${DB}.drill_events WHERE _id = 'coerce_me'`, + format: 'JSONEachRow', + }); + const [c] = await coerced.json<{ v: unknown; w: unknown }>(); + expect(Number(c.v)).toBe(9.2e25); + expect(String(c.w)).toBe('Infinity'); + + const stats = orchestrator.getStats(); + expect(stats.totalCoercions).toBeGreaterThanOrEqual(1); + expect(stats.chunksFailed).toBe(0); + expect(stats.status).toBe('completed'); + + const report = await orchestrator.getReport(); + expect((report.dlq as { byStatus: Record }).byStatus.pending).toBe(3); + }, 120_000); + + it('replayDlq keeps still-broken docs pending with an updated error', async () => { + const { replayed, stillFailing } = await orchestrator.replayDlq(); + expect(replayed).toBe(0); // bad ts still fails transform under same version + expect(stillFailing).toBe(3); + const pending = await dlq.listPending('e2e-1'); + expect(pending.length).toBe(3); + expect(pending[0].error).toContain('still fails transform'); + }, 60_000); + + it('waive is the terminal operator decision: pending drains, raw docs retained', async () => { + const waived = await dlq.waive('e2e-1'); + expect(waived).toBe(3); + expect((await dlq.listPending('e2e-1')).length).toBe(0); + const byStatus = await dlq.countByStatus('e2e-1'); + expect(byStatus.waived).toBe(3); // still in the DLQ as the record of what was excluded + }, 30_000); +}); diff --git a/tests/integration/live-parallel.test.ts b/tests/integration/live-parallel.test.ts new file mode 100644 index 0000000..254b63d --- /dev/null +++ b/tests/integration/live-parallel.test.ts @@ -0,0 +1,220 @@ +/** + * Migration under CONCURRENT live ingestion. + * + * The cutover-first playbook means live ingestion writes into drill_events + * the whole time the migrator runs. This suite runs a continuous writer + * (live-shaped rows: cd = now, mixed apps INCLUDING the same (a,e,n) scope + * the migrator is copying) against the live table during a full migration, + * with the invariant monitor enabled at a hot interval, and asserts: + * + * - migrated counts are exact (live rows never leak into window checks) + * - every live-written row survives (no purge/attach path touches them) + * - the invariant monitor never trips (no false violation from live rows) + * - verify passes and attributes any duplicate correctly + * - ATTACH lands into the CURRENT month partition while the writer is + * inserting into it (docs with recent ts but historical cd share the + * partition with live traffic — the hot-partition case). + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pino from 'pino'; +import { createHash } from 'node:crypto'; +import { MongoClient } from 'mongodb'; +import { createClient, type ClickHouseClient } from '@clickhouse/client'; + +import { LedgerStore } from '../../src/state/ledger-store.ts'; +import { DlqStore } from '../../src/state/dlq-store.ts'; +import { StagingManager } from '../../src/target/staging-manager.ts'; +import { MongoReader } from '../../src/source/mongo-reader.ts'; +import { RetryPolicy } from '../../src/runtime/retry-policy.ts'; +import { HashResolver } from '../../src/transform/hash-resolver.ts'; +import { ChunkOrchestrator } from '../../src/runtime/chunk-orchestrator.ts'; +import { loadConfig } from '../../src/config/loader.ts'; + +const MONGO_URI = 'mongodb://localhost:27017/?directConnection=true'; +const CH_URL = process.env.TEST_CLICKHOUSE_URL ?? 'http://localhost:8123'; +const CH_PASSWORD = process.env.TEST_CLICKHOUSE_PASSWORD ?? ''; +const DB = 'test_mig_live'; +const RUN = 'live-1'; +const logger = pino({ level: 'silent' }); + +const APP = 'app_live_test'; +const EV = 'checkout'; +const COLL = `drill_events${createHash('sha1').update(EV + APP).digest('hex')}`; +const DOCS = 30_000; +const BASE = Date.UTC(2026, 2, 1); + +describe('migration under concurrent live ingestion', () => { + let ch: ClickHouseClient; + let mc: MongoClient; + let orchestrator: ChunkOrchestrator; + const closers: Array<() => Promise> = []; + + beforeAll(async () => { + mc = new MongoClient(MONGO_URI); + await mc.connect(); + await mc.db(DB).dropDatabase(); + await mc.db(`${DB}_countly`).dropDatabase(); + await mc.db(`${DB}_countly`).collection('apps').insertOne({ _id: APP } as never); + await mc.db(`${DB}_countly`).collection('events').insertOne({ _id: APP, list: [EV] } as never); + + ch = createClient({ url: CH_URL, password: CH_PASSWORD }); + await ch.command({ query: `CREATE DATABASE IF NOT EXISTS ${DB}` }); + await ch.command({ query: `DROP TABLE IF EXISTS ${DB}.drill_events` }); + await ch.command({ + query: `CREATE TABLE ${DB}.drill_events ( + \`a\` LowCardinality(String), \`e\` LowCardinality(String), \`n\` String, + \`uid\` String, \`uid_canon\` Nullable(String), \`did\` String, \`lsid\` Nullable(String), + \`_id\` String, \`ts\` DateTime64(3), \`up\` JSON(max_dynamic_paths = 32), + \`custom\` Nullable(JSON(max_dynamic_paths = 0)), \`cmp\` Nullable(JSON(max_dynamic_paths = 0)), + \`sg\` JSON(max_dynamic_paths = 0), \`c\` UInt32, \`s\` Float64, \`dur\` Float64, + \`lu\` Nullable(DateTime64(3)), \`cd\` DateTime64(3) DEFAULT now64(3)) + ENGINE = MergeTree PARTITION BY toYYYYMM(ts, 'UTC') ORDER BY (a, e, n, ts)`, + }); + + // Source: most docs historical; the newest slice carries RECENT ts with + // historical cd, so its chunk attaches into the CURRENT month partition — + // the same partition live traffic inserts into. + const coll = mc.db(DB).collection(COLL); + const hotTsBase = Date.now() - 60 * 60_000; + let docs: Record[] = []; + for (let i = 0; i < DOCS; i++) { + const historical = i < DOCS - 2_000; + const ts = historical ? BASE + i * 60_000 : hotTsBase + (i - (DOCS - 2_000)) * 100; + const cd = BASE + i * 60_000; // cd strictly historical for ALL docs + docs.push({ _id: `m_${i}`, uid: String(i % 200), did: `d${i}`, ts, cd: new Date(cd), sg: { v: i }, c: 1 }); + if (docs.length === 5_000) { await coll.insertMany(docs as never[]); docs = []; } + } + if (docs.length) await coll.insertMany(docs as never[]); + await coll.createIndex({ cd: 1, _id: 1 }); + + process.env.SERVICE_NAME = 'live-e2e'; + process.env.MONGO_URI = MONGO_URI; + process.env.MONGO_DB = DB; + process.env.MONGO_COUNTLY_DB = `${DB}_countly`; + process.env.MANIFEST_DB = DB; + process.env.CLICKHOUSE_URL = CH_URL; + process.env.CLICKHOUSE_PASSWORD = CH_PASSWORD; + process.env.CLICKHOUSE_DB = DB; + process.env.LEDGER_RUN_ID = RUN; + process.env.LEDGER_CHUNK_DOCS_TARGET = '4000'; + process.env.MONGO_PAGE_SIZE = '1000'; + // Monitor HOT: every 150ms it spot-checks done chunks against the live + // table while the writer runs — a live row leaking into a window count + // would trip it and pause the engine (which the test would catch below). + process.env.LEDGER_MONITOR_INTERVAL_MS = '150'; + process.env.BACKPRESSURE_ENABLED = 'false'; + const config = loadConfig(); + + const mongoReader = new MongoReader({ + uri: MONGO_URI, database: DB, readPreference: 'primary', readConcern: 'local', + retryReads: true, appName: 'live-e2e', cursorBatchSize: 500, maxTimeMs: 60_000, + }, logger); + const ledger = new LedgerStore(MONGO_URI, DB, logger); + const dlq = new DlqStore(MONGO_URI, DB, logger); + const staging = new StagingManager({ + url: CH_URL, database: DB, table: 'drill_events', username: 'default', password: CH_PASSWORD, queryTimeoutMs: 60_000, + }, logger); + const hashResolver = new HashResolver({ uri: MONGO_URI, countlyDb: `${DB}_countly` }, logger); + + await mongoReader.connect(); + await ledger.connect(); + await dlq.connect(); + await staging.connect(); + await hashResolver.build(); + closers.push(() => mongoReader.close(), () => ledger.close(), () => dlq.close(), () => staging.close(), () => hashResolver.close()); + + orchestrator = new ChunkOrchestrator({ + config, logger, mongoReader, ledger, dlq, staging, + retryPolicy: new RetryPolicy({ maxRetries: 2, baseDelayMs: 50, maxDelayMs: 200 }), hashResolver, + }); + }, 120_000); + + afterAll(async () => { + for (const close of closers) await close().catch(() => {}); + await ch.command({ query: `DROP DATABASE IF EXISTS ${DB}` }).catch(() => {}); + await ch.close(); + await mc.db(DB).dropDatabase().catch(() => {}); + await mc.db(`${DB}_countly`).dropDatabase().catch(() => {}); + await mc.close(); + }); + + it('migrates exactly while live rows pour into the same table (and same hot partition)', async () => { + // Live writer: 40-row batches every 30ms — half into the SAME (a,e,n) + // scope the migrator is copying (the adversarial case for window + // counts), half into another app. cd left to the column default (insert + // time), ts = now → current month partition, shared with the migrated + // hot slice. + let liveWritten = 0; + let writerError: Error | null = null; + let stop = false; + const writer = (async () => { + while (!stop) { + const now = Date.now(); + const rows = Array.from({ length: 40 }, (_, j) => ({ + a: j % 2 === 0 ? APP : 'other_app', + e: '[CLY]_custom', + n: j % 2 === 0 ? EV : 'signup', + uid: `live_u${j}`, did: 'live_d', + _id: `live_${now}_${liveWritten + j}`, + ts: new Date(now).toISOString().replace('T', ' ').replace('Z', ''), + up: {}, sg: {}, c: 1, s: 0, dur: 0, + })); + try { + await ch.insert({ table: `${DB}.drill_events`, values: rows, format: 'JSONEachRow' }); + liveWritten += rows.length; + } catch (e) { writerError = e as Error; stop = true; } + await new Promise((r) => setTimeout(r, 30)); + } + })(); + + await new Promise((r) => setTimeout(r, 200)); // writer running before migration starts + await orchestrator.run(); + const writtenDuringRun = liveWritten; + await new Promise((r) => setTimeout(r, 300)); // writer keeps going after completion + stop = true; + await writer; + + expect(writerError).toBeNull(); + // Structural, not wall-clock-dependent: the writer demonstrably + // overlapped the migration (several batches landed while it ran) and + // kept going after — absolute volume varies with runner speed. + expect(writtenDuringRun).toBeGreaterThan(100); + expect(liveWritten).toBeGreaterThan(writtenDuringRun); + + // Engine finished cleanly: monitor never tripped (a trip pauses + flags) + const stats = orchestrator.getStats(); + expect(stats.status).toBe('completed'); + expect(stats.chunksFailed).toBe(0); + + // Migrated data exact: every source doc present exactly once + const mig = await ch.query({ + query: `SELECT count() AS c, uniqExact(_id) AS u FROM ${DB}.drill_events WHERE _id LIKE 'm_%'`, + format: 'JSONEachRow', + }); + const [m] = await mig.json<{ c: string; u: string }>(); + expect(Number(m.c)).toBe(DOCS); + expect(Number(m.u)).toBe(DOCS); + + // Every live-written row survived migration untouched + const live = await ch.query({ + query: `SELECT count() AS c FROM ${DB}.drill_events WHERE _id LIKE 'live_%'`, + format: 'JSONEachRow', + }); + expect(Number((await live.json<{ c: string }>())[0].c)).toBe(liveWritten); + + // The hot slice really did land in the live-traffic partition + const hot = await ch.query({ + query: `SELECT countIf(_id LIKE 'm_%') AS mig, countIf(_id LIKE 'live_%') AS live + FROM ${DB}.drill_events WHERE _partition_id = toString(toYYYYMM(now(), 'UTC'))`, + format: 'JSONEachRow', + }); + const [h] = await hot.json<{ mig: string; live: string }>(); + expect(Number(h.mig)).toBe(2_000); + expect(Number(h.live)).toBeGreaterThan(0); + + // Full verification passes with live data present + const verify = await orchestrator.verifyMigration(); + expect(verify.ok).toBe(true); + expect((verify.mismatches as unknown[]).length).toBe(0); + }, 180_000); +}); diff --git a/tests/integration/locks.test.ts b/tests/integration/locks.test.ts deleted file mode 100644 index a060b4c..0000000 --- a/tests/integration/locks.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Integration tests for the CollectionLock mechanism. - * - * Verifies Redis-based distributed locking: acquisition, contention, - * release, and TTL-based expiry reclaim. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import pino from "pino"; -import { - getRedis, - teardownRedis, - closeAll, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { - CollectionLock, - type CollectionLockConfig, -} from "../../src/state/collection-lock.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "silent" }); - -const baseLockConfig: CollectionLockConfig = { - lockTtlSec: 10, - renewIntervalMs: 60_000, // won't fire during short tests - podHeartbeatMs: 60_000, // won't fire during short tests - podDeadAfterSec: 2, // short TTL so we can test dead-pod steal - keyPrefix: TEST_REDIS_PREFIX, -}; - -const COLLECTION = "drill_events_lock_test"; - -// --------------------------------------------------------------------------- -// Lifecycle -// --------------------------------------------------------------------------- - -beforeAll(async () => { - await teardownRedis(); -}); - -afterAll(async () => { - await teardownRedis(); - await closeAll(); -}); - -beforeEach(async () => { - // Flush test-prefix keys between tests for isolation - await teardownRedis(); -}); - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -describe("CollectionLock", () => { - it("lock acquisition succeeds for unclaimed collection", async () => { - const redis = await getRedis(); - const lock = new CollectionLock(redis, "pod-A", baseLockConfig, logger); - - const result = await lock.tryAcquire(COLLECTION); - - expect(result).toBe("acquired"); - expect(lock.getHeldLocks()).toContain(COLLECTION); - - await lock.releaseAll(); - }); - - it("second pod cannot acquire same lock", async () => { - const redis = await getRedis(); - const lockA = new CollectionLock(redis, "pod-A", baseLockConfig, logger); - const lockB = new CollectionLock(redis, "pod-B", baseLockConfig, logger); - - // Pod A registers its heartbeat so it appears alive - await redis.set( - `${TEST_REDIS_PREFIX}:pod:pod-A`, - JSON.stringify({ podId: "pod-A", lastHeartbeat: new Date().toISOString(), collectionsActive: [] }), - "EX", - baseLockConfig.podDeadAfterSec, - ); - - const resultA = await lockA.tryAcquire(COLLECTION); - expect(resultA).toBe("acquired"); - - const resultB = await lockB.tryAcquire(COLLECTION); - expect(resultB).toBe("locked"); - - expect(lockB.getHeldLocks()).not.toContain(COLLECTION); - - await lockA.releaseAll(); - }); - - it("lock release allows reacquisition", async () => { - const redis = await getRedis(); - const lockA = new CollectionLock(redis, "pod-A", baseLockConfig, logger); - const lockB = new CollectionLock(redis, "pod-B", baseLockConfig, logger); - - // Acquire with pod-A - const r1 = await lockA.tryAcquire(COLLECTION); - expect(r1).toBe("acquired"); - - // Release with pod-A - await lockA.release(COLLECTION); - expect(lockA.getHeldLocks()).not.toContain(COLLECTION); - - // Now pod-B can acquire - const r2 = await lockB.tryAcquire(COLLECTION); - expect(r2).toBe("acquired"); - expect(lockB.getHeldLocks()).toContain(COLLECTION); - - await lockB.releaseAll(); - }); - - it("expired lock can be reclaimed", async () => { - const redis = await getRedis(); - - // Use a very short lock TTL (1 second) and short dead-after (1 second) - const shortConfig: CollectionLockConfig = { - ...baseLockConfig, - lockTtlSec: 1, - podDeadAfterSec: 1, - }; - - const lockA = new CollectionLock(redis, "pod-A", shortConfig, logger); - const lockB = new CollectionLock(redis, "pod-B", shortConfig, logger); - - // Pod-A acquires but registers a heartbeat with short TTL - await redis.set( - `${TEST_REDIS_PREFIX}:pod:pod-A`, - JSON.stringify({ podId: "pod-A", lastHeartbeat: new Date().toISOString(), collectionsActive: [] }), - "EX", - 1, // 1 second TTL - ); - - const r1 = await lockA.tryAcquire(COLLECTION); - expect(r1).toBe("acquired"); - - // Wait for both the lock and the pod heartbeat to expire - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Pod-B should be able to steal the lock from the dead pod - const r2 = await lockB.tryAcquire(COLLECTION); - // The lock key may have expired (TTL=1s), so result is "acquired", - // or if the key is still present but pod-A's heartbeat expired, result is "stolen" - expect(["acquired", "stolen"]).toContain(r2); - expect(lockB.getHeldLocks()).toContain(COLLECTION); - - await lockB.releaseAll(); - }); -}); diff --git a/tests/integration/manifest-store.test.ts b/tests/integration/manifest-store.test.ts deleted file mode 100644 index ad75334..0000000 --- a/tests/integration/manifest-store.test.ts +++ /dev/null @@ -1,287 +0,0 @@ -/** - * Integration tests for ManifestStore range-scoped query methods. - * - * Verifies that getLastBatch, getLastDoneBatch, getBatches, and - * sumCompletedBatchStats correctly filter by optional BatchSeqRange, - * and that empty ranges return null / zero as expected. - */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { getMongoDb, closeAll, TEST_MONGO_DB } from "../helpers/setup.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import type { Batch, BatchStatus } from "../../src/state/manifest-store.ts"; - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -const TEST_MONGO_URI = "mongodb://localhost:27017/?directConnection=true"; -const RUN_ID = "manifest-store-range-test-run"; -const SOURCE_NS = "countly_drill.drill_events_abc"; -const TARGET_TABLE = "drill_events"; -const TRANSFORM_VERSION = "v1-test"; - -// Range boundaries (matching the multi-pod range convention) -const RANGE_0 = { min: 0, max: 10_000 }; // batch_seq 0 .. 9999 -const RANGE_1 = { min: 10_000, max: 20_000 }; // batch_seq 10000 .. 19999 -const RANGE_2 = { min: 20_000, max: 30_000 }; // batch_seq 20000 .. 29999 -const EMPTY_RANGE = { min: 90_000, max: 100_000 }; // no batches here - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function makeBatch( - batchSeq: number, - status: BatchStatus, - docsRead: number, - rowsInserted: number, -): Omit { - return { - run_id: RUN_ID, - batch_seq: batchSeq, - lower_exclusive_cursor: `{"cd":${batchSeq * 100},"id":"lower_${batchSeq}"}`, - upper_inclusive_cursor: `{"cd":${(batchSeq + 1) * 100},"id":"upper_${batchSeq}"}`, - source_docs_read: docsRead, - docs_skipped: 0, - rows_to_insert: rowsInserted, - payload_digest: `digest_${batchSeq}`, - insert_dedup_token: `dedup_${batchSeq}`, - query_id: `qid_${batchSeq}`, - status, - retry_count: 0, - last_error: null, - started_at: new Date().toISOString(), - finished_at: status === "done" ? new Date().toISOString() : null, - }; -} - -// --------------------------------------------------------------------------- -// Lifecycle -// --------------------------------------------------------------------------- - -let store: ManifestStore; - -beforeAll(async () => { - store = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await store.connect(); - - // Clean up any previous test data for this run - await store.deleteRunData(RUN_ID); - - // Create the run record - await store.createRun({ - run_id: RUN_ID, - status: "active", - source_ns: SOURCE_NS, - target_table: TARGET_TABLE, - upper_bound_cursor: '{"cd":9999999999999,"id":"zzz"}', - transform_version: TRANSFORM_VERSION, - created_at: new Date().toISOString(), - }); - - // ── Seed batches across 3 ranges ──────────────────────────────────── - // Range 0: batch_seq 0..5 — all done, 100 docs / 90 rows each - for (let i = 0; i <= 5; i++) { - await store.insertBatch(makeBatch(i, "done", 100, 90)); - } - - // Range 1: batch_seq 10000..10005 — 4 done + 1 failed + 1 inflight - for (let i = 0; i <= 3; i++) { - await store.insertBatch(makeBatch(10_000 + i, "done", 200, 180)); - } - await store.insertBatch(makeBatch(10_004, "failed", 200, 0)); - await store.insertBatch(makeBatch(10_005, "inflight", 200, 0)); - - // Range 2: batch_seq 20000..20005 — all done, 50 docs / 45 rows each - for (let i = 0; i <= 5; i++) { - await store.insertBatch(makeBatch(20_000 + i, "done", 50, 45)); - } -}); - -afterAll(async () => { - if (store) { - await store.deleteRunData(RUN_ID); - await store.close(); - } - await closeAll(); -}); - -// --------------------------------------------------------------------------- -// getLastBatch -// --------------------------------------------------------------------------- - -describe("getLastBatch", () => { - it("returns the global last batch (highest batch_seq) when no range is provided", async () => { - const last = await store.getLastBatch(RUN_ID); - expect(last).not.toBeNull(); - // batch_seq 20005 is the highest across all ranges - expect(last!.batch_seq).toBe(20_005); - }); - - it("returns only range-0 last batch when scoped to {min:0, max:10000}", async () => { - const last = await store.getLastBatch(RUN_ID, RANGE_0); - expect(last).not.toBeNull(); - expect(last!.batch_seq).toBe(5); - expect(last!.run_id).toBe(RUN_ID); - }); - - it("returns only range-1 last batch when scoped to {min:10000, max:20000}", async () => { - const last = await store.getLastBatch(RUN_ID, RANGE_1); - expect(last).not.toBeNull(); - // 10005 is the highest in range 1 (inflight) - expect(last!.batch_seq).toBe(10_005); - }); - - it("returns null for an empty range", async () => { - const last = await store.getLastBatch(RUN_ID, EMPTY_RANGE); - expect(last).toBeNull(); - }); -}); - -// --------------------------------------------------------------------------- -// getLastDoneBatch -// --------------------------------------------------------------------------- - -describe("getLastDoneBatch", () => { - it("returns the global last done batch when no range is provided", async () => { - const last = await store.getLastDoneBatch(RUN_ID); - expect(last).not.toBeNull(); - expect(last!.status).toBe("done"); - // batch_seq 20005 is the highest done batch across all ranges - expect(last!.batch_seq).toBe(20_005); - }); - - it("returns only range-1 last done batch when scoped to {min:10000, max:20000}", async () => { - const last = await store.getLastDoneBatch(RUN_ID, RANGE_1); - expect(last).not.toBeNull(); - expect(last!.status).toBe("done"); - // In range 1, done batches are 10000..10003; 10004 is failed, 10005 is inflight - expect(last!.batch_seq).toBe(10_003); - }); - - it("returns range-0 last done batch correctly", async () => { - const last = await store.getLastDoneBatch(RUN_ID, RANGE_0); - expect(last).not.toBeNull(); - expect(last!.batch_seq).toBe(5); - expect(last!.status).toBe("done"); - }); - - it("returns null for an empty range", async () => { - const last = await store.getLastDoneBatch(RUN_ID, EMPTY_RANGE); - expect(last).toBeNull(); - }); -}); - -// --------------------------------------------------------------------------- -// getBatches -// --------------------------------------------------------------------------- - -describe("getBatches", () => { - it("returns only done batches in range 0 when filtered by status and batchSeqRange", async () => { - const batches = await store.getBatches(RUN_ID, { - status: "done", - batchSeqRange: RANGE_0, - }); - expect(batches).toHaveLength(6); // batch_seq 0..5, all done - for (const b of batches) { - expect(b.status).toBe("done"); - expect(b.batch_seq).toBeGreaterThanOrEqual(RANGE_0.min); - expect(b.batch_seq).toBeLessThan(RANGE_0.max); - } - // Verify sorted ascending by batch_seq - for (let i = 1; i < batches.length; i++) { - expect(batches[i].batch_seq).toBeGreaterThan(batches[i - 1].batch_seq); - } - }); - - it("returns only done batches in range 1 (excluding failed and inflight)", async () => { - const batches = await store.getBatches(RUN_ID, { - status: "done", - batchSeqRange: RANGE_1, - }); - expect(batches).toHaveLength(4); // 10000..10003 are done - for (const b of batches) { - expect(b.status).toBe("done"); - expect(b.batch_seq).toBeGreaterThanOrEqual(RANGE_1.min); - expect(b.batch_seq).toBeLessThan(RANGE_1.max); - } - }); - - it("returns all batches (any status) in range 1 when no status filter", async () => { - const batches = await store.getBatches(RUN_ID, { - batchSeqRange: RANGE_1, - }); - // 10000..10005 = 6 batches total (4 done + 1 failed + 1 inflight) - expect(batches).toHaveLength(6); - }); - - it("returns an empty array for an empty range", async () => { - const batches = await store.getBatches(RUN_ID, { - status: "done", - batchSeqRange: EMPTY_RANGE, - }); - expect(batches).toHaveLength(0); - }); - - it("respects the limit parameter", async () => { - const batches = await store.getBatches(RUN_ID, { - status: "done", - batchSeqRange: RANGE_0, - limit: 3, - }); - expect(batches).toHaveLength(3); - // Should return the first 3 (sorted ascending) - expect(batches[0].batch_seq).toBe(0); - expect(batches[1].batch_seq).toBe(1); - expect(batches[2].batch_seq).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// sumCompletedBatchStats -// --------------------------------------------------------------------------- - -describe("sumCompletedBatchStats", () => { - it("sums all done batches across all ranges when no range is provided", async () => { - const stats = await store.sumCompletedBatchStats(RUN_ID); - // Range 0: 6 done * 100 docs = 600 docs, 6 * 90 = 540 rows - // Range 1: 4 done * 200 docs = 800 docs, 4 * 180 = 720 rows - // Range 2: 6 done * 50 docs = 300 docs, 6 * 45 = 270 rows - // Total: 1700 docs, 1530 rows - expect(stats.docsRead).toBe(1700); - expect(stats.rowsInserted).toBe(1530); - }); - - it("sums only range-0 done batches when scoped to {min:0, max:10000}", async () => { - const stats = await store.sumCompletedBatchStats(RUN_ID, RANGE_0); - // 6 done * 100 = 600 docs, 6 * 90 = 540 rows - expect(stats.docsRead).toBe(600); - expect(stats.rowsInserted).toBe(540); - }); - - it("sums only range-1 done batches (excluding failed/inflight)", async () => { - const stats = await store.sumCompletedBatchStats(RUN_ID, RANGE_1); - // 4 done * 200 = 800 docs, 4 * 180 = 720 rows - expect(stats.docsRead).toBe(800); - expect(stats.rowsInserted).toBe(720); - }); - - it("sums only range-2 done batches when scoped", async () => { - const stats = await store.sumCompletedBatchStats(RUN_ID, RANGE_2); - // 6 done * 50 = 300 docs, 6 * 45 = 270 rows - expect(stats.docsRead).toBe(300); - expect(stats.rowsInserted).toBe(270); - }); - - it("returns zero for an empty range", async () => { - const stats = await store.sumCompletedBatchStats(RUN_ID, EMPTY_RANGE); - expect(stats.docsRead).toBe(0); - expect(stats.rowsInserted).toBe(0); - }); - - it("returns zero for a non-existent run", async () => { - const stats = await store.sumCompletedBatchStats("nonexistent-run-id"); - expect(stats.docsRead).toBe(0); - expect(stats.rowsInserted).toBe(0); - }); -}); diff --git a/tests/integration/multi-collection-and-rebuild.test.ts b/tests/integration/multi-collection-and-rebuild.test.ts new file mode 100644 index 0000000..3001431 --- /dev/null +++ b/tests/integration/multi-collection-and-rebuild.test.ts @@ -0,0 +1,509 @@ +/** + * Multi-collection correctness + ledger rebuild. + * + * Production Countly stores drill events in MANY hashed collections + * (drill_events{sha1(event+app)}), all overlapping in wall-clock time, while + * ClickHouse holds them in ONE table. Every cd-window query against the live + * table must therefore be scoped to the chunk's (a, e) — these tests pin that + * (a window purge or count without scope silently corrupts sibling + * collections), and exercise the ledger rebuild that regenerates mig_ranges + * from data when progress state is lost. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pino from 'pino'; +import { createHash } from 'node:crypto'; +import { MongoClient } from 'mongodb'; +import { createClient, type ClickHouseClient } from '@clickhouse/client'; + +import { LedgerStore } from '../../src/state/ledger-store.ts'; +import { DlqStore } from '../../src/state/dlq-store.ts'; +import { StagingManager } from '../../src/target/staging-manager.ts'; +import { MongoReader } from '../../src/source/mongo-reader.ts'; +import { RetryPolicy } from '../../src/runtime/retry-policy.ts'; +import { HashResolver } from '../../src/transform/hash-resolver.ts'; +import { ChunkOrchestrator } from '../../src/runtime/chunk-orchestrator.ts'; +import { rebuildLedger, newRebuildProgress } from '../../src/runtime/ledger-rebuild.ts'; +import { loadConfig } from '../../src/config/loader.ts'; +import type { Config } from '../../src/config/schema.ts'; + +const MONGO_URI = 'mongodb://localhost:27017/?directConnection=true'; +const CH_URL = process.env.TEST_CLICKHOUSE_URL ?? 'http://localhost:8123'; +const CH_PASSWORD = process.env.TEST_CLICKHOUSE_PASSWORD ?? ''; +const DB = 'test_mig_multi'; +const RUN = 'multi-1'; +const logger = pino({ level: 'silent' }); + +const APP = 'app_alpha'; +const EV1 = 'purchase'; +const EV2 = 'page_view'; +const hash = (ev: string) => createHash('sha1').update(ev + APP).digest('hex'); +const COLL1 = `drill_events${hash(EV1)}`; +const COLL2 = `drill_events${hash(EV2)}`; +const DOCS_PER_COLL = 600; +const BASE = Date.UTC(2026, 5, 1); + +describe('multi-collection scoping + ledger rebuild', () => { + let ch: ClickHouseClient; + let mc: MongoClient; + let orchestrator: ChunkOrchestrator; + let ledger: LedgerStore; + let hashResolver: HashResolver; + let staging: StagingManager; + let dlqStore: DlqStore; + let config: Config; + const closers: Array<() => Promise> = []; + + const liveCount = async (e: string): Promise => { + const res = await ch.query({ + query: `SELECT count() AS c FROM ${DB}.drill_events WHERE a = '${APP}' AND e = '[CLY]_custom' AND n = '${e}'`, + format: 'JSONEachRow', + }); + return Number((await res.json<{ c: string }>())[0].c); + }; + const totals = async (): Promise<{ t: number; u: number }> => { + const res = await ch.query({ + query: `SELECT count() AS t, uniqExact(_id) AS u FROM ${DB}.drill_events`, + format: 'JSONEachRow', + }); + const [row] = await res.json<{ t: string; u: string }>(); + return { t: Number(row.t), u: Number(row.u) }; + }; + + beforeAll(async () => { + mc = new MongoClient(MONGO_URI); + await mc.connect(); + await mc.db(DB).dropDatabase(); + await mc.db(`${DB}_countly`).dropDatabase(); + + // Countly meta so the resolver maps both hashed collections to (a, e) + await mc.db(`${DB}_countly`).collection('apps').insertOne({ _id: APP } as never); + await mc.db(`${DB}_countly`).collection('events').insertOne({ _id: APP, list: [EV1, EV2] } as never); + + ch = createClient({ url: CH_URL, password: CH_PASSWORD }); + await ch.command({ query: `CREATE DATABASE IF NOT EXISTS ${DB}` }); + await ch.command({ query: `DROP TABLE IF EXISTS ${DB}.drill_events` }); + await ch.command({ + query: `CREATE TABLE ${DB}.drill_events ( + \`a\` LowCardinality(String), \`e\` LowCardinality(String), \`n\` String, + \`uid\` String, \`uid_canon\` Nullable(String), \`did\` String, \`lsid\` Nullable(String), + \`_id\` String, \`ts\` DateTime64(3), \`up\` JSON(max_dynamic_paths = 32), + \`custom\` Nullable(JSON(max_dynamic_paths = 0)), \`cmp\` Nullable(JSON(max_dynamic_paths = 0)), + \`sg\` JSON(max_dynamic_paths = 0), \`c\` UInt32, \`s\` Float64, \`dur\` Float64, + \`lu\` Nullable(DateTime64(3)), \`cd\` DateTime64(3) DEFAULT now64(3)) + ENGINE = MergeTree PARTITION BY toYYYYMM(ts, 'UTC') ORDER BY (a, e, n, ts)`, + }); + + // Both collections cover the SAME wall-clock range — the production shape + // that makes unscoped cd-window queries dangerous. Docs in the hashed + // collections carry no a/e (implicit in the collection name). + for (const [coll, tag] of [[COLL1, 'p'], [COLL2, 'v']] as const) { + const docs: Record[] = []; + for (let i = 0; i < DOCS_PER_COLL; i++) { + const ts = BASE + i * 60_000; + docs.push({ _id: `${tag}_${i}`, uid: String(i % 40), did: `d${i}`, ts, cd: new Date(ts), sg: { v: i }, c: 1 }); + } + const c = mc.db(DB).collection(coll); + await c.insertMany(docs as never[]); + await c.createIndex({ cd: 1, _id: 1 }); + } + // Null-cd outliers in collection 1, ts INSIDE the regular range — their + // sweep rows land inside regular windows and must not confuse rebuild. + await mc.db(DB).collection(COLL1).insertMany([ + { _id: 'nocd_p_1', uid: 'u1', did: 'd', ts: BASE + 90_000 }, + { _id: 'nocd_p_2', uid: 'u2', did: 'd', ts: BASE + 150_000, cd: null }, + ] as never[]); + + process.env.SERVICE_NAME = 'multi-e2e'; + process.env.MONGO_URI = MONGO_URI; + process.env.MONGO_DB = DB; + process.env.MONGO_COUNTLY_DB = `${DB}_countly`; + process.env.MANIFEST_DB = DB; + process.env.CLICKHOUSE_URL = CH_URL; + process.env.CLICKHOUSE_PASSWORD = CH_PASSWORD; + process.env.CLICKHOUSE_DB = DB; + process.env.LEDGER_RUN_ID = RUN; + process.env.LEDGER_CHUNK_DOCS_TARGET = '250'; + process.env.LEDGER_MONITOR_INTERVAL_MS = '0'; + process.env.BACKPRESSURE_ENABLED = 'false'; + config = loadConfig(); + + const mongoReader = new MongoReader({ + uri: MONGO_URI, database: DB, readPreference: 'primary', readConcern: 'local', + retryReads: true, appName: 'multi-e2e', cursorBatchSize: 500, maxTimeMs: 60_000, + }, logger); + ledger = new LedgerStore(MONGO_URI, DB, logger); + dlqStore = new DlqStore(MONGO_URI, DB, logger); + const dlq = dlqStore; + staging = new StagingManager({ + url: CH_URL, database: DB, table: 'drill_events', username: 'default', password: CH_PASSWORD, queryTimeoutMs: 60_000, + }, logger); + const retryPolicy = new RetryPolicy({ maxRetries: 2, baseDelayMs: 50, maxDelayMs: 200 }); + hashResolver = new HashResolver({ uri: MONGO_URI, countlyDb: `${DB}_countly` }, logger); + + await mongoReader.connect(); + await ledger.connect(); + await dlq.connect(); + await staging.connect(); + await hashResolver.build(); + closers.push(() => mongoReader.close(), () => ledger.close(), () => dlq.close(), () => staging.close(), () => hashResolver.close()); + + orchestrator = new ChunkOrchestrator({ + config, logger, mongoReader, ledger, dlq, staging, retryPolicy, hashResolver, + }); + }, 60_000); + + afterAll(async () => { + for (const close of closers) await close().catch(() => {}); + if (process.env.KEEP_TEST_STATE) { await ch.close(); await mc.close(); return; } + await ch.command({ query: `DROP DATABASE IF EXISTS ${DB}` }).catch(() => {}); + await ch.close(); + await mc.db(DB).dropDatabase().catch(() => {}); + await mc.db(`${DB}_countly`).dropDatabase().catch(() => {}); + await mc.close(); + }); + + it('migrates overlapping collections exactly; chunks carry (a,e) scope; scoped verify passes', async () => { + await orchestrator.run(); + + expect(await liveCount(EV1)).toBe(DOCS_PER_COLL + 2); // + 2 null-cd sweep docs + expect(await liveCount(EV2)).toBe(DOCS_PER_COLL); + const { t, u } = await totals(); + expect(t).toBe(DOCS_PER_COLL * 2 + 2); + expect(u).toBe(t); // zero duplicates + + const all = await ledger.listAll(RUN); + expect(all.every((c) => c.scope_a === APP)).toBe(true); + expect(all.every((c) => c.scope_e === '[CLY]_custom')).toBe(true); // custom events: e is the bucket, n the name + expect(new Set(all.map((c) => c.scope_n))).toEqual(new Set([EV1, EV2])); + + // Pre-scoping, overlapping windows made per-chunk verify count BOTH + // collections' rows → guaranteed mismatch. Scoped, it must be clean. + const verify = await orchestrator.verifyMigration(); + expect(verify.ok).toBe(true); + expect((verify.mismatches as unknown[]).length).toBe(0); + expect(verify.unscopedSkipped).toBe(0); + }, 120_000); + + it('retryFailed purges ONLY the failed chunk\'s collection — siblings in the same window are untouched', async () => { + const all = await ledger.listAll(RUN); + const victim = all.find((c) => c.collection === COLL1 && c.status === 'done' && c.lower_cd >= 0)!; + await ledger.transition(victim._id, 'done', 'failed', { last_error: 'test: simulated invariant flag' }); + + await orchestrator.retryFailed(); + + // THE regression: before scoping, this purge deleted COLL2's rows in the + // same cd window. COLL2 must be complete while COLL1's window is redone. + expect(await liveCount(EV2)).toBe(DOCS_PER_COLL); + + await orchestrator.run(); + expect(await liveCount(EV1)).toBe(DOCS_PER_COLL + 2); + const { t, u } = await totals(); + expect(u).toBe(t); + expect(t).toBe(DOCS_PER_COLL * 2 + 2); + }, 120_000); + + it('rebuilds a lost ledger from data: all-done run, post-cutover live rows ignored', async () => { + // Disaster: the ledger vanishes. Meanwhile live ingestion (post-cutover) + // keeps writing rows with NEWER cd than anything in the frozen source. + await mc.db(DB).collection('mig_ranges').deleteMany({}); + await ch.command({ + query: `INSERT INTO ${DB}.drill_events (a, e, n, uid, did, _id, ts, cd) + VALUES ('${APP}', '[CLY]_custom', '${EV1}', 'u_live', 'd_live', 'live_1', ${BASE + 40 * 86_400_000}, fromUnixTimestamp64Milli(${BASE + 40 * 86_400_000})) + , ('${APP}', '[CLY]_custom', '${EV2}', 'u_live', 'd_live', 'live_2', ${BASE + 41 * 86_400_000}, fromUnixTimestamp64Milli(${BASE + 41 * 86_400_000}))`, + }); + + const progress = newRebuildProgress(); + progress.status = 'running'; + await rebuildLedger({ config, logger, ledger, dlq: dlqStore, hashResolver, progress }); + + const all = await ledger.listAll(RUN); + expect(all.length).toBeGreaterThan(0); + expect(all.every((c) => c.status === 'done')).toBe(true); + expect(all.every((c) => c.scope_a === APP)).toBe(true); + + const sentinel = all.find((c) => c.collection === COLL1 && c.lower_cd === -1)!; + expect(sentinel.status).toBe('done'); + expect(sentinel.rows_expected).toBe(2); + + const s1 = progress.summary.find((c) => c.collection === COLL1)!; + expect(s1.scoped).toBe(true); + expect(s1.nullCdDocs).toBe(2); + expect(s1.nullCdSwept).toBe(2); + // sweep rows subtracted per window; post-cutover rows outside all windows + expect(s1.mongoDocs).toBe(DOCS_PER_COLL); + expect(s1.liveRows).toBe(DOCS_PER_COLL); + + // Resuming after a rebuild that found everything done copies nothing new + await orchestrator.run(); + const { t, u } = await totals(); + expect(t).toBe(DOCS_PER_COLL * 2 + 2 + 2); // + the 2 live rows + expect(u).toBe(t); + }, 120_000); + + it('rebuild flags a half-migrated window as failed; retry + resume heal it exactly', async () => { + // Lose the ledger AND part of one window's rows (e.g. the crash that + // took the ledger also lost a ClickHouse part). + await mc.db(DB).collection('mig_ranges').deleteMany({}); + const goneIds = Array.from({ length: 50 }, (_, i) => `'p_${i + 10}'`).join(','); + await ch.command({ query: `DELETE FROM ${DB}.drill_events WHERE _id IN (${goneIds})` }); + + const progress = newRebuildProgress(); + progress.status = 'running'; + await rebuildLedger({ config, logger, ledger, dlq: dlqStore, hashResolver, progress }); + + const all = await ledger.listAll(RUN); + const failed = all.filter((c) => c.status === 'failed'); + expect(failed.length).toBe(1); + expect(failed[0].collection).toBe(COLL1); + expect(failed[0].last_error).toContain('rebuilt from data'); + // every other window checked out + expect(all.filter((c) => c.status === 'done').length).toBe(all.length - 1); + + await orchestrator.retryFailed(); + await orchestrator.run(); + + expect(await liveCount(EV1)).toBe(DOCS_PER_COLL + 2 + 1); // + live_1 + expect(await liveCount(EV2)).toBe(DOCS_PER_COLL + 1); // + live_2 + const { t, u } = await totals(); + expect(u).toBe(t); + + const verify = await orchestrator.verifyMigration(); + expect(verify.ok).toBe(true); + }, 120_000); + + it('verify attributes duplicates by cd boundary: live artifact / cross-cutover retry / migration defect', async () => { + const nowMs = Date.now(); + // A duplicate always shares its ts with the original (a retry RESENDS the + // same event) — that shared ts month is what makes per-partition + // duplicate scanning exact. + const mk = (id: string, tsMs: number, cdMs: number) => + `('${APP}', '[CLY]_custom', '${EV1}', 'u_dup', 'd_dup', '${id}', ${tsMs}, fromUnixTimestamp64Milli(${cdMs}))`; + // 1) live at-least-once redelivery: same _id twice, NO migrated copy + await ch.command({ + query: `INSERT INTO ${DB}.drill_events (a, e, n, uid, did, _id, ts, cd) + VALUES ${mk('redelivered_1', nowMs, nowMs)}, ${mk('redelivered_1', nowMs, nowMs + 500)}`, + }); + // 2) cross-cutover SDK retry: same event ts, post-cutover cd + await ch.command({ + query: `INSERT INTO ${DB}.drill_events (a, e, n, uid, did, _id, ts, cd) + VALUES ${mk('p_5', BASE + 5 * 60_000, nowMs)}`, + }); + + let verify = await orchestrator.verifyMigration(); + expect(verify.table.duplicates).toBe(2); + expect(verify.migrationDuplicates).toBe(0); + expect(verify.ok).toBe(true); // neither is a migration defect + const sample = (verify.duplicateSample as Array<{ _id: string; migratedCopies: number; verdict: string }>); + const byId = new Map(sample.map((d) => [d._id, d])); + expect(byId.get('redelivered_1')!.migratedCopies).toBe(0); + expect(byId.get('redelivered_1')!.verdict).toContain('live at-least-once artifact'); + expect(byId.get('p_5')!.migratedCopies).toBe(1); + expect(byId.get('p_5')!.verdict).toContain('cross-cutover retry'); + + // 3) a REAL migration defect looks like: two copies BELOW the boundary + // (both written by migration — same doc migrated twice) + await ch.command({ + query: `INSERT INTO ${DB}.drill_events (a, e, n, uid, did, _id, ts, cd) + VALUES ('${APP}', '[CLY]_custom', '${EV1}', 'u_dup', 'd_dup', 'p_6', ${BASE + 6 * 60_000}, fromUnixTimestamp64Milli(${BASE + 6 * 60_000 + 1}))`, + }); + verify = await orchestrator.verifyMigration(); + expect(verify.migrationDuplicates).toBe(1); + expect(verify.ok).toBe(false); + + await ch.command({ query: `DELETE FROM ${DB}.drill_events WHERE _id = 'redelivered_1' OR (_id = 'p_5' AND cd >= fromUnixTimestamp64Milli(${nowMs})) OR (_id = 'p_6' AND cd = fromUnixTimestamp64Milli(${BASE + 6 * 60_000 + 1}))` }); + }, 60_000); + + it('probeSourceFrozen detects writes landing between probes', async () => { + const db = mc.db(DB); + // frozen: nothing changes during the wait + const still = await orchestrator.probeSourceFrozen(db as never, [COLL1], 10); + expect(still.frozen).toBe(true); + + // not frozen: a write lands during the wait window + const busy = await orchestrator.probeSourceFrozen(db as never, [COLL1], 10, async () => { + await db.collection(COLL1).insertOne({ _id: 'late_arrival', uid: 'u', ts: Date.now(), cd: new Date() } as never); + }); + expect(busy.frozen).toBe(false); + expect(busy.grew).toContain(COLL1); + await db.collection(COLL1).deleteOne({ _id: 'late_arrival' } as never); + }, 30_000); + + it('DLQ pending listing paginates stably', async () => { + const { DlqStore } = await import('../../src/state/dlq-store.ts'); + const store = new DlqStore(MONGO_URI, DB, logger); + await store.connect(); + await store.add(Array.from({ length: 12 }, (_, i) => ({ + run_id: RUN, source_id: `pg_${String(i).padStart(2, '0')}`, collection: COLL1, + reason: 'skipped' as const, error: 'test', raw_doc: { i }, transform_version: 'v-test', + }))); + const page1 = await store.listPending(RUN, 5, 0); + const page2 = await store.listPending(RUN, 5, 5); + expect(page1.length).toBe(5); + expect(page2.length).toBe(5); + expect(new Set([...page1, ...page2].map((d) => d._id)).size).toBe(10); // no overlap + await store.waive(RUN); + await store.close(); + }, 30_000); + + it('replay skips DLQ entries whose rows are already live (redo-then-replay cannot duplicate)', async () => { + const { DlqStore } = await import('../../src/state/dlq-store.ts'); + const store = new DlqStore(MONGO_URI, DB, logger); + await store.connect(); + // p_100 is already migrated; its DLQ entry simulates a doc that failed + // once but was later migrated by a chunk redo with a fixed transform. + const srcTs = BASE + 100 * 60_000; + await store.add([{ + run_id: RUN, source_id: 'p_100', collection: COLL1, reason: 'insert_rejected', + error: 'old transform bug', transform_version: 'v-old', + raw_doc: { _id: 'p_100', uid: '100', did: 'd100', ts: srcTs, cd: new Date(srcTs), sg: { v: 100 }, c: 1 }, + }]); + + const res = await orchestrator.replayDlq(); + expect(res.alreadyLive).toBe(1); + expect(res.replayed).toBe(0); + const count = await ch.query({ + query: `SELECT count() AS c FROM ${DB}.drill_events WHERE _id = 'p_100'`, format: 'JSONEachRow', + }); + expect(Number((await count.json<{ c: string }>())[0].c)).toBe(1); // still exactly one copy + await store.close(); + }, 60_000); + + it('DLQ mass guard pauses the engine when pending crosses the threshold', async () => { + const { DlqStore } = await import('../../src/state/dlq-store.ts'); + const store = new DlqStore(MONGO_URI, DB, logger); + await store.connect(); + await store.add(Array.from({ length: 6 }, (_, i) => ({ + run_id: RUN, source_id: `mass_${i}`, collection: COLL1, reason: 'skipped' as const, + error: 'systematic', transform_version: 'v-test', raw_doc: { i }, + }))); + + const prev = config.ledger.dlqPauseThreshold; + config.ledger.dlqPauseThreshold = 5; + try { + expect(await orchestrator.checkDlqPressure(logger, true)).toBe(true); // tripped + paused + expect(await orchestrator.checkDlqPressure(logger, true)).toBe(false); // already paused — idempotent + } finally { + config.ledger.dlqPauseThreshold = prev; + orchestrator.resume(); + await store.waive(RUN); + await store.close(); + } + }, 30_000); + + it('audits close the count-blind spots: source recount and sampled content comparison', async () => { + const { rebuildLedger: rebuild, newRebuildProgress: newProgress } = await import('../../src/runtime/ledger-rebuild.ts'); + + // Clean state: both audits pass and the ledger is untouched by checkOnly + const before = await ledger.listAll(RUN); + let prog = newProgress(); + await rebuild({ config, logger, ledger, dlq: dlqStore, hashResolver, progress: prog, checkOnly: true }); + expect(prog.mismatchedWindows.length).toBe(0); + const after = await ledger.listAll(RUN); + expect(after.map((c) => c._id + c.status).join()).toBe(before.map((c) => c._id + c.status).join()); + + let audit = await orchestrator.contentAudit(100); + expect(audit.sampled).toBeGreaterThan(50); + expect(audit.missing).toBe(0); + expect(audit.different).toBe(0); + + // Corrupt one live row the sampler deterministically hits: p_80 gets a + // wrong uid (same _id and cd — invisible to every count and pair check). + const srcDoc = await mc.db(DB).collection(COLL1).findOne({ _id: 'p_80' } as never) as Record; + const cdMs = (srcDoc.cd as Date).getTime(); + await staging.deleteLiveByPairs([{ id: 'p_80', cdMs }]); + const { transformDocument } = await import('../../src/transform/normalize.ts'); + const defaults = hashResolver.resolveCollectionName(COLL1, config.source.collectionPrefix) ?? undefined; + const { row } = transformDocument(srcDoc as never, defaults); + await staging.insertIntoLive([{ ...row!, uid: 'EVIL' }], 'audit-corrupt'); + + audit = await orchestrator.contentAudit(600); // sample densely → must hit p_80 + expect(audit.different).toBeGreaterThanOrEqual(1); + const hit = audit.mismatches.find((m) => m._id === 'p_80'); + expect(hit?.fields).toContain('uid'); + // counts still agree everywhere — this class is invisible to the source audit + prog = newProgress(); + await rebuild({ config, logger, ledger, dlq: dlqStore, hashResolver, progress: prog, checkOnly: true }); + expect(prog.mismatchedWindows.length).toBe(0); + + // Now a LOSS: delete the row entirely — source audit flags the window + await staging.deleteLiveByPairs([{ id: 'p_80', cdMs }]); + prog = newProgress(); + await rebuild({ config, logger, ledger, dlq: dlqStore, hashResolver, progress: prog, checkOnly: true }); + expect(prog.mismatchedWindows.length).toBe(1); + expect(prog.mismatchedWindows[0].source - prog.mismatchedWindows[0].live).toBe(1); + + // Restore the true row; both audits green again + await staging.insertIntoLive([row!], 'audit-restore'); + prog = newProgress(); + await rebuild({ config, logger, ledger, dlq: dlqStore, hashResolver, progress: prog, checkOnly: true }); + expect(prog.mismatchedWindows.length).toBe(0); + audit = await orchestrator.contentAudit(600); + expect(audit.missing).toBe(0); + expect(audit.different).toBe(0); + }, 120_000); + + it('a waived DLQ doc explains its window shortfall — audit does not cry wolf', async () => { + const { rebuildLedger: rebuild, newRebuildProgress: newProgress } = await import('../../src/runtime/ledger-rebuild.ts'); + const { transformDocument } = await import('../../src/transform/normalize.ts'); + + // Simulate a doc that never migrated because it was DLQ'd and waived: + // remove its live row and record it as waived with its cd attributed. + const srcDoc = await mc.db(DB).collection(COLL1).findOne({ _id: 'p_40' } as never) as Record; + const cdMs = (srcDoc.cd as Date).getTime(); + await staging.deleteLiveByPairs([{ id: 'p_40', cdMs }]); + await dlqStore.add([{ + run_id: RUN, source_id: 'p_40', collection: COLL1, chunk_id: 'test', reason: 'insert_rejected', + error: 'unfixable by decision', transform_version: 'v-test', raw_doc: srcDoc, + } as never]); + await dlqStore.waive(RUN, [`${RUN}:p_40`]); + + // Source audit: source > live by exactly the waived doc → NOT a mismatch + const prog = newProgress(); + await rebuild({ config, logger, ledger, dlq: dlqStore, hashResolver, progress: prog, checkOnly: true }); + expect(prog.mismatchedWindows.length).toBe(0); + + // restore: un-waive bookkeeping + reinsert the true row + const defaults = hashResolver.resolveCollectionName(COLL1, config.source.collectionPrefix) ?? undefined; + const { row } = transformDocument(srcDoc as never, defaults); + await staging.insertIntoLive([row!], 'audit-waive-restore'); + await mc.db(DB).collection('mig_dlq_docs').deleteOne({ _id: `${RUN}:p_40` } as never); + const clean = newProgress(); + await rebuild({ config, logger, ledger, dlq: dlqStore, hashResolver, progress: clean, checkOnly: true }); + expect(clean.mismatchedWindows.length).toBe(0); + }, 60_000); + + it('attach-recovery pair check ignores live copies of the same _id (cross-cutover retry)', async () => { + // The mixing vector: a crash during attach + an SDK retry that landed the + // same _id in live (same ts → same month partition). Matching (_id, cd) + // pairs is exact: the retry copy's cd differs by construction. + const tsMs = BASE + 42 * 60_000; + const stagingTable = 'drill_events__stg_precision_test'; + await staging.createStaging(stagingTable); + await staging.insertBatch(stagingTable, [{ + _id: 'retry_victim', a: APP, e: '[CLY]_custom', n: EV1, uid: 'u', did: 'd', + ts: new Date(tsMs).toISOString().replace('T', ' ').replace('Z', ''), + c: 1, s: 0, dur: 0, + cd: new Date(tsMs).toISOString().replace('T', ' ').replace('Z', ''), + } as never], 'precision-test', 'precision-q1'); + const [partitionId] = await staging.listPartitions(stagingTable); + + // live retry copy: same _id, same ts (same partition), NOT migrated + await ch.command({ + query: `INSERT INTO ${DB}.drill_events (a, e, n, uid, did, _id, ts, cd) + VALUES ('${APP}', '[CLY]_custom', '${EV1}', 'u', 'd', 'retry_victim', ${tsMs}, fromUnixTimestamp64Milli(${Date.now()}))`, + }); + expect(await staging.countLiveByStagedIds(stagingTable, partitionId)).toBe(0); // id-only matching: 1 → skipped attach → data loss + + // once the migrated copy IS live, recovery correctly reports it + await staging.insertIntoLive([{ + _id: 'retry_victim', a: APP, e: '[CLY]_custom', n: EV1, uid: 'u', did: 'd', + ts: new Date(tsMs).toISOString().replace('T', ' ').replace('Z', ''), + c: 1, s: 0, dur: 0, + cd: new Date(tsMs).toISOString().replace('T', ' ').replace('Z', ''), + } as never], 'precision-test-live'); + expect(await staging.countLiveByStagedIds(stagingTable, partitionId)).toBe(1); + + await staging.dropStaging(stagingTable); + await ch.command({ query: `DELETE FROM ${DB}.drill_events WHERE _id = 'retry_victim'` }); + }, 60_000); +}); diff --git a/tests/integration/multi-collection.test.ts b/tests/integration/multi-collection.test.ts deleted file mode 100644 index ae7e961..0000000 --- a/tests/integration/multi-collection.test.ts +++ /dev/null @@ -1,572 +0,0 @@ -/** - * Integration test: multi-collection migration flows. - * - * Verifies that the migration service correctly processes multiple collections - * sequentially, skips already-completed collections on restart, and excludes - * APM collections. - */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter } from "../../src/target/clickhouse-writer.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { BatchRunner, type BatchRunnerDeps } from "../../src/runtime/batch-runner.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection, seedMultipleCollections } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -let chClientForPressure: ClickHouseClient | null = null; - -function getChClientForPressure(): ClickHouseClient { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return chClientForPressure; -} - -interface MigrationComponents { - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; - retryPolicy: RetryPolicy; - chPressure: ClickHousePressure; -} - -/** Build shared migration infrastructure components (single set, reused across collections). */ -async function buildComponents(): Promise { - const mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "multi-collection-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - - const chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }, - logger, - ); - await chWriter.connect(); - - const manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - const redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const chPressure = new ClickHousePressure( - getChClientForPressure(), - BACKPRESSURE_OFF, - logger, - ); - - const gcController = new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); - - const retryPolicy = new RetryPolicy({ - maxRetries: 3, - baseDelayMs: 100, - maxDelayMs: 1000, - }); - - return { mongoReader, chWriter, manifestStore, redisState, gcController, retryPolicy, chPressure }; -} - -async function cleanupComponents(c: MigrationComponents): Promise { - await c.mongoReader.close().catch(() => {}); - await c.chWriter.close().catch(() => {}); - await c.manifestStore.close().catch(() => {}); - await c.redisState.close().catch(() => {}); - c.gcController.dispose(); -} - -/** - * Run the migration for a single collection using a BatchRunner. - * Creates a run, resolves upper bound, and processes all batches. - * Returns the runId and final stats. - */ -async function migrateCollection( - c: MigrationComponents, - collName: string, - eventName: string, - appId: string = APP_ID, -): Promise<{ runId: string; docsRead: number; rowsInserted: number }> { - const { mongoReader, chWriter, manifestStore, redisState, chPressure, gcController, retryPolicy } = c; - - await mongoReader.switchCollection(collName); - - const upperBound = await mongoReader.getUpperBound(); - if (!upperBound) { - // Empty collection - return { runId: "", docsRead: 0, rowsInserted: 0 }; - } - - const upperBoundId = serializeCursor(upperBound); - const runId = randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - // Per-collection Redis prefix to avoid collisions - const collRedisState = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:${collName}`, - ); - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await collRedisState.setActiveRun(runId); - await collRedisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: collRedisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: appId, e: eventName }, - collectionName: collName, - }, - }; - - const runner = new BatchRunner(deps); - await runner.run(); - - const stats = runner.getStats(); - return { - runId, - docsRead: stats.totalDocsRead, - rowsInserted: stats.totalRowsInserted, - }; -} - -// --------------------------------------------------------------------------- -// Test suite -// --------------------------------------------------------------------------- - -describe("multi-collection", () => { - beforeAll(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - afterAll(async () => { - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await closeAll(); - }); - - // ----------------------------------------------------------------------- - // Test 1: processes 10 collections sequentially - // ----------------------------------------------------------------------- - - it("processes 10 collections sequentially", async () => { - // Fresh slate - await teardownClickHouse(); - await teardownMongo(); - await teardownRedis(); - await setupClickHouse(); - - const sizes = [500, 1000, 1500, 2000, 500, 800, 1200, 600, 900, 700]; - const eventNames = sizes.map((_, i) => `multi_event_${i}`); - - // Seed 10 collections with varying sizes - const seeded = await seedMultipleCollections( - sizes.map((count, i) => ({ - eventName: eventNames[i], - count, - appId: APP_ID, - })), - ); - - const expectedTotalRows = seeded.reduce((sum, s) => sum + s.expectedRows, 0); - - // Build shared components - const components = await buildComponents(); - - try { - // Migrate each collection sequentially - let totalDocsRead = 0; - let totalRowsInserted = 0; - - for (const seed of seeded) { - const result = await migrateCollection( - components, - seed.collName, - seed.eventName, - APP_ID, - ); - totalDocsRead += result.docsRead; - totalRowsInserted += result.rowsInserted; - } - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // Verify all data is in ClickHouse (min/max boundary tolerance ±2% per collection) - const count = await chRowCount(); - const tolerance = Math.ceil(expectedTotalRows * 0.02); - expect(count).toBeGreaterThanOrEqual(expectedTotalRows - tolerance); - expect(count).toBeLessThanOrEqual(expectedTotalRows + tolerance); - - // Verify total docs match (with same tolerance) - const totalExpected = sizes.reduce((a, b) => a + b, 0); - expect(totalDocsRead).toBeGreaterThanOrEqual(totalExpected - tolerance); - expect(totalRowsInserted).toBeGreaterThanOrEqual(expectedTotalRows - tolerance); - } finally { - await cleanupComponents(components); - } - }, 180_000); - - // ----------------------------------------------------------------------- - // Test 2: skips already-completed collections on restart - // ----------------------------------------------------------------------- - - it("skips already-completed collections on restart", async () => { - // Fresh slate - await teardownClickHouse(); - await teardownMongo(); - await teardownRedis(); - await setupClickHouse(); - - const eventNames = ["restart_a", "restart_b", "restart_c", "restart_d", "restart_e"]; - const sizes = [300, 400, 500, 600, 700]; - - // Seed 5 collections - const seeded = await seedMultipleCollections( - sizes.map((count, i) => ({ - eventName: eventNames[i], - count, - appId: APP_ID, - })), - ); - - const expectedTotal = seeded.reduce((sum, s) => sum + s.expectedRows, 0); - - // --- Session 1: process first 3 collections --- - const components1 = await buildComponents(); - const completedRunIds: string[] = []; - - try { - for (let i = 0; i < 3; i++) { - const result = await migrateCollection( - components1, - seeded[i].collName, - seeded[i].eventName, - APP_ID, - ); - completedRunIds.push(result.runId); - - // Mark the run as "completed" in the manifest so the next session skips it - const sourceNs = `${TEST_MONGO_DB}.${seeded[i].collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - await components1.manifestStore.writeSummary(result.runId, "completed", { - finished_at: new Date().toISOString(), - duration_ms: 0, - total_docs_read: result.docsRead, - total_rows_inserted: result.rowsInserted, - total_docs_skipped: 0, - avg_docs_per_second: 0, - avg_rows_per_second: 0, - total_batches: 1, - batches_done: 1, - batches_failed: 0, - batches_skipped_empty: 0, - skip_reasons: {}, - total_errors: 0, - failed_batch_seqs: [], - digest_mismatches: 0, - estimated_duplicate_rows: 0, - coverage_pct: 100, - }); - } - } finally { - await cleanupComponents(components1); - } - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - const countAfterSession1 = await chRowCount(); - const session1Expected = seeded.slice(0, 3).reduce((sum, s) => sum + s.expectedRows, 0); - expect(countAfterSession1).toBeGreaterThanOrEqual(session1Expected - 10); - expect(countAfterSession1).toBeLessThanOrEqual(session1Expected + 10); - - // --- Session 2: "restart" - new components, process remaining 2 --- - const components2 = await buildComponents(); - - try { - // Before migrating each remaining collection, verify the first 3 are already completed - for (let i = 0; i < 3; i++) { - const sourceNs = `${TEST_MONGO_DB}.${seeded[i].collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - const alreadyCompleted = await components2.manifestStore.existsCompletedRun(sourceNs, targetTable); - expect(alreadyCompleted).toBe(true); - } - - // Process collections 4 and 5 only (simulating the skip-and-continue logic) - for (let i = 3; i < 5; i++) { - const sourceNs = `${TEST_MONGO_DB}.${seeded[i].collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - const alreadyCompleted = await components2.manifestStore.existsCompletedRun(sourceNs, targetTable); - expect(alreadyCompleted).toBe(false); - - const result = await migrateCollection( - components2, - seeded[i].collName, - seeded[i].eventName, - APP_ID, - ); - expect(result.docsRead).toBeGreaterThanOrEqual(sizes[i] - 3); - expect(result.docsRead).toBeLessThanOrEqual(sizes[i] + 10); - } - } finally { - await cleanupComponents(components2); - } - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify total CH rows = sum of all 5 collections (with tolerance) - const finalCount = await chRowCount(); - expect(finalCount).toBeGreaterThanOrEqual(expectedTotal - 20); - expect(finalCount).toBeLessThanOrEqual(expectedTotal + 20); - }, 120_000); - - // ----------------------------------------------------------------------- - // Test 3: APM collections are excluded - // ----------------------------------------------------------------------- - - it("APM collections are excluded", async () => { - // Fresh slate - await teardownClickHouse(); - await teardownMongo(); - await teardownRedis(); - await setupClickHouse(); - - // Seed a normal collection - const normalEvent = "normal_event"; - const normalSeed = await seedCollection({ - count: 200, - appId: APP_ID, - eventName: normalEvent, - }); - - // Seed APM collections: [CLY]_apm_device and [CLY]_apm_network - const apmDeviceSeed = await seedCollection({ - count: 100, - appId: APP_ID, - eventName: "[CLY]_apm_device", - }); - - const apmNetworkSeed = await seedCollection({ - count: 100, - appId: APP_ID, - eventName: "[CLY]_apm_network", - }); - - // Seed another normal collection to confirm non-APM still works - const normalEvent2 = "normal_event_2"; - const normalSeed2 = await seedCollection({ - count: 150, - appId: APP_ID, - eventName: normalEvent2, - }); - - // The orchestrator filters APM by checking hashResolver.resolveCollectionName. - // We simulate this by checking the event names directly (the orchestrator uses - // the skipEventNames set: [CLY]_apm_device, [CLY]_apm_network). - const skipEventNames = new Set(["[CLY]_apm_device", "[CLY]_apm_network"]); - - // Gather all seeded collections and their event info - const allSeeded = [ - { ...normalSeed, eventName: normalEvent, isApm: false }, - { ...apmDeviceSeed, eventName: "[CLY]_apm_device", isApm: true }, - { ...apmNetworkSeed, eventName: "[CLY]_apm_network", isApm: true }, - { ...normalSeed2, eventName: normalEvent2, isApm: false }, - ]; - - // Filter: simulate orchestrator's APM exclusion - const nonApm = allSeeded.filter(s => !skipEventNames.has(s.eventName)); - const apmExcluded = allSeeded.filter(s => skipEventNames.has(s.eventName)); - - expect(nonApm.length).toBe(2); - expect(apmExcluded.length).toBe(2); - - // Migrate only non-APM collections - const components = await buildComponents(); - const results: Array<{ collName: string; eventName: string; status: string; rowsInserted: number }> = []; - - try { - for (const seed of nonApm) { - const result = await migrateCollection( - components, - seed.collName, - seed.eventName, - APP_ID, - ); - results.push({ - collName: seed.collName, - eventName: seed.eventName, - status: "completed", - rowsInserted: result.rowsInserted, - }); - } - - // Record APM collections as skipped (simulating orchestrator behavior) - for (const seed of apmExcluded) { - results.push({ - collName: seed.collName, - eventName: seed.eventName, - status: "skipped", - rowsInserted: 0, - }); - } - } finally { - await cleanupComponents(components); - } - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify: APM collections are NOT in results with status "completed" - const completedResults = results.filter(r => r.status === "completed"); - const skippedResults = results.filter(r => r.status === "skipped"); - - expect(completedResults.length).toBe(2); - expect(skippedResults.length).toBe(2); - - // No completed result should be an APM collection - for (const cr of completedResults) { - expect(skipEventNames.has(cr.eventName)).toBe(false); - } - - // All skipped should be APM - for (const sr of skippedResults) { - expect(skipEventNames.has(sr.eventName)).toBe(true); - } - - // ClickHouse should only have rows from the two normal collections - const expectedNonApmRows = normalSeed.expectedRows + normalSeed2.expectedRows; - const totalCh = await chRowCount(); - expect(totalCh).toBeGreaterThanOrEqual(expectedNonApmRows - 10); - expect(totalCh).toBeLessThanOrEqual(expectedNonApmRows + 10); - }, 60_000); -}); diff --git a/tests/integration/multi-pod-coordination.test.ts b/tests/integration/multi-pod-coordination.test.ts deleted file mode 100644 index 46ca949..0000000 --- a/tests/integration/multi-pod-coordination.test.ts +++ /dev/null @@ -1,1147 +0,0 @@ -/** - * Production-scale multi-pod coordination integration tests. - * - * Verifies that multiple pods (RangeCoordinators / BatchRunners) cooperate - * correctly through Redis-based distributed locking, range claiming, and - * heartbeat liveness checks. Each test seeds mixed data (null cd, missing - * uid, migrated, invalid ts) to exercise realistic code paths. - * - * Requirements: Docker containers for MongoDB:27017, ClickHouse:8123, Redis:6379. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader, type MongoReaderConfig } from "../../src/source/mongo-reader.ts"; -import { - ClickHouseWriter, - type ClickHouseWriterConfig, -} from "../../src/target/clickhouse-writer.ts"; -import { - ClickHousePressure, - type BackpressureConfig, -} from "../../src/target/clickhouse-pressure.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { CollectionLock, type CollectionLockConfig } from "../../src/state/collection-lock.ts"; -import { - RangeCoordinator, - type RangeCoordinatorConfig, - type RangeCoordinatorDeps, - type RangeEntry, -} from "../../src/runtime/range-coordinator.ts"; -import { - BatchRunner, - type BatchRunnerDeps, -} from "../../src/runtime/batch-runner.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { resolveRun } from "../../src/runtime/resolve-run.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - chQuery, - getRedis, - getMongoDb, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { - seedCollection, - seedNullCdCollection, - collectionName, -} from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared constants -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const TARGET_TABLE = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -/** Standard mixed-data seed profile for all tests. */ -const MIXED_DATA = { - nullCdFraction: 0.2, - missingUidFraction: 0.1, - migratedFraction: 0.1, - invalidTsFraction: 0.1, -}; - -// --------------------------------------------------------------------------- -// The MARK_RANGE_TERMINAL_LUA script (duplicated from range-coordinator.ts -// for direct Redis invocation in Test 1). This is a Lua script executed -// atomically on the Redis server via the ioredis .eval() method — it is -// NOT JavaScript eval. -// --------------------------------------------------------------------------- - -const MARK_RANGE_TERMINAL_LUA = ` -local raw = redis.call('HGET', KEYS[1], ARGV[1]) -if not raw then return 0 end -local data = cjson.decode(raw) -if data.status ~= 'processing' then return 0 end -if tostring(data.podId) ~= ARGV[3] then return 0 end -data.status = ARGV[2] -redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(data)) -return 1 -`; - -// --------------------------------------------------------------------------- -// Shared resources (single connection pool, reused across tests) -// --------------------------------------------------------------------------- - -let manifestStore: ManifestStore; -let redisState: RedisHotState; -let chWriter: ClickHouseWriter; -let chClientForPressure: ClickHouseClient | null = null; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -const MONGO_READER_CONFIG: MongoReaderConfig = { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "multi-pod-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, -}; - -/** - * Create a fresh MongoReader with its own connection. - * Required because MongoReader.switchCollection() mutates instance state, - * making it unsafe to share between concurrent RangeCoordinators. - */ -async function createMongoReader(): Promise { - const reader = new MongoReader(MONGO_READER_CONFIG, logger); - await reader.connect(); - return reader; -} - -function getChPressure(): ClickHousePressure { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return new ClickHousePressure(chClientForPressure, BACKPRESSURE_OFF, logger); -} - -function makeGc(): GcController { - return new GcController( - { - enabled: false, - rssSoftLimitBytes: 2e9, - rssHardLimitBytes: 3e9, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); -} - -function makeRetry(): RetryPolicy { - return new RetryPolicy({ maxRetries: 3, baseDelayMs: 50, maxDelayMs: 500 }); -} - -/** Build RangeCoordinatorDeps for a given pod and collection. */ -function buildRangeCoordinatorDeps( - collName: string, - eventName: string, - podId: string, - mongoReader: MongoReader, - opts?: { rangeCount?: number; batchRowsTarget?: number; rangeLeaseTtlSec?: number }, -): RangeCoordinatorDeps { - const config: RangeCoordinatorConfig = { - collectionName: collName, - sourceNs: `${TEST_MONGO_DB}.${collName}`, - targetTable: TARGET_TABLE, - transformVersion: "v1", - rangeCount: opts?.rangeCount ?? 8, - rangeLeaseTtlSec: opts?.rangeLeaseTtlSec ?? 5, - batchRowsTarget: opts?.batchRowsTarget ?? 500, - mongoPageSize: opts?.batchRowsTarget ?? 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 100, - collectionDefaults: { a: APP_ID, e: eventName }, - podId, - redisKeyPrefix: TEST_REDIS_PREFIX, - }; - - return { - redis: redisState.getRedisClient(), - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure: getChPressure(), - gcController: makeGc(), - retryPolicy: makeRetry(), - logger, - config, - }; -} - -/** Build BatchRunnerDeps for a single-collection run (not range-parallel). */ -async function buildBatchRunnerDeps( - collName: string, - eventName: string, - mongoReader: MongoReader, - opts?: { batchRowsTarget?: number }, -): Promise<{ - deps: BatchRunnerDeps; - runId: string; - upperBoundId: string; - collRedisState: RedisHotState; -}> { - await mongoReader.switchCollection(collName); - - const upperBound = await mongoReader.getUpperBound(); - const upperBoundId = upperBound ? serializeCursor(upperBound) : ""; - - const runId = randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - - const collRedisState = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:${collName}`, - ); - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: TARGET_TABLE, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await collRedisState.setActiveRun(runId); - await collRedisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable: TARGET_TABLE, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - - const batchRowsTarget = opts?.batchRowsTarget ?? 500; - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: collRedisState, - mongoReader, - chWriter, - chPressure: getChPressure(), - gcController: makeGc(), - retryPolicy: makeRetry(), - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable: TARGET_TABLE, - upperBoundId, - batchRowsTarget, - mongoPageSize: batchRowsTarget, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: APP_ID, e: eventName }, - collectionName: collName, - }, - }; - - return { deps, runId, upperBoundId, collRedisState }; -} - -/** Register a pod heartbeat in Redis so CLAIM_RANGE_LUA sees it as alive. */ -async function registerPodHeartbeat(podId: string, ttlSec = 300): Promise { - const redis = await getRedis(); - const podKey = `${TEST_REDIS_PREFIX}:pod:${podId}`; - await redis.set( - podKey, - JSON.stringify({ podId, lastHeartbeat: new Date().toISOString() }), - "EX", - ttlSec, - ); -} - -/** Count CH rows for a specific event name. */ -async function chCountByEvent(eventName: string): Promise { - const rows = await chQuery<{ cnt: string }>( - `SELECT count() AS cnt FROM ${TEST_CH_TABLE} WHERE n = '${eventName}'`, - ); - return Number(rows[0]?.cnt ?? 0); -} - -/** - * Migrate a single collection using BatchRunner, handling both normal - * and all-null-cd collections (mirrors the three-collection-nullcd test). - */ -async function migrateCollection( - collName: string, - eventName: string, - mongoReader: MongoReader, - opts?: { batchRowsTarget?: number }, -): Promise<{ runId: string; docsRead: number; rowsInserted: number }> { - await mongoReader.switchCollection(collName); - - const upperBound = await mongoReader.getUpperBound(); - if (!upperBound) { - // All-null-cd collection: use nullCdMode - const hasNullCd = await mongoReader.hasNullCdDocuments(); - if (!hasNullCd) { - return { runId: "", docsRead: 0, rowsInserted: 0 }; - } - - const bounds = await mongoReader.getNullCdBounds(); - if (!bounds) { - return { runId: "", docsRead: 0, rowsInserted: 0 }; - } - - const dummyUpperBound = JSON.stringify({ cd: 0, id: "000000000000000000000000" }); - const runId = randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - - const collRedisState = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:${collName}`, - ); - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: TARGET_TABLE, - upper_bound_cursor: dummyUpperBound, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await collRedisState.setActiveRun(runId); - await collRedisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable: TARGET_TABLE, - upperBoundCursor: dummyUpperBound, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: collRedisState, - mongoReader, - chWriter, - chPressure: getChPressure(), - gcController: makeGc(), - retryPolicy: makeRetry(), - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable: TARGET_TABLE, - upperBoundId: dummyUpperBound, - batchRowsTarget: opts?.batchRowsTarget ?? 500, - mongoPageSize: opts?.batchRowsTarget ?? 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: APP_ID, e: eventName }, - collectionName: collName, - nullCdMode: true, - nullCdUpperBound: bounds.upper, - }, - }; - - const runner = new BatchRunner(deps); - await runner.run(); - const stats = runner.getStats(); - return { runId, docsRead: stats.totalDocsRead, rowsInserted: stats.totalRowsInserted }; - } - - // Normal path: cursor-based migration (with auto null-cd sweep) - const { deps } = await buildBatchRunnerDeps(collName, eventName, mongoReader, opts); - const runner = new BatchRunner(deps); - await runner.run(); - const stats = runner.getStats(); - return { runId: deps.config.runId, docsRead: stats.totalDocsRead, rowsInserted: stats.totalRowsInserted }; -} - -// --------------------------------------------------------------------------- -// Suite -// --------------------------------------------------------------------------- - -describe("multi-pod-coordination", () => { - // Track MongoReaders created during tests so we can close them all - const openReaders: MongoReader[] = []; - - beforeAll(async () => { - manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const chWriterConfig: ClickHouseWriterConfig = { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }; - chWriter = new ClickHouseWriter(chWriterConfig, logger); - await chWriter.connect(); - }); - - afterAll(async () => { - // Close all MongoReaders created during tests - for (const reader of openReaders) { - await reader.close().catch(() => {}); - } - openReaders.length = 0; - - await chWriter.close().catch(() => {}); - await manifestStore.close().catch(() => {}); - await redisState.close().catch(() => {}); - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await closeAll(); - }); - - beforeEach(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - - // Re-create manifest store indexes after DB drop - const freshManifest = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await freshManifest.connect(); - await freshManifest.close(); - }); - - /** Helper to create + track a new MongoReader for cleanup in afterAll. */ - async function trackedMongoReader(): Promise { - const reader = await createMongoReader(); - openReaders.push(reader); - return reader; - } - - // ========================================================================= - // Test 1: markRangeDone atomicity (~30s) - // - // Pure Redis test. Seeds 2000 docs, inits 4 ranges via pod-A, then - // simulates a race: pod-A's range 0 goes stale, pod-B reclaims it, - // and pod-A's markRangeDone is rejected because podId no longer matches. - // ========================================================================= - - it("markRangeDone rejects when range was reclaimed by another pod", async () => { - const eventName = "atomicity_event"; - const collName = collectionName(eventName, APP_ID); - - // Seed 2000 mixed-data docs - await seedCollection({ - count: 2000, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - const redis = await getRedis(); - - // --- Pod-A initializes ranges --- - const readerA = await trackedMongoReader(); - await readerA.switchCollection(collName); - await registerPodHeartbeat("pod-A"); - - const depsA = buildRangeCoordinatorDeps(collName, eventName, "pod-A", readerA, { - rangeCount: 4, - rangeLeaseTtlSec: 2, - }); - const coordA = new RangeCoordinator(depsA); - - // Init ranges via pod-A (private method, accessed for testing) - const runId = await (coordA as any).initRanges(); - expect(runId).toBeTruthy(); - - // Verify 4 ranges were created in Redis - const rangesKey = `${TEST_REDIS_PREFIX}:ranges:${collName}`; - const allRanges = await redis.hgetall(rangesKey); - expect(Object.keys(allRanges).length).toBe(4); - - // Set ranges 1-3 as "done" so only range 0 is available - for (let i = 1; i <= 3; i++) { - const entry = JSON.parse(allRanges[String(i)]); - entry.status = "done"; - entry.podId = "pod-A"; - await redis.hset(rangesKey, String(i), JSON.stringify(entry)); - } - - // Manually claim range 0 as pod-A and set it to stale (10 seconds ago) - const range0 = JSON.parse(allRanges["0"]); - range0.status = "processing"; - range0.podId = "pod-A"; - range0.claimedAt = Math.floor(Date.now() / 1000) - 10; - await redis.hset(rangesKey, "0", JSON.stringify(range0)); - - // Kill pod-A heartbeat to make it appear dead - await redis.del(`${TEST_REDIS_PREFIX}:pod:pod-A`); - - // --- Pod-B reclaims range 0 via CLAIM_RANGE_LUA --- - await registerPodHeartbeat("pod-B"); - - const readerB = await trackedMongoReader(); - await readerB.switchCollection(collName); - - const depsB = buildRangeCoordinatorDeps(collName, eventName, "pod-B", readerB, { - rangeCount: 4, - rangeLeaseTtlSec: 2, - }); - const coordB = new RangeCoordinator(depsB); - - // Claim next range as pod-B (should reclaim stale range 0 from dead pod-A) - const claimed = await (coordB as any).claimNextRange(); - expect(claimed).toBeDefined(); - expect(claimed!.podId).toBe("pod-B"); - expect(claimed!.status).toBe("processing"); - - // --- Pod-A tries to markRangeDone (should be REJECTED) --- - // Invoke the MARK_RANGE_TERMINAL_LUA script directly on Redis. - // This is a Lua script that runs atomically on the Redis server - // via the ioredis .eval() method. - const markResult = await (redis as any).eval( - MARK_RANGE_TERMINAL_LUA, - 1, - rangesKey, - "0", // ARGV[1] = range index - "done", // ARGV[2] = target status - "pod-A", // ARGV[3] = pod-A's podId (no longer the owner) - ); - - // Pod-A's markRangeDone should return 0 (rejected — podId mismatch) - expect(markResult).toBe(0); - - // Verify range 0 is still "processing" owned by pod-B - const range0After = JSON.parse(await redis.hget(rangesKey, "0") ?? "{}"); - expect(range0After.status).toBe("processing"); - expect(range0After.podId).toBe("pod-B"); - }, 30_000); - - // ========================================================================= - // Test 2: Two pods claim ranges — all complete exactly once (~120s) - // - // Seeds 4000 docs, creates two RangeCoordinators with rangeCount=8 and - // separate MongoReaders. Both run concurrently via Promise.all. Verifies - // all 8 ranges complete with no failures and CH data matches expected. - // ========================================================================= - - it("two pods claim ranges concurrently and complete all 8 exactly once", async () => { - const eventName = "two_pods_event"; - const collName = collectionName(eventName, APP_ID); - - const seed = await seedCollection({ - count: 4000, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - // Register both pods - await registerPodHeartbeat("pod-A"); - await registerPodHeartbeat("pod-B"); - - // Create separate MongoReaders for each pod (critical for concurrency safety) - const readerA = await trackedMongoReader(); - await readerA.switchCollection(collName); - - const readerB = await trackedMongoReader(); - await readerB.switchCollection(collName); - - const depsA = buildRangeCoordinatorDeps(collName, eventName, "pod-A", readerA, { - rangeCount: 8, - }); - const depsB = buildRangeCoordinatorDeps(collName, eventName, "pod-B", readerB, { - rangeCount: 8, - }); - - const coordA = new RangeCoordinator(depsA); - const coordB = new RangeCoordinator(depsB); - - // Run both pods concurrently - const [resultA, resultB] = await Promise.all([ - coordA.run(), - coordB.run(), - ]); - - // All 8 ranges should be completed between the two pods - expect(resultA.completedRanges + resultB.completedRanges).toBe(8); - expect(resultA.failedRanges + resultB.failedRanges).toBe(0); - - // Allow ClickHouse async inserts to flush - await new Promise(r => setTimeout(r, 2000)); - - // Verify CH row count is within tolerance of expected - const totalCh = await chCountByEvent(eventName); - expect(totalCh).toBeGreaterThanOrEqual(seed.expectedRows - 30); - expect(totalCh).toBeLessThanOrEqual(seed.expectedRows + 30); - }, 120_000); - - // ========================================================================= - // Test 3: More pods than ranges — excess idle gracefully (~120s) - // - // Seeds 2000 docs, creates 4 RangeCoordinators for rangeCount=4. Excess - // pods that get 0 ranges should exit gracefully. Sum of completedRanges=4. - // ========================================================================= - - it("4 pods for 4 ranges — excess pods finish with 0 ranges gracefully", async () => { - const eventName = "excess_pods_event"; - const collName = collectionName(eventName, APP_ID); - - const seed = await seedCollection({ - count: 2000, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - // Register all 4 pods - for (let i = 0; i < 4; i++) { - await registerPodHeartbeat(`pod-${i}`); - } - - // Create separate MongoReaders and coordinators for all 4 pods - const coordinators: RangeCoordinator[] = []; - for (let i = 0; i < 4; i++) { - const reader = await trackedMongoReader(); - await reader.switchCollection(collName); - - const deps = buildRangeCoordinatorDeps(collName, eventName, `pod-${i}`, reader, { - rangeCount: 4, - }); - coordinators.push(new RangeCoordinator(deps)); - } - - // Run all 4 concurrently - const results = await Promise.all(coordinators.map(c => c.run())); - - // Sum of completed ranges must equal exactly 4 (the total range count) - const totalCompleted = results.reduce((sum, r) => sum + r.completedRanges, 0); - const totalFailed = results.reduce((sum, r) => sum + r.failedRanges, 0); - expect(totalCompleted).toBe(4); - expect(totalFailed).toBe(0); - - // Allow ClickHouse async inserts to flush - await new Promise(r => setTimeout(r, 2000)); - - const totalCh = await chCountByEvent(eventName); - expect(totalCh).toBeGreaterThanOrEqual(seed.expectedRows - 20); - expect(totalCh).toBeLessThanOrEqual(seed.expectedRows + 20); - }, 120_000); - - // ========================================================================= - // Test 4: Pod A stops, pod B resumes remaining ranges (~120s) - // - // Seeds 3000 docs with rangeCount=6. Pod-A starts, we poll until it - // completes >= 2 ranges, then stop it. Pod-B starts and finishes the rest. - // Total completedRanges across both pods must equal 6. - // ========================================================================= - - it("pod-A stops mid-flight, pod-B resumes and completes remaining ranges", async () => { - const eventName = "stop_resume_event"; - const collName = collectionName(eventName, APP_ID); - - const seed = await seedCollection({ - count: 3000, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - await registerPodHeartbeat("pod-A"); - - const readerA = await trackedMongoReader(); - await readerA.switchCollection(collName); - - const depsA = buildRangeCoordinatorDeps(collName, eventName, "pod-A", readerA, { - rangeCount: 6, - }); - const coordA = new RangeCoordinator(depsA); - - // Start pod-A, poll getRangeStatus() until done >= 2, then stop - const runPromiseA = coordA.run(); - - for (let i = 0; i < 200; i++) { - await new Promise(r => setTimeout(r, 300)); - try { - const status = await coordA.getRangeStatus(); - if (status.done >= 2) { - coordA.stop(); - break; - } - } catch { - // getRangeStatus may fail if ranges not yet initialized - } - } - - const resultA = await runPromiseA; - expect(resultA.completedRanges).toBeGreaterThanOrEqual(1); - - // Pod-B picks up remaining ranges. Kill pod-A heartbeat first so - // any stale "processing" ranges can be reclaimed. - const redis = await getRedis(); - await redis.del(`${TEST_REDIS_PREFIX}:pod:pod-A`); - await registerPodHeartbeat("pod-B"); - - const readerB = await trackedMongoReader(); - await readerB.switchCollection(collName); - - const depsB = buildRangeCoordinatorDeps(collName, eventName, "pod-B", readerB, { - rangeCount: 6, - }); - const coordB = new RangeCoordinator(depsB); - const resultB = await coordB.run(); - - // Between the two pods, all 6 ranges should be completed - expect(resultA.completedRanges + resultB.completedRanges).toBe(6); - - // Allow ClickHouse async inserts to flush - await new Promise(r => setTimeout(r, 2000)); - - const totalCh = await chCountByEvent(eventName); - expect(totalCh).toBeGreaterThanOrEqual(seed.expectedRows - 30); - expect(totalCh).toBeLessThanOrEqual(seed.expectedRows + 30); - }, 120_000); - - // ========================================================================= - // Test 5: Lock lost triggers abort (~60s) - // - // Verifies the onLockLost callback works. Uses a single-collection - // BatchRunner with tiny batch size (50). After 1+ batch, the lock key - // is deleted from Redis. The lock renewal detects the loss and calls - // runner.stopAfterBatch(). Verifies no data duplication. - // ========================================================================= - - it("onLockLost callback stops BatchRunner when lock is deleted", async () => { - const eventName = "lock_lost_event"; - const collName = collectionName(eventName, APP_ID); - - const seed = await seedCollection({ - count: 5000, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - const reader = await trackedMongoReader(); - - // Build BatchRunner deps with very tiny batch size for many batches (~200+) - const { deps } = await buildBatchRunnerDeps(collName, eventName, reader, { - batchRowsTarget: 25, - }); - deps.config.mongoPageSize = 25; - - const runner = new BatchRunner(deps); - - // Create a CollectionLock with short renewal interval and TTL - const redis = await getRedis(); - const lockConfig: CollectionLockConfig = { - lockTtlSec: 2, - renewIntervalMs: 500, - podHeartbeatMs: 30_000, - podDeadAfterSec: 180, - keyPrefix: TEST_REDIS_PREFIX, - }; - const lock = new CollectionLock(redis, "lock-test-pod", lockConfig, logger); - - // Wire onLockLost to stop the runner - lock.onLockLost = (_lostCollName: string) => { - runner.stopAfterBatch(); - }; - - // Register pod heartbeat and acquire lock - await registerPodHeartbeat("lock-test-pod"); - const acquireResult = await lock.tryAcquire(collName); - expect(acquireResult).toBe("acquired"); - - // Start heartbeat (handles lock renewal) - lock.startHeartbeat(); - - // Start processing in background - const runPromise = runner.run(); - - // Wait for runner to process at least 1 batch - for (let i = 0; i < 100; i++) { - await new Promise(r => setTimeout(r, 50)); - const status = runner.getStatus(); - if (status === "completed" || status === "failed") break; - if (runner.getStats().batchSeq >= 1) break; - } - - // Delete the lock key from Redis to simulate lock loss - const lockKey = `${TEST_REDIS_PREFIX}:lock:${collName}`; - await redis.del(lockKey); - - // Wait for lock renewal to detect the loss (renewIntervalMs=500ms). - // The onLockLost callback calls runner.stopAfterBatch(). - await runPromise; - - lock.stopHeartbeat(); - - const stats = runner.getStats(); - - // Runner should have stopped (or completed if it finished before the lock check) - expect(["stopped", "completed"]).toContain(stats.status); - - // If runner was stopped early, it should have inserted fewer rows than expected - if (stats.status === "stopped") { - expect(stats.totalRowsInserted).toBeLessThan(seed.expectedRows); - } - - // Verify no excessive duplication (small boundary overlap is expected - // from min/max cursor inclusivity — ~1 per batch boundary) - await new Promise(r => setTimeout(r, 2000)); - const duplicates = await chQuery<{ _id: string; cnt: string }>( - `SELECT _id, count() AS cnt FROM ${TEST_CH_TABLE} WHERE n = '${eventName}' GROUP BY _id HAVING cnt > 1`, - ); - // With batch size 25 and ~200 batches, up to ~200 boundary duplicates are possible. - // The key assertion is that the runner stopped, not that there are zero duplicates. - // In production, dedup tokens handle this at the ClickHouse level. - const totalRows = await chRowCount(`n = '${eventName}'`); - if (stats.status === "stopped") { - // Stopped early — should have partial data, not the full set - expect(totalRows).toBeLessThan(seed.expectedRows + 100); - } - }, 60_000); - - // ========================================================================= - // Test 6: Concurrent resolveRun (~30s) - // - // Seeds 500 docs, calls resolveRun twice concurrently for the same - // collection/sourceNs. Both should succeed. Documents whether the race - // produces duplicate active runs. - // ========================================================================= - - it("concurrent resolveRun calls for the same collection both succeed", async () => { - const eventName = "resolve_race_event"; - const collName = collectionName(eventName, APP_ID); - - await seedCollection({ - count: 500, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - - // Create two separate MongoReaders for the concurrent calls - const readerA = await trackedMongoReader(); - await readerA.switchCollection(collName); - - const readerB = await trackedMongoReader(); - await readerB.switchCollection(collName); - - // Create separate RedisHotState instances (sharing the underlying connection) - const redisStateA = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:resolveA`, - ); - const redisStateB = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:resolveB`, - ); - - // Call resolveRun twice concurrently - const [runA, runB] = await Promise.all([ - resolveRun({ - rerunMode: "resume", - manifestStore, - redisState: redisStateA, - mongoReader: readerA, - sourceNs, - targetTable: TARGET_TABLE, - transformVersion: "v1", - logger, - }), - resolveRun({ - rerunMode: "resume", - manifestStore, - redisState: redisStateB, - mongoReader: readerB, - sourceNs, - targetTable: TARGET_TABLE, - transformVersion: "v1", - logger, - }), - ]); - - // Both should succeed without crashing - expect(runA.runId).toBeTruthy(); - expect(runB.runId).toBeTruthy(); - expect(runA.upperBoundId).toBeTruthy(); - expect(runB.upperBoundId).toBeTruthy(); - - // Count active runs in manifest to document race behavior - const db = await getMongoDb(); - const activeRuns = await db - .collection("mig_runs") - .countDocuments({ status: "active", source_ns: sourceNs }); - - // At least 1 active run should exist; the race may produce 1 or 2 - expect(activeRuns).toBeGreaterThanOrEqual(1); - }, 30_000); - - // ========================================================================= - // Test 7: 20+ collections smoke test (~120s) - // - // Seeds 22 collections with varying profiles: 3 empty, 5 small (50-200), - // 8 medium (500-1500), 4 large (2000-4000), and 2 all-null-cd (300 each). - // Migrates all non-empty collections and verifies all have data in CH. - // ========================================================================= - - it("processes 22 collections of varying sizes and profiles", async () => { - const collConfigs: Array<{ - eventName: string; - count: number; - kind: "empty" | "small" | "medium" | "large" | "nullcd"; - }> = [ - // 3 empty collections - { eventName: "smoke_empty_1", count: 0, kind: "empty" }, - { eventName: "smoke_empty_2", count: 0, kind: "empty" }, - { eventName: "smoke_empty_3", count: 0, kind: "empty" }, - - // 5 small collections (50-200 docs) - { eventName: "smoke_small_1", count: 50, kind: "small" }, - { eventName: "smoke_small_2", count: 100, kind: "small" }, - { eventName: "smoke_small_3", count: 120, kind: "small" }, - { eventName: "smoke_small_4", count: 150, kind: "small" }, - { eventName: "smoke_small_5", count: 200, kind: "small" }, - - // 8 medium collections (500-1500 docs) - { eventName: "smoke_med_1", count: 500, kind: "medium" }, - { eventName: "smoke_med_2", count: 600, kind: "medium" }, - { eventName: "smoke_med_3", count: 750, kind: "medium" }, - { eventName: "smoke_med_4", count: 900, kind: "medium" }, - { eventName: "smoke_med_5", count: 1000, kind: "medium" }, - { eventName: "smoke_med_6", count: 1100, kind: "medium" }, - { eventName: "smoke_med_7", count: 1300, kind: "medium" }, - { eventName: "smoke_med_8", count: 1500, kind: "medium" }, - - // 4 large collections (2000-4000 docs) - { eventName: "smoke_large_1", count: 2000, kind: "large" }, - { eventName: "smoke_large_2", count: 2500, kind: "large" }, - { eventName: "smoke_large_3", count: 3000, kind: "large" }, - { eventName: "smoke_large_4", count: 4000, kind: "large" }, - - // 2 all-null-cd collections (300 docs each) - { eventName: "smoke_nullcd_1", count: 300, kind: "nullcd" }, - { eventName: "smoke_nullcd_2", count: 300, kind: "nullcd" }, - ]; - - // Seed all 22 collections - const seeded: Array<{ - eventName: string; - collName: string; - expectedRows: number; - kind: string; - }> = []; - - for (const cfg of collConfigs) { - if (cfg.kind === "empty") { - // Create an empty collection with the required index - const cName = collectionName(cfg.eventName, APP_ID); - const db = await getMongoDb(); - const coll = db.collection(cName); - await coll.drop().catch(() => {}); - await coll.insertOne({ _placeholder: true }); - await coll.deleteMany({}); - await coll.createIndex({ cd: 1, _id: 1 }); - seeded.push({ eventName: cfg.eventName, collName: cName, expectedRows: 0, kind: cfg.kind }); - } else if (cfg.kind === "nullcd") { - // Use seedNullCdCollection for all-null-cd collections - const s = await seedNullCdCollection({ - count: cfg.count, - appId: APP_ID, - eventName: cfg.eventName, - }); - seeded.push({ eventName: cfg.eventName, collName: s.collName, expectedRows: s.expectedRows, kind: cfg.kind }); - } else { - // Normal mixed-data seeding - const s = await seedCollection({ - count: cfg.count, - appId: APP_ID, - eventName: cfg.eventName, - ...MIXED_DATA, - }); - seeded.push({ eventName: cfg.eventName, collName: s.collName, expectedRows: s.expectedRows, kind: cfg.kind }); - } - } - - // Migrate each non-empty collection sequentially using a single MongoReader - // (sequential is fine here; each collection gets its own run) - const reader = await trackedMongoReader(); - - let totalExpected = 0; - let migratedCount = 0; - - for (const col of seeded) { - if (col.kind === "empty") continue; - - await migrateCollection(col.collName, col.eventName, reader); - totalExpected += col.expectedRows; - migratedCount++; - } - - // 22 total - 3 empty = 19 non-empty collections processed - expect(migratedCount).toBe(19); - - // Allow ClickHouse async inserts to flush - await new Promise(r => setTimeout(r, 2000)); - - // Verify all non-empty collections have data in CH - for (const col of seeded) { - if (col.kind === "empty") { - const count = await chCountByEvent(col.eventName); - expect(count).toBe(0); - continue; - } - - const count = await chCountByEvent(col.eventName); - expect(count).toBeGreaterThanOrEqual(col.expectedRows - 20); - expect(count).toBeLessThanOrEqual(col.expectedRows + 20); - // Every non-empty collection must have at least some rows - expect(count).toBeGreaterThan(0); - } - - // Verify aggregate count across all collections - const totalCh = await chRowCount(); - expect(totalCh).toBeGreaterThanOrEqual(totalExpected - 200); - expect(totalCh).toBeLessThanOrEqual(totalExpected + 200); - }, 120_000); - - // ========================================================================= - // Test 8: Collection transition under contention (~120s) - // - // 3 collections (1000, 1500, 2000 docs, all mixed data). Pod-A processes - // col-A then col-B sequentially. Pod-B processes col-C concurrently with - // pod-A. Both use separate MongoReaders. Verifies per-collection CH - // counts and no cross-contamination between events. - // ========================================================================= - - it("two pods process different collections concurrently without cross-contamination", async () => { - const events = ["contention_a", "contention_b", "contention_c"]; - const sizes = [1000, 1500, 2000]; - - // Seed 3 collections with mixed data - const seeds = await Promise.all( - events.map((eventName, i) => - seedCollection({ - count: sizes[i], - appId: APP_ID, - eventName, - ...MIXED_DATA, - }), - ), - ); - - // Pod-A: processes col-A then col-B sequentially - // Pod-B: processes col-C concurrently with pod-A - const readerA = await trackedMongoReader(); - const readerB = await trackedMongoReader(); - - const podAWork = async () => { - await migrateCollection(seeds[0].collName, events[0], readerA); - await migrateCollection(seeds[1].collName, events[1], readerA); - }; - - const podBWork = async () => { - await migrateCollection(seeds[2].collName, events[2], readerB); - }; - - // Run both pods concurrently - await Promise.all([podAWork(), podBWork()]); - - // Allow ClickHouse async inserts to flush - await new Promise(r => setTimeout(r, 2000)); - - // Verify per-collection CH counts — no cross-contamination - for (let i = 0; i < 3; i++) { - const count = await chCountByEvent(events[i]); - expect(count).toBeGreaterThanOrEqual(seeds[i].expectedRows - 20); - expect(count).toBeLessThanOrEqual(seeds[i].expectedRows + 20); - expect(count).toBeGreaterThan(0); - } - - // Verify aggregate count - const totalExpected = seeds.reduce((s, seed) => s + seed.expectedRows, 0); - const totalCh = await chRowCount(); - expect(totalCh).toBeGreaterThanOrEqual(totalExpected - 60); - expect(totalCh).toBeLessThanOrEqual(totalExpected + 60); - - // Verify only the 3 expected event names exist in CH (no cross-contamination) - const eventNames = await chQuery<{ n: string }>( - `SELECT DISTINCT n FROM ${TEST_CH_TABLE} ORDER BY n`, - ); - const chEvents = eventNames.map(r => r.n).sort(); - expect(chEvents).toEqual(events.sort()); - }, 120_000); -}); diff --git a/tests/integration/null-cd-sweep.test.ts b/tests/integration/null-cd-sweep.test.ts deleted file mode 100644 index 98d6265..0000000 --- a/tests/integration/null-cd-sweep.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Integration tests for null-cd document migration. - * - * Validates that documents with null/missing cd fields are: - * 1. Excluded from the cursor phase (readPage filter) - * 2. Swept in the null_cd phase after cursor phase completes - * 3. Correctly persisted to ClickHouse - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import { - setupClickHouse, teardownClickHouse, teardownMongo, teardownRedis, - closeAll, TEST_MONGO_URI, TEST_MONGO_DB, -} from "../helpers/setup.ts"; -import { seedCollection, seedNullCdCollection } from "../helpers/seed-mongo.ts"; -import { MongoReader } from "../../src/source/mongo-reader.ts"; - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; - -describe("null-cd sweep", () => { - let mongoReader: MongoReader; - - beforeAll(async () => { - await setupClickHouse(); - }); - - beforeEach(async () => { - await teardownMongo(); - await teardownRedis(); - await teardownClickHouse(); - await setupClickHouse(); - - mongoReader = new MongoReader({ - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "majority", - retryReads: true, - appName: "test", - batchRowsTarget: 1000, - cursorBatchSize: 1000, - maxTimeMs: 30_000, - }, (await import("pino")).default({ level: "silent" })); - await mongoReader.connect(); - }); - - afterAll(async () => { - if (mongoReader?.isConnected()) await mongoReader.close(); - await closeAll(); - }); - - // ── readPage filter ───────────────────────────────────────────────── - - it("readPage excludes null-cd documents", async () => { - const { collName } = await seedCollection({ - count: 20, - eventName: "test_read_page_filter", - nullCdFraction: 0.5, - }); - await mongoReader.switchCollection(collName); - - const upperBound = await mongoReader.getUpperBound(); - if (!upperBound) return; // All docs happened to be null-cd - - const page = await mongoReader.readPage(null, upperBound, 100); - for (const doc of page.docs) { - expect(doc.cd).not.toBeNull(); - expect(doc.cd).toBeDefined(); - } - }); - - // ── hasNullCdDocuments ────────────────────────────────────────────── - - it("hasNullCdDocuments returns true when null docs exist", async () => { - const { collName } = await seedNullCdCollection({ - count: 5, - eventName: "test_has_null", - }); - await mongoReader.switchCollection(collName); - expect(await mongoReader.hasNullCdDocuments()).toBe(true); - }); - - it("hasNullCdDocuments returns false when no null docs", async () => { - const { collName } = await seedCollection({ - count: 5, - eventName: "test_no_null", - }); - await mongoReader.switchCollection(collName); - expect(await mongoReader.hasNullCdDocuments()).toBe(false); - }); - - // ── getNullCdBounds ───────────────────────────────────────────────── - - it("getNullCdBounds returns min/max _id", async () => { - const { collName } = await seedNullCdCollection({ - count: 10, - eventName: "test_bounds", - }); - await mongoReader.switchCollection(collName); - const bounds = await mongoReader.getNullCdBounds(); - expect(bounds).not.toBeNull(); - expect(bounds!.lower).toBeDefined(); - expect(bounds!.upper).toBeDefined(); - expect(bounds!.lower < bounds!.upper).toBe(true); - }); - - it("getNullCdBounds returns null when no null docs", async () => { - const { collName } = await seedCollection({ - count: 5, - eventName: "test_no_null_bounds", - }); - await mongoReader.switchCollection(collName); - expect(await mongoReader.getNullCdBounds()).toBeNull(); - }); - - // ── readNullCdPage ────────────────────────────────────────────────── - - it("readNullCdPage paginates with _id and respects upper bound", async () => { - const { collName } = await seedNullCdCollection({ - count: 20, - eventName: "test_pagination", - }); - await mongoReader.switchCollection(collName); - - const bounds = await mongoReader.getNullCdBounds(); - expect(bounds).not.toBeNull(); - - // Read first page of 5 - const page1 = await mongoReader.readNullCdPage(null, bounds!.upper, 5); - expect(page1.docs.length).toBe(5); - expect(page1.lastCursor).not.toBeNull(); - expect(page1.lastCursor!.cd).toBe(0); - - // Read second page from last cursor - const page2 = await mongoReader.readNullCdPage(page1.lastCursor!.id, bounds!.upper, 5); - expect(page2.docs.length).toBe(5); - - // Ensure no overlap - const ids1 = new Set(page1.docs.map(d => d._id)); - for (const doc of page2.docs) { - expect(ids1.has(doc._id)).toBe(false); - } - }); - - it("readNullCdPage reads all docs when no limit", async () => { - const { collName } = await seedNullCdCollection({ - count: 15, - eventName: "test_no_limit", - }); - await mongoReader.switchCollection(collName); - - const bounds = await mongoReader.getNullCdBounds(); - expect(bounds).not.toBeNull(); - - const page = await mongoReader.readNullCdPage(null, bounds!.upper); - expect(page.docs.length).toBe(15); - }); - - // ── Bounds filtering ──────────────────────────────────────────────── - - it("getLowerBound and getUpperBound return null for all-null collections", async () => { - const { collName } = await seedNullCdCollection({ - count: 5, - eventName: "test_all_null_bounds", - }); - await mongoReader.switchCollection(collName); - expect(await mongoReader.getLowerBound()).toBeNull(); - expect(await mongoReader.getUpperBound()).toBeNull(); - }); - - it("getLowerBound and getUpperBound ignore null docs in mixed collections", async () => { - const { collName } = await seedCollection({ - count: 20, - eventName: "test_mixed_bounds", - nullCdFraction: 0.3, - }); - await mongoReader.switchCollection(collName); - - const lower = await mongoReader.getLowerBound(); - const upper = await mongoReader.getUpperBound(); - - // Should still find non-null bounds - if (lower && upper) { - expect(lower.cd).toBeGreaterThan(0); - expect(upper.cd).toBeGreaterThan(0); - } - }); -}); diff --git a/tests/integration/range-init-stale-lock.test.ts b/tests/integration/range-init-stale-lock.test.ts deleted file mode 100644 index ee230f8..0000000 --- a/tests/integration/range-init-stale-lock.test.ts +++ /dev/null @@ -1,239 +0,0 @@ -/** - * Integration test: RangeCoordinator stale initKey deadlock. - * - * Reproduces the scenario where a pod crashes after acquiring the - * range initialization lock but before writing runIdKey, leaving a - * stale Redis key that blocks all subsequent initialization attempts. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import pino from "pino"; -import { createClient } from "@clickhouse/client"; - -import { - RangeCoordinator, - type RangeCoordinatorConfig, - type RangeCoordinatorDeps, -} from "../../src/runtime/range-coordinator.ts"; -import { MongoReader, type MongoReaderConfig } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter, type ClickHouseWriterConfig } from "../../src/target/clickhouse-writer.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - getRedis, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection, collectionName } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const EVENT_NAME = "stale_lock_test"; -const COLL_NAME = collectionName(EVENT_NAME, APP_ID); - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Shared resources -// --------------------------------------------------------------------------- - -let manifestStore: ManifestStore; -let redisState: RedisHotState; -let mongoReader: MongoReader; -let chWriter: ClickHouseWriter; - -function buildDeps(podId: string): RangeCoordinatorDeps { - const config: RangeCoordinatorConfig = { - collectionName: COLL_NAME, - sourceNs: `${TEST_MONGO_DB}.${COLL_NAME}`, - targetTable: `${TEST_CH_DB}.${TEST_CH_TABLE}`, - transformVersion: "v1", - rangeCount: 4, - rangeLeaseTtlSec: 300, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: APP_ID, e: EVENT_NAME }, - podId, - redisKeyPrefix: TEST_REDIS_PREFIX, - }; - - return { - redis: redisState.getRedisClient(), - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure: new ClickHousePressure( - createClient({ url: TEST_CH_URL, database: TEST_CH_DB, username: "default", password: "" }), - BACKPRESSURE_OFF, - logger, - ), - gcController: new GcController( - { enabled: false, rssSoftLimitBytes: 2e9, rssHardLimitBytes: 3e9, heapUsedRatio: 0.9, everyNBatches: 999_999 }, - logger, - ), - retryPolicy: new RetryPolicy({ maxRetries: 2, baseDelayMs: 50, maxDelayMs: 200 }), - logger, - config, - }; -} - -// --------------------------------------------------------------------------- -// Suite -// --------------------------------------------------------------------------- - -describe("range-init-stale-lock", () => { - beforeAll(async () => { - manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "stale-lock-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - - chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }, - logger, - ); - await chWriter.connect(); - }); - - beforeEach(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - afterAll(async () => { - await mongoReader.close().catch(() => {}); - await chWriter.close().catch(() => {}); - await manifestStore.close().catch(() => {}); - await redisState.close().catch(() => {}); - await closeAll(); - }); - - // ----------------------------------------------------------------------- - // Test: stale initKey from crashed pod blocks new pod - // ----------------------------------------------------------------------- - - it("recovers from stale initKey left by a crashed pod", async () => { - // Seed a collection with enough data - await seedCollection({ - count: 2000, - appId: APP_ID, - eventName: EVENT_NAME, - }); - await mongoReader.switchCollection(COLL_NAME); - - const redis = await getRedis(); - const initKey = `${TEST_REDIS_PREFIX}:ranges:${COLL_NAME}:init`; - const runIdKey = `${TEST_REDIS_PREFIX}:ranges:${COLL_NAME}:runId`; - - // Simulate a crashed pod: set initKey (as if pod "dead-pod" acquired it) - // but never write runIdKey (pod crashed before completing init) - await redis.set(initKey, "dead-pod", "EX", 60, "NX"); - - // Verify the stale lock exists and runIdKey does NOT - expect(await redis.get(initKey)).toBe("dead-pod"); - expect(await redis.get(runIdKey)).toBeNull(); - - // Do NOT register a pod heartbeat for "dead-pod" — it's dead. - // A new pod ("recovery-pod") should detect the dead holder and reclaim. - - const deps = buildDeps("recovery-pod"); - const coordinator = new RangeCoordinator(deps); - - // This should NOT timeout — it should detect the dead pod and reclaim - const result = await coordinator.run(); - - expect(result.totalDocsRead).toBeGreaterThanOrEqual(2000); - expect(result.completedRanges).toBeGreaterThanOrEqual(1); - - // Verify data actually landed in ClickHouse - await new Promise(r => setTimeout(r, 2000)); - const count = await chRowCount(); - expect(count).toBeGreaterThanOrEqual(2000); - }, 120_000); - - // ----------------------------------------------------------------------- - // Test: initKey is cleaned up after successful initialization - // ----------------------------------------------------------------------- - - it("cleans up initKey after successful initialization", async () => { - await seedCollection({ - count: 1000, - appId: APP_ID, - eventName: EVENT_NAME, - }); - await mongoReader.switchCollection(COLL_NAME); - - const redis = await getRedis(); - const initKey = `${TEST_REDIS_PREFIX}:ranges:${COLL_NAME}:init`; - - const deps = buildDeps("cleanup-pod"); - const coordinator = new RangeCoordinator(deps); - await coordinator.run(); - - // After successful run, initKey should be gone (explicitly deleted, not just TTL) - const initKeyValue = await redis.get(initKey); - expect(initKeyValue).toBeNull(); - }, 120_000); -}); diff --git a/tests/integration/range-parallel.test.ts b/tests/integration/range-parallel.test.ts deleted file mode 100644 index 818b22f..0000000 --- a/tests/integration/range-parallel.test.ts +++ /dev/null @@ -1,488 +0,0 @@ -/** - * Integration test: range-parallel migration via RangeCoordinator. - * - * Verifies that: - * 1. All ranges complete and data lands in ClickHouse. - * 2. Boundary documents appear in exactly one range (no gaps or duplication). - * 3. The final range includes the maximum cd document. - * 4. A single range processes correctly when the collection is small. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import pino from "pino"; -import { createClient } from "@clickhouse/client"; -import { ObjectId, type Db } from "mongodb"; - -import { - RangeCoordinator, - type RangeCoordinatorConfig, - type RangeCoordinatorDeps, -} from "../../src/runtime/range-coordinator.ts"; -import { MongoReader, type MongoReaderConfig } from "../../src/source/mongo-reader.ts"; -import { - ClickHouseWriter, - type ClickHouseWriterConfig, -} from "../../src/target/clickhouse-writer.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - chQuery, - getMongoDb, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection, seedAtTimestamps, collectionName } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "silent" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const EVENT_NAME = "range_parallel_event"; -const COLL_NAME = collectionName(EVENT_NAME, APP_ID); -const SOURCE_NS = `${TEST_MONGO_DB}.${COLL_NAME}`; -const TARGET_TABLE = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 100_000, - partitionPctHigh: 0.8, - partitionPctLow: 0.6, - totalPctHigh: 0.8, - totalPctLow: 0.6, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Shared resources -// --------------------------------------------------------------------------- - -let manifestStore: ManifestStore; -let redisState: RedisHotState; -let mongoReader: MongoReader; -let chWriter: ClickHouseWriter; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function createChPressure(): ClickHousePressure { - const client = createClient({ - url: TEST_CH_URL, - username: "default", - password: "", - database: TEST_CH_DB, - }); - return new ClickHousePressure(client, BACKPRESSURE_OFF, logger); -} - -function createGcController(): GcController { - return new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.85, - everyNBatches: 999_999, - }, - logger, - ); -} - -function createRetryPolicy(): RetryPolicy { - return new RetryPolicy({ - maxRetries: 2, - baseDelayMs: 50, - maxDelayMs: 200, - }); -} - -function buildRangeCoordinatorDeps(overrides: { - rangeCount?: number; - batchRowsTarget?: number; - collectionNameOverride?: string; - eventNameOverride?: string; - podId?: string; -}): RangeCoordinatorDeps { - const collName = overrides.collectionNameOverride ?? COLL_NAME; - const eventName = overrides.eventNameOverride ?? EVENT_NAME; - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - - const config: RangeCoordinatorConfig = { - collectionName: collName, - sourceNs, - targetTable: TARGET_TABLE, - transformVersion: "v1", - rangeCount: overrides.rangeCount ?? 6, - rangeLeaseTtlSec: 300, - batchRowsTarget: overrides.batchRowsTarget ?? 500, - mongoPageSize: overrides.batchRowsTarget ?? 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 100, - collectionDefaults: { a: APP_ID, e: eventName }, - podId: overrides.podId ?? "test-pod-parallel", - redisKeyPrefix: TEST_REDIS_PREFIX, - }; - - return { - redis: redisState.getRedisClient(), - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure: createChPressure(), - gcController: createGcController(), - retryPolicy: createRetryPolicy(), - logger, - config, - }; -} - -// --------------------------------------------------------------------------- -// Suite -// --------------------------------------------------------------------------- - -describe("range-parallel", () => { - beforeAll(async () => { - // Connect shared resources - manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const mongoReaderConfig: MongoReaderConfig = { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "range-parallel-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }; - mongoReader = new MongoReader(mongoReaderConfig, logger); - await mongoReader.connect(); - - const chWriterConfig: ClickHouseWriterConfig = { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }; - chWriter = new ClickHouseWriter(chWriterConfig, logger); - await chWriter.connect(); - }); - - afterAll(async () => { - await mongoReader.close().catch(() => {}); - await chWriter.close().catch(() => {}); - await manifestStore.close().catch(() => {}); - await redisState.close().catch(() => {}); - await closeAll(); - }); - - beforeEach(async () => { - // Clean slate for each test - await teardownMongo(); - await teardownClickHouse(); - await setupClickHouse(); - await teardownRedis(); - - // Re-create manifest store indexes (teardownMongo drops the DB) - const freshManifest = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await freshManifest.connect(); - await freshManifest.close(); - }); - - // ------------------------------------------------------------------------- - // Test 1: all ranges complete and data lands in ClickHouse - // ------------------------------------------------------------------------- - - it("all ranges complete and data lands in ClickHouse", async () => { - // Seed 6000 docs spanning Jan-Dec 2024 - const { collName } = await seedCollection({ - count: 6000, - appId: APP_ID, - eventName: EVENT_NAME, - startDate: new Date("2024-01-01"), - endDate: new Date("2024-12-31"), - }); - - await mongoReader.switchCollection(collName); - - const deps = buildRangeCoordinatorDeps({ - rangeCount: 6, - batchRowsTarget: 500, - }); - - const coordinator = new RangeCoordinator(deps); - const result = await coordinator.run(); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // All 6 ranges should be completed - expect(result.totalRanges).toBe(6); - expect(result.completedRanges).toBe(6); - expect(result.failedRanges).toBe(0); - - // Verify all 6000 docs landed in ClickHouse (min() inclusivity adds small duplicates per range boundary) - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThanOrEqual(6000); - expect(totalRows).toBeLessThanOrEqual(6060); // 6 ranges × ~10 max duplicates - expect(result.totalRowsInserted).toBe(totalRows); - - // Run should be marked as "completed" - const run = await manifestStore.getRun(result.runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("completed"); - }, 120_000); - - // ------------------------------------------------------------------------- - // Test 2: boundary documents appear in exactly one range - // ------------------------------------------------------------------------- - - it("boundary documents appear in exactly one range", async () => { - const boundaryEventName = "range_boundary_event"; - const boundaryAppId = APP_ID; - const boundaryCollName = collectionName(boundaryEventName, boundaryAppId); - - // With rangeCount=4 and dates Jan-Dec 2024, range splits are at ~Apr, ~Jul, ~Oct. - // Compute the boundary timestamps. - const startMs = new Date("2024-01-01").getTime(); - const endMs = new Date("2024-12-31").getTime(); - const rangeCount = 4; - const spanMs = endMs - startMs; - const stepMs = Math.ceil(spanMs / rangeCount); - - // Boundary timestamps: start + stepMs, start + 2*stepMs, start + 3*stepMs - const boundaries = [ - new Date(startMs + stepMs), - new Date(startMs + 2 * stepMs), - new Date(startMs + 3 * stepMs), - ]; - - // Seed 10 docs at each boundary plus some docs spread across the full range - const allTimestamps: Date[] = []; - - // 10 docs per boundary = 30 boundary docs - for (const boundary of boundaries) { - for (let i = 0; i < 10; i++) { - allTimestamps.push(boundary); - } - } - - // Add spread docs to ensure enough data exists: 50 docs at start and 50 at end - for (let i = 0; i < 50; i++) { - const frac = i / 49; - allTimestamps.push(new Date(startMs + Math.floor(frac * spanMs))); - } - - const { collName, totalDocs } = await seedAtTimestamps( - boundaryEventName, - boundaryAppId, - allTimestamps, - ); - - await mongoReader.switchCollection(collName); - - const deps = buildRangeCoordinatorDeps({ - rangeCount, - batchRowsTarget: 500, - collectionNameOverride: collName, - eventNameOverride: boundaryEventName, - }); - - const coordinator = new RangeCoordinator(deps); - const result = await coordinator.run(); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // Verify total CH rows match total seeded docs (no gaps) - const totalRows = await chRowCount(); - expect(totalRows).toBe(totalDocs); - - // Verify no _id appears more than once (no duplication) - const duplicates = await chQuery<{ _id: string; cnt: string }>( - `SELECT _id, count() AS cnt FROM ${TEST_CH_TABLE} GROUP BY _id HAVING cnt > 1`, - ); - // Small duplication (1-2 docs) is acceptable due to min() inclusivity - expect(duplicates.length).toBeLessThanOrEqual(2); - - // All ranges should have completed - expect(result.completedRanges + result.failedRanges).toBe(rangeCount); - - const run = await manifestStore.getRun(result.runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("completed"); - }, 120_000); - - // ------------------------------------------------------------------------- - // Test 3: final range includes the maximum cd document - // ------------------------------------------------------------------------- - - it("final range includes the maximum cd document", async () => { - const maxEventName = "range_max_cd_event"; - const maxAppId = APP_ID; - const maxCollName = collectionName(maxEventName, maxAppId); - - // Seed docs with the last doc having cd = exactly the collection's max timestamp. - // We insert the docs manually to track the specific _id of the max-cd doc. - const db = await getMongoDb(); - const coll = db.collection(maxCollName); - await coll.drop().catch(() => {}); - - const startMs = new Date("2024-01-01").getTime(); - const endMs = new Date("2024-12-31T23:59:59.999Z").getTime(); - const maxDocId = new ObjectId().toHexString(); - - // Insert 500 docs spread across the range - const docs: Record[] = []; - for (let i = 0; i < 499; i++) { - const frac = i / 498; - const ts = startMs + Math.floor(frac * (endMs - startMs - 1)); - docs.push({ - _id: new ObjectId().toHexString(), - a: maxAppId, - e: maxEventName, - n: maxEventName, - uid: `user-${(i % 100).toString().padStart(4, "0")}`, - did: `device-${(i % 50).toString().padStart(4, "0")}`, - ts, - cd: new Date(ts), - c: 1, - s: 1.0, - dur: 0, - }); - } - - // The max-cd doc: cd = exactly endMs - docs.push({ - _id: maxDocId, - a: maxAppId, - e: maxEventName, - n: maxEventName, - uid: "user-max", - did: "device-max", - ts: endMs, - cd: new Date(endMs), - c: 1, - s: 1.0, - dur: 0, - }); - - const CHUNK = 5000; - for (let i = 0; i < docs.length; i += CHUNK) { - await coll.insertMany(docs.slice(i, i + CHUNK)); - } - await coll.createIndex({ cd: 1, _id: 1 }); - - await mongoReader.switchCollection(maxCollName); - - const deps = buildRangeCoordinatorDeps({ - rangeCount: 3, - batchRowsTarget: 200, - collectionNameOverride: maxCollName, - eventNameOverride: maxEventName, - }); - - const coordinator = new RangeCoordinator(deps); - const result = await coordinator.run(); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // Verify the max-cd doc's _id exists in ClickHouse - const maxDocRows = await chQuery<{ _id: string }>( - `SELECT _id FROM ${TEST_CH_TABLE} WHERE _id = '${maxDocId}'`, - ); - expect(maxDocRows.length).toBe(1); - expect(maxDocRows[0]._id).toBe(maxDocId); - - // Verify all 500 docs made it - const totalRows = await chRowCount(); - expect(totalRows).toBe(500); - - // Run should be completed - const run = await manifestStore.getRun(result.runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("completed"); - }, 120_000); - - // ------------------------------------------------------------------------- - // Test 4: single range processes correctly when collection is small - // ------------------------------------------------------------------------- - - it("single range processes correctly when collection is small", async () => { - const smallEventName = "range_small_event"; - const smallAppId = APP_ID; - const smallCollName = collectionName(smallEventName, smallAppId); - - // Seed 100 docs (below RANGE_PARALLEL_THRESHOLD) - const { collName } = await seedCollection({ - count: 100, - appId: smallAppId, - eventName: smallEventName, - startDate: new Date("2024-01-01"), - endDate: new Date("2024-12-31"), - }); - - await mongoReader.switchCollection(collName); - - // Force rangeCount=1 in the config - const deps = buildRangeCoordinatorDeps({ - rangeCount: 1, - batchRowsTarget: 500, - collectionNameOverride: collName, - eventNameOverride: smallEventName, - }); - - const coordinator = new RangeCoordinator(deps); - const result = await coordinator.run(); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // Single range should be completed - expect(result.totalRanges).toBe(1); - expect(result.completedRanges).toBe(1); - expect(result.failedRanges).toBe(0); - - // Verify all 100 docs in ClickHouse - const totalRows = await chRowCount(); - expect(totalRows).toBe(100); - expect(result.totalRowsInserted).toBe(100); - - // Run should be completed - const run = await manifestStore.getRun(result.runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("completed"); - }, 120_000); -}); diff --git a/tests/integration/range-retry.test.ts b/tests/integration/range-retry.test.ts deleted file mode 100644 index 8083cfb..0000000 --- a/tests/integration/range-retry.test.ts +++ /dev/null @@ -1,351 +0,0 @@ -/** - * Integration test: self-healing range retry mechanism in RangeCoordinator. - * - * Verifies that when a range fails during processing, the coordinator - * retries it up to MAX_RANGE_RETRIES times, and correctly marks the run - * as "failed" when retries are exhausted. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import pino from "pino"; -import { createClient } from "@clickhouse/client"; - -import { - RangeCoordinator, - type RangeCoordinatorConfig, - type RangeCoordinatorDeps, -} from "../../src/runtime/range-coordinator.ts"; -import { MongoReader, type MongoReaderConfig } from "../../src/source/mongo-reader.ts"; -import { - ClickHouseWriter, - type ClickHouseWriterConfig, - type InsertBatchParams, - type InsertResult, -} from "../../src/target/clickhouse-writer.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection, collectionName } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "silent" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const EVENT_NAME = "range_retry_event"; -const COLL_NAME = collectionName(EVENT_NAME, APP_ID); -const SOURCE_NS = `${TEST_MONGO_DB}.${COLL_NAME}`; -const TARGET_TABLE = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 100_000, - partitionPctHigh: 0.8, - partitionPctLow: 0.6, - totalPctHigh: 0.8, - totalPctLow: 0.6, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Shared resources -// --------------------------------------------------------------------------- - -let manifestStore: ManifestStore; -let redisState: RedisHotState; -let mongoReader: MongoReader; -let chWriter: ClickHouseWriter; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function createChPressure(): ClickHousePressure { - const client = createClient({ - url: TEST_CH_URL, - username: "default", - password: "", - database: TEST_CH_DB, - }); - return new ClickHousePressure(client, BACKPRESSURE_OFF, logger); -} - -function createGcController(): GcController { - return new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.85, - everyNBatches: 999_999, - }, - logger, - ); -} - -function createRetryPolicy(): RetryPolicy { - return new RetryPolicy({ - maxRetries: 2, - baseDelayMs: 50, - maxDelayMs: 200, - }); -} - -/** - * Creates a Proxy around ClickHouseWriter that intercepts insertBatch calls. - * The `shouldFail` predicate receives the InsertBatchParams and returns true - * if the call should throw. - */ -function proxyWriter( - real: ClickHouseWriter, - shouldFail: (params: InsertBatchParams) => boolean, -): ClickHouseWriter { - return new Proxy(real, { - get(target, prop, receiver) { - if (prop === "insertBatch") { - return async (params: InsertBatchParams): Promise => { - if (shouldFail(params)) { - throw new Error(`Injected failure for batchSeq=${params.batchSeq}`); - } - return target.insertBatch(params); - }; - } - return Reflect.get(target, prop, receiver); - }, - }); -} - -function buildRangeCoordinatorDeps(overrides: { - chWriterOverride?: ClickHouseWriter; - rangeCount?: number; -}): RangeCoordinatorDeps { - const config: RangeCoordinatorConfig = { - collectionName: COLL_NAME, - sourceNs: SOURCE_NS, - targetTable: TARGET_TABLE, - transformVersion: "v1", - rangeCount: overrides.rangeCount ?? 3, - rangeLeaseTtlSec: 300, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 100, - collectionDefaults: { a: APP_ID, e: EVENT_NAME }, - podId: "test-pod-retry", - redisKeyPrefix: TEST_REDIS_PREFIX, - }; - - return { - redis: redisState.getRedisClient(), - manifestStore, - redisState, - mongoReader, - chWriter: overrides.chWriterOverride ?? chWriter, - chPressure: createChPressure(), - gcController: createGcController(), - retryPolicy: createRetryPolicy(), - logger, - config, - }; -} - -// --------------------------------------------------------------------------- -// Suite -// --------------------------------------------------------------------------- - -describe("range-retry", () => { - beforeAll(async () => { - // Connect shared resources - manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const mongoReaderConfig: MongoReaderConfig = { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "range-retry-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }; - mongoReader = new MongoReader(mongoReaderConfig, logger); - await mongoReader.connect(); - - const chWriterConfig: ClickHouseWriterConfig = { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }; - chWriter = new ClickHouseWriter(chWriterConfig, logger); - await chWriter.connect(); - }); - - afterAll(async () => { - await mongoReader.close().catch(() => {}); - await chWriter.close().catch(() => {}); - await manifestStore.close().catch(() => {}); - await redisState.close().catch(() => {}); - await closeAll(); - }); - - beforeEach(async () => { - // Clean slate for each test - await teardownMongo(); - await teardownClickHouse(); - await setupClickHouse(); - await teardownRedis(); - - // Re-create manifest store indexes (teardownMongo drops the DB) - const freshManifest = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await freshManifest.connect(); - await freshManifest.close(); - }); - - // ------------------------------------------------------------------------- - // Test 1: failed range is retried and eventually succeeds - // ------------------------------------------------------------------------- - - it("failed range is retried and eventually succeeds", async () => { - // Seed enough docs for range-parallel mode across 3 ranges - const { collName } = await seedCollection({ - count: 3000, - appId: APP_ID, - eventName: EVENT_NAME, - startDate: new Date("2024-01-01"), - endDate: new Date("2024-12-31"), - }); - - await mongoReader.switchCollection(collName); - - // BATCH_SEQ_SLOTS_PER_RANGE = 10_000 - // Range 1 occupies batchSeq slots [10000, 20000). - // Fail insertBatch for the first attempt on any batchSeq in range 1's slot, - // then allow subsequent retries to succeed. - const failedOnce = new Set(); - const proxied = proxyWriter(chWriter, (params) => { - const batchSeq = params.batchSeq; - // Range 1 slots: [10000, 20000) - if (batchSeq >= 10_000 && batchSeq < 20_000) { - if (!failedOnce.has(batchSeq)) { - failedOnce.add(batchSeq); - return true; // fail the first attempt - } - } - return false; - }); - - const deps = buildRangeCoordinatorDeps({ - chWriterOverride: proxied, - rangeCount: 3, - }); - - const coordinator = new RangeCoordinator(deps); - const result = await coordinator.run(); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // Range 1 should have been retried (failed initially, then succeeded) - // All 3 ranges should be completed - expect(result.totalRanges).toBe(3); - expect(result.completedRanges + result.failedRanges).toBe(3); - - // The run should be completed (not failed) because range 1 succeeded on retry - const run = await manifestStore.getRun(result.runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("completed"); - - // All data from all 3 ranges should be in ClickHouse - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThan(0); - expect(totalRows).toBe(result.totalRowsInserted); - - // Verify range 1 was indeed retried (at least one batchSeq was failed then retried) - expect(failedOnce.size).toBeGreaterThan(0); - }); - - // ------------------------------------------------------------------------- - // Test 2: retry exhaustion marks run as failed - // ------------------------------------------------------------------------- - - it("retry exhaustion marks run as failed", async () => { - // Seed docs for 3 ranges - const { collName } = await seedCollection({ - count: 3000, - appId: APP_ID, - eventName: EVENT_NAME, - startDate: new Date("2024-01-01"), - endDate: new Date("2024-12-31"), - }); - - await mongoReader.switchCollection(collName); - - // Always fail insertBatch for range 1's batchSeq slots [10000, 20000). - // This means range 1 will never succeed, exhausting MAX_RANGE_RETRIES (3). - const proxied = proxyWriter(chWriter, (params) => { - const batchSeq = params.batchSeq; - return batchSeq >= 10_000 && batchSeq < 20_000; - }); - - const deps = buildRangeCoordinatorDeps({ - chWriterOverride: proxied, - rangeCount: 3, - }); - - const coordinator = new RangeCoordinator(deps); - const result = await coordinator.run(); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // With resilient batch handling, failed batches are skipped and the - // range still completes. All 3 ranges should finish (some with failed batches). - expect(result.totalRanges).toBe(3); - // All ranges complete — failed batches within a range are skipped, not retried at range level - expect(result.completedRanges).toBe(3); - - // The run completes (not "failed") because all ranges finished processing - const run = await manifestStore.getRun(result.runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("completed"); - - // Ranges 0 and 2 data should be in ClickHouse fully. - // Range 1 data is partially or fully missing (batches that failed were skipped). - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThan(0); - expect(totalRows).toBe(result.totalRowsInserted); - }); -}); diff --git a/tests/integration/resilience-comprehensive.test.ts b/tests/integration/resilience-comprehensive.test.ts deleted file mode 100644 index f84ac6f..0000000 --- a/tests/integration/resilience-comprehensive.test.ts +++ /dev/null @@ -1,741 +0,0 @@ -/** - * Comprehensive resilience tests for the migration system. - * - * Covers: dead-pod range reclaim, null-cd sweep lock reclaim, initKey error - * cleanup, pause/resume round-trips, and individual doc skip verification. - * Every test uses mixed data (valid cd, null cd, missing uid, migrated, - * invalid ts) to prove invalid docs are skipped individually — not at the - * collection level. - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter } from "../../src/target/clickhouse-writer.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { - BatchRunner, - type BatchRunnerDeps, -} from "../../src/runtime/batch-runner.ts"; -import { - RangeCoordinator, - type RangeCoordinatorConfig, - type RangeCoordinatorDeps, -} from "../../src/runtime/range-coordinator.ts"; -import { - ClickHousePressure, - type BackpressureConfig, -} from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - chQuery, - getRedis, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection, collectionName } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -/** Standard mixed-data seed options. */ -const MIXED_DATA = { - nullCdFraction: 0.2, - missingUidFraction: 0.1, - migratedFraction: 0.1, - invalidTsFraction: 0.1, -}; - -// --------------------------------------------------------------------------- -// Shared resources -// --------------------------------------------------------------------------- - -let manifestStore: ManifestStore; -let redisState: RedisHotState; -let mongoReader: MongoReader; -let chWriter: ClickHouseWriter; -let chClientForPressure: ClickHouseClient | null = null; - -function getChPressure(): ClickHousePressure { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return new ClickHousePressure(chClientForPressure, BACKPRESSURE_OFF, logger); -} - -function makeGc(): GcController { - return new GcController( - { - enabled: false, - rssSoftLimitBytes: 2e9, - rssHardLimitBytes: 3e9, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); -} - -function makeRetry(): RetryPolicy { - return new RetryPolicy({ maxRetries: 3, baseDelayMs: 50, maxDelayMs: 500 }); -} - -// --------------------------------------------------------------------------- -// BatchRunner helpers (single-collection mode) -// --------------------------------------------------------------------------- - -async function buildBatchRunnerDeps( - collName: string, - eventName: string, - opts?: { existingRunId?: string; existingUpperBoundId?: string }, -): Promise<{ - deps: BatchRunnerDeps; - runId: string; - upperBoundId: string; - collRedisState: RedisHotState; -}> { - await mongoReader.switchCollection(collName); - - const upperBound = await mongoReader.getUpperBound(); - const upperBoundId = - opts?.existingUpperBoundId ?? - (upperBound ? serializeCursor(upperBound) : ""); - - const runId = opts?.existingRunId ?? randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - const collRedisState = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:${collName}`, - ); - - if (!opts?.existingRunId) { - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await collRedisState.setActiveRun(runId); - await collRedisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - } - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: collRedisState, - mongoReader, - chWriter, - chPressure: getChPressure(), - gcController: makeGc(), - retryPolicy: makeRetry(), - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: APP_ID, e: eventName }, - collectionName: collName, - }, - }; - - return { deps, runId, upperBoundId, collRedisState }; -} - -// --------------------------------------------------------------------------- -// RangeCoordinator helpers -// --------------------------------------------------------------------------- - -function buildRangeCoordinatorDeps( - collName: string, - eventName: string, - podId: string, - rangeCount = 4, -): RangeCoordinatorDeps { - const config: RangeCoordinatorConfig = { - collectionName: collName, - sourceNs: `${TEST_MONGO_DB}.${collName}`, - targetTable: `${TEST_CH_DB}.${TEST_CH_TABLE}`, - transformVersion: "v1", - rangeCount, - rangeLeaseTtlSec: 5, // short TTL for tests - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: APP_ID, e: eventName }, - podId, - redisKeyPrefix: TEST_REDIS_PREFIX, - }; - - return { - redis: redisState.getRedisClient(), - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure: getChPressure(), - gcController: makeGc(), - retryPolicy: makeRetry(), - logger, - config, - }; -} - -/** Count CH rows for a specific event (custom events: n = eventName). */ -async function chCountByEvent(eventName: string): Promise { - const rows = await chQuery<{ cnt: string }>( - `SELECT count() AS cnt FROM ${TEST_CH_TABLE} WHERE n = '${eventName}'`, - ); - return Number(rows[0]?.cnt ?? 0); -} - -// --------------------------------------------------------------------------- -// Suite -// --------------------------------------------------------------------------- - -describe("resilience-comprehensive", () => { - beforeAll(async () => { - manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "resilience-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - - chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }, - logger, - ); - await chWriter.connect(); - }); - - beforeEach(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - afterAll(async () => { - await mongoReader.close().catch(() => {}); - await chWriter.close().catch(() => {}); - await manifestStore.close().catch(() => {}); - await redisState.close().catch(() => {}); - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await closeAll(); - }); - - // ----------------------------------------------------------------------- - // Test 1: Dead-pod range reclaim via CLAIM_RANGE_LUA - // ----------------------------------------------------------------------- - - it("reclaims stale range from dead pod and completes migration", async () => { - const eventName = "reclaim_range_event"; - const collName = collectionName(eventName, APP_ID); - - const seed = await seedCollection({ - count: 2000, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - await mongoReader.switchCollection(collName); - - // Pod-A initializes ranges and processes some, then "crashes" - const depsA = buildRangeCoordinatorDeps(collName, eventName, "pod-A", 4); - // Override rangeLeaseTtlSec to be very short so stale claims are detectable - (depsA.config as any).rangeLeaseTtlSec = 2; - - const coordA = new RangeCoordinator(depsA); - - // Register pod-A heartbeat so it can acquire the init lock - const redis = await getRedis(); - const podKeyA = `${TEST_REDIS_PREFIX}:pod:pod-A`; - await redis.set(podKeyA, JSON.stringify({ podId: "pod-A", lastHeartbeat: new Date().toISOString() }), "EX", 5); - - // Run pod-A (it will complete all ranges since it's the only pod) - // Instead, we simulate a partial run: let pod-A init ranges, then - // manually mark range 0 as "processing" by pod-A with an old claimedAt - // to simulate a crash. - - // First, init ranges via pod-A - const initRunId = await (coordA as any).initRanges(); - expect(initRunId).toBeTruthy(); - - // Verify ranges were created - const rangesKey = `${TEST_REDIS_PREFIX}:ranges:${collName}`; - const allRanges = await redis.hgetall(rangesKey); - expect(Object.keys(allRanges).length).toBe(4); - - // Simulate pod-A claiming range 0 and crashing: - // set range 0 to "processing" with old claimedAt, then expire pod-A heartbeat - const range0 = JSON.parse(allRanges["0"]); - range0.status = "processing"; - range0.podId = "pod-A"; - range0.claimedAt = Math.floor(Date.now() / 1000) - 10; // 10 seconds ago (stale) - await redis.hset(rangesKey, "0", JSON.stringify(range0)); - - // Expire pod-A heartbeat (simulate crash) - await redis.del(podKeyA); - - // Wait for lease TTL to be clearly exceeded - await new Promise(r => setTimeout(r, 500)); - - // Pod-B starts — should reclaim range 0 from dead pod-A and process everything - const depsB = buildRangeCoordinatorDeps(collName, eventName, "pod-B", 4); - (depsB.config as any).rangeLeaseTtlSec = 2; - - // Register pod-B heartbeat - const podKeyB = `${TEST_REDIS_PREFIX}:pod:pod-B`; - await redis.set(podKeyB, JSON.stringify({ podId: "pod-B", lastHeartbeat: new Date().toISOString() }), "EX", 300); - - const coordB = new RangeCoordinator(depsB); - const result = await coordB.run(); - - expect(result.completedRanges).toBe(4); - expect(result.totalDocsRead).toBeGreaterThan(0); - - await new Promise(r => setTimeout(r, 2000)); - - const count = await chCountByEvent(eventName); - // Valid + null-cd docs should all be in CH; invalid docs skipped individually - expect(count).toBeGreaterThanOrEqual(seed.expectedRows - 30); - expect(count).toBeLessThanOrEqual(seed.expectedRows + 30); - - // Verify no ranges stuck in "processing" - const finalStatus = await coordB.getRangeStatus(); - expect(finalStatus.processing).toBe(0); - }, 120_000); - - // ----------------------------------------------------------------------- - // Test 2: Null-cd sweep lock reclaim from dead pod - // ----------------------------------------------------------------------- - - it("reclaims stale null-cd sweep lock from dead pod", async () => { - const eventName = "sweep_reclaim_event"; - const collName = collectionName(eventName, APP_ID); - - const seed = await seedCollection({ - count: 1000, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - await mongoReader.switchCollection(collName); - - const redis = await getRedis(); - const rangesKey = `${TEST_REDIS_PREFIX}:ranges:${collName}`; - const sweepKey = `${rangesKey}:null_cd_sweep`; - - // Plant a stale sweep lock from a dead pod - await redis.set(sweepKey, "dead-sweep-pod", "EX", 3600, "NX"); - - // Do NOT register a heartbeat for "dead-sweep-pod" — it's dead - - // Register live pod heartbeat - const podKey = `${TEST_REDIS_PREFIX}:pod:live-pod`; - await redis.set(podKey, JSON.stringify({ podId: "live-pod" }), "EX", 300); - - // Run range coordinator — it should reclaim the stale sweep lock - const deps = buildRangeCoordinatorDeps(collName, eventName, "live-pod", 4); - const coord = new RangeCoordinator(deps); - const result = await coord.run(); - - expect(result.totalDocsRead).toBeGreaterThan(0); - - await new Promise(r => setTimeout(r, 2000)); - - const count = await chCountByEvent(eventName); - expect(count).toBeGreaterThanOrEqual(seed.expectedRows - 20); - expect(count).toBeLessThanOrEqual(seed.expectedRows + 20); - - // Verify sweep done key is set - const sweepDoneKey = `${sweepKey}:done`; - const sweepDone = await redis.get(sweepDoneKey); - expect(sweepDone).toBe("1"); - }, 120_000); - - // ----------------------------------------------------------------------- - // Test 3: initKey cleanup on error - // ----------------------------------------------------------------------- - - it("cleans up initKey when initialization errors", async () => { - const eventName = "init_error_event"; - const collName = collectionName(eventName, APP_ID); - - const redis = await getRedis(); - const initKey = `${TEST_REDIS_PREFIX}:ranges:${collName}:init`; - - // Register pod heartbeat - const podKey = `${TEST_REDIS_PREFIX}:pod:error-pod`; - await redis.set(podKey, JSON.stringify({ podId: "error-pod" }), "EX", 300); - - // Create an empty collection (no docs) — getUpperBound and hasNullCd - // both return null/false, so initRanges will throw "Collection is empty" - const db = await (await import("../helpers/setup.ts")).getMongoDb(); - const coll = db.collection(collName); - await coll.drop().catch(() => {}); - await coll.insertOne({ _placeholder: true }); // create collection - await coll.deleteMany({}); // but with zero docs - await coll.createIndex({ cd: 1, _id: 1 }); - - await mongoReader.switchCollection(collName); - - const deps = buildRangeCoordinatorDeps(collName, eventName, "error-pod", 4); - const coord = new RangeCoordinator(deps); - - // Should throw "Collection is empty" but clean up initKey - await expect(coord.run()).rejects.toThrow(/empty/i); - - // Verify initKey was cleaned up - const initKeyValue = await redis.get(initKey); - expect(initKeyValue).toBeNull(); - }, 30_000); - - // ----------------------------------------------------------------------- - // Test 4a: Pause/resume single collection - // ----------------------------------------------------------------------- - - it("pause and resume single collection with mixed data — no data loss", async () => { - const eventName = "pause_resume_single"; - const collName = collectionName(eventName, APP_ID); - - const seed = await seedCollection({ - count: 1500, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - const { deps } = await buildBatchRunnerDeps(collName, eventName); - - // Use tiny batch size so there are many batches (~30), giving time to pause - deps.config.batchRowsTarget = 50; - deps.config.mongoPageSize = 50; - - const runner = new BatchRunner(deps); - const runPromise = runner.run(); - - // Poll until runner is running and has processed at least one batch, then pause - let didPause = false; - for (let i = 0; i < 100; i++) { - await new Promise(r => setTimeout(r, 50)); - const status = runner.getStatus(); - if (status === "completed" || status === "failed") break; - if (status === "running" && runner.getStats().batchSeq >= 1) { - runner.pause(); - didPause = true; - break; - } - } - - if (didPause) { - // Verify paused state held - await new Promise(r => setTimeout(r, 200)); - expect(runner.getStatus()).toBe("paused"); - - // Resume - runner.resume(); - } - // If runner completed before pause took effect, that's ok — still verify data - - await runPromise; - - const stats = runner.getStats(); - expect(stats.status).toBe("completed"); - expect(stats.totalDocsRead).toBeGreaterThan(0); - - await new Promise(r => setTimeout(r, 2000)); - - const count = await chCountByEvent(eventName); - expect(count).toBeGreaterThanOrEqual(seed.expectedRows - 20); - expect(count).toBeLessThanOrEqual(seed.expectedRows + 20); - }, 60_000); - - // ----------------------------------------------------------------------- - // Test 4b: Pause/resume multi-collection - // ----------------------------------------------------------------------- - - it("pause and resume across 3 collections with mixed data — no data loss", async () => { - const events = ["pause_multi_a", "pause_multi_b", "pause_multi_c"]; - const sizes = [1000, 1500, 2000]; - - const seeds = await Promise.all( - events.map((eventName, i) => - seedCollection({ - count: sizes[i], - appId: APP_ID, - eventName, - ...MIXED_DATA, - }), - ), - ); - - for (let i = 0; i < 3; i++) { - const { deps } = await buildBatchRunnerDeps( - seeds[i].collName, - events[i], - ); - - // Tiny batches for collection 2 so pause has time to take effect - if (i === 1) { - deps.config.batchRowsTarget = 50; - deps.config.mongoPageSize = 50; - } - - const runner = new BatchRunner(deps); - const runPromise = runner.run(); - - // Pause/resume during collection 2 (index 1) - if (i === 1) { - let didPause = false; - for (let j = 0; j < 100; j++) { - await new Promise(r => setTimeout(r, 50)); - const status = runner.getStatus(); - if (status === "completed" || status === "failed") break; - if (status === "running" && runner.getStats().batchSeq >= 1) { - runner.pause(); - didPause = true; - break; - } - } - - if (didPause) { - await new Promise(r => setTimeout(r, 200)); - expect(runner.getStatus()).toBe("paused"); - runner.resume(); - } - } - - await runPromise; - expect(runner.getStats().status).toBe("completed"); - } - - await new Promise(r => setTimeout(r, 2000)); - - // Verify per-collection counts - for (let i = 0; i < 3; i++) { - const count = await chCountByEvent(events[i]); - expect(count).toBeGreaterThanOrEqual(seeds[i].expectedRows - 20); - expect(count).toBeLessThanOrEqual(seeds[i].expectedRows + 20); - } - - // Verify aggregate - const totalExpected = seeds.reduce((s, seed) => s + seed.expectedRows, 0); - const totalCh = await chRowCount(); - expect(totalCh).toBeGreaterThanOrEqual(totalExpected - 60); - expect(totalCh).toBeLessThanOrEqual(totalExpected + 60); - }, 120_000); - - // ----------------------------------------------------------------------- - // Test 5a: Mixed data individual skip — single collection - // ----------------------------------------------------------------------- - - it("skips invalid docs individually in single collection (not entire collection)", async () => { - const eventName = "skip_single"; - const collName = collectionName(eventName, APP_ID); - - const seed = await seedCollection({ - count: 1000, - appId: APP_ID, - eventName, - ...MIXED_DATA, - }); - - // expectedRows should be less than 1000 (invalid docs deducted) - // but greater than 0 (collection NOT entirely skipped) - expect(seed.expectedRows).toBeLessThan(1000); - expect(seed.expectedRows).toBeGreaterThan(0); - - const { deps } = await buildBatchRunnerDeps(collName, eventName); - const runner = new BatchRunner(deps); - await runner.run(); - - const stats = runner.getStats(); - expect(stats.status).toBe("completed"); - expect(stats.totalDocsSkipped).toBeGreaterThan(0); - - // Verify skip reasons are populated (individual skips happened) - const skips = stats.skipsByReason; - const totalSkips = - (skips.missing_uid ?? 0) + - (skips.already_marked_migrated ?? 0) + - (skips.invalid_ts ?? 0); - expect(totalSkips).toBeGreaterThan(0); - - await new Promise(r => setTimeout(r, 2000)); - - const count = await chCountByEvent(eventName); - // Valid + null-cd docs present; invalid docs absent - expect(count).toBeGreaterThanOrEqual(seed.expectedRows - 20); - expect(count).toBeLessThanOrEqual(seed.expectedRows + 20); - - // Collection was NOT skipped — there are rows - expect(count).toBeGreaterThan(0); - }, 60_000); - - // ----------------------------------------------------------------------- - // Test 5b: Mixed data individual skip — multi-collection - // ----------------------------------------------------------------------- - - it("skips invalid docs individually across 3 collections (none skipped entirely)", async () => { - const events = [ - "skip_multi_a", // heavy null-cd - "skip_multi_b", // heavy missing uid - "skip_multi_c", // heavy migrated + invalid ts - ]; - - // Different invalid-data profiles per collection - const seedConfigs = [ - { count: 1000, nullCdFraction: 0.4, missingUidFraction: 0.05, migratedFraction: 0.05, invalidTsFraction: 0.05 }, - { count: 1500, nullCdFraction: 0.1, missingUidFraction: 0.2, migratedFraction: 0.05, invalidTsFraction: 0.05 }, - { count: 2000, nullCdFraction: 0.1, missingUidFraction: 0.05, migratedFraction: 0.15, invalidTsFraction: 0.15 }, - ]; - - const seeds = await Promise.all( - events.map((eventName, i) => - seedCollection({ - appId: APP_ID, - eventName, - ...seedConfigs[i], - }), - ), - ); - - // Verify none have expectedRows = 0 (all have some valid docs) - for (const seed of seeds) { - expect(seed.expectedRows).toBeGreaterThan(0); - expect(seed.expectedRows).toBeLessThan(seed.totalDocs); - } - - // Migrate all 3 - for (let i = 0; i < 3; i++) { - const { deps } = await buildBatchRunnerDeps( - seeds[i].collName, - events[i], - ); - const runner = new BatchRunner(deps); - await runner.run(); - - const stats = runner.getStats(); - expect(stats.status).toBe("completed"); - // Each collection had skips but was NOT skipped entirely - expect(stats.totalDocsSkipped).toBeGreaterThan(0); - expect(stats.totalRowsInserted).toBeGreaterThan(0); - } - - await new Promise(r => setTimeout(r, 2000)); - - // Verify per-collection counts - for (let i = 0; i < 3; i++) { - const count = await chCountByEvent(events[i]); - expect(count).toBeGreaterThanOrEqual(seeds[i].expectedRows - 30); - expect(count).toBeLessThanOrEqual(seeds[i].expectedRows + 30); - // NOT zero — collection was processed - expect(count).toBeGreaterThan(0); - } - - // Verify aggregate - const totalExpected = seeds.reduce((s, seed) => s + seed.expectedRows, 0); - const totalCh = await chRowCount(); - expect(totalExpected).toBeGreaterThan(0); - expect(totalCh).toBeGreaterThanOrEqual(totalExpected - 90); - expect(totalCh).toBeLessThanOrEqual(totalExpected + 90); - }, 120_000); -}); diff --git a/tests/integration/resume-after-stop.test.ts b/tests/integration/resume-after-stop.test.ts deleted file mode 100644 index a24207a..0000000 --- a/tests/integration/resume-after-stop.test.ts +++ /dev/null @@ -1,559 +0,0 @@ -/** - * Integration test: pause/stop/resume scenarios for BatchRunner. - * - * Validates that: - * 1. Stopping mid-migration and resuming completes all data with no duplicates - * 2. Counter recovery is correct after resume (totalDocsRead is full count) - * 3. batchSeq continues from the correct offset after resume - */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter } from "../../src/target/clickhouse-writer.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { BatchRunner, type BatchRunnerDeps } from "../../src/runtime/batch-runner.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - chQuery, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const EVENT_NAME = "resume_test_event"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -let chClientForPressure: ClickHouseClient | null = null; - -function getChClientForPressure(): ClickHouseClient { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return chClientForPressure; -} - -interface BuildDepsResult { - deps: BatchRunnerDeps; - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; - runId: string; - upperBoundId: string; - sourceNs: string; - targetTable: string; -} - -/** - * Build a full set of BatchRunner dependencies for a given collection. - * Optionally reuse an existing runId and upperBoundId (for resume scenarios). - */ -async function buildDeps(opts: { - collName: string; - appId?: string; - eventName?: string; - batchRowsTarget?: number; - existingRunId?: string; - existingUpperBoundId?: string; - skipRunCreation?: boolean; -}): Promise { - const appId = opts.appId ?? APP_ID; - const eventName = opts.eventName ?? EVENT_NAME; - const collName = opts.collName; - const batchRowsTarget = opts.batchRowsTarget ?? 500; - - // MongoReader - const mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "resume-test", - batchRowsTarget, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - await mongoReader.switchCollection(collName); - - // ClickHouseWriter - const chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: true, - }, - logger, - ); - await chWriter.connect(); - - // ManifestStore - const manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - // RedisHotState - const redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - // ClickHousePressure (disabled) - const chPressure = new ClickHousePressure( - getChClientForPressure(), - BACKPRESSURE_OFF, - logger, - ); - - // GcController (no-op) - const gcController = new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); - - // RetryPolicy - const retryPolicy = new RetryPolicy({ - maxRetries: 3, - baseDelayMs: 100, - maxDelayMs: 1000, - }); - - // Determine upper bound - let upperBoundId: string; - if (opts.existingUpperBoundId) { - upperBoundId = opts.existingUpperBoundId; - } else { - const upperBound = await mongoReader.getUpperBound(); - upperBoundId = upperBound ? serializeCursor(upperBound) : ""; - } - - const runId = opts.existingRunId ?? randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - if (!opts.skipRunCreation) { - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await redisState.setActiveRun(runId); - await redisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - } - - const deps: BatchRunnerDeps = { - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget, - mongoPageSize: batchRowsTarget, - backpressure: BACKPRESSURE_OFF, - useDedupToken: true, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: appId, e: eventName }, - }, - }; - - return { - deps, - mongoReader, - chWriter, - manifestStore, - redisState, - gcController, - runId, - upperBoundId, - sourceNs, - targetTable, - }; -} - -async function cleanupDeps(parts: { - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; -}): Promise { - await parts.mongoReader.close().catch(() => {}); - await parts.chWriter.close().catch(() => {}); - await parts.manifestStore.close().catch(() => {}); - await parts.redisState.close().catch(() => {}); - parts.gcController.dispose(); -} - -// --------------------------------------------------------------------------- -// Test suite -// --------------------------------------------------------------------------- - -describe("resume-after-stop", () => { - beforeAll(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - beforeEach(async () => { - // Clean CH table + Redis between tests - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - afterAll(async () => { - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await closeAll(); - }); - - // ----------------------------------------------------------------------- - // Test 1: stop mid-migration and resume completes all data - // ----------------------------------------------------------------------- - - it("stop mid-migration and resume completes all data", async () => { - // Seed 5000 docs - const { collName, expectedRows } = await seedCollection({ - count: 5000, - appId: APP_ID, - eventName: "stop_resume", - }); - expect(expectedRows).toBe(5000); - - // --- Phase 1: Run and stop after first batch --- - const parts1 = await buildDeps({ - collName, - eventName: "stop_resume", - batchRowsTarget: 500, - }); - - const runner1 = new BatchRunner(parts1.deps); - - // Wrap CH writer to call stopAfterBatch() after first successful insert - let insertCount = 0; - const origInsert = parts1.chWriter.insertBatch.bind(parts1.chWriter); - parts1.chWriter.insertBatch = async (params) => { - const result = await origInsert(params); - insertCount++; - if (insertCount >= 1) { - runner1.stopAfterBatch(); - } - return result; - }; - - await runner1.run(); - - // Wait for the runner to stop - expect(runner1.getStatus()).toBe("stopped"); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 1500)); - - const rowsAfterPhase1 = await chRowCount(); - expect(rowsAfterPhase1).toBeGreaterThan(0); - expect(rowsAfterPhase1).toBeLessThan(expectedRows); - - const savedRunId = parts1.runId; - const savedUpperBoundId = parts1.upperBoundId; - await cleanupDeps(parts1); - - // --- Phase 2: Create NEW BatchRunner and resume --- - const parts2 = await buildDeps({ - collName, - eventName: "stop_resume", - batchRowsTarget: 500, - existingRunId: savedRunId, - existingUpperBoundId: savedUpperBoundId, - skipRunCreation: true, - }); - - // Re-activate the run - await parts2.manifestStore.updateRunStatus(savedRunId, "active"); - - const runner2 = new BatchRunner(parts2.deps); - await runner2.run(); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify: total ClickHouse rows = all expected docs - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThanOrEqual(expectedRows); - expect(totalRows).toBeLessThanOrEqual(expectedRows + 20); - - // Verify: no duplicate rows (COUNT DISTINCT _id should equal total rows) - const distinctResult = await chQuery<{ cnt: string }>( - `SELECT count(DISTINCT _id) AS cnt FROM ${TEST_CH_TABLE}`, - ); - const distinctCount = Number(distinctResult[0]?.cnt ?? 0); - // max() is exclusive so 1 doc at the exact upper bound may be missed - expect(distinctCount).toBeGreaterThanOrEqual(expectedRows - 1); - expect(distinctCount).toBeLessThanOrEqual(expectedRows); - - await cleanupDeps(parts2); - }); - - // ----------------------------------------------------------------------- - // Test 2: counter recovery is correct after resume - // ----------------------------------------------------------------------- - - it("counter recovery is correct after resume", async () => { - // Seed 3000 docs - const { collName, expectedRows } = await seedCollection({ - count: 3000, - appId: APP_ID, - eventName: "counter_resume", - }); - expect(expectedRows).toBe(3000); - - // --- Phase 1: Run and stop after 2 batches --- - const parts1 = await buildDeps({ - collName, - eventName: "counter_resume", - batchRowsTarget: 500, - }); - - const runner1 = new BatchRunner(parts1.deps); - - let batchCount = 0; - const origInsert = parts1.chWriter.insertBatch.bind(parts1.chWriter); - parts1.chWriter.insertBatch = async (params) => { - const result = await origInsert(params); - batchCount++; - if (batchCount >= 2) { - runner1.stopAfterBatch(); - } - return result; - }; - - await runner1.run(); - await new Promise((r) => setTimeout(r, 1500)); - - const phase1Stats = runner1.getStats(); - expect(phase1Stats.totalDocsRead).toBeGreaterThan(0); - - const savedRunId = parts1.runId; - const savedUpperBoundId = parts1.upperBoundId; - await cleanupDeps(parts1); - - // --- Phase 2: Resume --- - const parts2 = await buildDeps({ - collName, - eventName: "counter_resume", - batchRowsTarget: 500, - existingRunId: savedRunId, - existingUpperBoundId: savedUpperBoundId, - skipRunCreation: true, - }); - - await parts2.manifestStore.updateRunStatus(savedRunId, "active"); - - const runner2 = new BatchRunner(parts2.deps); - await runner2.run(); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify: totalDocsRead should be the full count, not just the resumed portion - const phase2Stats = runner2.getStats(); - expect(phase2Stats.totalDocsRead).toBeGreaterThanOrEqual(expectedRows); - expect(phase2Stats.totalDocsRead).toBeLessThanOrEqual(expectedRows + 20); - - // It should not be less than what phase 1 had processed - // (proving it recovered counters rather than starting from 0) - expect(phase2Stats.totalDocsRead).toBeGreaterThanOrEqual(phase1Stats.totalDocsRead); - - // All rows in CH - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThanOrEqual(expectedRows); - expect(totalRows).toBeLessThanOrEqual(expectedRows + 20); - - await cleanupDeps(parts2); - }); - - // ----------------------------------------------------------------------- - // Test 3: batchSeq continues from correct offset - // ----------------------------------------------------------------------- - - it("batchSeq continues from correct offset", async () => { - // Seed 2000 docs - const { collName, expectedRows } = await seedCollection({ - count: 2000, - appId: APP_ID, - eventName: "batchseq_resume", - }); - expect(expectedRows).toBe(2000); - - // --- Phase 1: Run a few batches (expect batch_seq 1-3 or so), then stop --- - const parts1 = await buildDeps({ - collName, - eventName: "batchseq_resume", - batchRowsTarget: 500, - }); - - const runner1 = new BatchRunner(parts1.deps); - - // Stop after 3 batches - let batchCount = 0; - const origInsert = parts1.chWriter.insertBatch.bind(parts1.chWriter); - parts1.chWriter.insertBatch = async (params) => { - const result = await origInsert(params); - batchCount++; - if (batchCount >= 3) { - runner1.stopAfterBatch(); - } - return result; - }; - - await runner1.run(); - await new Promise((r) => setTimeout(r, 1500)); - - // Record the batchSeq at stop - const phase1FinalBatchSeq = runner1.getCurrentBatchSeq(); - expect(phase1FinalBatchSeq).toBeGreaterThanOrEqual(3); - - // Verify the manifest has these batch records - const phase1Batches = await parts1.manifestStore.getBatches(parts1.runId, { status: "done" }); - expect(phase1Batches.length).toBeGreaterThanOrEqual(3); - - // Record max batch_seq from phase 1 - const maxBatchSeqPhase1 = Math.max(...phase1Batches.map((b) => b.batch_seq)); - - const savedRunId = parts1.runId; - const savedUpperBoundId = parts1.upperBoundId; - await cleanupDeps(parts1); - - // --- Phase 2: Resume --- - const parts2 = await buildDeps({ - collName, - eventName: "batchseq_resume", - batchRowsTarget: 500, - existingRunId: savedRunId, - existingUpperBoundId: savedUpperBoundId, - skipRunCreation: true, - }); - - await parts2.manifestStore.updateRunStatus(savedRunId, "active"); - - const runner2 = new BatchRunner(parts2.deps); - await runner2.run(); - - // Allow async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // Verify: new batches start from batch_seq > maxBatchSeqPhase1 (not 0) - const allBatches = await parts2.manifestStore.getBatches(savedRunId); - const phase2Batches = allBatches.filter((b) => b.batch_seq > maxBatchSeqPhase1); - - // There should be new batches from phase 2 - expect(phase2Batches.length).toBeGreaterThan(0); - - // The first new batch should start right after maxBatchSeqPhase1 - const minBatchSeqPhase2 = Math.min(...phase2Batches.map((b) => b.batch_seq)); - expect(minBatchSeqPhase2).toBe(maxBatchSeqPhase1 + 1); - - // No batch_seq gaps between phase 1 and phase 2 - const allBatchSeqs = allBatches.map((b) => b.batch_seq).sort((a, b) => a - b); - for (let i = 1; i < allBatchSeqs.length; i++) { - expect(allBatchSeqs[i]).toBe(allBatchSeqs[i - 1] + 1); - } - - // All rows should be in ClickHouse - const totalRows = await chRowCount(); - expect(totalRows).toBeGreaterThanOrEqual(expectedRows); - expect(totalRows).toBeLessThanOrEqual(expectedRows + 20); - - await cleanupDeps(parts2); - }); -}); diff --git a/tests/integration/schema-compliance.test.ts b/tests/integration/schema-compliance.test.ts deleted file mode 100644 index 5e9a76c..0000000 --- a/tests/integration/schema-compliance.test.ts +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Integration tests for schema compliance. - * - * Verifies that output rows produced by the transform layer match the - * ClickHouse table schema: required fields, timestamp format, event name - * derivation, numeric types, null handling, and end-to-end insert. - */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { ObjectId } from "mongodb"; - -import { - transformDocument, - type SourceDocument, -} from "../../src/transform/normalize.ts"; -import { - setupClickHouse, - teardownClickHouse, - closeAll, - getClickHouseClient, - chQuery, - TEST_CH_DB, - TEST_CH_TABLE, -} from "../helpers/setup.ts"; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/** Build a minimal valid source document. */ -function makeValidDoc(overrides: Partial = {}): SourceDocument { - return { - _id: new ObjectId().toHexString(), - a: "test_app_id", - e: "test_event", - uid: "user-001", - did: "device-001", - ts: 1711525200123, // 2024-03-27 09:00:00.123 UTC - cd: new Date("2024-03-27T09:00:00.123Z"), - c: 3, - s: 1.5, - dur: 12.75, - n: "test_event", - ...overrides, - }; -} - -// --------------------------------------------------------------------------- -// Lifecycle -// --------------------------------------------------------------------------- - -beforeAll(async () => { - await teardownClickHouse(); - await setupClickHouse(); -}); - -afterAll(async () => { - await teardownClickHouse(); - await closeAll(); -}); - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -describe("schema-compliance", () => { - it("all required fields present in output", () => { - const doc = makeValidDoc(); - const { row, skipReason } = transformDocument(doc); - - expect(skipReason).toBeNull(); - expect(row).not.toBeNull(); - - // All required fields from the ClickHouse schema must be present - expect(row!._id).toBeDefined(); - expect(row!.a).toBeDefined(); - expect(row!.e).toBeDefined(); - expect(row!.n).toBeDefined(); - expect(row!.uid).toBeDefined(); - expect(row!.ts).toBeDefined(); - expect(row!.c).toBeDefined(); - expect(row!.s).toBeDefined(); - expect(row!.dur).toBeDefined(); - expect(row!.cd).toBeDefined(); - }); - - it("cd preserved from source document", () => { - const doc = makeValidDoc({ cd: new Date("2024-03-27T09:00:00.123Z") }); - const { row } = transformDocument(doc); - - expect(row).not.toBeNull(); - expect(row!.cd).toBe("2024-03-27 09:00:00.123"); - }); - - it("cd accepts epoch millis and epoch seconds", () => { - // Older drill documents store cd as a raw epoch rather than a BSON Date. - const millis = transformDocument(makeValidDoc({ cd: 1711530000123 })); - expect(millis.row!.cd).toBe("2024-03-27 09:00:00.123"); - - const seconds = transformDocument(makeValidDoc({ cd: 1711530000 })); - expect(seconds.row!.cd).toBe("2024-03-27 09:00:00.000"); - }); - - it("cd falls back to ts when source cd is missing or null", () => { - // Documents predating the introduction of `cd` -- the population the - // null-cd sweep exists to pick up. They must still get a deterministic - // value rather than ClickHouse's now64(3) default. - const missing = makeValidDoc(); - delete (missing as any).cd; - const fromMissing = transformDocument(missing); - expect(fromMissing.row).not.toBeNull(); - expect(fromMissing.row!.cd).toBe(fromMissing.row!.ts); - - const nulled = makeValidDoc({ cd: null }); - const fromNull = transformDocument(nulled); - expect(fromNull.row).not.toBeNull(); - expect(fromNull.row!.cd).toBe(fromNull.row!.ts); - }); - - it("timestamp formatted as DateTime64(3)", () => { - const doc = makeValidDoc({ ts: 1711525200123 }); - const { row } = transformDocument(doc); - - expect(row).not.toBeNull(); - - // formatTimestamp produces 'yyyy-MM-dd HH:mm:ss.SSS' (UTC) - expect(row!.ts).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/); - expect(row!.ts).toContain(".123"); - - // Verify the millisecond component is preserved - expect(row!.ts).toMatch(/\.\d{3}$/); - }); - - it("custom event normalized to [CLY]_custom", () => { - const doc = makeValidDoc({ e: "my_custom_event" }); - const { row } = transformDocument(doc); - - expect(row).not.toBeNull(); - - // Custom events (not starting with [CLY]_) get e=[CLY]_custom, n=original name - expect(row!.e).toBe("[CLY]_custom"); - expect(row!.n).toBe("my_custom_event"); - }); - - it("view event derives name from sg.name", () => { - const doc = makeValidDoc({ - e: "[CLY]_view", - sg: { name: "Home" }, - }); - const { row } = transformDocument(doc); - - expect(row).not.toBeNull(); - expect(row!.e).toBe("[CLY]_view"); - expect(row!.n).toBe("Home"); - }); - - it("numeric fields have correct types", () => { - const doc = makeValidDoc({ c: 5, s: 3.14, dur: 99.9 }); - const { row } = transformDocument(doc); - - expect(row).not.toBeNull(); - - // c: non-negative integer - expect(typeof row!.c).toBe("number"); - expect(Number.isInteger(row!.c)).toBe(true); - expect(row!.c).toBeGreaterThanOrEqual(0); - - // s: float - expect(typeof row!.s).toBe("number"); - - // dur: float - expect(typeof row!.dur).toBe("number"); - }); - - it("null/missing optional fields handled", () => { - // Document with no uid_canon, lsid, cmp - const doc = makeValidDoc(); - delete (doc as any).uid_canon; - delete (doc as any).lsid; - delete (doc as any).cmp; - - const { row, skipReason } = transformDocument(doc); - - // Transform should succeed without errors - expect(skipReason).toBeNull(); - expect(row).not.toBeNull(); - - // Optional fields should be undefined or null, not cause errors - // (the transform only copies fields that exist in the source document) - const rowAny = row as Record; - expect(rowAny.uid_canon === undefined || rowAny.uid_canon === null).toBe(true); - expect(rowAny.lsid === undefined || rowAny.lsid === null).toBe(true); - expect(rowAny.cmp === undefined || rowAny.cmp === null).toBe(true); - }); - - it("end-to-end: transformed row inserts into ClickHouse without errors", async () => { - // Transform a doc - const docId = new ObjectId().toHexString(); - const doc = makeValidDoc({ - _id: docId, - e: "e2e_schema_test", - ts: 1711525200456, - }); - const { row } = transformDocument(doc); - expect(row).not.toBeNull(); - - // Insert into ClickHouse directly - const ch = await getClickHouseClient(); - await ch.insert({ - table: TEST_CH_TABLE, - values: [row!], - format: "JSONEachRow", - clickhouse_settings: { - date_time_input_format: "best_effort", - }, - }); - - // Wait for async insert to flush - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Query back and verify the data - const rows = await chQuery<{ - _id: string; - a: string; - e: string; - n: string; - uid: string; - c: string; - s: string; - dur: string; - }>(`SELECT _id, a, e, n, uid, c, s, dur FROM ${TEST_CH_TABLE} WHERE _id = '${docId}'`); - - expect(rows.length).toBe(1); - expect(rows[0]._id).toBe(docId); - expect(rows[0].a).toBe(row!.a); - expect(rows[0].e).toBe("[CLY]_custom"); - expect(rows[0].n).toBe("e2e_schema_test"); - expect(rows[0].uid).toBe(row!.uid); - expect(Number(rows[0].c)).toBe(row!.c); - expect(Number(rows[0].s)).toBeCloseTo(row!.s, 2); - expect(Number(rows[0].dur)).toBeCloseTo(row!.dur, 2); - }); - - it("end-to-end: cd survives the insert instead of defaulting to now64(3)", async () => { - // Regression guard. The column is declared `cd DateTime64(3) DEFAULT - // now64(3)`, so a row that omits `cd` from its JSONEachRow payload is - // stamped with the migration's wall-clock time -- silently relocating the - // whole migrated history into the term the migration ran in. - const docId = new ObjectId().toHexString(); - const sourceCd = new Date("2021-06-15T12:34:56.789Z"); - const doc = makeValidDoc({ - _id: docId, - e: "cd_preservation_test", - cd: sourceCd, - }); - const { row } = transformDocument(doc); - expect(row).not.toBeNull(); - - const ch = await getClickHouseClient(); - await ch.insert({ - table: TEST_CH_TABLE, - values: [row!], - format: "JSONEachRow", - clickhouse_settings: { - date_time_input_format: "best_effort", - }, - }); - - await new Promise(resolve => setTimeout(resolve, 2000)); - - const rows = await chQuery<{ cd_ms: string }>( - `SELECT toUnixTimestamp64Milli(cd) AS cd_ms FROM ${TEST_CH_TABLE} WHERE _id = '${docId}'`, - ); - - expect(rows.length).toBe(1); - expect(Number(rows[0].cd_ms)).toBe(sourceCd.getTime()); - }); -}); diff --git a/tests/integration/stats-accuracy.test.ts b/tests/integration/stats-accuracy.test.ts deleted file mode 100644 index e0a2134..0000000 --- a/tests/integration/stats-accuracy.test.ts +++ /dev/null @@ -1,490 +0,0 @@ -/** - * Integration test: migration statistics accuracy. - * - * Verifies that migration metrics (docsRead, rowsInserted, docsSkipped) - * match actual data in MongoDB and ClickHouse, and that cluster-aware - * status derivation logic is correct. - */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter } from "../../src/target/clickhouse-writer.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { BatchRunner, type BatchRunnerDeps, type BatchRunnerStats } from "../../src/runtime/batch-runner.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - getMongoDb, - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection, seedMultipleCollections } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -let chClientForPressure: ClickHouseClient | null = null; - -function getChClientForPressure(): ClickHouseClient { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return chClientForPressure; -} - -interface MigrationResult { - runId: string; - stats: BatchRunnerStats; -} - -/** - * Build deps and run migration for a single collection. - * Returns the run ID and the BatchRunner stats snapshot. - */ -async function buildAndRunMigration(opts: { - collName: string; - appId?: string; - eventName: string; -}): Promise<{ - result: MigrationResult; - cleanup: () => Promise; -}> { - const appId = opts.appId ?? APP_ID; - const { collName, eventName } = opts; - - const mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "stats-accuracy-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - await mongoReader.switchCollection(collName); - - const chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }, - logger, - ); - await chWriter.connect(); - - const manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - const redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const collRedisState = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:${collName}`, - ); - - const chPressure = new ClickHousePressure( - getChClientForPressure(), - BACKPRESSURE_OFF, - logger, - ); - - const gcController = new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); - - const retryPolicy = new RetryPolicy({ - maxRetries: 3, - baseDelayMs: 100, - maxDelayMs: 1000, - }); - - const upperBound = await mongoReader.getUpperBound(); - const upperBoundId = upperBound ? serializeCursor(upperBound) : ""; - - const runId = randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await collRedisState.setActiveRun(runId); - await collRedisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: collRedisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: appId, e: eventName }, - collectionName: collName, - }, - }; - - const runner = new BatchRunner(deps); - await runner.run(); - - const stats = runner.getStats(); - - const cleanup = async () => { - await mongoReader.close().catch(() => {}); - await chWriter.close().catch(() => {}); - await manifestStore.close().catch(() => {}); - await redisState.close().catch(() => {}); - gcController.dispose(); - }; - - return { - result: { runId, stats }, - cleanup, - }; -} - -// --------------------------------------------------------------------------- -// Test suite -// --------------------------------------------------------------------------- - -describe("stats-accuracy", () => { - beforeAll(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - afterAll(async () => { - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await closeAll(); - }); - - // ----------------------------------------------------------------------- - // Test 1: estimatedCounts match estimatedDocumentCount - // ----------------------------------------------------------------------- - - it("estimatedCounts match estimatedDocumentCount", async () => { - // Fresh slate - await teardownMongo(); - await teardownRedis(); - - const sizes = [1000, 2000, 3000]; - const eventNames = ["est_event_a", "est_event_b", "est_event_c"]; - - const seeded = await seedMultipleCollections( - sizes.map((count, i) => ({ - eventName: eventNames[i], - count, - appId: APP_ID, - })), - ); - - // Verify estimatedDocumentCount matches seeded count - const db = await getMongoDb(); - for (let i = 0; i < seeded.length; i++) { - const estimated = await db.collection(seeded[i].collName).estimatedDocumentCount(); - // estimatedDocumentCount can be slightly off, allow +/-1 tolerance - expect(estimated).toBeGreaterThanOrEqual(sizes[i] - 1); - expect(estimated).toBeLessThanOrEqual(sizes[i] + 1); - } - }); - - // ----------------------------------------------------------------------- - // Test 2: docsRead matches actual docs processed - // ----------------------------------------------------------------------- - - it("docsRead matches actual docs processed", async () => { - // Fresh slate for CH - await teardownClickHouse(); - await teardownMongo(); - await teardownRedis(); - await setupClickHouse(); - - // Seed 2000 docs with 10% migrated (should be skipped at transform, not at read) - const eventName = "docs_read_test"; - const seed = await seedCollection({ - count: 2000, - appId: APP_ID, - eventName, - migratedFraction: 0.1, - }); - - const { result, cleanup } = await buildAndRunMigration({ - collName: seed.collName, - eventName, - }); - - try { - const { stats } = result; - - // BatchRunner reads ALL docs including migrated ones (min() inclusivity adds small duplicates) - expect(stats.totalDocsRead).toBeGreaterThanOrEqual(2000); - expect(stats.totalDocsRead).toBeLessThanOrEqual(2020); - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // CH rows should only include non-migrated docs (with small min() tolerance) - const chCount = await chRowCount(); - expect(chCount).toBeGreaterThanOrEqual(seed.expectedRows); - expect(chCount).toBeLessThanOrEqual(seed.expectedRows + 20); - - // expectedRows is approximately 1800 (2000 - ~10% migrated) - expect(seed.expectedRows).toBeLessThan(2000); - expect(seed.expectedRows).toBeGreaterThan(1700); - - // totalDocsSkipped should account for the migrated documents - // Migrated docs are skipped with reason "already_marked_migrated" - const migratedSkipped = stats.totalDocsRead - seed.expectedRows; - expect(stats.totalDocsSkipped).toBeGreaterThanOrEqual(migratedSkipped - 10); - expect(stats.totalDocsSkipped).toBeLessThanOrEqual(migratedSkipped + 10); - } finally { - await cleanup(); - } - }, 60_000); - - // ----------------------------------------------------------------------- - // Test 3: rowsInserted matches ClickHouse count - // ----------------------------------------------------------------------- - - it("rowsInserted matches ClickHouse count", async () => { - // Fresh slate - await teardownClickHouse(); - await teardownMongo(); - await teardownRedis(); - await setupClickHouse(); - - const eventName = "rows_inserted_test"; - const seed = await seedCollection({ - count: 3000, - appId: APP_ID, - eventName, - }); - - // No migrated or missing uid docs, so expectedRows = 3000 - expect(seed.expectedRows).toBe(3000); - - const { result, cleanup } = await buildAndRunMigration({ - collName: seed.collName, - eventName, - }); - - try { - const { stats } = result; - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 2000)); - - // stats.totalRowsInserted should match SELECT count() from ClickHouse (with min() tolerance) - const chCount = await chRowCount(); - expect(chCount).toBeGreaterThanOrEqual(3000); - expect(chCount).toBeLessThanOrEqual(3020); - // The runner's count should match CH - expect(stats.totalRowsInserted).toBe(chCount); - } finally { - await cleanup(); - } - }, 60_000); - - // ----------------------------------------------------------------------- - // Test 4: cluster-aware status shows running while processing - // ----------------------------------------------------------------------- - - it("cluster-aware status shows running while processing", () => { - // This tests the status derivation IIFE logic from stats-route.ts: - // - // status: (() => { - // if (!clusterData) return runnerStatus; - // if (clusterProcessing > 0) return "running"; - // if ((clusterDone + clusterFailed) >= clusterTotal && clusterTotal > 0) return "completed"; - // return runnerStatus; - // })() - // - // We test this as a pure unit test of the derivation logic. - - function deriveClusterStatus(opts: { - clusterData: boolean; - runnerStatus: string; - clusterProcessing: number; - clusterDone: number; - clusterFailed: number; - clusterTotal: number; - }): string { - if (!opts.clusterData) return opts.runnerStatus; - if (opts.clusterProcessing > 0) return "running"; - if ((opts.clusterDone + opts.clusterFailed) >= opts.clusterTotal && opts.clusterTotal > 0) return "completed"; - return opts.runnerStatus; - } - - // Case 1: No cluster data - returns runner status directly - expect(deriveClusterStatus({ - clusterData: false, - runnerStatus: "idle", - clusterProcessing: 0, - clusterDone: 0, - clusterFailed: 0, - clusterTotal: 10, - })).toBe("idle"); - - // Case 2: Cluster processing > 0 - returns "running" - expect(deriveClusterStatus({ - clusterData: true, - runnerStatus: "idle", - clusterProcessing: 3, - clusterDone: 5, - clusterFailed: 0, - clusterTotal: 10, - })).toBe("running"); - - // Case 3: All collections done (no failures) - returns "completed" - expect(deriveClusterStatus({ - clusterData: true, - runnerStatus: "idle", - clusterProcessing: 0, - clusterDone: 10, - clusterFailed: 0, - clusterTotal: 10, - })).toBe("completed"); - - // Case 4: All collections terminal (some failed) - returns "completed" - expect(deriveClusterStatus({ - clusterData: true, - runnerStatus: "idle", - clusterProcessing: 0, - clusterDone: 8, - clusterFailed: 2, - clusterTotal: 10, - })).toBe("completed"); - - // Case 5: Still processing with some failures - returns "running" - expect(deriveClusterStatus({ - clusterData: true, - runnerStatus: "idle", - clusterProcessing: 1, - clusterDone: 7, - clusterFailed: 2, - clusterTotal: 10, - })).toBe("running"); - - // Case 6: Zero total collections (edge case) - falls through to runnerStatus - expect(deriveClusterStatus({ - clusterData: true, - runnerStatus: "idle", - clusterProcessing: 0, - clusterDone: 0, - clusterFailed: 0, - clusterTotal: 0, - })).toBe("idle"); - - // Case 7: Partial completion, no processing - falls through to runnerStatus - expect(deriveClusterStatus({ - clusterData: true, - runnerStatus: "waiting_for_index", - clusterProcessing: 0, - clusterDone: 5, - clusterFailed: 0, - clusterTotal: 10, - })).toBe("waiting_for_index"); - }); -}); diff --git a/tests/integration/three-collection-nullcd-e2e.test.ts b/tests/integration/three-collection-nullcd-e2e.test.ts deleted file mode 100644 index 1add823..0000000 --- a/tests/integration/three-collection-nullcd-e2e.test.ts +++ /dev/null @@ -1,442 +0,0 @@ -/** - * End-to-end test: 3 collections with varying null-cd fractions. - * - * Seeds 3 collections (1000, 1500, 2000 docs) with 0%, 30%, and 100% - * null-cd documents, runs the full migration pipeline, and verifies - * all rows land in ClickHouse — including the null-cd sweep phase. - */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter } from "../../src/target/clickhouse-writer.ts"; -import { ManifestStore } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { BatchRunner, type BatchRunnerDeps } from "../../src/runtime/batch-runner.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - chRowCount, - chQuery, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection, seedNullCdCollection } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "warn" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -let chClientForPressure: ClickHouseClient | null = null; - -function getChClientForPressure(): ClickHouseClient { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return chClientForPressure; -} - -interface MigrationComponents { - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; - retryPolicy: RetryPolicy; - chPressure: ClickHousePressure; -} - -async function buildComponents(): Promise { - const mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "three-coll-e2e-test", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - - const chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: false, - }, - logger, - ); - await chWriter.connect(); - - const manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - const redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const chPressure = new ClickHousePressure( - getChClientForPressure(), - BACKPRESSURE_OFF, - logger, - ); - - const gcController = new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); - - const retryPolicy = new RetryPolicy({ - maxRetries: 3, - baseDelayMs: 100, - maxDelayMs: 1000, - }); - - return { mongoReader, chWriter, manifestStore, redisState, gcController, retryPolicy, chPressure }; -} - -async function cleanupComponents(c: MigrationComponents): Promise { - await c.mongoReader.close().catch(() => {}); - await c.chWriter.close().catch(() => {}); - await c.manifestStore.close().catch(() => {}); - await c.redisState.close().catch(() => {}); - c.gcController.dispose(); -} - -async function migrateCollection( - c: MigrationComponents, - collName: string, - eventName: string, - appId: string = APP_ID, -): Promise<{ runId: string; docsRead: number; rowsInserted: number }> { - const { mongoReader, chWriter, manifestStore, redisState, chPressure, gcController, retryPolicy } = c; - - await mongoReader.switchCollection(collName); - - const upperBound = await mongoReader.getUpperBound(); - if (!upperBound) { - // Empty collection or all-null-cd — check for null-cd docs - const hasNullCd = await mongoReader.hasNullCdDocuments(); - if (!hasNullCd) { - return { runId: "", docsRead: 0, rowsInserted: 0 }; - } - - // All-null-cd collection: create a run with nullCdMode enabled. - // BatchRunner still deserializes upperBoundId in its loop, so we - // provide a dummy cursor that won't actually be used for reads. - const bounds = await mongoReader.getNullCdBounds(); - if (!bounds) { - return { runId: "", docsRead: 0, rowsInserted: 0 }; - } - - const dummyUpperBound = JSON.stringify({ cd: 0, id: "000000000000000000000000" }); - - const runId = randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - const collRedisState = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:${collName}`, - ); - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: dummyUpperBound, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await collRedisState.setActiveRun(runId); - await collRedisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: dummyUpperBound, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: collRedisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId: dummyUpperBound, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: appId, e: eventName }, - collectionName: collName, - nullCdMode: true, - nullCdUpperBound: bounds.upper, - }, - }; - - const runner = new BatchRunner(deps); - await runner.run(); - - const stats = runner.getStats(); - return { runId, docsRead: stats.totalDocsRead, rowsInserted: stats.totalRowsInserted }; - } - - // Normal path: cursor-based migration (with auto null-cd sweep if needed) - const upperBoundId = serializeCursor(upperBound); - const runId = randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - const collRedisState = RedisHotState.fromExistingConnection( - redisState.getRedisClient(), - `${TEST_REDIS_PREFIX}:${collName}`, - ); - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await collRedisState.setActiveRun(runId); - await collRedisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - - const deps: BatchRunnerDeps = { - manifestStore, - redisState: collRedisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: false, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: appId, e: eventName }, - collectionName: collName, - }, - }; - - const runner = new BatchRunner(deps); - await runner.run(); - - const stats = runner.getStats(); - return { runId, docsRead: stats.totalDocsRead, rowsInserted: stats.totalRowsInserted }; -} - -/** Count rows in ClickHouse for a specific event name (custom events have n = eventName). */ -async function chCountByEvent(eventName: string): Promise { - const rows = await chQuery<{ cnt: string }>( - `SELECT count() AS cnt FROM ${TEST_CH_TABLE} WHERE n = '${eventName}'`, - ); - return Number(rows[0]?.cnt ?? 0); -} - -// --------------------------------------------------------------------------- -// Test suite -// --------------------------------------------------------------------------- - -describe("three-collection-nullcd-e2e", () => { - beforeAll(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); - }); - - afterAll(async () => { - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await closeAll(); - }); - - it("migrates 3 collections with 0%, 30%, and 100% null-cd docs to ClickHouse", async () => { - // ── Seed ──────────────────────────────────────────────────────────── - - // Collection 1: 1000 docs, all with valid cd - const seed1 = await seedCollection({ - count: 1000, - appId: APP_ID, - eventName: "e2e_all_valid", - nullCdFraction: 0, - }); - - // Collection 2: 1500 docs, 30% null-cd - const seed2 = await seedCollection({ - count: 1500, - appId: APP_ID, - eventName: "e2e_mixed_cd", - nullCdFraction: 0.3, - }); - - // Collection 3: 2000 docs, ALL null-cd - const seed3 = await seedNullCdCollection({ - count: 2000, - appId: APP_ID, - eventName: "e2e_all_null_cd", - }); - - logger.info({ - seed1: { collName: seed1.collName, total: seed1.totalDocs, expected: seed1.expectedRows }, - seed2: { collName: seed2.collName, total: seed2.totalDocs, expected: seed2.expectedRows }, - seed3: { collName: seed3.collName, total: seed3.totalDocs, expected: seed3.expectedRows }, - }, "Seeded 3 collections"); - - // ── Migrate ───────────────────────────────────────────────────────── - - const components = await buildComponents(); - - try { - // Collection 1: all-valid (cursor phase only) - const result1 = await migrateCollection(components, seed1.collName, "e2e_all_valid"); - expect(result1.docsRead).toBeGreaterThanOrEqual(1000); - - // Collection 2: mixed (cursor phase + null-cd sweep) - const result2 = await migrateCollection(components, seed2.collName, "e2e_mixed_cd"); - expect(result2.docsRead).toBeGreaterThanOrEqual(1); - - // Collection 3: all-null-cd (null-cd mode only) - const result3 = await migrateCollection(components, seed3.collName, "e2e_all_null_cd"); - expect(result3.docsRead).toBeGreaterThanOrEqual(1); - - // ── Verify per-collection ─────────────────────────────────────── - - // Allow ClickHouse async inserts to flush - await new Promise((r) => setTimeout(r, 3000)); - - // Collection 1: 1000 docs, 0% null-cd → all 1000 expected - const count1 = await chCountByEvent("e2e_all_valid"); - expect(count1).toBeGreaterThanOrEqual(1000); - expect(count1).toBeLessThanOrEqual(1020); - - // Collection 2: 1500 docs, 30% null-cd → all 1500 expected (cursor + sweep) - const count2 = await chCountByEvent("e2e_mixed_cd"); - expect(count2).toBeGreaterThanOrEqual(seed2.expectedRows); - expect(count2).toBeLessThanOrEqual(seed2.expectedRows + 30); - - // Collection 3: 2000 docs, 100% null-cd → all 2000 expected (sweep only) - const count3 = await chCountByEvent("e2e_all_null_cd"); - expect(count3).toBeGreaterThanOrEqual(2000); - expect(count3).toBeLessThanOrEqual(2020); - - // ── Verify aggregate total ────────────────────────────────────── - - const totalExpected = seed1.expectedRows + seed2.expectedRows + seed3.expectedRows; - const totalCh = await chRowCount(); - expect(totalCh).toBeGreaterThanOrEqual(totalExpected); - expect(totalCh).toBeLessThanOrEqual(totalExpected + 70); - - logger.info({ - count1, count2, count3, - totalCh, totalExpected, - }, "Verification complete"); - } finally { - await cleanupComponents(components); - } - }, 180_000); -}); diff --git a/tests/integration/write-summary.test.ts b/tests/integration/write-summary.test.ts deleted file mode 100644 index c48999e..0000000 --- a/tests/integration/write-summary.test.ts +++ /dev/null @@ -1,391 +0,0 @@ -/** - * Integration tests for run finalization and summary writing. - * - * Verifies that ManifestStore.writeSummary correctly persists run summaries, - * that failed runs have the correct status, and that the RangeCoordinator's - * SETNX-based finalization ensures only one pod finalizes a run. - */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { randomUUID } from "node:crypto"; -import pino from "pino"; -import { createClient, type ClickHouseClient } from "@clickhouse/client"; - -import { MongoReader } from "../../src/source/mongo-reader.ts"; -import { ClickHouseWriter } from "../../src/target/clickhouse-writer.ts"; -import { ManifestStore, type RunSummary } from "../../src/state/manifest-store.ts"; -import { RedisHotState } from "../../src/state/redis-hot-state.ts"; -import { BatchRunner, type BatchRunnerDeps } from "../../src/runtime/batch-runner.ts"; -import { ClickHousePressure, type BackpressureConfig } from "../../src/target/clickhouse-pressure.ts"; -import { GcController } from "../../src/runtime/gc-controller.ts"; -import { RetryPolicy } from "../../src/runtime/retry-policy.ts"; -import { serializeCursor } from "../../src/types/cursor.ts"; - -import { - getRedis, - setupClickHouse, - teardownClickHouse, - teardownMongo, - teardownRedis, - closeAll, - TEST_MONGO_URI, - TEST_MONGO_DB, - TEST_CH_URL, - TEST_CH_DB, - TEST_CH_TABLE, - TEST_REDIS_URL, - TEST_REDIS_PREFIX, -} from "../helpers/setup.ts"; -import { seedCollection } from "../helpers/seed-mongo.ts"; - -// --------------------------------------------------------------------------- -// Shared config -// --------------------------------------------------------------------------- - -const logger = pino({ level: "silent" }); - -const APP_ID = "aaaaaaaaaaaaaaaaaaaaaaaa"; -const EVENT_NAME = "summary_test_event"; - -const BACKPRESSURE_OFF: BackpressureConfig = { - enabled: false, - partsToThrowInsert: 300, - maxPartsInTotal: 500, - partitionPctHigh: 0.7, - partitionPctLow: 0.55, - totalPctHigh: 0.7, - totalPctLow: 0.55, - pollIntervalMs: 5000, - maxPauseEpisodeMs: 180_000, -}; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -let chClientForPressure: ClickHouseClient | null = null; - -function getChClientForPressure(): ClickHouseClient { - if (!chClientForPressure) { - chClientForPressure = createClient({ - url: TEST_CH_URL, - database: TEST_CH_DB, - username: "default", - password: "", - }); - } - return chClientForPressure; -} - -async function buildDeps(collName: string): Promise<{ - deps: BatchRunnerDeps; - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; - runId: string; -}> { - const mongoReader = new MongoReader( - { - uri: TEST_MONGO_URI, - database: TEST_MONGO_DB, - readPreference: "primary", - readConcern: "local", - retryReads: true, - appName: "integration-test-summary", - batchRowsTarget: 500, - cursorBatchSize: 500, - maxTimeMs: 30_000, - }, - logger, - ); - await mongoReader.connect(); - await mongoReader.switchCollection(collName); - - const chWriter = new ClickHouseWriter( - { - url: TEST_CH_URL, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - username: "default", - password: "", - queryTimeoutMs: 30_000, - useDedupToken: true, - }, - logger, - ); - await chWriter.connect(); - - const manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - const redisState = new RedisHotState(TEST_REDIS_URL, TEST_REDIS_PREFIX); - await redisState.connect(); - - const chPressure = new ClickHousePressure( - getChClientForPressure(), - BACKPRESSURE_OFF, - logger, - ); - - const gcController = new GcController( - { - enabled: false, - rssSoftLimitBytes: 2 * 1024 * 1024 * 1024, - rssHardLimitBytes: 3 * 1024 * 1024 * 1024, - heapUsedRatio: 0.9, - everyNBatches: 999_999, - }, - logger, - ); - - const retryPolicy = new RetryPolicy({ - maxRetries: 3, - baseDelayMs: 100, - maxDelayMs: 1000, - }); - - const upperBound = await mongoReader.getUpperBound(); - const upperBoundId = upperBound ? serializeCursor(upperBound) : ""; - - const runId = randomUUID(); - const now = new Date().toISOString(); - const sourceNs = `${TEST_MONGO_DB}.${collName}`; - const targetTable = `${TEST_CH_DB}.${TEST_CH_TABLE}`; - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: sourceNs, - target_table: targetTable, - upper_bound_cursor: upperBoundId, - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - await redisState.setActiveRun(runId); - await redisState.setState(runId, { - runId, - status: "active", - sourceNs, - targetTable, - upperBoundCursor: upperBoundId, - lastCommittedCursor: null, - transformVersion: "v1", - totalBatches: 0, - completedBatches: 0, - startedAt: now, - }); - - const deps: BatchRunnerDeps = { - manifestStore, - redisState, - mongoReader, - chWriter, - chPressure, - gcController, - retryPolicy, - logger, - config: { - runId, - transformVersion: "v1", - sourceNs, - targetTable, - upperBoundId, - batchRowsTarget: 500, - mongoPageSize: 500, - backpressure: BACKPRESSURE_OFF, - useDedupToken: true, - database: TEST_CH_DB, - table: TEST_CH_TABLE, - snapshotInterval: 10, - collectionDefaults: { a: APP_ID, e: EVENT_NAME }, - }, - }; - - return { deps, mongoReader, chWriter, manifestStore, redisState, gcController, runId }; -} - -async function cleanupDeps(parts: { - mongoReader: MongoReader; - chWriter: ClickHouseWriter; - manifestStore: ManifestStore; - redisState: RedisHotState; - gcController: GcController; -}): Promise { - await parts.mongoReader.close().catch(() => {}); - await parts.chWriter.close().catch(() => {}); - await parts.manifestStore.close().catch(() => {}); - await parts.redisState.close().catch(() => {}); - parts.gcController.dispose(); -} - -// --------------------------------------------------------------------------- -// Lifecycle -// --------------------------------------------------------------------------- - -beforeAll(async () => { - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await setupClickHouse(); -}); - -afterAll(async () => { - if (chClientForPressure) { - await chClientForPressure.close(); - chClientForPressure = null; - } - await teardownMongo(); - await teardownClickHouse(); - await teardownRedis(); - await closeAll(); -}); - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -describe("write-summary", () => { - it("completed run has correct summary stats", async () => { - // Seed a small collection - const { collName, expectedRows } = await seedCollection({ - count: 200, - appId: APP_ID, - eventName: EVENT_NAME, - }); - expect(expectedRows).toBe(200); - - const parts = await buildDeps(collName); - - try { - // Run migration - const runner = new BatchRunner(parts.deps); - await runner.run(); - - // Allow async inserts to flush - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Get batch runner stats to build a summary - const stats = runner.getStats(); - - // Write a summary via ManifestStore - const summary: RunSummary = { - finished_at: new Date().toISOString(), - duration_ms: stats.elapsedMs, - total_docs_read: stats.totalDocsRead, - total_rows_inserted: stats.totalRowsInserted, - total_docs_skipped: stats.totalDocsSkipped, - avg_docs_per_second: stats.docsPerSecond, - avg_rows_per_second: stats.rowsPerSecond, - total_batches: stats.batchSeq, - batches_done: stats.batchSeq - stats.batchesFailed, - batches_failed: stats.batchesFailed, - batches_skipped_empty: 0, - skip_reasons: stats.skipsByReason, - total_errors: stats.batchesFailed, - failed_batch_seqs: [], - digest_mismatches: stats.digestMismatches, - estimated_duplicate_rows: stats.estimatedDuplicateRows, - coverage_pct: 100, - }; - - await parts.manifestStore.writeSummary(parts.runId, "completed", summary); - - // Verify the run document in MongoDB - const run = await parts.manifestStore.getRun(parts.runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("completed"); - expect(run!.summary).not.toBeNull(); - expect(run!.summary!.total_docs_read).toBe(stats.totalDocsRead); - expect(run!.summary!.total_rows_inserted).toBe(stats.totalRowsInserted); - // max() is exclusive so the upper-bound doc may be missed (off by 1) - expect(run!.summary!.total_docs_read).toBeGreaterThanOrEqual(198); - expect(run!.summary!.total_rows_inserted).toBeGreaterThanOrEqual(198); - } finally { - await cleanupDeps(parts); - } - }); - - it("failed run has status 'failed' not 'completed'", async () => { - // Create a run and force-write a failed status with summary - const manifestStore = new ManifestStore(TEST_MONGO_URI, TEST_MONGO_DB); - await manifestStore.connect(); - - try { - const runId = randomUUID(); - const now = new Date().toISOString(); - - await manifestStore.createRun({ - run_id: runId, - status: "active", - source_ns: `${TEST_MONGO_DB}.test_coll`, - target_table: `${TEST_CH_DB}.${TEST_CH_TABLE}`, - upper_bound_cursor: '{"cd":9999999999999,"id":"zzz"}', - transform_version: "v1", - created_at: now, - updated_at: now, - }); - - // Write a "failed" summary - const failedSummary: RunSummary = { - finished_at: now, - duration_ms: 1000, - total_docs_read: 50, - total_rows_inserted: 30, - total_docs_skipped: 5, - avg_docs_per_second: 50, - avg_rows_per_second: 30, - total_batches: 3, - batches_done: 2, - batches_failed: 1, - batches_skipped_empty: 0, - skip_reasons: {}, - total_errors: 1, - failed_batch_seqs: [2], - digest_mismatches: 0, - estimated_duplicate_rows: 0, - coverage_pct: 66.7, - }; - - await manifestStore.writeSummary(runId, "failed", failedSummary); - - // Verify the run is marked as failed - const run = await manifestStore.getRun(runId); - expect(run).toBeDefined(); - expect(run!.status).toBe("failed"); - expect(run!.status).not.toBe("completed"); - expect(run!.summary).not.toBeNull(); - expect(run!.summary!.batches_failed).toBe(1); - expect(run!.summary!.failed_batch_seqs).toEqual([2]); - } finally { - await manifestStore.close(); - } - }); - - it("run finalized exactly once in range mode (SETNX)", async () => { - const redis = await getRedis(); - const collName = "test_finalize_once"; - const prefix = TEST_REDIS_PREFIX; - const finalizeKey = `${prefix}:ranges:${collName}:finalized`; - - // Clean up the key first - await redis.del(finalizeKey); - - // Simulate two pods racing to finalize via SETNX - const podAResult = await redis.set(finalizeKey, "pod-A", "EX", 60, "NX"); - const podBResult = await redis.set(finalizeKey, "pod-B", "EX", 60, "NX"); - - // Only one pod should succeed - expect(podAResult).toBe("OK"); - expect(podBResult).toBeNull(); - - // The value should be from the first pod - const value = await redis.get(finalizeKey); - expect(value).toBe("pod-A"); - - // Clean up - await redis.del(finalizeKey); - }); -});