Ledger engine: crash-safe, resumable drill_events migration — no Redis

.env.example

@@ -1,67 +1,32 @@
-# ─── Service ───
-SERVICE_NAME=mongo-to-clickhouse-migrator
-SERVICE_PORT=8080
-SERVICE_HOST=0.0.0.0
-GRACEFUL_SHUTDOWN_TIMEOUT_MS=60000
-RERUN_MODE=resume
-
-# ─── MongoDB Source ───
-MONGO_URI=mongodb://localhost:27017/?replicaSet=rs0
-MONGO_DB=countly_drill
-MONGO_COLLECTION_PREFIX=drill_events
-MONGO_READ_PREFERENCE=primary
-MONGO_READ_CONCERN=majority
-MONGO_RETRY_READS=true
-MONGO_APP_NAME=mongo-to-clickhouse-migrator
-MONGO_BATCH_ROWS_TARGET=10000
-MONGO_CURSOR_BATCH_SIZE=2000
-MONGO_MAX_TIME_MS=120000
-
-# ─── Transform ───
-TRANSFORM_VERSION=v1
-
-# ─── ClickHouse Target ───
+# ─── Required (the only two you must set) ───
+MONGO_URI=mongodb://localhost:27017
 CLICKHOUSE_URL=http://localhost:8123
+
+# ─── Common ───
+#SERVICE_NAME=drill-migrator
+SERVICE_PORT=8080
+MONGO_DB=countly_drill              # source database with drill_events* collections
+MONGO_COUNTLY_DB=countly            # for per-event collection-hash resolution
 CLICKHOUSE_DB=countly_drill
 CLICKHOUSE_TABLE=drill_events
-CLICKHOUSE_USERNAME=default
-CLICKHOUSE_PASSWORD=
-CLICKHOUSE_QUERY_TIMEOUT_MS=120000
-CLICKHOUSE_MAX_RETRIES=8
-CLICKHOUSE_RETRY_BASE_DELAY_MS=1000
-CLICKHOUSE_RETRY_MAX_DELAY_MS=30000
-CLICKHOUSE_USE_DEDUP_TOKEN=true
-
-# ─── Backpressure ───
-BACKPRESSURE_ENABLED=true
-BACKPRESSURE_PARTS_TO_THROW_INSERT=300
-BACKPRESSURE_MAX_PARTS_IN_TOTAL=500
-BACKPRESSURE_PARTITION_PCT_HIGH=0.50
-BACKPRESSURE_PARTITION_PCT_LOW=0.35
-BACKPRESSURE_TOTAL_PCT_HIGH=0.50
-BACKPRESSURE_TOTAL_PCT_LOW=0.40
-BACKPRESSURE_POLL_INTERVAL_MS=15000
-BACKPRESSURE_MAX_PAUSE_EPISODE_MS=180000
-
-# ─── State ───
-MANIFEST_DB=countly_drill
-REDIS_URL=redis://redis:6379
-REDIS_KEY_PREFIX=mig
-TIMELINE_SNAPSHOT_INTERVAL=10
-
-# ─── Memory / GC ───
-GC_ENABLED=true
-GC_RSS_SOFT_LIMIT_MB=1536
-GC_RSS_HARD_LIMIT_MB=2048
-GC_HEAP_USED_RATIO=0.70
-GC_EVERY_N_BATCHES=10
-
-# ─── Logging ───
-LOG_LEVEL=info
-
-# ─── Lifecycle ───
-# When true, the service shuts down cleanly (exit 0) once all collections
-# finish migrating. Used by one-shot orchestration (e.g. docker compose
-# --abort-on-container-exit). Default false: service stays running with
-# dashboard available.
-EXIT_ON_COMPLETE=false
+MANIFEST_DB=countly_drill           # where progress state lives: mig_ranges + mig_dlq_docs
+                                    # (recoverable if lost: dashboard → Help & Recovery → Rebuild ledger from data)
+# Read preference is AUTO: on a replica set the engine picks
+# secondaryPreferred by itself (source is frozen after cutover, so secondary
+# reads are exact). Set explicitly only to override:
+#MONGO_READ_PREFERENCE=primary
+
+# ─── Run identity & scaling ───
+LEDGER_RUN_ID=migration-1           # stable resume key — keep it the same across restarts
+#POD_ID=pod-1                       # unique per instance when running multiple pods
+#EXIT_ON_COMPLETE=true              # exit 0 when all chunks are done (one-shot orchestration)
+
+# ─── Sizing (see the UI's Configuration card for guidance) ───
+#LEDGER_CHUNK_DOCS_TARGET=2000000
+#LEDGER_MAX_CHUNK_DAYS=7
+#MONGO_PAGE_SIZE=10000
+#LEDGER_INSERT_INFLIGHT=3
+
+# ─── Rehearsal ───
+#DRY_RUN=true                       # sampled rehearsal, nothing stored (or use the UI button)
+#DRY_RUN_SAMPLE_PCT=2

.github/workflows/ci.yml

@@ -0,0 +1,78 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit:
+    name: Typecheck + differential harness
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '25'
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Typecheck
+        run: npm run typecheck
+
+      # Asserts src/transform/ reproduces countly-platform's live ingestion
+      # normalization exactly (vendored goldens in tests/differential/).
+      # Pure unit tests — no MongoDB/ClickHouse/Redis required.
+      - name: Ingestion-matching differential harness
+        run: npx vitest run tests/differential
+
+  integration:
+    name: Integration (MongoDB 7 + ClickHouse 26.4)
+    runs-on: ubuntu-latest
+    services:
+      mongo:
+        image: mongo:7
+        ports: ['27017:27017']
+        options: >-
+          --health-cmd "mongosh --quiet --eval 'db.runCommand({ping:1})'"
+          --health-interval 5s --health-timeout 5s --health-retries 30
+      clickhouse:
+        # Production ClickHouse version (deploy/compose/images.standard.env
+        # in countly-platform). 26.4 requires a password for non-localhost
+        # clients — the tests read TEST_CLICKHOUSE_PASSWORD.
+        image: clickhouse/clickhouse-server:26.4
+        env:
+          CLICKHOUSE_PASSWORD: ci-pass
+        ports: ['8123:8123']
+        options: >-
+          --health-cmd "clickhouse-client --password ci-pass --query 'SELECT 1'"
+          --health-interval 5s --health-timeout 5s --health-retries 30
+    env:
+      TEST_CLICKHOUSE_PASSWORD: ci-pass
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '25'
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      # Full engine suite: ledger store semantics, e2e chunk pipeline with
+      # DLQ/coercions/null-cd sweep, multi-collection scoping regressions,
+      # ledger rebuild, duplicate attribution, pair-based attach recovery.
+      - name: Integration tests
+        run: npx vitest run tests/integration