Cut Metro's memory and CPU footprint during harness runs

.nx/version-plans/cut-metro-memory-footprint.md

@@ -0,0 +1,5 @@
+---
+__default__: patch
+---
+
+Harness test runs now use markedly less memory while Metro bundles, and Metro configs keep the project's own `resolver.blockList` instead of having it discarded.

apps/playground/metro.config.js

@@ -1,5 +1,7 @@
 const { withNxMetro } = require('@nx/react-native');
 const { getDefaultConfig, mergeConfig } = require('@react-native/metro-config');
+const exclusionList =
+  require('metro-config/private/defaults/exclusionList').default;
 const path = require('path');
 const fs = require('fs');
 
@@ -41,6 +43,11 @@ const customConfig = {
   cacheVersion: '@react-native-harness/playground',
   resolver: {
     unstable_enablePackageExports: true,
+    // `withNxMetro` puts the monorepo root in watchFolders, so Nx's local
+    // cache and workspace data (~6k files with watched extensions) would
+    // otherwise be crawled and held in Metro's file map. Nothing resolves
+    // through it.
+    blockList: exclusionList([/\.nx\/.*/]),
   },
   server: {
     ...(defaultConfig.server || {}),

packages/bundler-metro/src/__tests__/metro-block-list.test.ts

@@ -0,0 +1,198 @@
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { describe, expect, it } from 'vitest';
+import type { MetroConfig } from 'metro-config';
+import { getHarnessBlockList } from '../metro-block-list.js';
+import { getHarnessManifestPath } from '../paths.js';
+
+// `exclusionList` is what produces Metro's stock `resolver.blockList`, but it
+// is not reachable through metro-config's `exports` map -- load it by path so
+// these tests assert against Metro's real output rather than a copy of it.
+const require = createRequire(import.meta.url);
+const exclusionList = require(
+  path.join(
+    path.dirname(require.resolve('metro-config/package.json')),
+    'src/defaults/exclusionList.js'
+  )
+).default as (extras?: (RegExp | string)[]) => RegExp;
+
+const withBlockList = (
+  blockList: NonNullable<MetroConfig['resolver']>['blockList']
+): MetroConfig => ({ resolver: { blockList } }) as MetroConfig;
+
+describe('getHarnessBlockList', () => {
+  describe('harness-owned exclusions', () => {
+    it('excludes the cache harness creates under .harness', () => {
+      const { blockList } = getHarnessBlockList(withBlockList(undefined));
+
+      expect(blockList.test('/p/.harness/cache/metro/ab/cdef')).toBe(true);
+      expect(blockList.test('/p/.harness/cache/metro-file-map/map.v1')).toBe(
+        true
+      );
+    });
+
+    it('keeps the harness manifest crawlable', () => {
+      // The manifest is injected via `serializer.getPolyfills`, so a module
+      // missing from the file map fails with `Failed to get the SHA-1`.
+      const { blockList } = getHarnessBlockList(withBlockList(undefined));
+
+      expect(blockList.test(getHarnessManifestPath('/p'))).toBe(false);
+      expect(blockList.test('/p/.harness/manifest.js')).toBe(false);
+    });
+
+    it('adds nothing else of its own', () => {
+      const { blockList } = getHarnessBlockList(withBlockList(undefined));
+
+      for (const path of [
+        '/p/.nx/cache/a.js',
+        '/p/.gradle/caches/a.json',
+        '/p/ios/Pods/Foo/foo.json',
+        '/p/ios/build/Release/x.json',
+        '/p/android/build/outputs/y.json',
+        '/p/src/__image_snapshots__/a.png',
+        '/p/packages/runtime/dist/index.js',
+        '/p/node_modules/@jest/globals/build/index.js',
+      ]) {
+        expect(blockList.test(path), path).toBe(false);
+      }
+    });
+  });
+
+  describe('inheriting the project blockList', () => {
+    it('honours a plain project pattern', () => {
+      const { blockList, dropped } = getHarnessBlockList(
+        withBlockList(/[/\\]fixtures[/\\]/)
+      );
+
+      expect(dropped).toEqual([]);
+      expect(blockList.test('/p/src/fixtures/big.json')).toBe(true);
+      expect(blockList.test('/p/src/app.tsx')).toBe(false);
+    });
+
+    it('honours an array of project patterns', () => {
+      const { blockList, dropped } = getHarnessBlockList(
+        withBlockList([/[/\\]fixtures[/\\]/, /\.snap$/])
+      );
+
+      expect(dropped).toEqual([]);
+      expect(blockList.test('/p/src/fixtures/big.json')).toBe(true);
+      expect(blockList.test('/p/src/a.snap')).toBe(true);
+    });
+
+    it('honours an anchored project pattern', () => {
+      const { blockList } = getHarnessBlockList(
+        withBlockList(/^\/p\/vendor\//)
+      );
+
+      expect(blockList.test('/p/vendor/lib.js')).toBe(true);
+      expect(blockList.test('/other/p/vendor/lib.js')).toBe(false);
+    });
+
+    it('preserves group numbering so backreferences still work', () => {
+      const { blockList } = getHarnessBlockList(
+        withBlockList(/([/\\])dup\1/)
+      );
+
+      expect(blockList.test('/p/dup/dup')).toBe(true);
+      expect(blockList.test('/p/dup\\dup')).toBe(false);
+    });
+
+    it('adopts project flags so a case-insensitive pattern keeps working', () => {
+      const { blockList, dropped } = getHarnessBlockList(
+        withBlockList(/[/\\]FIXTURES[/\\]/i)
+      );
+
+      expect(dropped).toEqual([]);
+      expect(blockList.flags).toBe('i');
+      expect(blockList.test('/p/src/fixtures/big.json')).toBe(true);
+    });
+
+    it('drops minority-flag patterns instead of letting Metro throw', () => {
+      // Metro's array handling throws when combining mismatched flags.
+      const minority = /[/\\]other[/\\]/i;
+      const { blockList, dropped } = getHarnessBlockList(
+        withBlockList([/[/\\]a[/\\]/, /[/\\]b[/\\]/, minority])
+      );
+
+      expect(dropped).toEqual([
+        { pattern: minority, reason: 'incompatible-flags' },
+      ]);
+      expect(blockList.flags).toBe('');
+      expect(blockList.test('/p/a/x.js')).toBe(true);
+      expect(blockList.test('/p/b/x.js')).toBe(true);
+    });
+
+    it('is stateless across calls when given a global pattern', () => {
+      const { blockList } = getHarnessBlockList(
+        withBlockList(/[/\\]fixtures[/\\]/g)
+      );
+
+      expect(blockList.test('/p/src/fixtures/a.json')).toBe(true);
+      expect(blockList.test('/p/src/fixtures/a.json')).toBe(true);
+    });
+  });
+
+  describe('carving __tests__ out of the inherited blockList', () => {
+    it("keeps tests crawlable under Metro's stock blockList", () => {
+      const { blockList, dropped } = getHarnessBlockList(
+        withBlockList(exclusionList())
+      );
+
+      expect(dropped).toEqual([]);
+      expect(blockList.test('/p/src/__tests__/smoke.harness.ts')).toBe(false);
+    });
+
+    it("keeps a project's own exclusions while still crawling tests", () => {
+      // exclusionList fuses the project's patterns and the __tests__ rule into
+      // a single alternation, so the two cannot be separated by inspection.
+      const { blockList, dropped } = getHarnessBlockList(
+        withBlockList(exclusionList([/ios\/build\/.*/]))
+      );
+
+      expect(dropped).toEqual([]);
+      expect(blockList.test('/p/ios/build/Release/x.json')).toBe(true);
+      expect(blockList.test('/p/src/__tests__/smoke.harness.ts')).toBe(false);
+      expect(blockList.test(getHarnessManifestPath('/p'))).toBe(false);
+    });
+
+    it('keeps tests crawlable even inside an otherwise excluded directory', () => {
+      const { blockList } = getHarnessBlockList(
+        withBlockList(/[/\\]generated[/\\]/)
+      );
+
+      expect(blockList.test('/p/generated/a.js')).toBe(true);
+      expect(blockList.test('/p/generated/__tests__/a.harness.ts')).toBe(false);
+    });
+
+    it('matches `inherited && !protected` for every pattern shape', () => {
+      const protectedPaths = /[/\\]__tests__[/\\]/;
+      const patterns = [
+        exclusionList(),
+        exclusionList([/ios\/build\/.*/]),
+        exclusionList(['ios/build']),
+        /[/\\]fixtures[/\\]/,
+        /^\/p\/vendor\//,
+      ];
+      const paths = [
+        '/p/src/__tests__/a.harness.ts',
+        '/p/ios/build/Release/x.json',
+        '/p/src/fixtures/big.json',
+        '/p/vendor/lib.js',
+        '/p/src/app.tsx',
+        '/p/ios/build/__tests__/nested.harness.ts',
+      ];
+
+      for (const pattern of patterns) {
+        const { blockList } = getHarnessBlockList(withBlockList(pattern));
+
+        for (const path of paths) {
+          const expected =
+            new RegExp(pattern.source, pattern.flags).test(path) &&
+            !protectedPaths.test(path);
+
+          expect(blockList.test(path), `${pattern} :: ${path}`).toBe(expected);
+        }
+      }
+    });
+  });
+});

packages/bundler-metro/src/__tests__/metro-workers.test.ts

@@ -2,51 +2,63 @@ import { describe, expect, it } from 'vitest';
 import { getCappedMaxWorkers } from '../metro-workers.js';
 
 describe('getCappedMaxWorkers', () => {
-  it('caps to 1 worker when only 2 host cores are available', () => {
+  it('runs in-band on a 2-core host', () => {
     expect(
-      getCappedMaxWorkers({ configuredMaxWorkers: undefined, hostParallelism: 2 })
+      getCappedMaxWorkers({ configuredMaxWorkers: 1, hostParallelism: 2 })
     ).toBe(1);
   });
 
-  it('caps to 1 worker when only 3 host cores are available', () => {
+  it('runs in-band on a 3-vCPU CI runner, leaving cores for the device', () => {
+    // Metro's own default here is 2; the device budget claims 2 of 3 cores.
     expect(
-      getCappedMaxWorkers({ configuredMaxWorkers: undefined, hostParallelism: 3 })
+      getCappedMaxWorkers({ configuredMaxWorkers: 2, hostParallelism: 3 })
     ).toBe(1);
   });
 
-  it('caps to 2 workers when 4 host cores are available', () => {
+  it("halves Metro's default on a 12-core host", () => {
     expect(
-      getCappedMaxWorkers({ configuredMaxWorkers: undefined, hostParallelism: 4 })
-    ).toBe(2);
+      getCappedMaxWorkers({ configuredMaxWorkers: 8, hostParallelism: 12 })
+    ).toBe(4);
   });
 
-  it('reserves the four cores granted to the device on an 8-core host', () => {
+  it("halves Metro's default on a 16-core host", () => {
     expect(
-      getCappedMaxWorkers({ configuredMaxWorkers: 6, hostParallelism: 8 })
-    ).toBe(4);
+      getCappedMaxWorkers({ configuredMaxWorkers: 10, hostParallelism: 16 })
+    ).toBe(5);
   });
 
-  it('respects a configured value well below the cap on a large host', () => {
+  it('applies the device-core reservation when it binds harder than halving', () => {
+    // 8 cores: Metro's default is 6 (halved: 3), device cap is 8 - 4 = 4.
     expect(
-      getCappedMaxWorkers({ configuredMaxWorkers: 10, hostParallelism: 16 })
-    ).toBe(10);
+      getCappedMaxWorkers({ configuredMaxWorkers: 6, hostParallelism: 8 })
+    ).toBe(3);
   });
 
-  it('never lowers a configured value of 1', () => {
+  it('never raises a configured value of 1', () => {
     expect(
-      getCappedMaxWorkers({ configuredMaxWorkers: 1, hostParallelism: 8 })
+      getCappedMaxWorkers({ configuredMaxWorkers: 1, hostParallelism: 16 })
     ).toBe(1);
   });
 
-  it('falls back to the cap when maxWorkers is undefined', () => {
+  it('lowers a configured value that exceeds both limits', () => {
     expect(
-      getCappedMaxWorkers({ configuredMaxWorkers: undefined, hostParallelism: 8 })
-    ).toBe(4);
+      getCappedMaxWorkers({ configuredMaxWorkers: 64, hostParallelism: 4 })
+    ).toBe(2);
   });
 
-  it('lowers a configured value that exceeds the cap', () => {
+  it('never returns less than one worker', () => {
     expect(
-      getCappedMaxWorkers({ configuredMaxWorkers: 8, hostParallelism: 4 })
-    ).toBe(2);
+      getCappedMaxWorkers({ configuredMaxWorkers: 1, hostParallelism: 1 })
+    ).toBe(1);
+  });
+
+  it("falls back to Metro's own default when maxWorkers is unset", () => {
+    // Mirrors the 12-core case above, deriving 8 rather than being handed it.
+    expect(
+      getCappedMaxWorkers({
+        configuredMaxWorkers: undefined,
+        hostParallelism: 12,
+      })
+    ).toBe(4);
   });
 });

packages/bundler-metro/src/__tests__/withRnHarness.test.ts

@@ -17,6 +17,15 @@ type MinimalMetroConfig = {
       collapse: boolean;
     }>;
   };
+  resolver?: {
+    blockList?: RegExp;
+  };
+  server?: {
+    useGlobalHotkey?: boolean;
+  };
+  transformer?: {
+    unstable_workerThreads?: boolean;
+  };
 };
 
 const { ensureDomainDirectories } = vi.hoisted(() => ({
@@ -124,6 +133,60 @@ describe('withRnHarness', () => {
     });
   });
 
+  it('trims Metro functionality a harness run does not need', async () => {
+    const { withRnHarness } = await import('../withRnHarness.js');
+
+    const config = (await withRnHarness(
+      {
+        projectRoot: '/tmp/app',
+        serializer: {},
+        symbolicator: {
+          async customizeFrame() {
+            return {};
+          },
+        },
+      },
+      true,
+    )()) as unknown as MinimalMetroConfig;
+
+    expect(config.server?.useGlobalHotkey).toBe(false);
+    expect(config.transformer?.unstable_workerThreads).toBe(true);
+  });
+
+  it('inherits the project blockList while keeping test files crawlable', async () => {
+    const { withRnHarness } = await import('../withRnHarness.js');
+
+    const config = (await withRnHarness(
+      {
+        projectRoot: '/tmp/app',
+        // What `exclusionList([/ios\/build\/.*/])` produces: the project's own
+        // exclusion fused with the __tests__ rule that would hide every test.
+        resolver: { blockList: /(ios\/build\/.*|\/__tests__\/.*)$/ },
+        serializer: {},
+        symbolicator: {
+          async customizeFrame() {
+            return {};
+          },
+        },
+      },
+      true,
+    )()) as unknown as MinimalMetroConfig;
+
+    const blockList = config.resolver?.blockList;
+
+    expect(blockList).toBeInstanceOf(RegExp);
+    // The project's own exclusion survives.
+    expect(blockList?.test('/tmp/app/ios/build/Release/a.json')).toBe(true);
+    // Harness's own cache is excluded; its manifest is not.
+    expect(blockList?.test('/tmp/app/.harness/cache/metro/ab')).toBe(true);
+    expect(blockList?.test('/tmp/app/.harness/manifest.js')).toBe(false);
+    // Tests stay crawlable, and nothing else is excluded on harness's behalf.
+    expect(blockList?.test('/tmp/app/src/__tests__/smoke.harness.ts')).toBe(
+      false,
+    );
+    expect(blockList?.test('/tmp/app/.nx/cache/a.js')).toBe(false);
+  });
+
   it('caps maxWorkers to leave host cores free for the device under test', async () => {
     const { withRnHarness } = await import('../withRnHarness.js');
     const { getCappedMaxWorkers } = await import('../metro-workers.js');