test: support inspecting WPTs in child processes

test/common/wpt.js

@@ -754,6 +754,8 @@ function runSpecOnThread(execArgv, workerData, handlers) {
  * @returns {SpecHandle}
  */
 function runSpecInProcess(execArgv, workerData, handlers) {
+  const forwardStderr = execArgv.some(
+    (flag) => flag === '--inspect-brk' || flag.startsWith('--inspect-brk='));
   const child = fork(workerPath, {
     execArgv,
     // Status files may skip subtests by regular expression, which JSON
@@ -767,6 +769,9 @@ function runSpecInProcess(execArgv, workerData, handlers) {
   child.stderr.setEncoding('utf8');
   child.stderr.on('data', (chunk) => {
     stderr += chunk;
+    if (forwardStderr) {
+      process.stderr.write(chunk);
+    }
   });
 
   child.on('message', (message) => {
@@ -789,7 +794,8 @@ function runSpecInProcess(execArgv, workerData, handlers) {
     const name = signal ?
       `Test process was killed by signal ${signal}` :
       `Test process exited with code ${code}`;
-    if (!handlers.failure({ name, message: name, stack: stderr }) && stderr) {
+    if (!handlers.failure({ name, message: name, stack: stderr }) &&
+        stderr && !forwardStderr) {
       process.stderr.write(stderr);
     }
   });
@@ -823,9 +829,15 @@ class WPTRunner {
       concurrency = Math.min(10, concurrency);
     }
 
+    this.inspectBrk = process.env.WPT_INSPECT !== undefined;
+
     // The override exists so that every suite can be run either way without
     // editing the drivers, which is how the two backends are kept compatible.
-    backend = process.env.WPT_BACKEND || backend;
+    if (this.inspectBrk) {
+      backend = 'process';
+    } else {
+      backend = process.env.WPT_BACKEND || backend;
+    }
     this.runSpec = backends[backend];
     if (this.runSpec === undefined) {
       throw new Error(`Invalid WPT backend ${backend}, expected one of ` +
@@ -841,6 +853,9 @@ class WPTRunner {
     // we enable the API globally. This has no practical
     // effect on the non-web-worker tests, however.
     this.flags = ['--experimental-web-worker'];
+    if (this.inspectBrk) {
+      this.flags.push('--inspect-brk=0');
+    }
     this.globalThisInitScripts = [];
     this.initScript = null;
 
@@ -1299,6 +1314,9 @@ class WPTRunner {
     const queue = [];
     this.skippedSpecCount = 0;
     const arg = process.argv[2];
+    if (this.inspectBrk && !arg) {
+      throw new Error('WPT_INSPECT requires a WPT test path');
+    }
     for (const spec of this.specs) {
       if (arg) {
         if (spec.isSelectedBy(arg)) {
@@ -1330,6 +1348,18 @@ class WPTRunner {
     if (arg && queue.length === 0) {
       throw new Error(`${arg} not found!`);
     }
+    if (this.inspectBrk && queue.length !== 1) {
+      const matches = queue.map((spec) => spec.getTestPath()).join('\n');
+      throw new Error(
+        `WPT_INSPECT requires exactly one generated WPT test path; ` +
+        `${arg} matched ${queue.length}:\n${matches}`,
+      );
+    }
+    if (this.inspectBrk && queue[0].isWebWorkerTest()) {
+      throw new Error(
+        `WPT_INSPECT does not support worker tests: ${queue[0].getTestPath()}`,
+      );
+    }
 
     return queue;
   }

test/parallel/test-common-wpt-backends.js

@@ -13,6 +13,7 @@ const { spawnSync } = require('child_process');
 const { backends, WPTRunner } = require('../common/wpt');
 
 const queueProbe = process.env.NODE_TEST_WPT_QUEUE_PROBE === '1';
+const backendProbe = process.env.NODE_TEST_WPT_BACKEND_PROBE;
 
 const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
 const specPath = fixtures.path('wpt-backends-spec.js');
@@ -156,7 +157,74 @@ function checkQueuedSpecsKeepRunnerAlive() {
   assert.strictEqual(status, 0, `Queued WPT probe failed:\n${stdout}${stderr}`);
 }
 
+function runDefaultBackendProbe() {
+  const runner = new WPTRunner('compression');
+  assert.strictEqual(runner.runSpec, backends[backendProbe]);
+}
+
+function checkBackendSelection() {
+  const env = { ...process.env };
+  delete env.WPT_BACKEND;
+  delete env.WPT_INSPECT;
+
+  for (const [name, expected, args, overrides] of [
+    ['default', 'thread', [__filename]],
+    ['environment override', 'process', [__filename], { WPT_BACKEND: 'process' }],
+    ['inspect override', 'process', [__filename], {
+      WPT_BACKEND: 'thread',
+      WPT_INSPECT: '1',
+    }],
+  ]) {
+    const result = spawnSync(process.execPath, args, {
+      env: {
+        ...env,
+        ...overrides,
+        NODE_TEST_WPT_BACKEND_PROBE: expected,
+      },
+      encoding: 'utf8',
+      timeout: common.platformTimeout(10_000),
+    });
+    const { error, status, stdout, stderr } = result;
+    assert.ifError(error);
+    assert.strictEqual(
+      status,
+      0,
+      `${name} WPT backend probe failed:\n${stdout}${stderr}`,
+    );
+  }
+}
+
+function checkInspectSelection() {
+  const driver = path.join(__dirname, '../wpt/test-compression.js');
+  const env = { ...process.env, WPT_INSPECT: '1' };
+  delete env.WPT_BACKEND;
+
+  const runFailure = common.mustCall((args, expected) => {
+    const result = spawnSync(process.execPath, [driver, ...args], {
+      env,
+      encoding: 'utf8',
+      timeout: common.platformTimeout(10_000),
+    });
+    const { error, status, stdout, stderr } = result;
+    assert.ifError(error);
+    assert.strictEqual(status, 1, `WPT inspect probe passed:\n${stdout}${stderr}`);
+    assert.match(stderr, expected);
+  }, 3);
+
+  runFailure([], /WPT_INSPECT requires a WPT test path/);
+  runFailure(
+    ['compression-bad-chunks.any.js'],
+    /WPT_INSPECT requires exactly one generated WPT test path; .* matched 2:\r?\ncompression\/compression-bad-chunks\.any\.html\r?\ncompression\/compression-bad-chunks\.any\.worker\.html/,
+  );
+  runFailure(
+    ['compression/compression-bad-chunks.any.worker.html'],
+    /WPT_INSPECT does not support worker tests: compression\/compression-bad-chunks\.any\.worker\.html/,
+  );
+}
+
 async function main() {
+  checkBackendSelection();
+  checkInspectSelection();
   checkQueuedSpecsKeepRunnerAlive();
 
   const completed = await compare(false);
@@ -197,7 +265,9 @@ async function main() {
   assert.deepStrictEqual(workerResults, windowResults);
 }
 
-if (queueProbe) {
+if (backendProbe) {
+  runDefaultBackendProbe();
+} else if (queueProbe) {
   runQueueProbe();
 } else {
   main().then(common.mustCall());

test/parallel/test-common-wpt-inspect.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const common = require('../common');
+common.skipIfInspectorDisabled();
+
+const assert = require('assert');
+const path = require('path');
+const { NodeInstance } = require('../common/inspector-helper');
+
+const driver = path.join(__dirname, '../wpt/test-compression.js');
+
+async function main() {
+  const parent = new NodeInstance([], `
+    delete process.env.WPT_BACKEND;
+    process.env.WPT_INSPECT = '1';
+    process.argv[2] = 'compression/compression-bad-chunks.any.html';
+    require(${JSON.stringify(driver)});
+  `, '', {
+    log() {},
+    error() {},
+  });
+  const stderr = [];
+  parent.on('stderr', (line) => stderr.push(line));
+
+  const session = await parent.connectInspectorSession();
+  await session.send([
+    { method: 'Runtime.enable' },
+    { method: 'Debugger.enable' },
+    { method: 'Runtime.runIfWaitingForDebugger' },
+  ]);
+  await session.waitForNotification('Debugger.paused');
+  await session.send({ method: 'Debugger.resume' });
+  await session.disconnect();
+
+  const { exitCode, signal } = await parent.expectShutdown();
+  assert.strictEqual(signal, null);
+  assert.strictEqual(exitCode, 0);
+  assert.strictEqual(
+    stderr.filter((line) => line.startsWith('Debugger listening on ')).length,
+    1,
+  );
+}
+
+main().then(common.mustCall());

test/wpt/README.md

@@ -12,6 +12,50 @@ declares build requirements, expected failures, and tests to skip.
 See [`test/fixtures/wpt/README.md`][] for the pinned WPT commit
 hashes for each module.
 
+## Running tests
+
+Run a WPT module through the Python test runner:
+
+```bash
+tools/test.py wpt/test-url
+```
+
+Pass a source file to its module runner to run all globals and variants
+generated from that file:
+
+```bash
+node test/wpt/test-url.js url-searchparams.any.js
+```
+
+Pass a generated path, as printed in the test output, to run only that test:
+
+```bash
+node test/wpt/test-url.js 'url/url-searchparams.any.html'
+```
+
+### Execution backends
+
+By default, each test runs in a worker thread. Set `WPT_BACKEND=process` to run
+tests in child processes instead:
+
+```bash
+WPT_BACKEND=process tools/test.py wpt
+```
+
+### Debugging a test
+
+Set `WPT_INSPECT=1` to run one generated main-thread test in a child process
+with `--inspect-brk` on an available port:
+
+```bash
+WPT_INSPECT=1 out/Release/node test/wpt/test-compression.js \
+  'compression/compression-bad-chunks.any.html'
+```
+
+Connect an inspector client to the URL printed on stderr. A source file that
+generates multiple tests is rejected with the exact paths to choose from.
+Worker tests are not supported by inspect mode.
+
 <a id="add-tests"></a>
 
 ## How to add tests for a new module
@@ -53,8 +97,8 @@ runner.runJsTests();
 ```
 
 The runner loads the tests from `test/fixtures/wpt/url`, applies the
-status rules from `test/wpt/status/url.cjs`, and runs them using
-worker threads.
+status rules from `test/wpt/status/url.cjs`, and runs them using the
+selected backend.
 
 #### `new WPTRunner(path[, options])`
 
@@ -64,15 +108,18 @@ worker threads.
   * `concurrency` {number} Number of tests to run in parallel.
     Defaults to `os.availableParallelism() - 1`. Set to `1` for tests
     that require sequential execution (e.g. web-locks, webstorage).
+  * `backend` {string} Test execution backend. Must be either `'thread'` or
+    `'process'`. Defaults to `'thread'`. `WPT_BACKEND` overrides this option.
+    `WPT_INSPECT` always uses `'process'`.
 
 #### `runner.setFlags(flags)`
 
-* `flags` {string\[]} Node.js CLI flags passed to each worker thread
+* `flags` {string\[]} Node.js CLI flags passed to each test process or worker
   (e.g. `['--expose-internals']`).
 
 #### `runner.setInitScript(script)`
 
-* `script` {string} JavaScript code executed in the worker before
+* `script` {string} JavaScript code executed before
   the tests run. Useful for setting up globals needed by the tests.
 
 #### `runner.setScriptModifier(modifier)`
@@ -94,19 +141,7 @@ Starts running the tests. Must be called last, after all configuration.
 
 ### 4. Run the tests
 
-Run the test using `tools/test.py` and see if there are any failures.
-For example, to run all the URL tests under `test/fixtures/wpt/url`:
-
-```bash
-tools/test.py wpt/test-url
-```
-
-To run a specific test in WPT, for example, `url/url-searchparams.any.js`,
-pass the file name as argument to the corresponding test runner:
-
-```bash
-node test/wpt/test-url.js url-searchparams.any.js
-```
+Run the module as described in [Running tests](#running-tests).
 
 If there are any failures, update the corresponding status file
 (in this case, `test/wpt/status/url.cjs`) to make the test pass.