diff --git a/test/common/wpt.js b/test/common/wpt.js index 82b0777de5d5..4ad788ae029d 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -754,6 +754,8 @@ function runSpecOnThread(execArgv, workerData, handlers) { * @returns {SpecHandle} */ function runSpecInProcess(execArgv, workerData, handlers) { + const forwardStderr = execArgv.some( + (flag) => flag === '--inspect-brk' || flag.startsWith('--inspect-brk=')); const child = fork(workerPath, { execArgv, // Status files may skip subtests by regular expression, which JSON @@ -767,6 +769,9 @@ function runSpecInProcess(execArgv, workerData, handlers) { child.stderr.setEncoding('utf8'); child.stderr.on('data', (chunk) => { stderr += chunk; + if (forwardStderr) { + process.stderr.write(chunk); + } }); child.on('message', (message) => { @@ -789,7 +794,8 @@ function runSpecInProcess(execArgv, workerData, handlers) { const name = signal ? `Test process was killed by signal ${signal}` : `Test process exited with code ${code}`; - if (!handlers.failure({ name, message: name, stack: stderr }) && stderr) { + if (!handlers.failure({ name, message: name, stack: stderr }) && + stderr && !forwardStderr) { process.stderr.write(stderr); } }); @@ -823,9 +829,15 @@ class WPTRunner { concurrency = Math.min(10, concurrency); } + this.inspectBrk = process.env.WPT_INSPECT !== undefined; + // The override exists so that every suite can be run either way without // editing the drivers, which is how the two backends are kept compatible. - backend = process.env.WPT_BACKEND || backend; + if (this.inspectBrk) { + backend = 'process'; + } else { + backend = process.env.WPT_BACKEND || backend; + } this.runSpec = backends[backend]; if (this.runSpec === undefined) { throw new Error(`Invalid WPT backend ${backend}, expected one of ` + @@ -841,6 +853,9 @@ class WPTRunner { // we enable the API globally. This has no practical // effect on the non-web-worker tests, however. this.flags = ['--experimental-web-worker']; + if (this.inspectBrk) { + this.flags.push('--inspect-brk=0'); + } this.globalThisInitScripts = []; this.initScript = null; @@ -1299,6 +1314,9 @@ class WPTRunner { const queue = []; this.skippedSpecCount = 0; const arg = process.argv[2]; + if (this.inspectBrk && !arg) { + throw new Error('WPT_INSPECT requires a WPT test path'); + } for (const spec of this.specs) { if (arg) { if (spec.isSelectedBy(arg)) { @@ -1330,6 +1348,18 @@ class WPTRunner { if (arg && queue.length === 0) { throw new Error(`${arg} not found!`); } + if (this.inspectBrk && queue.length !== 1) { + const matches = queue.map((spec) => spec.getTestPath()).join('\n'); + throw new Error( + `WPT_INSPECT requires exactly one generated WPT test path; ` + + `${arg} matched ${queue.length}:\n${matches}`, + ); + } + if (this.inspectBrk && queue[0].isWebWorkerTest()) { + throw new Error( + `WPT_INSPECT does not support worker tests: ${queue[0].getTestPath()}`, + ); + } return queue; } diff --git a/test/parallel/test-common-wpt-backends.js b/test/parallel/test-common-wpt-backends.js index 10aa619bc8d5..d3e99b2895b2 100644 --- a/test/parallel/test-common-wpt-backends.js +++ b/test/parallel/test-common-wpt-backends.js @@ -13,6 +13,7 @@ const { spawnSync } = require('child_process'); const { backends, WPTRunner } = require('../common/wpt'); const queueProbe = process.env.NODE_TEST_WPT_QUEUE_PROBE === '1'; +const backendProbe = process.env.NODE_TEST_WPT_BACKEND_PROBE; const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js'); const specPath = fixtures.path('wpt-backends-spec.js'); @@ -156,7 +157,74 @@ function checkQueuedSpecsKeepRunnerAlive() { assert.strictEqual(status, 0, `Queued WPT probe failed:\n${stdout}${stderr}`); } +function runDefaultBackendProbe() { + const runner = new WPTRunner('compression'); + assert.strictEqual(runner.runSpec, backends[backendProbe]); +} + +function checkBackendSelection() { + const env = { ...process.env }; + delete env.WPT_BACKEND; + delete env.WPT_INSPECT; + + for (const [name, expected, args, overrides] of [ + ['default', 'thread', [__filename]], + ['environment override', 'process', [__filename], { WPT_BACKEND: 'process' }], + ['inspect override', 'process', [__filename], { + WPT_BACKEND: 'thread', + WPT_INSPECT: '1', + }], + ]) { + const result = spawnSync(process.execPath, args, { + env: { + ...env, + ...overrides, + NODE_TEST_WPT_BACKEND_PROBE: expected, + }, + encoding: 'utf8', + timeout: common.platformTimeout(10_000), + }); + const { error, status, stdout, stderr } = result; + assert.ifError(error); + assert.strictEqual( + status, + 0, + `${name} WPT backend probe failed:\n${stdout}${stderr}`, + ); + } +} + +function checkInspectSelection() { + const driver = path.join(__dirname, '../wpt/test-compression.js'); + const env = { ...process.env, WPT_INSPECT: '1' }; + delete env.WPT_BACKEND; + + const runFailure = common.mustCall((args, expected) => { + const result = spawnSync(process.execPath, [driver, ...args], { + env, + encoding: 'utf8', + timeout: common.platformTimeout(10_000), + }); + const { error, status, stdout, stderr } = result; + assert.ifError(error); + assert.strictEqual(status, 1, `WPT inspect probe passed:\n${stdout}${stderr}`); + assert.match(stderr, expected); + }, 3); + + runFailure([], /WPT_INSPECT requires a WPT test path/); + runFailure( + ['compression-bad-chunks.any.js'], + /WPT_INSPECT requires exactly one generated WPT test path; .* matched 2:\r?\ncompression\/compression-bad-chunks\.any\.html\r?\ncompression\/compression-bad-chunks\.any\.worker\.html/, + ); + runFailure( + ['compression/compression-bad-chunks.any.worker.html'], + /WPT_INSPECT does not support worker tests: compression\/compression-bad-chunks\.any\.worker\.html/, + ); +} + async function main() { + checkBackendSelection(); + checkInspectSelection(); checkQueuedSpecsKeepRunnerAlive(); const completed = await compare(false); @@ -197,7 +265,9 @@ async function main() { assert.deepStrictEqual(workerResults, windowResults); } -if (queueProbe) { +if (backendProbe) { + runDefaultBackendProbe(); +} else if (queueProbe) { runQueueProbe(); } else { main().then(common.mustCall()); diff --git a/test/parallel/test-common-wpt-inspect.js b/test/parallel/test-common-wpt-inspect.js new file mode 100644 index 000000000000..ff53902972c5 --- /dev/null +++ b/test/parallel/test-common-wpt-inspect.js @@ -0,0 +1,44 @@ +'use strict'; + +const common = require('../common'); +common.skipIfInspectorDisabled(); + +const assert = require('assert'); +const path = require('path'); +const { NodeInstance } = require('../common/inspector-helper'); + +const driver = path.join(__dirname, '../wpt/test-compression.js'); + +async function main() { + const parent = new NodeInstance([], ` + delete process.env.WPT_BACKEND; + process.env.WPT_INSPECT = '1'; + process.argv[2] = 'compression/compression-bad-chunks.any.html'; + require(${JSON.stringify(driver)}); + `, '', { + log() {}, + error() {}, + }); + const stderr = []; + parent.on('stderr', (line) => stderr.push(line)); + + const session = await parent.connectInspectorSession(); + await session.send([ + { method: 'Runtime.enable' }, + { method: 'Debugger.enable' }, + { method: 'Runtime.runIfWaitingForDebugger' }, + ]); + await session.waitForNotification('Debugger.paused'); + await session.send({ method: 'Debugger.resume' }); + await session.disconnect(); + + const { exitCode, signal } = await parent.expectShutdown(); + assert.strictEqual(signal, null); + assert.strictEqual(exitCode, 0); + assert.strictEqual( + stderr.filter((line) => line.startsWith('Debugger listening on ')).length, + 1, + ); +} + +main().then(common.mustCall()); diff --git a/test/wpt/README.md b/test/wpt/README.md index bdd0b851fe31..ac96e3b247ba 100644 --- a/test/wpt/README.md +++ b/test/wpt/README.md @@ -12,6 +12,50 @@ declares build requirements, expected failures, and tests to skip. See [`test/fixtures/wpt/README.md`][] for the pinned WPT commit hashes for each module. +## Running tests + +Run a WPT module through the Python test runner: + +```bash +tools/test.py wpt/test-url +``` + +Pass a source file to its module runner to run all globals and variants +generated from that file: + +```bash +node test/wpt/test-url.js url-searchparams.any.js +``` + +Pass a generated path, as printed in the test output, to run only that test: + +```bash +node test/wpt/test-url.js 'url/url-searchparams.any.html' +``` + +### Execution backends + +By default, each test runs in a worker thread. Set `WPT_BACKEND=process` to run +tests in child processes instead: + +```bash +WPT_BACKEND=process tools/test.py wpt +``` + +### Debugging a test + +Set `WPT_INSPECT=1` to run one generated main-thread test in a child process +with `--inspect-brk` on an available port: + +```bash +WPT_INSPECT=1 out/Release/node test/wpt/test-compression.js \ + 'compression/compression-bad-chunks.any.html' +``` + +Connect an inspector client to the URL printed on stderr. A source file that +generates multiple tests is rejected with the exact paths to choose from. +Worker tests are not supported by inspect mode. + ## How to add tests for a new module @@ -53,8 +97,8 @@ runner.runJsTests(); ``` The runner loads the tests from `test/fixtures/wpt/url`, applies the -status rules from `test/wpt/status/url.cjs`, and runs them using -worker threads. +status rules from `test/wpt/status/url.cjs`, and runs them using the +selected backend. #### `new WPTRunner(path[, options])` @@ -64,15 +108,18 @@ worker threads. * `concurrency` {number} Number of tests to run in parallel. Defaults to `os.availableParallelism() - 1`. Set to `1` for tests that require sequential execution (e.g. web-locks, webstorage). + * `backend` {string} Test execution backend. Must be either `'thread'` or + `'process'`. Defaults to `'thread'`. `WPT_BACKEND` overrides this option. + `WPT_INSPECT` always uses `'process'`. #### `runner.setFlags(flags)` -* `flags` {string\[]} Node.js CLI flags passed to each worker thread +* `flags` {string\[]} Node.js CLI flags passed to each test process or worker (e.g. `['--expose-internals']`). #### `runner.setInitScript(script)` -* `script` {string} JavaScript code executed in the worker before +* `script` {string} JavaScript code executed before the tests run. Useful for setting up globals needed by the tests. #### `runner.setScriptModifier(modifier)` @@ -94,19 +141,7 @@ Starts running the tests. Must be called last, after all configuration. ### 4. Run the tests -Run the test using `tools/test.py` and see if there are any failures. -For example, to run all the URL tests under `test/fixtures/wpt/url`: - -```bash -tools/test.py wpt/test-url -``` - -To run a specific test in WPT, for example, `url/url-searchparams.any.js`, -pass the file name as argument to the corresponding test runner: - -```bash -node test/wpt/test-url.js url-searchparams.any.js -``` +Run the module as described in [Running tests](#running-tests). If there are any failures, update the corresponding status file (in this case, `test/wpt/status/url.cjs`) to make the test pass.