From 030f8a321e9f5cadd36d5724ef35268bc6efc549 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Mon, 10 Jul 2023 16:44:11 -0400 Subject: [PATCH 01/30] Add d2l-test binary --- src/server/wtr-config.js => bin/test.js | 84 +++++++++++-------- package-lock.json | 3 + package.json | 5 +- src/server/headed-mode-plugin.js | 1 + src/server/index.js | 2 +- src/server/visual-diff-plugin.js | 1 + src/server/visual-diff-reporter.js | 2 +- ...wtr-vdiff.config.js => d2l-test.config.js} | 10 +-- test/server/wtr-config.test.js | 6 +- 9 files changed, 68 insertions(+), 46 deletions(-) rename src/server/wtr-config.js => bin/test.js (76%) mode change 100644 => 100755 rename test/browser/{wtr-vdiff.config.js => d2l-test.config.js} (60%) diff --git a/src/server/wtr-config.js b/bin/test.js old mode 100644 new mode 100755 similarity index 76% rename from src/server/wtr-config.js rename to bin/test.js index 05d68208..17b2d1c9 --- a/src/server/wtr-config.js +++ b/bin/test.js @@ -1,32 +1,44 @@ +#!/usr/bin/env node + +import { ConfigLoaderError, readConfig } from '@web/config-loader'; +import { defaultReporter, startTestRunner } from '@web/test-runner'; import commandLineArgs from 'command-line-args'; -import { defaultReporter } from '@web/test-runner'; -import { headedMode } from './headed-mode-plugin.js'; +import { headedMode } from '../src/server/headed-mode-plugin.js'; import { playwrightLauncher } from '@web/test-runner-playwright'; -import { visualDiff } from './visual-diff-plugin.js'; -import { visualDiffReporter } from './visual-diff-reporter.js'; +import { visualDiff } from '../src/server/visual-diff-plugin.js'; +import { visualDiffReporter } from '../src/server/visual-diff-reporter.js'; const optionDefinitions = [ // @web/test-runner options { name: 'files', type: String, multiple: true }, - { name: 'group', type: String }, + { name: 'group', type: String, defaultOption: true }, { name: 'manual', type: Boolean }, - { name: 'playwright', type: Boolean }, + { name: 'config', type: String }, { name: 'watch', type: Boolean }, // custom options - { name: 'chromium', type: Boolean }, + { name: 'chrome', type: Boolean }, { name: 'filter', alias: 'f', type: String, multiple: true }, { name: 'firefox', type: Boolean }, { name: 'golden', type: Boolean }, { name: 'grep', alias: 'g', type: String }, { name: 'timeout', type: Number }, - { name: 'webkit', type: Boolean }, + { name: 'safari', type: Boolean }, ]; const cliArgs = commandLineArgs(optionDefinitions, { partial: true }); +const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err => { + if (err instanceof ConfigLoaderError) { + throw new Error(err.message); + } else { + throw err; + } +}) || {}; + +//const DISALLOWED_ARGS = ['browsers', 'playwright', '_unknown']; const DEFAULT_PATTERN = type => `./test/**/*.${type}.js`; -const DEFAULT_VDIFF = false; -const ALLOWED_BROWSERS = ['chromium', 'firefox', 'webkit']; +const ALLOWED_BROWSERS = ['chrome', 'firefox', 'safari']; +const BROWSER_MAP = { chrome: 'chromium', safari: 'webkit' }; export class WTRConfig { @@ -35,13 +47,13 @@ export class WTRConfig { constructor(cliArgs) { this.#cliArgs = cliArgs || {}; - const requestedBrowsers = ALLOWED_BROWSERS.filter(b => cliArgs?.[b]); + this.#cliArgs.group ??= 'unit'; + const requestedBrowsers = ALLOWED_BROWSERS.filter(b => cliArgs?.[b]).map(b => BROWSER_MAP[b] || b); this.#requestedBrowsers = requestedBrowsers.length && requestedBrowsers; } get #defaultConfig() { return { - files: this.#getPattern('test'), nodeResolve: true, testRunnerHtml: testFramework => ` @@ -64,7 +76,7 @@ export class WTRConfig { return { name: 'vdiff', files: this.#getPattern('vdiff'), - browsers: this.getBrowsers(['chromium']), + browsers: ['chrome'], testRunnerHtml: testFramework => ` @@ -126,7 +138,7 @@ export class WTRConfig { } #getPattern(type) { - const pattern = this.#cliArgs.files || this.pattern(type); + const pattern = [].concat(this.#cliArgs.files || this.pattern(type)); // replace filename wildcards with all filter strings // e.g. If filter is ['button', 'list'], pattern './test/*.test.*' becomes: @@ -134,37 +146,27 @@ export class WTRConfig { if (this.#cliArgs.filter) { return this.#cliArgs.filter.map(filterStr => { // replace everything after the last forward slash - return pattern.replace(/[^/]*$/, fileGlob => { + return pattern.map(p => p.replace(/[^/]*$/, fileGlob => { // create a new glob for each wildcard const fileGlobs = Array.from(fileGlob.matchAll(/(? { const arr = fileGlob.split(''); - arr.splice(index, 1, `*${filterStr}*`); + arr.splice(index, 1, filterStr); return arr.join(''); }); - return `+(${fileGlobs.join('|')})`; - }); - }); + return `+(${fileGlobs.join('|') || fileGlob})`; + })); + }).flat(); } return pattern; } create({ pattern = DEFAULT_PATTERN, - vdiff = DEFAULT_VDIFF, timeout, ...passthroughConfig } = {}) { - const { files, filter, golden, grep, group, manual, playwright, watch } = this.#cliArgs; - if (!group || group === 'default') { - if (playwright) { - console.warn('Warning: reducedMotion disabled. Use the unit group to enable reducedMotion.'); - } else { - console.warn('Warning: Running with puppeteer, reducedMotion disabled. Use the unit group to use playwright with reducedMotion enabled'); - } - } - delete passthroughConfig.browsers; if (typeof pattern !== 'function') throw new TypeError('pattern must be a function'); @@ -179,11 +181,10 @@ export class WTRConfig { config.groups ??= []; config.groups.push({ name: 'unit', - files: this.#getPattern('test'), - browsers: this.getBrowsers() + files: this.#getPattern('test') }); - if (vdiff) { + if (group === 'vdiff') { config.reporters ??= [ defaultReporter() ]; config.reporters.push(visualDiffReporter({ reportResults: !golden })); @@ -193,6 +194,9 @@ export class WTRConfig { config.groups.push(this.visualDiffGroup); } + // convert all browsers to playwright + config.groups.forEach(g => g.browsers = this.getBrowsers(g.browsers)); + if (watch || manual) { config.plugins ??= []; const currentPattern = files || config.groups.find(g => g.name === group)?.files || config.files; @@ -221,12 +225,26 @@ export class WTRConfig { } -export function createConfig(...args) { +/* +function createConfig(...args) { const wtrConfig = new WTRConfig(cliArgs); return wtrConfig.create(...args); } +*/ +/* export function getBrowsers(browsers) { const wtrConfig = new WTRConfig(cliArgs); return wtrConfig.getBrowsers(browsers); } +*/ + +const wtrConfig = new WTRConfig(cliArgs); +const config = wtrConfig.create(testConfig); + +await startTestRunner({ + readCliArgs: true, + argv: [ '--group', cliArgs.group, ...(cliArgs._unknown || []) ], + readFileConfig: false, + config +}); diff --git a/package-lock.json b/package-lock.json index b7be36d5..a8cb6db8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,9 @@ "pngjs": "^7", "rollup": "^3" }, + "bin": { + "d2l-test": "bin/test.js" + }, "devDependencies": { "@web/dev-server": "^0.2", "chai": "^4", diff --git a/package.json b/package.json index 78171d6d..df042f61 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,12 @@ "test": "npm run lint && npm run test:server && npm run test:browser", "test:browser": "web-test-runner --files \"./test/browser/**/*.test.js\" --node-resolve --playwright", "test:server": "mocha ./test/server/**/*.test.js", - "test:vdiff": "web-test-runner --config ./test/browser/wtr-vdiff.config.js --group vdiff", + "test:vdiff": "d2l-test --config ./test/browser/d2l-test.config.js --group vdiff", "test:vdiff:golden": "npm run test:vdiff -- --golden" }, + "bin": { + "d2l-test": "./bin/test.js" + }, "author": "D2L Corporation", "license": "Apache-2.0", "devDependencies": { diff --git a/src/server/headed-mode-plugin.js b/src/server/headed-mode-plugin.js index cc4c757d..ec6912b1 100644 --- a/src/server/headed-mode-plugin.js +++ b/src/server/headed-mode-plugin.js @@ -8,6 +8,7 @@ export function headedMode({ manual, watch, pattern }) { name: 'brightspace-headed-mode', async transform(context) { if ((watch || manual) && files.includes(context.path.slice(1))) { + // allow time to open devtools in firefox and webkit watch && await new Promise(r => setTimeout(r, 2000)); return `debugger;\n${context.body}`; } diff --git a/src/server/index.js b/src/server/index.js index 0c69be54..3462b1c5 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -1 +1 @@ -export { createConfig, getBrowsers } from './wtr-config.js'; +//export { getBrowsers } from './wtr-config.js'; diff --git a/src/server/visual-diff-plugin.js b/src/server/visual-diff-plugin.js index 7bf33e6d..1e9059ce 100644 --- a/src/server/visual-diff-plugin.js +++ b/src/server/visual-diff-plugin.js @@ -88,6 +88,7 @@ async function tryMoveFile(srcFileName, destFileName) { await rename(srcFileName, destFileName); return true; } catch (e) { + console.log('HEY!'); console.warn(e); return false; } diff --git a/src/server/visual-diff-reporter.js b/src/server/visual-diff-reporter.js index 10f81bc3..6c552f44 100644 --- a/src/server/visual-diff-reporter.js +++ b/src/server/visual-diff-reporter.js @@ -112,7 +112,7 @@ export function visualDiffReporter({ reportResults = true } = {}) { cpSync(inputDir, tempDir, { force: true, recursive: true }); writeFileSync(join(tempDir, 'data.js'), `export default ${json};`); - execSync(`rollup -c ${join(__dirname, './rollup.config.js')}`); + execSync(`npx rollup -c ${join(__dirname, './rollup.config.js')}`, { stdio: 'pipe' }); rmSync(tempDir, { recursive: true }); diff --git a/test/browser/wtr-vdiff.config.js b/test/browser/d2l-test.config.js similarity index 60% rename from test/browser/wtr-vdiff.config.js rename to test/browser/d2l-test.config.js index 1ee0a3d7..62d4bd85 100644 --- a/test/browser/wtr-vdiff.config.js +++ b/test/browser/d2l-test.config.js @@ -1,7 +1,4 @@ import { argv } from 'node:process'; -import { createConfig } from '../../src/server/wtr-config.js'; - -const pattern = type => `test/browser/**/*.${type}.js`; function getGoldenFlag() { return { @@ -13,8 +10,7 @@ function getGoldenFlag() { }; } -export default createConfig({ - pattern, - vdiff: true, +export default { + pattern: type => `test/browser/**/*.${type}.js`, plugins: [getGoldenFlag()] -}); +}; diff --git a/test/server/wtr-config.test.js b/test/server/wtr-config.test.js index 3f573d9b..8e6977b1 100644 --- a/test/server/wtr-config.test.js +++ b/test/server/wtr-config.test.js @@ -169,14 +169,14 @@ describe('createWtrConfig', () => { }); it('should use browsers passed as arguments', () => { - const browsers = wtrConfig.getBrowsers(['webkit']); + const browsers = wtrConfig.getBrowsers(['safari']); expect(browsers).to.have.length(1); expect(browsers[0].name).to.equal('Webkit'); }); it('should only use CLI browsers when provided', () => { - const wtrConfig = new WTRConfig({ firefox: true, webkit: true }); - const browsers = wtrConfig.getBrowsers(['chromium']); + const wtrConfig = new WTRConfig({ firefox: true, safari: true }); + const browsers = wtrConfig.getBrowsers(['chrome']); expect(browsers).to.have.length(2); expect(browsers.map(b => b.name)).to.have.members(['Webkit', 'Firefox']); }); From fde5e92e774a654f0d982e8fa7d07b8d1a593c10 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 13:54:29 -0400 Subject: [PATCH 02/30] Map browser names in getBrowsers --- bin/test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/test.js b/bin/test.js index 17b2d1c9..44bfb556 100755 --- a/bin/test.js +++ b/bin/test.js @@ -38,7 +38,7 @@ const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err //const DISALLOWED_ARGS = ['browsers', 'playwright', '_unknown']; const DEFAULT_PATTERN = type => `./test/**/*.${type}.js`; const ALLOWED_BROWSERS = ['chrome', 'firefox', 'safari']; -const BROWSER_MAP = { chrome: 'chromium', safari: 'webkit' }; +const BROWSER_MAP = { chrome: 'chromium', firefox: 'firefox', safari: 'webkit' }; export class WTRConfig { @@ -48,7 +48,7 @@ export class WTRConfig { constructor(cliArgs) { this.#cliArgs = cliArgs || {}; this.#cliArgs.group ??= 'unit'; - const requestedBrowsers = ALLOWED_BROWSERS.filter(b => cliArgs?.[b]).map(b => BROWSER_MAP[b] || b); + const requestedBrowsers = ALLOWED_BROWSERS.filter(b => cliArgs?.[b]); this.#requestedBrowsers = requestedBrowsers.length && requestedBrowsers; } @@ -218,7 +218,7 @@ export class WTRConfig { return browsers.map((b) => playwrightLauncher({ concurrency: b === 'firefox' ? 1 : undefined, // focus in Firefox unreliable if concurrency > 1 (https://github.com/modernweb-dev/web/issues/238) - product: b, + product: BROWSER_MAP[b], createBrowserContext: ({ browser }) => browser.newContext({ deviceScaleFactor: 2, reducedMotion: 'reduce' }) })); } From 0b8bb58c0ecbd96a1b1d0435d8a27c0fc0c483a6 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 13:55:19 -0400 Subject: [PATCH 03/30] Cleanup --- bin/test.js | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/bin/test.js b/bin/test.js index 44bfb556..5c60021e 100755 --- a/bin/test.js +++ b/bin/test.js @@ -165,7 +165,7 @@ export class WTRConfig { timeout, ...passthroughConfig } = {}) { - const { files, filter, golden, grep, group, manual, playwright, watch } = this.#cliArgs; + const { files, filter, golden, grep, group, manual, watch } = this.#cliArgs; delete passthroughConfig.browsers; @@ -225,20 +225,6 @@ export class WTRConfig { } -/* -function createConfig(...args) { - const wtrConfig = new WTRConfig(cliArgs); - return wtrConfig.create(...args); -} -*/ - -/* -export function getBrowsers(browsers) { - const wtrConfig = new WTRConfig(cliArgs); - return wtrConfig.getBrowsers(browsers); -} -*/ - const wtrConfig = new WTRConfig(cliArgs); const config = wtrConfig.create(testConfig); From ba6172addcddabeb90247738baec76ce60aa493b Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 14:17:26 -0400 Subject: [PATCH 04/30] Fix option handling --- bin/test.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/bin/test.js b/bin/test.js index 5c60021e..78309b65 100755 --- a/bin/test.js +++ b/bin/test.js @@ -10,10 +10,10 @@ import { visualDiffReporter } from '../src/server/visual-diff-reporter.js'; const optionDefinitions = [ // @web/test-runner options + { name: 'config', type: String }, { name: 'files', type: String, multiple: true }, { name: 'group', type: String, defaultOption: true }, { name: 'manual', type: Boolean }, - { name: 'config', type: String }, { name: 'watch', type: Boolean }, // custom options { name: 'chrome', type: Boolean }, @@ -21,8 +21,8 @@ const optionDefinitions = [ { name: 'firefox', type: Boolean }, { name: 'golden', type: Boolean }, { name: 'grep', alias: 'g', type: String }, - { name: 'timeout', type: Number }, { name: 'safari', type: Boolean }, + { name: 'timeout', type: Number }, ]; const cliArgs = commandLineArgs(optionDefinitions, { partial: true }); @@ -228,9 +228,16 @@ export class WTRConfig { const wtrConfig = new WTRConfig(cliArgs); const config = wtrConfig.create(testConfig); +const argv = [ + '--group', cliArgs.group, + ...(cliArgs._unknown || []) +]; +// copy cli-only wtr options back to argv to be processed +cliArgs.watch && argv.push('--watch'); +cliArgs.manual && argv.push('--manual'); + await startTestRunner({ - readCliArgs: true, - argv: [ '--group', cliArgs.group, ...(cliArgs._unknown || []) ], - readFileConfig: false, - config + argv, + config, + readFileConfig: false }); From 50274ed3fd7fef0391fbf1ebb5b120ee86101751 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 16:18:45 -0400 Subject: [PATCH 05/30] Move WTRConfig class back out of test binary --- bin/test.js | 198 +-------------------------------------- src/server/wtr-config.js | 195 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+), 196 deletions(-) create mode 100644 src/server/wtr-config.js diff --git a/bin/test.js b/bin/test.js index 78309b65..1e6ad5fa 100755 --- a/bin/test.js +++ b/bin/test.js @@ -1,12 +1,8 @@ #!/usr/bin/env node - import { ConfigLoaderError, readConfig } from '@web/config-loader'; -import { defaultReporter, startTestRunner } from '@web/test-runner'; import commandLineArgs from 'command-line-args'; -import { headedMode } from '../src/server/headed-mode-plugin.js'; -import { playwrightLauncher } from '@web/test-runner-playwright'; -import { visualDiff } from '../src/server/visual-diff-plugin.js'; -import { visualDiffReporter } from '../src/server/visual-diff-reporter.js'; +import { startTestRunner } from '@web/test-runner'; +import { WTRConfig } from '../src/server/wtr-config.js'; const optionDefinitions = [ // @web/test-runner options @@ -35,196 +31,6 @@ const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err } }) || {}; -//const DISALLOWED_ARGS = ['browsers', 'playwright', '_unknown']; -const DEFAULT_PATTERN = type => `./test/**/*.${type}.js`; -const ALLOWED_BROWSERS = ['chrome', 'firefox', 'safari']; -const BROWSER_MAP = { chrome: 'chromium', firefox: 'firefox', safari: 'webkit' }; - -export class WTRConfig { - - #cliArgs; - #requestedBrowsers; - - constructor(cliArgs) { - this.#cliArgs = cliArgs || {}; - this.#cliArgs.group ??= 'unit'; - const requestedBrowsers = ALLOWED_BROWSERS.filter(b => cliArgs?.[b]); - this.#requestedBrowsers = requestedBrowsers.length && requestedBrowsers; - } - - get #defaultConfig() { - return { - nodeResolve: true, - testRunnerHtml: testFramework => - ` - - - - - - `, - }; - } - - get visualDiffGroup() { - return { - name: 'vdiff', - files: this.#getPattern('vdiff'), - browsers: ['chrome'], - testRunnerHtml: testFramework => - ` - - - - - - - - - - ` - }; - } - - #getMochaConfig(timeoutConfig) { - const { - timeout = timeoutConfig, - grep, - watch, - manual - } = this.#cliArgs; - - if (typeof timeout !== 'undefined' && typeof timeout !== 'number') throw new TypeError('timeout must be a number'); - - const config = {}; - - if (timeout) config.timeout = String(timeout); - if (watch || manual) config.timeout = '0'; - if (grep) config.grep = grep; - - return Object.keys(config).length && { testFramework: { config } }; - } - - #getPattern(type) { - const pattern = [].concat(this.#cliArgs.files || this.pattern(type)); - - // replace filename wildcards with all filter strings - // e.g. If filter is ['button', 'list'], pattern './test/*.test.*' becomes: - // [ './test/+(*button*.test.*|*.test.*button*)', './test/+(*list*.test.*|*.test.*list*)' ] - if (this.#cliArgs.filter) { - return this.#cliArgs.filter.map(filterStr => { - // replace everything after the last forward slash - return pattern.map(p => p.replace(/[^/]*$/, fileGlob => { - // create a new glob for each wildcard - const fileGlobs = Array.from(fileGlob.matchAll(/(? { - const arr = fileGlob.split(''); - arr.splice(index, 1, filterStr); - return arr.join(''); - }); - return `+(${fileGlobs.join('|') || fileGlob})`; - })); - }).flat(); - } - return pattern; - } - - create({ - pattern = DEFAULT_PATTERN, - timeout, - ...passthroughConfig - } = {}) { - const { files, filter, golden, grep, group, manual, watch } = this.#cliArgs; - - delete passthroughConfig.browsers; - - if (typeof pattern !== 'function') throw new TypeError('pattern must be a function'); - this.pattern = pattern; - - const config = { - ...this.#defaultConfig, - ...this.#getMochaConfig(timeout), - ...passthroughConfig - }; - - config.groups ??= []; - config.groups.push({ - name: 'unit', - files: this.#getPattern('test') - }); - - if (group === 'vdiff') { - config.reporters ??= [ defaultReporter() ]; - config.reporters.push(visualDiffReporter({ reportResults: !golden })); - - config.plugins ??= []; - config.plugins.push(visualDiff({ updateGoldens: golden, runSubset: !!(filter || grep) })); - - config.groups.push(this.visualDiffGroup); - } - - // convert all browsers to playwright - config.groups.forEach(g => g.browsers = this.getBrowsers(g.browsers)); - - if (watch || manual) { - config.plugins ??= []; - const currentPattern = files || config.groups.find(g => g.name === group)?.files || config.files; - - config.plugins.push(headedMode({ - pattern: currentPattern, - manual, - watch - })); - } - - return config; - } - - getBrowsers(browsers) { - browsers = this.#requestedBrowsers || browsers || ALLOWED_BROWSERS; - - if (!Array.isArray(browsers)) throw new TypeError('browsers must be an array'); - - return browsers.map((b) => playwrightLauncher({ - concurrency: b === 'firefox' ? 1 : undefined, // focus in Firefox unreliable if concurrency > 1 (https://github.com/modernweb-dev/web/issues/238) - product: BROWSER_MAP[b], - createBrowserContext: ({ browser }) => browser.newContext({ deviceScaleFactor: 2, reducedMotion: 'reduce' }) - })); - } - -} - const wtrConfig = new WTRConfig(cliArgs); const config = wtrConfig.create(testConfig); diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js new file mode 100644 index 00000000..44849382 --- /dev/null +++ b/src/server/wtr-config.js @@ -0,0 +1,195 @@ +import { defaultReporter } from '@web/test-runner'; +import { headedMode } from '../src/server/headed-mode-plugin.js'; +import { playwrightLauncher } from '@web/test-runner-playwright'; +import { visualDiff } from '../src/server/visual-diff-plugin.js'; +import { visualDiffReporter } from '../src/server/visual-diff-reporter.js'; + +//const DISALLOWED_ARGS = ['browsers', 'playwright', '_unknown']; +const DEFAULT_PATTERN = type => `./test/**/*.${type}.js`; +const ALLOWED_BROWSERS = ['chrome', 'firefox', 'safari']; +const BROWSER_MAP = { chrome: 'chromium', firefox: 'firefox', safari: 'webkit' }; + +export class WTRConfig { + + #cliArgs; + #requestedBrowsers; + + constructor(cliArgs) { + this.#cliArgs = cliArgs || {}; + this.#cliArgs.group ??= 'unit'; + const requestedBrowsers = ALLOWED_BROWSERS.filter(b => cliArgs?.[b]); + this.#requestedBrowsers = requestedBrowsers.length && requestedBrowsers; + } + + get #defaultConfig() { + return { + nodeResolve: true, + testRunnerHtml: testFramework => + ` + + + + + + `, + }; + } + + get visualDiffGroup() { + return { + name: 'vdiff', + files: this.#getPattern('vdiff'), + browsers: ['chrome'], + testRunnerHtml: testFramework => + ` + + + + + + + + + + ` + }; + } + + #getMochaConfig(timeoutConfig) { + const { + timeout = timeoutConfig, + grep, + watch, + manual + } = this.#cliArgs; + + if (typeof timeout !== 'undefined' && typeof timeout !== 'number') throw new TypeError('timeout must be a number'); + + const config = {}; + + if (timeout) config.timeout = String(timeout); + if (watch || manual) config.timeout = '0'; + if (grep) config.grep = grep; + + return Object.keys(config).length && { testFramework: { config } }; + } + + #getPattern(type) { + const pattern = [].concat(this.#cliArgs.files || this.pattern(type)); + + // replace filename wildcards with all filter strings + // e.g. If filter is ['button', 'list'], pattern './test/*.test.*' becomes: + // [ './test/+(*button*.test.*|*.test.*button*)', './test/+(*list*.test.*|*.test.*list*)' ] + if (this.#cliArgs.filter) { + return this.#cliArgs.filter.map(filterStr => { + // replace everything after the last forward slash + return pattern.map(p => p.replace(/[^/]*$/, fileGlob => { + // create a new glob for each wildcard + const fileGlobs = Array.from(fileGlob.matchAll(/(? { + const arr = fileGlob.split(''); + arr.splice(index, 1, filterStr); + return arr.join(''); + }); + return `+(${fileGlobs.join('|') || fileGlob})`; + })); + }).flat(); + } + return pattern; + } + + create({ + pattern = DEFAULT_PATTERN, + timeout, + ...passthroughConfig + } = {}) { + const { files, filter, golden, grep, group, manual, watch } = this.#cliArgs; + + delete passthroughConfig.browsers; + + if (typeof pattern !== 'function') throw new TypeError('pattern must be a function'); + this.pattern = pattern; + + const config = { + ...this.#defaultConfig, + ...this.#getMochaConfig(timeout), + ...passthroughConfig + }; + + config.groups ??= []; + config.groups.push({ + name: 'unit', + files: this.#getPattern('test') + }); + + if (group === 'vdiff') { + config.reporters ??= [ defaultReporter() ]; + config.reporters.push(visualDiffReporter({ reportResults: !golden })); + + config.plugins ??= []; + config.plugins.push(visualDiff({ updateGoldens: golden, runSubset: !!(filter || grep) })); + + config.groups.push(this.visualDiffGroup); + } + + // convert all browsers to playwright + config.groups.forEach(g => g.browsers = this.getBrowsers(g.browsers)); + + if (watch || manual) { + config.plugins ??= []; + const currentPattern = files || config.groups.find(g => g.name === group)?.files || config.files; + + config.plugins.push(headedMode({ + pattern: currentPattern, + manual, + watch + })); + } + + return config; + } + + getBrowsers(browsers) { + browsers = this.#requestedBrowsers || browsers || ALLOWED_BROWSERS; + + if (!Array.isArray(browsers)) throw new TypeError('browsers must be an array'); + + return browsers.map((b) => playwrightLauncher({ + concurrency: b === 'firefox' ? 1 : undefined, // focus in Firefox unreliable if concurrency > 1 (https://github.com/modernweb-dev/web/issues/238) + product: BROWSER_MAP[b], + createBrowserContext: ({ browser }) => browser.newContext({ deviceScaleFactor: 2, reducedMotion: 'reduce' }) + })); + } + +} From f9a1ab085e9e0fa31d72ca2d0fd1294ee66e89ec Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 16:19:20 -0400 Subject: [PATCH 06/30] Add option aliases --- bin/test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/test.js b/bin/test.js index 1e6ad5fa..df4244fc 100755 --- a/bin/test.js +++ b/bin/test.js @@ -6,19 +6,19 @@ import { WTRConfig } from '../src/server/wtr-config.js'; const optionDefinitions = [ // @web/test-runner options - { name: 'config', type: String }, { name: 'files', type: String, multiple: true }, { name: 'group', type: String, defaultOption: true }, { name: 'manual', type: Boolean }, { name: 'watch', type: Boolean }, // custom options { name: 'chrome', type: Boolean }, + { name: 'config', alias: 'c', type: String }, // disabled for wtr { name: 'filter', alias: 'f', type: String, multiple: true }, { name: 'firefox', type: Boolean }, { name: 'golden', type: Boolean }, { name: 'grep', alias: 'g', type: String }, { name: 'safari', type: Boolean }, - { name: 'timeout', type: Number }, + { name: 'timeout', alias: 't', type: Number }, ]; const cliArgs = commandLineArgs(optionDefinitions, { partial: true }); From 639cd8a9f5f0dbee2d050d0e05071765753bf542 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 16:28:24 -0400 Subject: [PATCH 07/30] Fix imports --- src/server/wtr-config.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index 44849382..a3f796ca 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -1,8 +1,8 @@ import { defaultReporter } from '@web/test-runner'; -import { headedMode } from '../src/server/headed-mode-plugin.js'; +import { headedMode } from './headed-mode-plugin.js'; import { playwrightLauncher } from '@web/test-runner-playwright'; -import { visualDiff } from '../src/server/visual-diff-plugin.js'; -import { visualDiffReporter } from '../src/server/visual-diff-reporter.js'; +import { visualDiff } from './visual-diff-plugin.js'; +import { visualDiffReporter } from './visual-diff-reporter.js'; //const DISALLOWED_ARGS = ['browsers', 'playwright', '_unknown']; const DEFAULT_PATTERN = type => `./test/**/*.${type}.js`; From f69e8315e5257b230467bfa4425fd5d4b05eb571 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 16:28:49 -0400 Subject: [PATCH 08/30] Delete server/index.js --- package.json | 3 +-- src/server/index.js | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 src/server/index.js diff --git a/package.json b/package.json index df042f61..b002a9a7 100644 --- a/package.json +++ b/package.json @@ -27,8 +27,7 @@ "sinon": "^15" }, "exports": { - ".": "./src/browser/index.js", - "./wtr-config.js": "./src/server/index.js" + ".": "./src/browser/index.js" }, "files": [ "/src" diff --git a/src/server/index.js b/src/server/index.js deleted file mode 100644 index 3462b1c5..00000000 --- a/src/server/index.js +++ /dev/null @@ -1 +0,0 @@ -//export { getBrowsers } from './wtr-config.js'; From 754e0fc574a831ba47de334561c6fa76b1954266 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 16:36:13 -0400 Subject: [PATCH 09/30] Remove HEY! --- src/server/visual-diff-plugin.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/visual-diff-plugin.js b/src/server/visual-diff-plugin.js index 1e9059ce..7bf33e6d 100644 --- a/src/server/visual-diff-plugin.js +++ b/src/server/visual-diff-plugin.js @@ -88,7 +88,6 @@ async function tryMoveFile(srcFileName, destFileName) { await rename(srcFileName, destFileName); return true; } catch (e) { - console.log('HEY!'); console.warn(e); return false; } From f098c0333038ad9f4d8e39c84f8769bbc019a37c Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 22:28:37 -0400 Subject: [PATCH 10/30] Install config-loader --- package-lock.json | 1 + package.json | 1 + 2 files changed, 2 insertions(+) diff --git a/package-lock.json b/package-lock.json index a8cb6db8..c6f9d05e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@open-wc/testing": "^3", "@rollup/plugin-node-resolve": "^15", + "@web/config-loader": "^0.2", "@web/rollup-plugin-html": "^2", "@web/test-runner": "^0.16", "@web/test-runner-commands": "^0.7", diff --git a/package.json b/package.json index b002a9a7..28d26935 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "dependencies": { "@rollup/plugin-node-resolve": "^15", "@open-wc/testing": "^3", + "@web/config-loader": "^0.2", "@web/rollup-plugin-html": "^2", "@web/test-runner": "^0.16", "@web/test-runner-commands": "^0.7", From 31fe6f8a74432e3cd7bdb5eef4dc77cb322d333b Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 22:33:00 -0400 Subject: [PATCH 11/30] Remove options that wtr shouldn't use --- bin/test.js | 4 ++++ src/server/wtr-config.js | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/test.js b/bin/test.js index df4244fc..07b105c8 100755 --- a/bin/test.js +++ b/bin/test.js @@ -4,6 +4,8 @@ import commandLineArgs from 'command-line-args'; import { startTestRunner } from '@web/test-runner'; import { WTRConfig } from '../src/server/wtr-config.js'; +const DISALLOWED_OPTIONS = ['--browsers', '--playwright', '--puppeteer', '--groups']; + const optionDefinitions = [ // @web/test-runner options { name: 'files', type: String, multiple: true }, @@ -23,6 +25,8 @@ const optionDefinitions = [ const cliArgs = commandLineArgs(optionDefinitions, { partial: true }); +cliArgs._unknown = cliArgs._unknown?.filter(o => !DISALLOWED_OPTIONS.includes(o)); + const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err => { if (err instanceof ConfigLoaderError) { throw new Error(err.message); diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index a3f796ca..214c2f7c 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -4,7 +4,6 @@ import { playwrightLauncher } from '@web/test-runner-playwright'; import { visualDiff } from './visual-diff-plugin.js'; import { visualDiffReporter } from './visual-diff-reporter.js'; -//const DISALLOWED_ARGS = ['browsers', 'playwright', '_unknown']; const DEFAULT_PATTERN = type => `./test/**/*.${type}.js`; const ALLOWED_BROWSERS = ['chrome', 'firefox', 'safari']; const BROWSER_MAP = { chrome: 'chromium', firefox: 'firefox', safari: 'webkit' }; From 719c7b5b420bd0640a1439d12ef6aeee55ee1739 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 22:34:03 -0400 Subject: [PATCH 12/30] Add --help --- bin/test.js | 22 ++++++++++++++++++++++ package-lock.json | 1 + package.json | 1 + 3 files changed, 24 insertions(+) diff --git a/bin/test.js b/bin/test.js index 07b105c8..d2e12d32 100755 --- a/bin/test.js +++ b/bin/test.js @@ -1,6 +1,8 @@ #!/usr/bin/env node import { ConfigLoaderError, readConfig } from '@web/config-loader'; import commandLineArgs from 'command-line-args'; +import commandLineUsage from 'command-line-usage'; +import process from 'node:process'; import { startTestRunner } from '@web/test-runner'; import { WTRConfig } from '../src/server/wtr-config.js'; @@ -19,12 +21,32 @@ const optionDefinitions = [ { name: 'firefox', type: Boolean }, { name: 'golden', type: Boolean }, { name: 'grep', alias: 'g', type: String }, + { name: 'help', alias: 'h', type: Boolean }, { name: 'safari', type: Boolean }, { name: 'timeout', alias: 't', type: Number }, ]; const cliArgs = commandLineArgs(optionDefinitions, { partial: true }); +if (cliArgs.help) { + const help = commandLineUsage([ + { + header: 'D2L Test', + content: 'Test runner for D2L components and applications' + }, + { + header: 'Usage', + content: 'd2l-test [options]', + }, + { + header: 'Options', + optionList: optionDefinitions + } + ]); + process.stdout.write(help); + process.exit(); +} + cliArgs._unknown = cliArgs._unknown?.filter(o => !DISALLOWED_OPTIONS.includes(o)); const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err => { diff --git a/package-lock.json b/package-lock.json index c6f9d05e..f65cfa05 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@web/test-runner-commands": "^0.7", "@web/test-runner-playwright": "^0.10", "command-line-args": "^5", + "command-line-usage": "^7", "glob": "^10", "lit": "^2", "page": "^1", diff --git a/package.json b/package.json index 28d26935..38170154 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@web/test-runner-commands": "^0.7", "@web/test-runner-playwright": "^0.10", "command-line-args": "^5", + "command-line-usage": "^7", "glob": "^10", "lit": "^2", "page": "^1", From 4e18fb1373add3f1d2bed929493cb86f2c266dc9 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 22:37:10 -0400 Subject: [PATCH 13/30] Add unit group when used --- src/server/wtr-config.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index 214c2f7c..ad196be2 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -22,6 +22,7 @@ export class WTRConfig { get #defaultConfig() { return { + groups: [], nodeResolve: true, testRunnerHtml: testFramework => ` @@ -146,13 +147,12 @@ export class WTRConfig { ...passthroughConfig }; - config.groups ??= []; - config.groups.push({ - name: 'unit', - files: this.#getPattern('test') - }); - - if (group === 'vdiff') { + if (group === 'unit') { + config.groups.push({ + name: 'unit', + files: this.#getPattern('test') + }); + } else if (group === 'vdiff') { config.reporters ??= [ defaultReporter() ]; config.reporters.push(visualDiffReporter({ reportResults: !golden })); From 71332f50f322b12dedfe2958ae7e4688e768f5c6 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Tue, 11 Jul 2023 22:37:31 -0400 Subject: [PATCH 14/30] Update filter comment --- src/server/wtr-config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index ad196be2..1c15e55e 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -110,8 +110,8 @@ export class WTRConfig { const pattern = [].concat(this.#cliArgs.files || this.pattern(type)); // replace filename wildcards with all filter strings - // e.g. If filter is ['button', 'list'], pattern './test/*.test.*' becomes: - // [ './test/+(*button*.test.*|*.test.*button*)', './test/+(*list*.test.*|*.test.*list*)' ] + // e.g. If filter is ['button', 'list*'], pattern './test/*.test.*' becomes: + // [ './test/+(button.test.*|*.test.button)', './test/+(list*.test.*|*.test.list*)' ] if (this.#cliArgs.filter) { return this.#cliArgs.filter.map(filterStr => { // replace everything after the last forward slash From 65a67c7d29d7aacde49437093af813efcdab409b Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 12 Jul 2023 00:49:08 -0400 Subject: [PATCH 15/30] Add --help details --- bin/test.js | 106 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 91 insertions(+), 15 deletions(-) diff --git a/bin/test.js b/bin/test.js index d2e12d32..cf379f58 100755 --- a/bin/test.js +++ b/bin/test.js @@ -10,20 +10,94 @@ const DISALLOWED_OPTIONS = ['--browsers', '--playwright', '--puppeteer', '--grou const optionDefinitions = [ // @web/test-runner options - { name: 'files', type: String, multiple: true }, - { name: 'group', type: String, defaultOption: true }, - { name: 'manual', type: Boolean }, - { name: 'watch', type: Boolean }, - // custom options - { name: 'chrome', type: Boolean }, - { name: 'config', alias: 'c', type: String }, // disabled for wtr - { name: 'filter', alias: 'f', type: String, multiple: true }, - { name: 'firefox', type: Boolean }, - { name: 'golden', type: Boolean }, - { name: 'grep', alias: 'g', type: String }, - { name: 'help', alias: 'h', type: Boolean }, - { name: 'safari', type: Boolean }, - { name: 'timeout', alias: 't', type: Number }, + { + name: 'files', + type: String, + multiple: true, + description: 'Test files to run. Path or glob.\n[Default: ./test/**/*..js]', + order: 8 + }, + { + name: 'group', + type: String, + required: true, + defaultOption: true, + description: 'Name of the group to run tests for\n[Default: unit]', + order: 1 + }, + { + name: 'manual', + type: Boolean, + description: 'Starts test runner in manual testing mode. Ignores browser options and prints manual testing URL.\n{underline Not compatible with automated browser interactions}\nConsider using --watch to debug in the browser instead', + order: 11 + }, + { + name: 'watch', + type: Boolean, + description: 'Reload tests on file changes. Allows debugging in all browsers.', + order: 9 + }, + + // d2l-test options + { + name: 'chrome', + type: Boolean, + description: 'Run tests in Chromium', + order: 2 + }, + { + name: 'config', + alias: 'c', + type: String, + description: 'Location to read config file from\n[Default: ./d2l-test.config.js]', + order: 9 + }, + { + name: 'filter', + alias: 'f', + type: String, + multiple: true, + description: 'Filter test files by replacing wildcards with this glob', + order: 6 + }, + { + name: 'firefox', + type: Boolean, + description: 'Run tests in Firefox', + order: 3 + }, + { + name: 'golden', + type: Boolean, + description: 'Generate new golden screenshots', + order: 10 + }, + { + name: 'grep', + alias: 'g', + type: String, + description: 'Only run tests matching this string or regexp', + order: 7 + }, + { + name: 'help', + type: Boolean, + description: 'Print usage information and exit', + order: 12 + }, + { + name: 'safari', + type: Boolean, + description: 'Run tests in Webkit', + order: 4 + }, + { + name: 'timeout', + alias: 't', + type: Number, + description: 'Test timeout threshold in ms\n[Default: 2000]', + order: 5 + }, ]; const cliArgs = commandLineArgs(optionDefinitions, { partial: true }); @@ -36,11 +110,13 @@ if (cliArgs.help) { }, { header: 'Usage', - content: 'd2l-test [options]', + content: 'd2l-test [options]', }, { header: 'Options', optionList: optionDefinitions + .map(o => (o.description += '\n') && o) + .sort((a, b) => (a.order > b.order ? 1 : -1)) } ]); process.stdout.write(help); From b4ff12661aafa79053f68653c72818e752267d79 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 12 Jul 2023 00:49:39 -0400 Subject: [PATCH 16/30] Use proper node imports --- src/server/visual-diff-reporter.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/visual-diff-reporter.js b/src/server/visual-diff-reporter.js index 6c552f44..9f397199 100644 --- a/src/server/visual-diff-reporter.js +++ b/src/server/visual-diff-reporter.js @@ -1,8 +1,8 @@ -import { cpSync, mkdirSync, rmSync, writeFileSync } from 'fs'; -import { dirname, join } from 'path'; +import { cpSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; import { getTestInfo, PATHS } from './visual-diff-plugin.js'; -import { execSync } from 'child_process'; -import { fileURLToPath } from 'url'; +import { execSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); From e4719e9585e032bde5069429917e0a8897e3a916 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 12 Jul 2023 10:26:35 -0400 Subject: [PATCH 17/30] Add support for filtering custom group files --- src/server/wtr-config.js | 42 ++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index 1c15e55e..c8513dff 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -87,6 +87,21 @@ export class WTRConfig { }; } + #filterFiles(files) { + return this.#cliArgs.filter.map(filterStr => { + // replace everything after the last forward slash + return files.map(f => f.replace(/[^/]*$/, fileGlob => { + // create a new glob for each wildcard + const fileGlobs = Array.from(fileGlob.matchAll(/(? { + const arr = fileGlob.split(''); + arr.splice(index, 1, filterStr); + return arr.join(''); + }); + return `+(${fileGlobs.join('|') || fileGlob})`; + })); + }).flat(); + } + #getMochaConfig(timeoutConfig) { const { timeout = timeoutConfig, @@ -107,26 +122,13 @@ export class WTRConfig { } #getPattern(type) { - const pattern = [].concat(this.#cliArgs.files || this.pattern(type)); + const files = this.#cliArgs.files || [ this.pattern(type) ]; - // replace filename wildcards with all filter strings - // e.g. If filter is ['button', 'list*'], pattern './test/*.test.*' becomes: - // [ './test/+(button.test.*|*.test.button)', './test/+(list*.test.*|*.test.list*)' ] if (this.#cliArgs.filter) { - return this.#cliArgs.filter.map(filterStr => { - // replace everything after the last forward slash - return pattern.map(p => p.replace(/[^/]*$/, fileGlob => { - // create a new glob for each wildcard - const fileGlobs = Array.from(fileGlob.matchAll(/(? { - const arr = fileGlob.split(''); - arr.splice(index, 1, filterStr); - return arr.join(''); - }); - return `+(${fileGlobs.join('|') || fileGlob})`; - })); - }).flat(); + return this.#filterFiles(files); } - return pattern; + + return files; } create({ @@ -147,6 +149,12 @@ export class WTRConfig { ...passthroughConfig }; + if (filter) { + config.groups.forEach(group => { + group.files = this.#filterFiles([ group.files ].flat()); + }); + } + if (group === 'unit') { config.groups.push({ name: 'unit', From 14d85d28f02edd963511ff8a7815741c602b23fc Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 12 Jul 2023 21:15:16 -0400 Subject: [PATCH 18/30] unit -> test --- bin/test.js | 4 ++-- src/server/wtr-config.js | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/bin/test.js b/bin/test.js index cf379f58..041b4d51 100755 --- a/bin/test.js +++ b/bin/test.js @@ -14,7 +14,7 @@ const optionDefinitions = [ name: 'files', type: String, multiple: true, - description: 'Test files to run. Path or glob.\n[Default: ./test/**/*..js]', + description: 'Test files to run. Path or glob.\n[Default: ./test/**/*..js]', order: 8 }, { @@ -22,7 +22,7 @@ const optionDefinitions = [ type: String, required: true, defaultOption: true, - description: 'Name of the group to run tests for\n[Default: unit]', + description: 'Name of the group to run tests for\n[Default: test]', order: 1 }, { diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index c8513dff..c93d6c97 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -15,7 +15,7 @@ export class WTRConfig { constructor(cliArgs) { this.#cliArgs = cliArgs || {}; - this.#cliArgs.group ??= 'unit'; + this.#cliArgs.group ??= 'test'; const requestedBrowsers = ALLOWED_BROWSERS.filter(b => cliArgs?.[b]); this.#requestedBrowsers = requestedBrowsers.length && requestedBrowsers; } @@ -41,10 +41,20 @@ export class WTRConfig { }; } + get #pattern() { + const files = this.#cliArgs.files || [ this.pattern(this.#cliArgs.group) ]; + + if (this.#cliArgs.filter) { + return this.#filterFiles(files); + } + + return files; + } + get visualDiffGroup() { return { name: 'vdiff', - files: this.#getPattern('vdiff'), + files: this.#pattern, browsers: ['chrome'], testRunnerHtml: testFramework => ` @@ -121,16 +131,6 @@ export class WTRConfig { return Object.keys(config).length && { testFramework: { config } }; } - #getPattern(type) { - const files = this.#cliArgs.files || [ this.pattern(type) ]; - - if (this.#cliArgs.filter) { - return this.#filterFiles(files); - } - - return files; - } - create({ pattern = DEFAULT_PATTERN, timeout, @@ -155,10 +155,10 @@ export class WTRConfig { }); } - if (group === 'unit') { + if (group === 'test') { config.groups.push({ - name: 'unit', - files: this.#getPattern('test') + name: 'test', + files: this.#pattern }); } else if (group === 'vdiff') { config.reporters ??= [ defaultReporter() ]; @@ -175,7 +175,7 @@ export class WTRConfig { if (watch || manual) { config.plugins ??= []; - const currentPattern = files || config.groups.find(g => g.name === group)?.files || config.files; + const currentPattern = files || config.groups.find(g => g.name === group)?.files; config.plugins.push(headedMode({ pattern: currentPattern, From 1858d00cb1fd2cf28212224cf520d8a73e00f923 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 12 Jul 2023 21:27:32 -0400 Subject: [PATCH 19/30] Check for unknown group --- src/server/wtr-config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index c93d6c97..f7579437 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -138,6 +138,10 @@ export class WTRConfig { } = {}) { const { files, filter, golden, grep, group, manual, watch } = this.#cliArgs; + if (!['test', 'vdiff', ...passthroughConfig.groups.map(g => g.name)].includes(group)) { + return {}; // allow wtr to error + } + delete passthroughConfig.browsers; if (typeof pattern !== 'function') throw new TypeError('pattern must be a function'); From 0236ee448df08e810c9eca226f869f7ec1aae058 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 19 Jul 2023 07:39:54 -0400 Subject: [PATCH 20/30] Reorganize and fix/add tests --- bin/test.js | 152 +---------------------------- package.json | 7 +- src/server/cli/test.js | 159 +++++++++++++++++++++++++++++++ src/server/wtr-config.js | 4 +- test/{server => }/.eslintrc.json | 0 test/bin/test.test.js | 18 ++++ test/server/cli/test.test.js | 56 +++++++++++ test/server/wtr-config.test.js | 109 +++++++-------------- 8 files changed, 277 insertions(+), 228 deletions(-) create mode 100755 src/server/cli/test.js rename test/{server => }/.eslintrc.json (100%) create mode 100644 test/bin/test.test.js create mode 100644 test/server/cli/test.test.js diff --git a/bin/test.js b/bin/test.js index 041b4d51..b37aa2c5 100755 --- a/bin/test.js +++ b/bin/test.js @@ -1,151 +1,7 @@ #!/usr/bin/env node -import { ConfigLoaderError, readConfig } from '@web/config-loader'; -import commandLineArgs from 'command-line-args'; -import commandLineUsage from 'command-line-usage'; -import process from 'node:process'; -import { startTestRunner } from '@web/test-runner'; -import { WTRConfig } from '../src/server/wtr-config.js'; +import { argv } from 'node:process'; +import { runner } from '../src/server/cli/test.js'; -const DISALLOWED_OPTIONS = ['--browsers', '--playwright', '--puppeteer', '--groups']; +const options = await runner.getOptions(argv); -const optionDefinitions = [ - // @web/test-runner options - { - name: 'files', - type: String, - multiple: true, - description: 'Test files to run. Path or glob.\n[Default: ./test/**/*..js]', - order: 8 - }, - { - name: 'group', - type: String, - required: true, - defaultOption: true, - description: 'Name of the group to run tests for\n[Default: test]', - order: 1 - }, - { - name: 'manual', - type: Boolean, - description: 'Starts test runner in manual testing mode. Ignores browser options and prints manual testing URL.\n{underline Not compatible with automated browser interactions}\nConsider using --watch to debug in the browser instead', - order: 11 - }, - { - name: 'watch', - type: Boolean, - description: 'Reload tests on file changes. Allows debugging in all browsers.', - order: 9 - }, - - // d2l-test options - { - name: 'chrome', - type: Boolean, - description: 'Run tests in Chromium', - order: 2 - }, - { - name: 'config', - alias: 'c', - type: String, - description: 'Location to read config file from\n[Default: ./d2l-test.config.js]', - order: 9 - }, - { - name: 'filter', - alias: 'f', - type: String, - multiple: true, - description: 'Filter test files by replacing wildcards with this glob', - order: 6 - }, - { - name: 'firefox', - type: Boolean, - description: 'Run tests in Firefox', - order: 3 - }, - { - name: 'golden', - type: Boolean, - description: 'Generate new golden screenshots', - order: 10 - }, - { - name: 'grep', - alias: 'g', - type: String, - description: 'Only run tests matching this string or regexp', - order: 7 - }, - { - name: 'help', - type: Boolean, - description: 'Print usage information and exit', - order: 12 - }, - { - name: 'safari', - type: Boolean, - description: 'Run tests in Webkit', - order: 4 - }, - { - name: 'timeout', - alias: 't', - type: Number, - description: 'Test timeout threshold in ms\n[Default: 2000]', - order: 5 - }, -]; - -const cliArgs = commandLineArgs(optionDefinitions, { partial: true }); - -if (cliArgs.help) { - const help = commandLineUsage([ - { - header: 'D2L Test', - content: 'Test runner for D2L components and applications' - }, - { - header: 'Usage', - content: 'd2l-test [options]', - }, - { - header: 'Options', - optionList: optionDefinitions - .map(o => (o.description += '\n') && o) - .sort((a, b) => (a.order > b.order ? 1 : -1)) - } - ]); - process.stdout.write(help); - process.exit(); -} - -cliArgs._unknown = cliArgs._unknown?.filter(o => !DISALLOWED_OPTIONS.includes(o)); - -const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err => { - if (err instanceof ConfigLoaderError) { - throw new Error(err.message); - } else { - throw err; - } -}) || {}; - -const wtrConfig = new WTRConfig(cliArgs); -const config = wtrConfig.create(testConfig); - -const argv = [ - '--group', cliArgs.group, - ...(cliArgs._unknown || []) -]; -// copy cli-only wtr options back to argv to be processed -cliArgs.watch && argv.push('--watch'); -cliArgs.manual && argv.push('--manual'); - -await startTestRunner({ - argv, - config, - readFileConfig: false -}); +await runner.start(options); diff --git a/package.json b/package.json index 38170154..dfa245ea 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,10 @@ "scripts": { "lint": "eslint . --ext .js", "start": "web-dev-server --root-dir ./.vdiff --open ./.report/", - "test": "npm run lint && npm run test:server && npm run test:browser", - "test:browser": "web-test-runner --files \"./test/browser/**/*.test.js\" --node-resolve --playwright", - "test:server": "mocha ./test/server/**/*.test.js", + "test": "npm run lint && npm run test:bin && npm run test:server && npm run test:browser", + "test:bin": "mocha './test/bin/**/*.test.js'", + "test:browser": "web-test-runner --files './test/browser/**/*.test.js' --node-resolve --playwright", + "test:server": "mocha './test/server/**/*.test.js'", "test:vdiff": "d2l-test --config ./test/browser/d2l-test.config.js --group vdiff", "test:vdiff:golden": "npm run test:vdiff -- --golden" }, diff --git a/src/server/cli/test.js b/src/server/cli/test.js new file mode 100755 index 00000000..91eb2171 --- /dev/null +++ b/src/server/cli/test.js @@ -0,0 +1,159 @@ +import { ConfigLoaderError, readConfig } from '@web/config-loader'; +import commandLineArgs from 'command-line-args'; +import commandLineUsage from 'command-line-usage'; +import process from 'node:process'; +import { startTestRunner } from '@web/test-runner'; +import { WTRConfig } from '../wtr-config.js'; + +async function getTestRunnerOptions(argv = []) { + + const DISALLOWED_OPTIONS = ['--browsers', '--playwright', '--puppeteer', '--groups']; + + const optionDefinitions = [ + // @web/test-runner options + { + name: 'files', + type: String, + multiple: true, + description: 'Test files to run. Path or glob.\n[Default: ./test/**/*..js]', + order: 8 + }, + { + name: 'group', + type: String, + required: true, + defaultOption: true, + description: 'Name of the group to run tests for\n[Default: test]', + order: 1 + }, + { + name: 'manual', + type: Boolean, + description: 'Starts test runner in manual testing mode. Ignores browser options and prints manual testing URL.\n{underline Not compatible with automated browser interactions}\nConsider using --watch to debug in the browser instead', + order: 11 + }, + { + name: 'watch', + type: Boolean, + description: 'Reload tests on file changes. Allows debugging in all browsers.', + order: 9 + }, + + // d2l-test options + { + name: 'chrome', + type: Boolean, + description: 'Run tests in Chromium', + order: 2 + }, + { + name: 'config', + alias: 'c', + type: String, + description: 'Location to read config file from\n[Default: ./d2l-test.config.js]', + order: 9 + }, + { + name: 'filter', + alias: 'f', + type: String, + multiple: true, + description: 'Filter test files by replacing wildcards with this glob', + order: 6 + }, + { + name: 'firefox', + type: Boolean, + description: 'Run tests in Firefox', + order: 3 + }, + { + name: 'golden', + type: Boolean, + description: 'Generate new golden screenshots', + order: 10 + }, + { + name: 'grep', + alias: 'g', + type: String, + description: 'Only run tests matching this string or regexp', + order: 7 + }, + { + name: 'help', + type: Boolean, + description: 'Print usage information and exit', + order: 12 + }, + { + name: 'safari', + type: Boolean, + description: 'Run tests in Webkit', + order: 4 + }, + { + name: 'timeout', + alias: 't', + type: Number, + description: 'Test timeout threshold in ms\n[Default: 2000]', + order: 5 + }, + ]; + + const cliArgs = commandLineArgs(optionDefinitions, { partial: true, argv }); + + if (cliArgs.help) { + const help = commandLineUsage([ + { + header: 'D2L Test', + content: 'Test runner for D2L components and applications' + }, + { + header: 'Usage', + content: 'd2l-test [options]', + }, + { + header: 'Options', + optionList: optionDefinitions + .map(o => (o.description += '\n') && o) + .filter(o => 'order' in o) + .sort((a, b) => (a.order > b.order ? 1 : -1)) + } + ]); + process.stdout.write(help); + process.exit(); + } + + cliArgs._unknown = cliArgs._unknown?.filter(o => !DISALLOWED_OPTIONS.includes(o)); + + const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err => { + if (err instanceof ConfigLoaderError) { + throw new Error(err.message); + } else { + throw err; + } + }) || {}; + + const wtrConfig = new WTRConfig(cliArgs); + const config = wtrConfig.create(testConfig); + + argv = [ + '--group', cliArgs.group, + ...(cliArgs._unknown || []) + ]; + // copy cli-only wtr options back to argv to be processed + cliArgs.watch && argv.push('--watch'); + cliArgs.manual && argv.push('--manual'); + + return { + argv, + config, + readFileConfig: false + }; +} + +export const runner = { + getOptions: getTestRunnerOptions, + start: startTestRunner +}; diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index f7579437..1e461ef4 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -136,9 +136,11 @@ export class WTRConfig { timeout, ...passthroughConfig } = {}) { + const { files, filter, golden, grep, group, manual, watch } = this.#cliArgs; + const passthroughGroupNames = passthroughConfig.groups?.map(g => g.name) ?? []; - if (!['test', 'vdiff', ...passthroughConfig.groups.map(g => g.name)].includes(group)) { + if (!['test', 'vdiff', ...passthroughGroupNames].includes(group)) { return {}; // allow wtr to error } diff --git a/test/server/.eslintrc.json b/test/.eslintrc.json similarity index 100% rename from test/server/.eslintrc.json rename to test/.eslintrc.json diff --git a/test/bin/test.test.js b/test/bin/test.test.js new file mode 100644 index 00000000..9cbaddf3 --- /dev/null +++ b/test/bin/test.test.js @@ -0,0 +1,18 @@ +import { assert, restore, stub } from 'sinon'; +import { argv } from 'node:process'; +import { runner } from '../../src/server/cli/test.js'; + +describe('d2l-test', () => { + + it('starts test runner with options', async() => { + const opts = { my: 'options' }; + const optionsStub = stub(runner, 'getOptions').returns(opts); + const startStub = stub(runner, 'start'); + await import('../../bin/test.js'); + + assert.calledOnceWithExactly(optionsStub, argv); + assert.calledOnceWithExactly(startStub, opts); + + restore(); + }); +}); diff --git a/test/server/cli/test.test.js b/test/server/cli/test.test.js new file mode 100644 index 00000000..3f851323 --- /dev/null +++ b/test/server/cli/test.test.js @@ -0,0 +1,56 @@ +import { expect } from 'chai'; +import { runner } from '../../../src/server/cli/test.js'; + +describe('runner.getOptions()', () => { + + it('should create default options correctly', async() => { + const opts = await runner.getOptions(); + expect(opts).to.deep.include({ + argv: ['--group', 'test'], + readFileConfig: false, + }); + expect(opts.config).to.deep.include({ nodeResolve: true }); + expect(opts.config).to.have.property('testRunnerHtml').that.is.a('function'); + expect(opts.config.groups).to.be.an('array').with.length(1); + expect(opts.config.groups[0]).to.deep.include({ + name: 'test', + files: ['./test/**/*.test.js'] + }); + }); + + it('should use --group for the default option', async() => { + const opts = await runner.getOptions(['group-name']); + expect(opts.argv).to.deep.equal(['--group', 'group-name']); + }); + + it('should not forward disallowed or stolen options', async() => { + const disallowed = ['--browsers', '--playwright', '--puppeteer', '--groups']; + const stolen = ['--config']; + const opts = await runner.getOptions([ ...disallowed, ...stolen, '--unknown-allowed']); + expect(opts.argv).to.deep.equal(['--group', 'test', '--unknown-allowed']); + }); + + it('should forward borrowed options', async() => { + const borrowed = ['--watch', '--manual']; + const opts = await runner.getOptions(borrowed); + expect(opts.argv).to.deep.equal(['--group', 'test', '--watch', '--manual']); + }); + + it('should convert aliases', async() => { + const opts = await runner.getOptions([ + '-c', './test/browser/d2l-test.config.js', + '-f', 'abc', + '-g', 'ghi', + '-t', 123] + ); + expect(opts.argv).to.deep.equal(['--group', 'test']); + expect(opts.config.groups[0]).to.deep.include({ + name: 'test', + files: [ 'test/browser/**/+(abc.test.js)' ] + }); + expect(opts.config.testFramework).to.deep.include({ + config: { timeout: '123', grep: 'ghi' } + }); + }); + +}); diff --git a/test/server/wtr-config.test.js b/test/server/wtr-config.test.js index 8e6977b1..e238b5e7 100644 --- a/test/server/wtr-config.test.js +++ b/test/server/wtr-config.test.js @@ -1,38 +1,9 @@ -import { assert, match, restore, spy, stub } from 'sinon/pkg/sinon-esm.js'; -import { createConfig, getBrowsers, WTRConfig } from '../../src/server/wtr-config.js'; -import { createConfig as createConfigPublic, getBrowsers as getBrowsersPublic } from '../../src/server/index.js'; import { expect } from 'chai'; +import { WTRConfig } from '../../src/server/wtr-config.js'; -describe('createWtrConfig', () => { +describe('WTRConfig', () => { - afterEach(() => { - restore(); - }); - - describe('createConfig()', () => { - - it('should be exported from index', () => { - expect(createConfig).to.equal(createConfigPublic); - }); - - it('calls WTRConfig.create', () => { - const createStub = stub(WTRConfig.prototype, 'create').callsFake(num => num / 2); - const res = createConfig(10); - - assert.calledOnce(createStub); - assert.calledWith(createStub, 10); - expect(res).to.equal(5); - }); - }); - - describe('WTRConfig()', () => { - it('should not be exported from index', async() => { - const res = (await import('../../src/server/index.js')).WTRConfig; - expect(res).to.be.undefined; - }); - }); - - describe('WTRConfig.create()', () => { + describe('create()', () => { let config, wtrConfig; before(() => { @@ -40,27 +11,27 @@ describe('createWtrConfig', () => { config = wtrConfig.create(); }); - it('should warn about not using a group', () => { - const consoleSpy = spy(console, 'warn'); - wtrConfig.create(); - assert.calledOnce(consoleSpy); - assert.calledWith(consoleSpy, match('puppeteer')); + it('should return an empty object given invalid group', () => { + const wtrConfig = new WTRConfig({ group: 'default' }); + const config = wtrConfig.create(); + expect(config).to.deep.equal({}); }); - it('should warn about using the default group', () => { - const consoleSpy = spy(console, 'warn'); - const wtrConfig = new WTRConfig({ group: 'default' }); - wtrConfig.create(); - assert.calledOnce(consoleSpy); - assert.calledWith(consoleSpy, match('puppeteer')); + it('should remove browsers passthrough config', () => { + const config = wtrConfig.create({ browsers: [ 1, 2, 3 ] }); + expect(config).to.be.an('object'); + expect(config.browsers).to.be.undefined; }); - it('should warn about not using a group with playwright', () => { - const consoleSpy = spy(console, 'warn'); - const wtrConfig = new WTRConfig({ playwright: true }); - wtrConfig.create(); - assert.calledOnce(consoleSpy); - assert.calledWith(consoleSpy, match('Warning: reducedMotion disabled.')); + it('should convert passthrough group browsers to playwright', () => { + const wtrConfig = new WTRConfig({ group: 'a-group' }); + const config = wtrConfig.create({ groups: [{ name: 'a-group', browsers: ['chrome', 'firefox', 'safari', 'webkit'] }] }); + const group = config.groups[0]; + expect(config.groups).to.be.an('array').that.has.length(1); + expect(group.name).to.equal('a-group'); + expect(group.browsers).to.be.an('array').that.has.length(4); + expect(group.browsers.filter(b => b.name === 'Chromium')).to.have.length(2); + group.browsers.forEach(b => expect(b.constructor.name).to.equal('PlaywrightLauncher')); }); it('should enable nodeResolve', () => { @@ -94,17 +65,17 @@ describe('createWtrConfig', () => { }); it('should set a common default files pattern', () => { - expect(config.files).to.equal('./test/**/*.test.js'); + expect(config.groups[0].files).to.deep.equal(['./test/**/*.test.js']); }); it('should run a given pattern function to set files', () => { const config = wtrConfig.create({ pattern: type => `./a/b.${type}.js` }); - expect(config.files).to.equal('./a/b.test.js'); + expect(config.groups[0].files).to.deep.equal(['./a/b.test.js']); }); - it('should create a unit test group by default', () => { + it('should create a "test" group by default', () => { expect(config.groups).to.be.an('array'); - expect(config.groups.map(g => g.name)).to.have.members(['unit']); + expect(config.groups.map(g => g.name)).to.have.members(['test']); }); it('should not configure vdiff by default', () => { @@ -113,18 +84,20 @@ describe('createWtrConfig', () => { expect(config.groups.find(g => g.name === 'vdiff')).to.be.undefined; }); - it('should configure vdiff when enabled', () => { - const config = wtrConfig.create({ vdiff: true }); - expect(config.groups).to.be.an('array').that.has.length(2); + it('should configure vdiff when group is "vdiff"', () => { + const wtrConfig = new WTRConfig({ group: 'vdiff' }); + const config = wtrConfig.create(); + expect(config.groups).to.be.an('array').that.has.length(1); expect(config.plugins).to.be.an('array').that.has.length(1); - expect(config.groups[1]).to.have.property('name', 'vdiff'); + expect(config.groups[0]).to.have.property('name', 'vdiff'); }); it('should filter test files using --filter values', () => { - const wtrConfig = new WTRConfig({ filter: ['subset', 'subset2'] }); + const wtrConfig = new WTRConfig({ filter: ['subset', 'subset*'] }); const config = wtrConfig.create({ pattern: type => `./test/**/*/*.${type}.*` }); - expect(config.files).to.have.length(2); - expect(config.files).to.have.members(['./test/**/*/+(*subset*.test.*|*.test.*subset*)', './test/**/*/+(*subset2*.test.*|*.test.*subset2*)']); + expect(config.files).to.be.undefined; + expect(config.groups[0].files).to.have.length(2); + expect(config.groups[0].files).to.have.members(['./test/**/*/+(subset.test.*|*.test.subset)', './test/**/*/+(subset*.test.*|*.test.subset*)']); }); it('should add --grep value to testFramework config', () => { @@ -137,22 +110,6 @@ describe('createWtrConfig', () => { describe('getBrowsers()', () => { - it('should be exported from index', () => { - expect(getBrowsers).to.equal(getBrowsersPublic); - }); - - it('calls WTRConfig.getBrowsers', () => { - const getBrowsersStub = stub(WTRConfig.prototype, 'getBrowsers').callsFake(num => num / 2); - const res = getBrowsers(10); - - assert.calledOnce(getBrowsersStub); - assert.calledWith(getBrowsersStub, 10); - expect(res).to.equal(5); - }); - }); - - describe('WTRConfig.getBrowsers()', () => { - let browsers, wtrConfig; before(() => { wtrConfig = new WTRConfig(); From 92609c2a3d8c3c58cf6955c0b1f29bf63d787504 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 19 Jul 2023 08:16:48 -0400 Subject: [PATCH 21/30] Clarify --golden description --- src/server/cli/test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/cli/test.js b/src/server/cli/test.js index 91eb2171..fb2529eb 100755 --- a/src/server/cli/test.js +++ b/src/server/cli/test.js @@ -70,7 +70,7 @@ async function getTestRunnerOptions(argv = []) { { name: 'golden', type: Boolean, - description: 'Generate new golden screenshots', + description: 'Generate new golden screenshots. Ignored unless group is "vdiff"', order: 10 }, { From b2d4060f313e8b5af0d5ab218794f97593112da1 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 19 Jul 2023 08:20:25 -0400 Subject: [PATCH 22/30] Simplify pattern --- test/browser/d2l-test.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/browser/d2l-test.config.js b/test/browser/d2l-test.config.js index 62d4bd85..33c53411 100644 --- a/test/browser/d2l-test.config.js +++ b/test/browser/d2l-test.config.js @@ -11,6 +11,6 @@ function getGoldenFlag() { } export default { - pattern: type => `test/browser/**/*.${type}.js`, + pattern: () => `test/browser/**/*.vdiff.js`, plugins: [getGoldenFlag()] }; From 904434c7f963890139ad8888bb702d5cc93bf501 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 19 Jul 2023 08:35:26 -0400 Subject: [PATCH 23/30] Fix lint --- test/browser/d2l-test.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/browser/d2l-test.config.js b/test/browser/d2l-test.config.js index 33c53411..2bf09440 100644 --- a/test/browser/d2l-test.config.js +++ b/test/browser/d2l-test.config.js @@ -11,6 +11,6 @@ function getGoldenFlag() { } export default { - pattern: () => `test/browser/**/*.vdiff.js`, + pattern: () => 'test/browser/**/*.vdiff.js', plugins: [getGoldenFlag()] }; From 0a68bd0b2315ecf7c67d75c1a5b4b34bb589349c Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Wed, 19 Jul 2023 09:37:09 -0400 Subject: [PATCH 24/30] Fix test --- test/server/cli/test.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/server/cli/test.test.js b/test/server/cli/test.test.js index 3f851323..c28b6886 100644 --- a/test/server/cli/test.test.js +++ b/test/server/cli/test.test.js @@ -46,7 +46,7 @@ describe('runner.getOptions()', () => { expect(opts.argv).to.deep.equal(['--group', 'test']); expect(opts.config.groups[0]).to.deep.include({ name: 'test', - files: [ 'test/browser/**/+(abc.test.js)' ] + files: [ 'test/browser/**/+(abc.vdiff.js)' ] }); expect(opts.config.testFramework).to.deep.include({ config: { timeout: '123', grep: 'ghi' } From 7a87801f9a77fd5c3fa60c49e5d532a9761632a0 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 20 Jul 2023 14:11:40 -0400 Subject: [PATCH 25/30] Fix option descriptions --- src/server/cli/test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/cli/test.js b/src/server/cli/test.js index fb2529eb..a7f8ca41 100755 --- a/src/server/cli/test.js +++ b/src/server/cli/test.js @@ -29,7 +29,7 @@ async function getTestRunnerOptions(argv = []) { { name: 'manual', type: Boolean, - description: 'Starts test runner in manual testing mode. Ignores browser options and prints manual testing URL.\n{underline Not compatible with automated browser interactions}\nConsider using --watch to debug in the browser instead', + description: 'Starts test runner in manual testing mode. Ignores browser options and prints manual testing URL.\n{underline Not compatible with automated browser interactions}\nConsider using --watch to debug in the browser instead.', order: 11 }, { @@ -70,7 +70,7 @@ async function getTestRunnerOptions(argv = []) { { name: 'golden', type: Boolean, - description: 'Generate new golden screenshots. Ignored unless group is "vdiff"', + description: 'Generate new golden screenshots. Ignored unless group is "vdiff".', order: 10 }, { From a131243270726e6902d8c110b400b05972201945 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 20 Jul 2023 14:20:13 -0400 Subject: [PATCH 26/30] Rename things --- bin/{test.js => d2l-test-runner.js} | 2 +- package-lock.json | 2 +- package.json | 4 ++-- src/server/cli/{test.js => test-runner.js} | 10 +++++----- test/bin/{test.test.js => d2l-test-runner.test.js} | 6 +++--- test/browser/{d2l-test.config.js => vdiff.config.js} | 0 test/server/cli/{test.test.js => test-runner.test.js} | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) rename bin/{test.js => d2l-test-runner.js} (69%) rename src/server/cli/{test.js => test-runner.js} (94%) rename test/bin/{test.test.js => d2l-test-runner.test.js} (73%) rename test/browser/{d2l-test.config.js => vdiff.config.js} (100%) rename test/server/cli/{test.test.js => test-runner.test.js} (94%) diff --git a/bin/test.js b/bin/d2l-test-runner.js similarity index 69% rename from bin/test.js rename to bin/d2l-test-runner.js index b37aa2c5..c4d07e25 100755 --- a/bin/test.js +++ b/bin/d2l-test-runner.js @@ -1,6 +1,6 @@ #!/usr/bin/env node import { argv } from 'node:process'; -import { runner } from '../src/server/cli/test.js'; +import { runner } from '../src/server/cli/test-runner.js'; const options = await runner.getOptions(argv); diff --git a/package-lock.json b/package-lock.json index f65cfa05..69b2dcc1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,7 @@ "rollup": "^3" }, "bin": { - "d2l-test": "bin/test.js" + "d2l-test-runner": "bin/d2l-test-runner.js" }, "devDependencies": { "@web/dev-server": "^0.2", diff --git a/package.json b/package.json index dfa245ea..e12afd43 100644 --- a/package.json +++ b/package.json @@ -11,11 +11,11 @@ "test:bin": "mocha './test/bin/**/*.test.js'", "test:browser": "web-test-runner --files './test/browser/**/*.test.js' --node-resolve --playwright", "test:server": "mocha './test/server/**/*.test.js'", - "test:vdiff": "d2l-test --config ./test/browser/d2l-test.config.js --group vdiff", + "test:vdiff": "npx d2l-test-runner--config ./test/browser/d2l-test.config.js --group vdiff", "test:vdiff:golden": "npm run test:vdiff -- --golden" }, "bin": { - "d2l-test": "./bin/test.js" + "d2l-test-runner": "./bin/d2l-test-runner.js" }, "author": "D2L Corporation", "license": "Apache-2.0", diff --git a/src/server/cli/test.js b/src/server/cli/test-runner.js similarity index 94% rename from src/server/cli/test.js rename to src/server/cli/test-runner.js index a7f8ca41..79ea98a7 100755 --- a/src/server/cli/test.js +++ b/src/server/cli/test-runner.js @@ -10,7 +10,7 @@ async function getTestRunnerOptions(argv = []) { const DISALLOWED_OPTIONS = ['--browsers', '--playwright', '--puppeteer', '--groups']; const optionDefinitions = [ - // @web/test-runner options + // web-test-runner options { name: 'files', type: String, @@ -39,7 +39,7 @@ async function getTestRunnerOptions(argv = []) { order: 9 }, - // d2l-test options + // d2l-test-runner options { name: 'chrome', type: Boolean, @@ -50,7 +50,7 @@ async function getTestRunnerOptions(argv = []) { name: 'config', alias: 'c', type: String, - description: 'Location to read config file from\n[Default: ./d2l-test.config.js]', + description: 'Location to read config file from\n[Default: ./d2l-test-runner.config.js]', order: 9 }, { @@ -111,7 +111,7 @@ async function getTestRunnerOptions(argv = []) { }, { header: 'Usage', - content: 'd2l-test [options]', + content: 'd2l-test-runner [options]', }, { header: 'Options', @@ -127,7 +127,7 @@ async function getTestRunnerOptions(argv = []) { cliArgs._unknown = cliArgs._unknown?.filter(o => !DISALLOWED_OPTIONS.includes(o)); - const testConfig = await readConfig('d2l-test.config', cliArgs.config).catch(err => { + const testConfig = await readConfig('d2l-test-runner.config', cliArgs.config).catch(err => { if (err instanceof ConfigLoaderError) { throw new Error(err.message); } else { diff --git a/test/bin/test.test.js b/test/bin/d2l-test-runner.test.js similarity index 73% rename from test/bin/test.test.js rename to test/bin/d2l-test-runner.test.js index 9cbaddf3..425751ec 100644 --- a/test/bin/test.test.js +++ b/test/bin/d2l-test-runner.test.js @@ -1,14 +1,14 @@ import { assert, restore, stub } from 'sinon'; import { argv } from 'node:process'; -import { runner } from '../../src/server/cli/test.js'; +import { runner } from '../../src/server/cli/test-runner.js'; -describe('d2l-test', () => { +describe('d2l-test-runner', () => { it('starts test runner with options', async() => { const opts = { my: 'options' }; const optionsStub = stub(runner, 'getOptions').returns(opts); const startStub = stub(runner, 'start'); - await import('../../bin/test.js'); + await import('../../bin/d2l-test-runner.js'); assert.calledOnceWithExactly(optionsStub, argv); assert.calledOnceWithExactly(startStub, opts); diff --git a/test/browser/d2l-test.config.js b/test/browser/vdiff.config.js similarity index 100% rename from test/browser/d2l-test.config.js rename to test/browser/vdiff.config.js diff --git a/test/server/cli/test.test.js b/test/server/cli/test-runner.test.js similarity index 94% rename from test/server/cli/test.test.js rename to test/server/cli/test-runner.test.js index c28b6886..37c36842 100644 --- a/test/server/cli/test.test.js +++ b/test/server/cli/test-runner.test.js @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { runner } from '../../../src/server/cli/test.js'; +import { runner } from '../../../src/server/cli/test-runner.js'; describe('runner.getOptions()', () => { @@ -38,7 +38,7 @@ describe('runner.getOptions()', () => { it('should convert aliases', async() => { const opts = await runner.getOptions([ - '-c', './test/browser/d2l-test.config.js', + '-c', './test/browser/vdiff.config.js', '-f', 'abc', '-g', 'ghi', '-t', 123] From b90739b0376ae0af1efa0843c3042a6a6ef7d736 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 20 Jul 2023 14:37:07 -0400 Subject: [PATCH 27/30] Fix test:vdiff script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e12afd43..c5e13bf4 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "test:bin": "mocha './test/bin/**/*.test.js'", "test:browser": "web-test-runner --files './test/browser/**/*.test.js' --node-resolve --playwright", "test:server": "mocha './test/server/**/*.test.js'", - "test:vdiff": "npx d2l-test-runner--config ./test/browser/d2l-test.config.js --group vdiff", + "test:vdiff": "npx d2l-test-runner --config ./test/browser/vdiff.config.js --group vdiff", "test:vdiff:golden": "npm run test:vdiff -- --golden" }, "bin": { From 674aab246ddb7b38106add3492a88dfcd9daf1a8 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 20 Jul 2023 15:39:45 -0400 Subject: [PATCH 28/30] Accept all browser names --- src/server/cli/test-runner.js | 25 ++++++++++++++++++++----- src/server/wtr-config.js | 10 ++++++++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/server/cli/test-runner.js b/src/server/cli/test-runner.js index 79ea98a7..f84e59f0 100755 --- a/src/server/cli/test-runner.js +++ b/src/server/cli/test-runner.js @@ -41,11 +41,16 @@ async function getTestRunnerOptions(argv = []) { // d2l-test-runner options { - name: 'chrome', + name: 'chromium', type: Boolean, description: 'Run tests in Chromium', order: 2 }, + { + name: 'chrome', + type: Boolean, + longAlias: 'chromium' + }, { name: 'config', alias: 'c', @@ -88,9 +93,8 @@ async function getTestRunnerOptions(argv = []) { }, { name: 'safari', - type: Boolean, - description: 'Run tests in Webkit', - order: 4 + longAlias: 'webkit', + type: Boolean }, { name: 'timeout', @@ -99,6 +103,12 @@ async function getTestRunnerOptions(argv = []) { description: 'Test timeout threshold in ms\n[Default: 2000]', order: 5 }, + { + name: 'webkit', + type: Boolean, + description: 'Run tests in Webkit', + order: 4 + }, ]; const cliArgs = commandLineArgs(optionDefinitions, { partial: true, argv }); @@ -116,7 +126,12 @@ async function getTestRunnerOptions(argv = []) { { header: 'Options', optionList: optionDefinitions - .map(o => (o.description += '\n') && o) + .map(o => { + o.description += '\n'; + const longAlias = optionDefinitions.find(clone => clone !== o && clone.longAlias === o.name)?.name; + if (longAlias) o.name += `, --${longAlias}`; + return o; + }) .filter(o => 'order' in o) .sort((a, b) => (a.order > b.order ? 1 : -1)) } diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index 1e461ef4..481199e0 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -5,8 +5,14 @@ import { visualDiff } from './visual-diff-plugin.js'; import { visualDiffReporter } from './visual-diff-reporter.js'; const DEFAULT_PATTERN = type => `./test/**/*.${type}.js`; -const ALLOWED_BROWSERS = ['chrome', 'firefox', 'safari']; -const BROWSER_MAP = { chrome: 'chromium', firefox: 'firefox', safari: 'webkit' }; +const ALLOWED_BROWSERS = ['chrome', 'chromium', 'firefox', 'safari', 'webkit']; +const BROWSER_MAP = { + chrome: 'chromium', + chromium: 'chromium', + firefox: 'firefox', + safari: 'webkit', + webkit: 'webkit' +}; export class WTRConfig { From b972f3d59f5029eba89f501f9249e8ad14102580 Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 20 Jul 2023 15:45:39 -0400 Subject: [PATCH 29/30] Fix browser tests --- src/server/wtr-config.js | 3 ++- test/server/wtr-config.test.js | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index 481199e0..963ed4b7 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -6,6 +6,7 @@ import { visualDiffReporter } from './visual-diff-reporter.js'; const DEFAULT_PATTERN = type => `./test/**/*.${type}.js`; const ALLOWED_BROWSERS = ['chrome', 'chromium', 'firefox', 'safari', 'webkit']; +const DEFAULT_BROWSERS = ['chromium', 'firefox', 'webkit']; const BROWSER_MAP = { chrome: 'chromium', chromium: 'chromium', @@ -200,7 +201,7 @@ export class WTRConfig { } getBrowsers(browsers) { - browsers = this.#requestedBrowsers || browsers || ALLOWED_BROWSERS; + browsers = this.#requestedBrowsers || browsers || DEFAULT_BROWSERS; if (!Array.isArray(browsers)) throw new TypeError('browsers must be an array'); diff --git a/test/server/wtr-config.test.js b/test/server/wtr-config.test.js index e238b5e7..332a02fc 100644 --- a/test/server/wtr-config.test.js +++ b/test/server/wtr-config.test.js @@ -25,12 +25,12 @@ describe('WTRConfig', () => { it('should convert passthrough group browsers to playwright', () => { const wtrConfig = new WTRConfig({ group: 'a-group' }); - const config = wtrConfig.create({ groups: [{ name: 'a-group', browsers: ['chrome', 'firefox', 'safari', 'webkit'] }] }); + const config = wtrConfig.create({ groups: [{ name: 'a-group', browsers: ['chromium', 'chrome', 'firefox', 'gecko', 'safari', 'webkit'] }] }); const group = config.groups[0]; expect(config.groups).to.be.an('array').that.has.length(1); expect(group.name).to.equal('a-group'); - expect(group.browsers).to.be.an('array').that.has.length(4); - expect(group.browsers.filter(b => b.name === 'Chromium')).to.have.length(2); + expect(group.browsers).to.be.an('array').that.has.length(6); + expect(group.browsers.filter(b => b.name === 'Chromium')).to.have.length(3); group.browsers.forEach(b => expect(b.constructor.name).to.equal('PlaywrightLauncher')); }); From c2784fbba4b3dbadae61d787054a0ea5331dcacd Mon Sep 17 00:00:00 2001 From: Danny Gleckler Date: Thu, 20 Jul 2023 16:58:42 -0400 Subject: [PATCH 30/30] Dedupe browsers --- src/server/wtr-config.js | 6 +++--- test/server/wtr-config.test.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/wtr-config.js b/src/server/wtr-config.js index 963ed4b7..f25315c8 100644 --- a/src/server/wtr-config.js +++ b/src/server/wtr-config.js @@ -201,13 +201,13 @@ export class WTRConfig { } getBrowsers(browsers) { - browsers = this.#requestedBrowsers || browsers || DEFAULT_BROWSERS; + browsers = (this.#requestedBrowsers || browsers || DEFAULT_BROWSERS).map(b => BROWSER_MAP[b] || 'chromium'); if (!Array.isArray(browsers)) throw new TypeError('browsers must be an array'); - return browsers.map((b) => playwrightLauncher({ + return [...new Set(browsers)].map((b) => playwrightLauncher({ concurrency: b === 'firefox' ? 1 : undefined, // focus in Firefox unreliable if concurrency > 1 (https://github.com/modernweb-dev/web/issues/238) - product: BROWSER_MAP[b], + product: b, createBrowserContext: ({ browser }) => browser.newContext({ deviceScaleFactor: 2, reducedMotion: 'reduce' }) })); } diff --git a/test/server/wtr-config.test.js b/test/server/wtr-config.test.js index 332a02fc..532442f8 100644 --- a/test/server/wtr-config.test.js +++ b/test/server/wtr-config.test.js @@ -29,8 +29,8 @@ describe('WTRConfig', () => { const group = config.groups[0]; expect(config.groups).to.be.an('array').that.has.length(1); expect(group.name).to.equal('a-group'); - expect(group.browsers).to.be.an('array').that.has.length(6); - expect(group.browsers.filter(b => b.name === 'Chromium')).to.have.length(3); + expect(group.browsers).to.be.an('array').that.has.length(3); + expect(group.browsers.filter(b => b.name === 'Chromium')).to.have.length(1); group.browsers.forEach(b => expect(b.constructor.name).to.equal('PlaywrightLauncher')); });