diff --git a/package-lock.json b/package-lock.json index 77db4b79..489638c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,9 @@ "@web/test-runner-commands": "^0.7", "@web/test-runner-playwright": "^0.10", "command-line-args": "^5", - "glob": "^10" + "glob": "^10", + "pixelmatch": "^5", + "pngjs": "^7" }, "devDependencies": { "chai": "^4", @@ -5618,6 +5620,25 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pixelmatch": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.3.0.tgz", + "integrity": "sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==", + "dependencies": { + "pngjs": "^6.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, + "node_modules/pixelmatch/node_modules/pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "engines": { + "node": ">=12.13.0" + } + }, "node_modules/playwright": { "version": "1.35.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.35.1.tgz", @@ -5644,6 +5665,14 @@ "node": ">=16" } }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "engines": { + "node": ">=14.19.0" + } + }, "node_modules/portfinder": { "version": "1.0.32", "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.32.tgz", diff --git a/package.json b/package.json index bccc1d73..40959528 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,8 @@ "@web/test-runner-commands": "^0.7", "@web/test-runner-playwright": "^0.10", "command-line-args": "^5", - "glob": "^10" + "glob": "^10", + "pixelmatch": "^5", + "pngjs": "^7" } } diff --git a/src/browser/vdiff.js b/src/browser/vdiff.js index f86fbdfc..ca704a55 100644 --- a/src/browser/vdiff.js +++ b/src/browser/vdiff.js @@ -17,8 +17,12 @@ mocha.setup({ // eslint-disable-line no-undef async function ScreenshotAndCompare(opts) { const name = this.test.fullTitle(); const rect = this.elem.getBoundingClientRect(); - const { pass, message } = await executeServerCommand('brightspace-visual-diff', { name, rect, opts }); - if (!pass) { - expect.fail(message); + let result = await executeServerCommand('brightspace-visual-diff-compare', { name, rect, opts }); + if (result.resizeRequired) { + this.test.timeout(0); + result = await executeServerCommand('brightspace-visual-diff-compare-resize', { name }); + } + if (!result.pass) { + expect.fail(result.message); } } diff --git a/src/server/visual-diff-plugin.js b/src/server/visual-diff-plugin.js index 24467a49..639ed288 100644 --- a/src/server/visual-diff-plugin.js +++ b/src/server/visual-diff-plugin.js @@ -1,9 +1,12 @@ -import { access, constants, mkdir, readdir, rename, rm, stat } from 'node:fs/promises'; +import { access, constants, mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { basename, dirname, join } from 'node:path'; import { env } from 'node:process'; +import pixelmatch from 'pixelmatch'; +import { PNG } from 'pngjs'; const isCI = !!env['CI']; const DEFAULT_MARGIN = 10; +const DEFAULT_TOLERANCE = 0; // TODO: Support tolerance override? const PATHS = { FAIL: 'fail', GOLDEN: 'golden', @@ -56,6 +59,31 @@ async function clearDiffPaths(dir) { } } +async function createComparisonPNGs(original, newSize) { + const resizedPNGs = []; + [ + { name: 'top', coord: 0 }, + { name: 'center', coord: Math.floor((newSize.height - original.height) / 2) }, + { name: 'bottom', coord: newSize.height - original.height } + ].forEach(y => { + [ + { name: 'left', coord: 0 }, + { name: 'center', coord: Math.floor((newSize.width - original.width) / 2) }, + { name: 'right', coord: newSize.width - original.width } + ].forEach(x => { // TODO: position added for reports, remove/adjust as needed + if (original.width === newSize.width && original.height === newSize.height) { + resizedPNGs.push({ png: original, position: `${y.name}-${x.name}` }); + } else { + const resized = new PNG(newSize); + PNG.bitblt(original, resized, 0, 0, original.width, original.height, x.coord, y.coord); + resizedPNGs.push({ png: resized, position: `${y.name}-${x.name}` }); + } + }); + }); + + return resizedPNGs; +} + async function tryMoveFile(srcFileName, destFileName) { await mkdir(dirname(destFileName), { recursive: true }); try { @@ -101,9 +129,6 @@ export function visualDiff({ updateGoldens = false, runSubset = false } = {}) { }, async executeCommand({ command, payload, session }) { - if (command !== 'brightspace-visual-diff') { - return; - } if (session.browser.type !== 'playwright') { throw new Error('Visual-diff is only supported for browser type Playwright.'); } @@ -114,56 +139,100 @@ export function visualDiff({ updateGoldens = false, runSubset = false } = {}) { const newPath = join(rootDir, PATHS.VDIFF_ROOT, testPath, dir); const goldenFileName = `${join(newPath, PATHS.GOLDEN, browser, newName)}.png`; const passFileName = `${join(newPath, PATHS.PASS, browser, newName)}.png`; - const screenshotFileName = `${join(newPath, PATHS.FAIL, browser, newName)}.png`; - - if (!isCI) { // CI will be a fresh .vdiff folder each time and only one run - if (session.testRun !== currentRun) { - currentRun = session.testRun; - clearedDirs.clear(); - } + const screenshotFile = join(newPath, PATHS.FAIL, browser, newName); + const screenshotFileName = `${screenshotFile}.png`; + + if (command === 'brightspace-visual-diff-compare') { + if (!isCI) { // CI will be a fresh .vdiff folder each time and only one run + if (session.testRun !== currentRun) { + currentRun = session.testRun; + clearedDirs.clear(); + } - if (runSubset || currentRun > 0) { - if (!clearedDirs.has(newPath)) { - clearedDirs.set(newPath, clearDir(updateGoldens, newPath)); + if (runSubset || currentRun > 0) { + if (!clearedDirs.has(newPath)) { + clearedDirs.set(newPath, clearDir(updateGoldens, newPath)); + } + await clearedDirs.get(newPath); } - await clearedDirs.get(newPath); } - } - const opts = { margin: DEFAULT_MARGIN, ...payload.opts }; - - const page = session.browser.getPage(session.id); - await page.screenshot({ - animations: 'disabled', - clip: { - x: payload.rect.x - opts.margin, - y: payload.rect.y - opts.margin, - width: payload.rect.width + (opts.margin * 2), - height: payload.rect.height + (opts.margin * 2) - }, - path: updateGoldens ? goldenFileName : screenshotFileName - }); - - if (updateGoldens) { - return { pass: true }; - } + const opts = { margin: DEFAULT_MARGIN, ...payload.opts }; + + const page = session.browser.getPage(session.id); + await page.screenshot({ + animations: 'disabled', + clip: { + x: payload.rect.x - opts.margin, + y: payload.rect.y - opts.margin, + width: payload.rect.width + (opts.margin * 2), + height: payload.rect.height + (opts.margin * 2) + }, + path: updateGoldens ? goldenFileName : screenshotFileName + }); + + if (updateGoldens) { + return { pass: true }; + } - const goldenExists = await checkFileExists(goldenFileName); - if (!goldenExists) { - return { pass: false, message: 'No golden exists. Use the "--golden" CLI flag to re-run and re-generate goldens.' }; - } + const goldenExists = await checkFileExists(goldenFileName); + if (!goldenExists) { + return { pass: false, message: 'No golden exists. Use the "--golden" CLI flag to re-run and re-generate goldens.' }; + } - const screenshotInfo = await stat(screenshotFileName); - const goldenInfo = await stat(goldenFileName); + const screenshotImage = PNG.sync.read(await readFile(screenshotFileName)); + const goldenImage = PNG.sync.read(await readFile(goldenFileName)); + + if (screenshotImage.width === goldenImage.width && screenshotImage.height === goldenImage.height) { + const diff = new PNG({ width: screenshotImage.width, height: screenshotImage.height }); + const pixelsDiff = pixelmatch( + screenshotImage.data, goldenImage.data, diff.data, screenshotImage.width, screenshotImage.height, { diffMask: true, threshold: DEFAULT_TOLERANCE } + ); + + if (pixelsDiff !== 0) { + await writeFile(`${screenshotFile}-diff.png`, PNG.sync.write(diff)); + return { pass: false, message: `Image does not match golden. ${pixelsDiff} pixels are different.` }; + } else { + const success = await tryMoveFile(screenshotFileName, passFileName); + if (!success) return { pass: false, message: 'Problem moving file to "pass" directory.' }; + return { pass: true }; + } + } else { + return { resizeRequired: true }; + } + } else if (command === 'brightspace-visual-diff-compare-resize') { + const screenshotImage = PNG.sync.read(await readFile(screenshotFileName)); + const goldenImage = PNG.sync.read(await readFile(goldenFileName)); + + const newWidth = Math.max(screenshotImage.width, goldenImage.width); + const newHeight = Math.max(screenshotImage.height, goldenImage.height); + const newSize = { width: newWidth, height: newHeight }; + + const newScreenshots = await createComparisonPNGs(screenshotImage, newSize); + const newGoldens = await createComparisonPNGs(goldenImage, newSize); + + let bestIndex = -1; + let bestDiffImage = null; + let pixelsDiff = Number.MAX_SAFE_INTEGER; + for (let i = 0; i < newScreenshots.length; i++) { + const currentDiff = new PNG(newSize); + const currentPixelsDiff = pixelmatch( + newScreenshots[i].png.data, newGoldens[i].png.data, currentDiff.data, currentDiff.width, currentDiff.height, { diffMask: true, threshold: DEFAULT_TOLERANCE } + ); + + if (currentPixelsDiff < pixelsDiff) { + bestIndex = i; + bestDiffImage = currentDiff; + pixelsDiff = currentPixelsDiff; + } + } - // TODO: obviously this isn't how to diff against the golden! Use pixelmatch here. - const same = (screenshotInfo.size === goldenInfo.size); + await writeFile(`${screenshotFile}-resized-screenshot.png`, PNG.sync.write(newScreenshots[bestIndex].png)); + await writeFile(`${screenshotFile}-resized-golden.png`, PNG.sync.write(newGoldens[bestIndex].png)); + await writeFile(`${screenshotFile}-diff.png`, PNG.sync.write(bestDiffImage)); - if (same) { - const success = await tryMoveFile(screenshotFileName, passFileName); - if (!success) return { pass: false, message: 'Problem moving file to pass directory.' }; + return { pass: false, message: `Images are not the same size. When resized, ${pixelsDiff} pixels are different.` }; } - return { pass: same, message: 'Does not match golden.' }; // TODO: Add more details once actually diff-ing } }; diff --git a/test/browser/element.vdiff.js b/test/browser/element.vdiff.js index df36cbbf..ecf20596 100644 --- a/test/browser/element.vdiff.js +++ b/test/browser/element.vdiff.js @@ -56,14 +56,24 @@ describe('element-different', () => { elem.style.borderColor = 'black'; elem.text = 'Different Text'; } }, - /*{ name: 'smaller', action: elem => { + { name: 'smaller', action: elem => { elem.style.width = '200px'; elem.style.height = '50px'; } }, { name: 'larger', action: elem => { elem.style.width = '350px'; elem.style.height = '70px'; - } }*/ + } }, + { name: 'slimer-taller', action: elem => { + elem.style.width = '200px'; + elem.style.height = '70px'; + elem.style.textAlign = 'end'; + } }, + { name: 'wider-shorter', action: elem => { + elem.style.width = '350px'; + elem.style.height = '50px'; + elem.style.textAlign = 'end'; + } } ].forEach(({ name, action }) => { it(name, async() => { const elem = await fixture(`<${elementTag} text="Visual Difference">`); diff --git a/test/browser/nested/element.vdiff.js b/test/browser/nested/element.vdiff.js new file mode 100644 index 00000000..8bc90f35 --- /dev/null +++ b/test/browser/nested/element.vdiff.js @@ -0,0 +1 @@ +import '../element.vdiff.js'; diff --git a/test/browser/wtr-vdiff.config.js b/test/browser/wtr-vdiff.config.js index a1a9adee..1ee0a3d7 100644 --- a/test/browser/wtr-vdiff.config.js +++ b/test/browser/wtr-vdiff.config.js @@ -1,7 +1,7 @@ import { argv } from 'node:process'; import { createConfig } from '../../src/server/wtr-config.js'; -const pattern = type => `test/browser/*.${type}.js`; +const pattern = type => `test/browser/**/*.${type}.js`; function getGoldenFlag() { return {