From 38bc49118aa51ea520e6b5afc6946692016361f9 Mon Sep 17 00:00:00 2001 From: Dave Lockhart Date: Mon, 12 Aug 2024 09:14:51 -0400 Subject: [PATCH] feat: separate out tests with byte differences in the report (#459) --- src/server/report/app.js | 13 +++++++++---- src/server/report/common.js | 3 ++- src/server/report/result.js | 2 +- src/server/visual-diff-reporter.js | 23 ++++++++++++++++++----- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/server/report/app.js b/src/server/report/app.js index 98ff9325..3ec20b12 100644 --- a/src/server/report/app.js +++ b/src/server/report/app.js @@ -121,7 +121,7 @@ class App extends LitElement { constructor() { super(); this._filterBrowsers = data.browsers.map(b => b.name); - this._filterStatus = data.numFailed > 0 ? FILTER_STATUS.FAILED : FILTER_STATUS.ALL; + this._filterStatus = data.numFailed > 0 ? FILTER_STATUS.FAILED : (data.numByteDiff > 0 ? FILTER_STATUS.BYTEDIFF : FILTER_STATUS.ALL); this._fullMode = FULL_MODE.GOLDEN.value; this._layout = LAYOUTS.SPLIT.value; this._overlay = true; @@ -146,7 +146,10 @@ class App extends LitElement { } if (searchParams.has('status')) { let filterStatus = searchParams.get('status'); - if (filterStatus === FILTER_STATUS.FAILED && data.numFailed === 0) filterStatus = FILTER_STATUS.ALL; + if (filterStatus === FILTER_STATUS.FAILED && data.numFailed === 0 || + filterStatus === FILTER_STATUS.BYTEDIFF && data.numByteDiff === 0) { + filterStatus = FILTER_STATUS.ALL; + } this._filterStatus = filterStatus; } if (searchParams.has('browsers')) { @@ -198,7 +201,8 @@ class App extends LitElement { const statusFilters = [ { name: FILTER_STATUS.FAILED, count: data.numFailed }, - { name: FILTER_STATUS.PASSED, count: data.numTests - data.numFailed }, + { name: FILTER_STATUS.BYTEDIFF, count: data.numByteDiff }, + { name: FILTER_STATUS.PASSED, count: data.numTests - data.numFailed - data.numByteDiff }, { name: FILTER_STATUS.ALL, count: data.numTests } ]; @@ -434,7 +438,8 @@ class App extends LitElement { if (this._filterBrowsers.includes(r.name) && (this._filterStatus === FILTER_STATUS.ALL || r.passed && this._filterStatus === FILTER_STATUS.PASSED || - !r.passed && this._filterStatus === FILTER_STATUS.FAILED)) numStatusMatch++; + r.bytediff && this._filterStatus === FILTER_STATUS.BYTEDIFF || + !r.bytediff && !r.passed && this._filterStatus === FILTER_STATUS.FAILED)) numStatusMatch++; }); if (numStatusMatch > 0) { if (lookingForNextTest) { diff --git a/src/server/report/common.js b/src/server/report/common.js index 23aeca93..d0d9722d 100644 --- a/src/server/report/common.js +++ b/src/server/report/common.js @@ -77,7 +77,8 @@ export const COMMON_STYLE = css` export const FILTER_STATUS = { ALL: 'All', PASSED: 'Passed', - FAILED: 'Failed' + FAILED: 'Failed', + BYTEDIFF: 'Byte Diff' }; export const FULL_MODE = { diff --git a/src/server/report/result.js b/src/server/report/result.js index f1245f94..ecdbf147 100644 --- a/src/server/report/result.js +++ b/src/server/report/result.js @@ -175,7 +175,7 @@ function renderResult(resultData, options) { let newPart; if (resultData.passed) { newPart = html`
${ICON_TADA}

Hooray! No changes here.

`; - } else if (!resultData.info.diff && resultData.info.pixelsDiff === 0) { + } else if (resultData.bytediff) { newPart = html`
${ICON_BYTES}

No pixels have changed, but the bytes are different.

diff --git a/src/server/visual-diff-reporter.js b/src/server/visual-diff-reporter.js index 27a05591..39eb998d 100644 --- a/src/server/visual-diff-reporter.js +++ b/src/server/visual-diff-reporter.js @@ -27,6 +27,7 @@ function createData(rootDir, updateGoldens, sessions) { const prevBrowser = metadata.browsers?.find(b => b.name === browserName); browsers.set(browserName, { name: browserName, + numByteDiff: 0, numFailed: 0, version: parseInt(s.browser.browser.version()), previousVersion: prevBrowser?.version @@ -36,14 +37,14 @@ function createData(rootDir, updateGoldens, sessions) { const fileName = s.testFile.substring(rootDir.length + 1); if (!files.has(fileName)) { - files.set(fileName, { name: fileName, numFailed: 0, tests: new Map() }); + files.set(fileName, { name: fileName, numByteDiff: 0, numFailed: 0, tests: new Map() }); } const fileData = files.get(fileName); flattenResults(s, browserData, fileData); }); - let numTests = 0, numFailed = 0; + let numTests = 0, numFailed = 0, numByteDiff = 0; files.forEach(f => { numTests += f.tests.size; f.tests.forEach(t => { @@ -51,6 +52,10 @@ function createData(rootDir, updateGoldens, sessions) { f.numFailed++; numFailed++; } + if (t.numByteDiff > 0) { + f.numByteDiff++; + numByteDiff++; + } }); }); @@ -61,7 +66,7 @@ function createData(rootDir, updateGoldens, sessions) { writeFileSync(metadataPath, `${JSON.stringify(metadata, undefined, '\t')}\n`); } - return { browsers, files, numFailed, numTests }; + return { browsers, files, numByteDiff, numFailed, numTests }; } @@ -79,20 +84,28 @@ function flattenResults(session, browserData, fileData) { if (!fileData.tests.has(testName)) { fileData.tests.set(testName, { name: testName, + numByteDiff: 0, numFailed: 0, results: [] }); } const testData = fileData.tests.get(testName); + const bytediff = !t.passed && !info.diff && info.pixelsDiff === 0; if (!t.passed) { - browserData.numFailed++; - testData.numFailed++; + if (bytediff) { + browserData.numByteDiff++; + testData.numByteDiff++; + } else { + browserData.numFailed++; + testData.numFailed++; + } } testData.results.push({ name: browserData.name, duration: t.duration, error: t.error?.message, passed: t.passed, + bytediff, info: info }); });