feat: separate out tests with byte differences in the report (#459)

BrightspaceUI · Aug 12, 2024 · 38bc491 · 38bc491
1 parent fecd2e1
commit 38bc491
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 11 deletions.
diff --git a/src/server/report/app.js b/src/server/report/app.js
@@ -121,7 +121,7 @@ class App extends LitElement {
 	constructor() {
 		super();
 		this._filterBrowsers = data.browsers.map(b => b.name);
-		this._filterStatus = data.numFailed > 0 ? FILTER_STATUS.FAILED : FILTER_STATUS.ALL;
+		this._filterStatus = data.numFailed > 0 ? FILTER_STATUS.FAILED : (data.numByteDiff > 0 ? FILTER_STATUS.BYTEDIFF : FILTER_STATUS.ALL);
 		this._fullMode = FULL_MODE.GOLDEN.value;
 		this._layout = LAYOUTS.SPLIT.value;
 		this._overlay = true;
@@ -146,7 +146,10 @@ class App extends LitElement {
 			}
 			if (searchParams.has('status')) {
 				let filterStatus = searchParams.get('status');
-				if (filterStatus === FILTER_STATUS.FAILED && data.numFailed === 0) filterStatus = FILTER_STATUS.ALL;
+				if (filterStatus === FILTER_STATUS.FAILED && data.numFailed === 0 ||
+					filterStatus === FILTER_STATUS.BYTEDIFF && data.numByteDiff === 0) {
+					filterStatus = FILTER_STATUS.ALL;
+				}
 				this._filterStatus = filterStatus;
 			}
 			if (searchParams.has('browsers')) {
@@ -198,7 +201,8 @@ class App extends LitElement {
 
 		const statusFilters = [
 			{ name: FILTER_STATUS.FAILED, count: data.numFailed },
-			{ name: FILTER_STATUS.PASSED, count: data.numTests - data.numFailed },
+			{ name: FILTER_STATUS.BYTEDIFF, count: data.numByteDiff },
+			{ name: FILTER_STATUS.PASSED, count: data.numTests - data.numFailed - data.numByteDiff },
 			{ name: FILTER_STATUS.ALL, count: data.numTests }
 		];
 
@@ -434,7 +438,8 @@ class App extends LitElement {
 					if (this._filterBrowsers.includes(r.name) &&
 						(this._filterStatus === FILTER_STATUS.ALL ||
 						r.passed && this._filterStatus === FILTER_STATUS.PASSED ||
-						!r.passed && this._filterStatus === FILTER_STATUS.FAILED)) numStatusMatch++;
+						r.bytediff && this._filterStatus === FILTER_STATUS.BYTEDIFF ||
+						!r.bytediff && !r.passed && this._filterStatus === FILTER_STATUS.FAILED)) numStatusMatch++;
 				});
 				if (numStatusMatch > 0) {
 					if (lookingForNextTest) {

diff --git a/src/server/report/common.js b/src/server/report/common.js
@@ -77,7 +77,8 @@ export const COMMON_STYLE = css`
 export const FILTER_STATUS = {
 	ALL: 'All',
 	PASSED: 'Passed',
-	FAILED: 'Failed'
+	FAILED: 'Failed',
+	BYTEDIFF: 'Byte Diff'
 };
 
 export const FULL_MODE = {

diff --git a/src/server/report/result.js b/src/server/report/result.js
@@ -175,7 +175,7 @@ function renderResult(resultData, options) {
 		let newPart;
 		if (resultData.passed) {
 			newPart = html`<div class="result-graphic padding">${ICON_TADA}<p>Hooray! No changes here.</p></div>`;
-		} else if (!resultData.info.diff && resultData.info.pixelsDiff === 0) {
+		} else if (resultData.bytediff) {
 			newPart = html`<div class="result-graphic padding">${ICON_BYTES}
 				<p>No pixels have changed, but the bytes are different.</p>
 				<p class="details">

diff --git a/src/server/visual-diff-reporter.js b/src/server/visual-diff-reporter.js
@@ -27,6 +27,7 @@ function createData(rootDir, updateGoldens, sessions) {
 			const prevBrowser = metadata.browsers?.find(b => b.name === browserName);
 			browsers.set(browserName, {
 				name: browserName,
+				numByteDiff: 0,
 				numFailed: 0,
 				version: parseInt(s.browser.browser.version()),
 				previousVersion: prevBrowser?.version
@@ -36,21 +37,25 @@ function createData(rootDir, updateGoldens, sessions) {
 
 		const fileName = s.testFile.substring(rootDir.length + 1);
 		if (!files.has(fileName)) {
-			files.set(fileName, { name: fileName, numFailed: 0, tests: new Map() });
+			files.set(fileName, { name: fileName, numByteDiff: 0, numFailed: 0, tests: new Map() });
 		}
 		const fileData = files.get(fileName);
 		flattenResults(s, browserData, fileData);
 
 	});
 
-	let numTests = 0, numFailed = 0;
+	let numTests = 0, numFailed = 0, numByteDiff = 0;
 	files.forEach(f => {
 		numTests += f.tests.size;
 		f.tests.forEach(t => {
 			if (t.numFailed > 0) {
 				f.numFailed++;
 				numFailed++;
 			}
+			if (t.numByteDiff > 0) {
+				f.numByteDiff++;
+				numByteDiff++;
+			}
 		});
 	});
 
@@ -61,7 +66,7 @@ function createData(rootDir, updateGoldens, sessions) {
 		writeFileSync(metadataPath, `${JSON.stringify(metadata, undefined, '\t')}\n`);
 	}
 
-	return { browsers, files, numFailed, numTests };
+	return { browsers, files, numByteDiff, numFailed, numTests };
 
 }
 
@@ -79,20 +84,28 @@ function flattenResults(session, browserData, fileData) {
 			if (!fileData.tests.has(testName)) {
 				fileData.tests.set(testName, {
 					name: testName,
+					numByteDiff: 0,
 					numFailed: 0,
 					results: []
 				});
 			}
 			const testData = fileData.tests.get(testName);
+			const bytediff = !t.passed && !info.diff && info.pixelsDiff === 0;
 			if (!t.passed) {
-				browserData.numFailed++;
-				testData.numFailed++;
+				if (bytediff) {
+					browserData.numByteDiff++;
+					testData.numByteDiff++;
+				} else {
+					browserData.numFailed++;
+					testData.numFailed++;
+				}
 			}
 			testData.results.push({
 				name: browserData.name,
 				duration: t.duration,
 				error: t.error?.message,
 				passed: t.passed,
+				bytediff,
 				info: info
 			});
 		});