Compare actual/expected screenshots in acceptance tests

javascript/.gitignore

@@ -15,3 +15,5 @@ storybook-static
 *-go
 *.iml
 .vscode-test
+test/examples/**/*.actual.png
+test/examples/**/*.diff.png

javascript/package.json

@@ -38,6 +38,8 @@
     "@types/glob": "^7.2.0",
     "@types/mocha": "9.1.1",
     "@types/node": "17.0.23",
+    "@types/pixelmatch": "^5.2.4",
+    "@types/pngjs": "^6.0.1",
     "@types/react": "17.0.45",
     "@types/react-dom": "18.0.5",
     "@typescript-eslint/eslint-plugin": "^5.18.0",
@@ -55,6 +57,8 @@
     "glob": "^8.0.0",
     "mini-css-extract-plugin": "2.6.0",
     "mocha": "10.0.0",
+    "pixelmatch": "^5.3.0",
+    "pngjs": "^6.0.0",
     "prettier": "^2.6.2",
     "puppeteer": "^14.0.0",
     "react": "17.0.2",

javascript/test/acceptance.ts

@@ -3,37 +3,35 @@ import assert from 'assert'
 import fs from 'fs'
 import glob from 'glob'
 import path from 'path'
+import pixelmatch from 'pixelmatch'
+import { PNG } from 'pngjs'
 import puppeteer from 'puppeteer'
 import { PassThrough, pipeline } from 'stream'
 
 import CucumberHtmlStream from '../src/CucumberHtmlStream'
 
-async function canRenderHtml(html: string): Promise<boolean> {
+async function renderHtml(
+  html: string,
+  block: (page: puppeteer.Page) => Promise<void>
+): Promise<void> {
   const browser = await puppeteer.launch({
     headless: true,
     args: ['--no-sandbox'],
   })
   const page = await browser.newPage()
   await page.setContent(html)
-  const dynamicHTML = await page.evaluate(() => {
-    const content = document.querySelector('[data-testid="cucumber-react"]')
-    return content && content.innerHTML
-  })
+  await page.waitForSelector('[data-testid="cucumber-react"]')
+  await block(page)
   await browser.close()
-
-  if (!dynamicHTML) {
-    return false
-  }
-
-  return true
 }
 
 describe('html-formatter', () => {
   const files = glob.sync(
     `./node_modules/@cucumber/compatibility-kit/features/**/*.ndjson`
   )
   for (const ndjson of files) {
-    it(`can render ${path.basename(ndjson, '.ndjson')}`, async () => {
+    const example = path.basename(ndjson, '.ndjson')
+    it(`can render ${example}`, async () => {
       const ndjsonData = fs.createReadStream(ndjson, { encoding: 'utf-8' })
       const toMessageStream = new NdjsonToMessageStream()
       const htmlData = await new Promise<string>((resolve, reject) => {
@@ -56,7 +54,46 @@ describe('html-formatter', () => {
           }
         )
       })
-      assert.ok(await canRenderHtml(htmlData.toString()))
+      const dir = path.join(__dirname, 'examples', 'cck', example)
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true })
+      }
+      const actualScreenshotPath = path.join(dir, `${example}.actual.png`)
+      await renderHtml(htmlData.toString(), async (page) => {
+        await page.screenshot({
+          path: actualScreenshotPath,
+          fullPage: true,
+        })
+      })
+
+      const actualScreenshot = PNG.sync.read(
+        fs.readFileSync(actualScreenshotPath)
+      )
+      const expectedScreenshotPath = path.join(dir, `${example}.expected.png`)
+      const expectedScreenshot = PNG.sync.read(
+        fs.readFileSync(expectedScreenshotPath)
+      )
+      const { width, height } = expectedScreenshot
+      const diff = new PNG({ width, height })
+      const pixels = pixelmatch(
+        actualScreenshot.data,
+        expectedScreenshot.data,
+        diff.data,
+        width,
+        height,
+        {
+          threshold: 0.1,
+        }
+      )
+      const diffScreenshotPath = path.join(dir, `${example}.diff.png`)
+      fs.writeFileSync(diffScreenshotPath, PNG.sync.write(diff))
+      assert.ok(
+        pixels == 0,
+        `Screenshot of rendered browser did not look as expected.
+actual:   ${actualScreenshotPath}
+expected: ${expectedScreenshotPath}
+diff:     ${diffScreenshotPath}`
+      )
     })
   }
 })