From ffb0d5484a436e96837992d55925876621790a11 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 22 Sep 2024 16:44:22 +0300 Subject: [PATCH 1/4] feat: match test case IDs from test annotations --- README.md | 34 ++ package.json | 9 +- src/playwright-azure-reporter.ts | 98 +++- tests/reporter/reporter-constructor.spec.ts | 572 ++++++++++++++++---- tests/reporter/reporterPath.ts | 10 +- 5 files changed, 576 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index 4b7640e..d11ab15 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,27 @@ Reporter options (\* - required): // This will throw an error: "Invalid testCaseIdMatcher. Must be a string or RegExp. Actual: 1234" ``` +- `testCaseIdZone` [string] - Specifies where to look for the test case IDs. It can be either `'title'` or `'annotation'`. When set to `'title'`, the reporter will extract test case IDs from the test title and tag test section also. When set to `'annotation'`, it will extract test case IDs only from the test annotations. Default: `'title'`. + + **Pay attention that if you use `testCaseIdZone: 'annotation'` and `testCaseIdMatcher` is not defined, the reporter will not extract test case IDs from the test annotations. You should define `testCaseIdMatcher` to extract test case IDs from the test annotations. Matcher should match the annotation type not the annotation description!** + + #### Example Usage + + - Test title: `Test case [12345]` + + - `testCaseIdZone: 'title'` + - Extracted tags: `['12345']` + + - Test annotations: + ```typescript + test('Test case', { annotations: [{ type: 'TestCase', description: '12345' }] }, () => { + expect(true).toBe(true); + }); + ``` + - `testCaseIdZone: 'annotation'` + - `testCaseIdMatcher: /(TestCase)/` + - Extracted tags: `['12345']`] + ## Usefulness - **AZURE_PW_TEST_RUN_ID** - Id of current test run. It will be set in environment variables after test run created. Can be accessed by `process.env.AZURE_PW_TEST_RUN_ID`. Pay attention what `publishTestResultsMode` configuration you use. If you use `testResult` mode - this variable will be set when test run created, at the start of tests execution, if you use `testRun` mode - this variable will be set when test run completed, at the end of tests execution. @@ -271,3 +292,16 @@ Reporter options (\* - required): - script: echo $(playwright.AZURE_PW_TEST_RUN_ID) displayName: 'Print test run id' ``` + +- **AZUREPWDEBUG** - Enable debug logging from reporter `0` - disabled, `1` - enabled. Default: `0`. + + Example of usage in Azure DevOps pipeline: + + ```yaml + - script: npx playwright test + displayName: 'Run Playwright tests' + name: 'playwright' + env: + CI: 'true' + AZUREPWDEBUG: '1' + ``` diff --git a/package.json b/package.json index 166b74e..431dc32 100644 --- a/package.json +++ b/package.json @@ -14,11 +14,18 @@ "clean": "rm -rf dist || true", "prepublishOnly": "tsc", "dev": "yarn run lint && tsc", - "dev:watch": "nodemon --watch './src/*' -e ts --exec \"yarn run build\"", + "dev:watch": "nodemon", "version": "conventional-changelog -p angular -i CHANGELOG.md -s && git add CHANGELOG.md", "release": "conventional-github-releaser -p angular", "prepare": "husky install" }, + "nodemonConfig": { + "watch": ["src"], + "ext": "ts", + "ignore": ["**/test/**", "**/docs/**", "**/dist/**", "**/node_modules/**"], + "delay": 2500, + "exec": "yarn run build" + }, "keywords": [ "playwright", "azure", diff --git a/src/playwright-azure-reporter.ts b/src/playwright-azure-reporter.ts index 30d7121..4f8b4b2 100644 --- a/src/playwright-azure-reporter.ts +++ b/src/playwright-azure-reporter.ts @@ -34,6 +34,7 @@ type TAttachmentType = Array<(typeof attachmentTypesArray)[number] | RegExp>; type TTestRunConfig = Omit | undefined; type TTestResultsToBePublished = { testCase: ITestCaseExtended; testResult: TestResult }; type TPublishTestResults = 'testResult' | 'testRun'; +type TTestCaseIdZone = 'title' | 'annotation'; interface ITestCaseExtended extends TestCase { testAlias: string; @@ -57,6 +58,7 @@ export interface AzureReporterOptions { isExistingTestRun?: boolean; testRunId?: number; testCaseIdMatcher?: string | RegExp | Array; + testCaseIdZone?: TTestCaseIdZone; } interface TestResultsToTestRun { @@ -145,6 +147,7 @@ class AzureDevOpsReporter implements Reporter { private _testRunId: number | undefined; private _isExistingTestRun = false; private _testCaseIdMatcher: string | RegExp | Array = new RegExp(/\[([\d,\s]+)\]/, 'g'); + private _testCaseIdZone: TTestCaseIdZone = 'title'; public constructor(options: AzureReporterOptions) { this._runIdPromise = new Promise((resolve, reject) => { @@ -263,6 +266,19 @@ class AzureDevOpsReporter implements Reporter { } this._isExistingTestRun = options.isExistingTestRun || false; this._testCaseIdMatcher = options.testCaseIdMatcher || new RegExp(/\[([\d,\s]+)\]/, 'g'); + const validZones: TTestCaseIdZone[] = ['title', 'annotation']; + if (options.testCaseIdZone && validZones.includes(options.testCaseIdZone as TTestCaseIdZone)) { + this._testCaseIdZone = options.testCaseIdZone as TTestCaseIdZone; + } else { + this._testCaseIdZone = 'title'; + } + + if (this._testCaseIdZone === 'annotation' && !options.testCaseIdMatcher) { + this._logger?.warn("'testCaseIdMatcher' is not set. The default matcher is set to '\\[([\\d,\\s]+)\\]'."); + this._logger?.warn( + 'This means you need to define your own testCaseIdMatcher, specifically for the "annotation" area' + ); + } } async onBegin(): Promise { @@ -414,17 +430,19 @@ class AzureDevOpsReporter implements Reporter { return result; } - private _extractMatches(text: string): string[] { - const reList = (Array.isArray(this._testCaseIdMatcher) ? this._testCaseIdMatcher : [this._testCaseIdMatcher]).map( - (re) => { - if (typeof re === 'string') { - return new RegExp(re, 'g'); - } else if (!isRegExp(re)) { - throw new Error(`Invalid testCaseIdMatcher. Must be a string or RegExp. Actual: ${re}`); - } - return re; + private _prepareExtractMatches(testCaseIdMatcher: string | RegExp | (string | RegExp)[]): RegExp[] { + return (Array.isArray(testCaseIdMatcher) ? testCaseIdMatcher : [testCaseIdMatcher]).map((re) => { + if (typeof re === 'string') { + return new RegExp(re, 'g'); + } else if (!isRegExp(re)) { + throw new Error(`Invalid testCaseIdMatcher. Must be a string or RegExp. Actual: ${re}`); } - ); + return re; + }); + } + + private _extractMatchesFromText(text: string): string[] { + const reList = this._prepareExtractMatches(this._testCaseIdMatcher); this._logger?.debug(`Extracting matches from text: ${text}`); this._logger?.debug(`Using matchers: ${reList}`); @@ -444,22 +462,56 @@ class AzureDevOpsReporter implements Reporter { return matchesResult; } + private _extractMatchesFromObject(obj: Record): string[] { + if (!obj) return []; + if (Object.keys(obj).length === 0) return []; + if (Array.isArray(obj)) throw new Error('Object must be a key-value pair'); + this._logger?.debug(`Extracting matches from object: \n${JSON.stringify(obj, null, 2)}`); + const matchesResult: string[] = []; + for (const key in obj) { + if (key === 'type') { + this._logger?.debug(`[_extractMatches] Checking key ${key}`); + for (const re of this._prepareExtractMatches(this._testCaseIdMatcher)) { + const matches = obj[key].match(re); + if (matches && matches.length > 1) { + this._logger?.debug(`[_extractMatches] Matches found: ${key} - ${matches[1]}`); + matchesResult.push(obj['description']); + } + } + } + } + return matchesResult; + } + private _getCaseIds(test: TestCase): string[] { const result: string[] = []; - const matches = this._extractMatches(test.title); - this._logger?.debug(`[_getCaseIds] Matches found: ${matches}`); - matches.forEach((match) => { - const ids = match.split(',').map((id) => id.trim()); - result.push(...ids); - }); - if (test.tags) { - test.tags.forEach((tag) => { - const ids = this._extractMatches(tag); - this._logger?.debug(`[_getCaseIds] Matches found in tag: ${ids}`); - ids.forEach((id) => { - result.push(id); - }); + if (this._testCaseIdZone === 'title') { + const matches = this._extractMatchesFromText(test.title); + this._logger?.debug(`[_getCaseIds] Matches found: ${matches}`); + matches.forEach((match) => { + const ids = match.split(',').map((id) => id.trim()); + result.push(...ids); }); + if (test.tags) { + test.tags.forEach((tag) => { + const ids = this._extractMatchesFromText(tag); + this._logger?.debug(`[_getCaseIds] Matches found in tag: ${ids}`); + ids.forEach((id) => { + result.push(id); + }); + }); + } + } else { + if (test.annotations) { + test.annotations.forEach((annotation) => { + const matches = this._extractMatchesFromObject(annotation); + this._logger?.debug(`[_getCaseIds] Matches found in annotation: ${matches}`); + matches.forEach((id) => { + const ids = id.split(',').map((id) => id.trim()); + result.push(...ids); + }); + }); + } } return [...new Set(result)]; } diff --git a/tests/reporter/reporter-constructor.spec.ts b/tests/reporter/reporter-constructor.spec.ts index 5626bc8..51b0a7d 100644 --- a/tests/reporter/reporter-constructor.spec.ts +++ b/tests/reporter/reporter-constructor.spec.ts @@ -378,60 +378,429 @@ test.describe('Reporter constructor', () => { }); }); -let testCaseIdMatchers: { - testCaseIdMatcher?: RegExp | RegExp[] | string | string[] | undefined; - testTitle: string; - tagsSection?: string; - expected: string[]; -}[] = [ - { - testCaseIdMatcher: /@tag1=(\d+)/, - testTitle: 'Test case @tag1=123', - tagsSection: "['@tag1=7']", - expected: ['123'], - }, - { - testCaseIdMatcher: /@TestCase=(\d+)/, - testTitle: 'Test case @TestCase=123 [@TestCase=456]', - tagsSection: "['@TestCase=7', '@TestCase=7']", - expected: ['123', '456'], - }, - { - testCaseIdMatcher: [/[a-z]+(\d+)/, /[A-Z]+(\d+)/], - testTitle: 'Test case test123 TEST456', - tagsSection: "['@test7', '@TEST7']", - expected: ['123', '456'], - }, - { - testCaseIdMatcher: ['@tag1=(\\d+)', '@tag2=(\\d+)'], - testTitle: 'Test case @tag1=123 @tag2=456', - tagsSection: "['@tag1=7', '@tag2=7']", - expected: ['123', '456'], - }, - { - testCaseIdMatcher: undefined, - testTitle: 'Test case [12345]', - tagsSection: "['@[7]']", - expected: ['12345'], - }, -]; - -testCaseIdMatchers.forEach((item) => { - test(`_extractMatches should return ${item.expected} for ${item.testTitle}`, () => { - const reporter = new AzureDevOpsReporter({ - orgUrl: 'http://localhost:4000', - projectName: 'SampleSample', - planId: 4, - token: 'token', - isDisabled: false, - testCaseIdMatcher: item.testCaseIdMatcher, +test.describe('Test Case ID matcher in title and tag section', () => { + let testCaseIdMatchers: { + testCaseIdMatcher?: RegExp | RegExp[] | string | string[] | undefined; + testTitle: string; + tagsSection?: string; + expected: string[]; + }[] = [ + { + testCaseIdMatcher: /@tag1=(\d+)/, + testTitle: 'Test case @tag1=123', + tagsSection: "['@tag1=7']", + expected: ['123'], + }, + { + testCaseIdMatcher: /@TestCase=(\d+)/, + testTitle: 'Test case @TestCase=123 [@TestCase=456]', + tagsSection: "['@TestCase=7', '@TestCase=7']", + expected: ['123', '456'], + }, + { + testCaseIdMatcher: [/[a-z]+(\d+)/, /[A-Z]+(\d+)/], + testTitle: 'Test case test123 TEST456', + tagsSection: "['@test7', '@TEST7']", + expected: ['123', '456'], + }, + { + testCaseIdMatcher: ['@tag1=(\\d+)', '@tag2=(\\d+)'], + testTitle: 'Test case @tag1=123 @tag2=456', + tagsSection: "['@tag1=7', '@tag2=7']", + expected: ['123', '456'], + }, + { + testCaseIdMatcher: undefined, + testTitle: 'Test case [12345]', + tagsSection: "['@[7]']", + expected: ['12345'], + }, + ]; + + testCaseIdMatchers.forEach((item) => { + test(`_extractMatchesFromText should return ${item.expected} for ${item.testTitle}`, () => { + const reporter = new AzureDevOpsReporter({ + orgUrl: 'http://localhost:4000', + projectName: 'SampleSample', + planId: 4, + token: 'token', + isDisabled: false, + testCaseIdMatcher: item.testCaseIdMatcher, + }); + + const matches = (reporter as any)._extractMatchesFromText(item.testTitle); + expect.soft(matches).toEqual(item.expected); + }); + + test(`match tags with own tags matcher for ${item.tagsSection}`, async ({ runInlineTest, server }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_INVALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', async (req, res) => { + const body = await getRequestBody(req); + setHeaders(res, headers); + expect(body.pointsFilter?.testcaseIds[0]).toBeDefined(); + if (body.pointsFilter?.testcaseIds[0] === 3) server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + else if (body.pointsFilter?.testcaseIds[0] === 7) server.serveFile(req, res, POINTS_7_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', async (req, res) => { + const body = await getRequestBody(req); + setHeaders(res, headers); + expect(body[0].testPoint?.id).toBeDefined(); + if (body[0].testPoint?.id === '1') server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + else if (body[0].testPoint?.id === '2') server.serveFile(req, res, TEST_RUN_RESULTS_7_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results/100001/Attachments', async (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, RUN_RESULTS_ATTACHMENTS_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + use: { + screenshot: 'only-on-failure', + trace: 'retain-on-failure', + video: 'retain-on-failure', + }, + reporter: [ + ['dot'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + uploadAttachments: true, + logging: true, + ${ + item.testCaseIdMatcher + ? `testCaseIdMatcher: ${ + Array.isArray(item.testCaseIdMatcher) + ? `[${item.testCaseIdMatcher + .map((tag) => `${isRegExp(tag) ? tag.toString() : `'${tag.replace(/\\/g, '\\\\')}'`}`) + .join(',')}]` + : `[${item.testCaseIdMatcher}]` + }` + : '' + } + }] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('[7] with screenshot', ${ + item.testCaseIdMatcher !== undefined ? `{ tag: ${item.tagsSection} }, ` : '' + } async ({ page }) => { + await page.goto('https://playwright.dev/') + await page.locator('text=Get started').click() + await expect(page).toHaveTitle(/Getting sttttarted/) + }); + `, + }, + { reporter: '' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).toContain( + "'attachmentsType' is not set. Attachments Type will be set to 'screenshot' by default." + ); + expect(result.output).toMatch(/azure:pw:log Using run (\d.*) to publish test results/); + expect(result.output).toContain(`azure:pw:log [7] with screenshot - failed`); + expect(result.output).toContain('azure:pw:log Start publishing: [7] with screenshot'); + expect(result.output).toContain('azure:pw:log Uploading attachments for test: [7] with screenshot'); + expect(result.output).toContain('azure:pw:log Uploaded attachment'); + expect(result.output).toContain('azure:pw:log Result published: [7] with screenshot'); + expect(result.output).toMatch(/azure:pw:log Run (\d.*) - Completed/); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); }); + }); - const matches = (reporter as any)._extractMatches(item.testTitle); - expect.soft(matches).toEqual(item.expected); + testCaseIdMatchers = [ + { + //@ts-ignore + testCaseIdMatcher: [3432, 3944], + testTitle: 'Test case [12345]', + expected: ['12345'], + }, + ]; + + testCaseIdMatchers.forEach((testCaseIdMatcher) => { + test(`Test should throw an error for invalid testCaseIdMatcher: ${testCaseIdMatcher.testCaseIdMatcher}`, () => { + try { + new AzureDevOpsReporter({ + orgUrl: 'http://localhost:4000', + projectName: 'SampleSample', + planId: 4, + token: 'token', + isDisabled: false, + testCaseIdMatcher: testCaseIdMatcher.testCaseIdMatcher, + }); + } catch (error) { + expect(error.message).toContain('Invalid testCaseIdMatcher. Must be a string or RegExp. Actual: 3432'); + } + }); }); +}); + +type TestCaseAnnotation = { + type: string; + description: string; +}; + +test.describe('Test Case ID matcher in annotation section', () => { + let annotationObjects: { annotations: TestCaseAnnotation[]; testCaseIdMatcher: RegExp; expected: string[] }[] = [ + { + annotations: [ + { + type: '@Tag1', + description: '123', + }, + ], + testCaseIdMatcher: /(@Tag1)/, + expected: ['123'], + }, + { + annotations: [ + { + type: 'TestCase', + description: '456', + }, + ], + testCaseIdMatcher: /(TestCase)/, + expected: ['456'], + }, + { + annotations: [ + { + type: 'tag2', + description: '789', + }, + { + type: 'tag3', + description: '101112', + }, + ], + testCaseIdMatcher: /(tag2)|(tag3)/, + expected: ['789', '101112'], + }, + ]; + + annotationObjects.forEach((item) => { + test(`_extractMatchesFromObject should return ${item.expected} for ${item.testCaseIdMatcher}`, () => { + const reporter = new AzureDevOpsReporter({ + orgUrl: 'http://localhost:4000', + projectName: 'SampleSample', + planId: 4, + token: 'token', + isDisabled: false, + testCaseIdMatcher: item.testCaseIdMatcher, + testCaseIdZone: 'annotation', + }); - test(`match tags with own tags matcher for ${item.tagsSection}`, async ({ runInlineTest, server }) => { + const matches: string[] = []; + for (const annotation of item.annotations) { + matches.push(...(reporter as any)._extractMatchesFromObject(annotation)); + } + expect.soft(matches).toEqual(item.expected); + }); + }); + + annotationObjects = [ + { + annotations: [ + { + type: '@tag1', + description: '7', + }, + ], + testCaseIdMatcher: /(@tag1)/, + expected: ['7'], + }, + { + annotations: [ + { + type: 'TestCase', + description: '7', + }, + ], + testCaseIdMatcher: /(TestCase)/, + expected: ['7'], + }, + { + annotations: [ + { + type: 'tag3', + description: '7', + }, + ], + testCaseIdMatcher: /(tag3)/, + expected: ['7'], + }, + ]; + + annotationObjects.forEach((item) => { + test(`match tags with own tags matcher for ${item.testCaseIdMatcher} in annotations section`, async ({ + runInlineTest, + server, + }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_INVALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', async (req, res) => { + const body = await getRequestBody(req); + setHeaders(res, headers); + expect(body.pointsFilter?.testcaseIds[0]).toBeDefined(); + if (body.pointsFilter?.testcaseIds[0] === 3) server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + else if (body.pointsFilter?.testcaseIds[0] === 7) server.serveFile(req, res, POINTS_7_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', async (req, res) => { + const body = await getRequestBody(req); + setHeaders(res, headers); + expect(body[0].testPoint?.id).toBeDefined(); + if (body[0].testPoint?.id === '1') server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + else if (body[0].testPoint?.id === '2') server.serveFile(req, res, TEST_RUN_RESULTS_7_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results/100001/Attachments', async (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, RUN_RESULTS_ATTACHMENTS_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + use: { + screenshot: 'only-on-failure', + trace: 'retain-on-failure', + video: 'retain-on-failure', + }, + reporter: [ + ['dot'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + uploadAttachments: true, + logging: true, + ${ + item.testCaseIdMatcher + ? `testCaseIdMatcher: ${ + Array.isArray(item.testCaseIdMatcher) + ? `[${item.testCaseIdMatcher + .map((tag) => `${isRegExp(tag) ? tag.toString() : `'${tag.replace(/\\/g, '\\\\')}'`}`) + .join(',')}]` + : `[${item.testCaseIdMatcher}]` + }` + : '' + }, + testCaseIdZone: 'annotation' + }] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('with screenshot', { + annotation: ${JSON.stringify(item.annotations)} + }, async ({ page }) => { + await page.goto('https://playwright.dev/') + await page.locator('text=Get started').click() + await expect(page).toHaveTitle(/Getting sttttarted/) + }); + `, + }, + { reporter: '' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).toContain( + "'attachmentsType' is not set. Attachments Type will be set to 'screenshot' by default." + ); + expect(result.output).toMatch(/azure:pw:log Using run (\d.*) to publish test results/); + expect(result.output).toContain(`azure:pw:log with screenshot - failed`); + expect(result.output).toContain('azure:pw:log Start publishing: with screenshot'); + expect(result.output).toContain('azure:pw:log Uploading attachments for test: with screenshot'); + expect(result.output).toContain('azure:pw:log Uploaded attachment'); + expect(result.output).toContain('azure:pw:log Result published: with screenshot'); + expect(result.output).toMatch(/azure:pw:log Run (\d.*) - Completed/); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); + }); + + test("'testCaseIdMatcher' is not set", async ({ runInlineTest, server }) => { server.setRoute('/_apis/Location', (_, res) => { setHeaders(res, headers); res.end(JSON.stringify(location)); @@ -491,46 +860,34 @@ testCaseIdMatchers.forEach((item) => { const result = await runInlineTest( { 'playwright.config.ts': ` - module.exports = { - use: { - screenshot: 'only-on-failure', - trace: 'retain-on-failure', - video: 'retain-on-failure', - }, - reporter: [ - ['dot'], - ['${reporterPath}', { - orgUrl: 'http://localhost:${server.PORT}', - projectName: 'SampleSample', - planId: 4, - token: 'token', - uploadAttachments: true, - logging: true, - ${ - item.testCaseIdMatcher - ? `testCaseIdMatcher: ${ - Array.isArray(item.testCaseIdMatcher) - ? `[${item.testCaseIdMatcher - .map((tag) => `${isRegExp(tag) ? tag.toString() : `'${tag.replace(/\\/g, '\\\\')}'`}`) - .join(',')}]` - : `[${item.testCaseIdMatcher}]` - }` - : '' - } - }] - ] - }; - `, + module.exports = { + use: { + screenshot: 'only-on-failure', + trace: 'retain-on-failure', + video: 'retain-on-failure', + }, + reporter: [ + ['dot'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + uploadAttachments: true, + logging: true, + testCaseIdZone: 'annotation' + }] + ] + }; + `, 'a.spec.js': ` - import { test, expect } from '@playwright/test'; - test('[7] with screenshot', ${ - item.testCaseIdMatcher !== undefined ? `{ tag: ${item.tagsSection} }, ` : '' - } async ({ page }) => { - await page.goto('https://playwright.dev/') - await page.locator('text=Get started').click() - await expect(page).toHaveTitle(/Getting sttttarted/) - }); - `, + import { test, expect } from '@playwright/test'; + test('with screenshot', async ({ page }) => { + await page.goto('https://playwright.dev/') + await page.locator('text=Get started').click() + await expect(page).toHaveTitle(/Getting sttttarted/) + }); + `, }, { reporter: '' } ); @@ -540,39 +897,14 @@ testCaseIdMatchers.forEach((item) => { "'attachmentsType' is not set. Attachments Type will be set to 'screenshot' by default." ); expect(result.output).toMatch(/azure:pw:log Using run (\d.*) to publish test results/); - expect(result.output).toContain(`azure:pw:log [7] with screenshot - failed`); - expect(result.output).toContain('azure:pw:log Start publishing: [7] with screenshot'); - expect(result.output).toContain('azure:pw:log Uploading attachments for test: [7] with screenshot'); - expect(result.output).toContain('azure:pw:log Uploaded attachment'); - expect(result.output).toContain('azure:pw:log Result published: [7] with screenshot'); + expect(result.output).toContain( + `azure:pw:warn 'testCaseIdMatcher' is not set. The default matcher is set to '\\[([\\d,\\s]+)\\]'.` + ); + expect(result.output).toContain( + `This means you need to define your own testCaseIdMatcher, specifically for the "annotation" area` + ); expect(result.output).toMatch(/azure:pw:log Run (\d.*) - Completed/); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); }); }); - -testCaseIdMatchers = [ - { - //@ts-ignore - testCaseIdMatcher: [3432, 3944], - testTitle: 'Test case [12345]', - expected: ['12345'], - }, -]; - -testCaseIdMatchers.forEach((testCaseIdMatcher) => { - test(`Test should throw an error for invalid testCaseIdMatcher: ${testCaseIdMatcher.testCaseIdMatcher}`, () => { - try { - new AzureDevOpsReporter({ - orgUrl: 'http://localhost:4000', - projectName: 'SampleSample', - planId: 4, - token: 'token', - isDisabled: false, - testCaseIdMatcher: testCaseIdMatcher.testCaseIdMatcher, - }); - } catch (error) { - expect(error.message).toContain('Invalid testCaseIdMatcher. Must be a string or RegExp. Actual: 3432'); - } - }); -}); diff --git a/tests/reporter/reporterPath.ts b/tests/reporter/reporterPath.ts index 06c5c72..38398ba 100644 --- a/tests/reporter/reporterPath.ts +++ b/tests/reporter/reporterPath.ts @@ -1,5 +1,9 @@ import path from 'path'; -export const reporterPath = path.join(__dirname, '../../dist/playwright-azure-reporter.js'); -export const customReporterTestRunPath = path.join(__dirname, './assets/custom-reporter-testRun.ts'); -export const customReporterTestResultPath = path.join(__dirname, './assets/custom-reporter-testResult.ts'); +export const reporterPath = path.resolve(__dirname, '../../dist/playwright-azure-reporter.js').replace(/\\/g, '/'); +export const customReporterTestRunPath = path + .resolve(__dirname, './assets/custom-reporter-testRun.ts') + .replace(/\\/g, '/'); +export const customReporterTestResultPath = path + .resolve(__dirname, './assets/custom-reporter-testResult.ts') + .replace(/\\/g, '/'); From f0945cc23d333da3905b5cc45d74188c9ae037dd Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 22 Sep 2024 16:50:50 +0300 Subject: [PATCH 2/4] format files --- package.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 431dc32..fe4c1fd 100644 --- a/package.json +++ b/package.json @@ -20,9 +20,16 @@ "prepare": "husky install" }, "nodemonConfig": { - "watch": ["src"], + "watch": [ + "src" + ], "ext": "ts", - "ignore": ["**/test/**", "**/docs/**", "**/dist/**", "**/node_modules/**"], + "ignore": [ + "**/test/**", + "**/docs/**", + "**/dist/**", + "**/node_modules/**" + ], "delay": 2500, "exec": "yarn run build" }, From ff9eee69a8ea37dbfdf5b254634fdd180c5e3fcd Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 22 Sep 2024 16:53:34 +0300 Subject: [PATCH 3/4] v1.12.0-beta.0 --- CHANGELOG.md | 9 +++++++++ package.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f4f1a2..52a5290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# [1.12.0-beta.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.11.0...v1.12.0-beta.0) (2024-09-22) + + +### Features + +* match test case IDs from test annotations ([ffb0d54](https://github.com/alexneo2003/playwright-azure-reporter/commit/ffb0d5484a436e96837992d55925876621790a11)) + + + # [1.11.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.11.0-beta.0...v1.11.0) (2024-09-19) diff --git a/package.json b/package.json index fe4c1fd..16c8551 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex_neo/playwright-azure-reporter", - "version": "1.11.0", + "version": "1.12.0-beta.0", "description": "Playwright Azure Reporter", "main": "./dist/playwright-azure-reporter.js", "types": "./dist/playwright-azure-reporter.d.js", From 7ab8fc36263ed6c7217a76bf678f853731b92e2c Mon Sep 17 00:00:00 2001 From: Alex Neo Date: Thu, 26 Sep 2024 17:24:27 +0300 Subject: [PATCH 4/4] v1.12.0 --- CHANGELOG.md | 9 +++++++++ package.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52a5290..609216a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# [1.12.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.11.0...v1.12.0) (2024-09-26) + + +### Features + +* match test case IDs from test annotations ([ffb0d54](https://github.com/alexneo2003/playwright-azure-reporter/commit/ffb0d5484a436e96837992d55925876621790a11)) + + + # [1.12.0-beta.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.11.0...v1.12.0-beta.0) (2024-09-22) diff --git a/package.json b/package.json index 16c8551..7368913 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex_neo/playwright-azure-reporter", - "version": "1.12.0-beta.0", + "version": "1.12.0", "description": "Playwright Azure Reporter", "main": "./dist/playwright-azure-reporter.js", "types": "./dist/playwright-azure-reporter.d.js",