From c60349a0b894f58a2e88df162ff8214b424ad17d Mon Sep 17 00:00:00 2001 From: Semen Syzov Date: Tue, 21 Nov 2023 12:11:36 +0200 Subject: [PATCH 1/4] add existingTestRun publishing mode --- README.md | 3 +++ src/playwright-azure-reporter.ts | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 02b0447..eb517d4 100644 --- a/README.md +++ b/README.md @@ -164,3 +164,6 @@ Reporter options (\* - required): - `testResult` - Published results of tests, at the end of each test, parallel to test run.. - `testRun` - Published test results to test run, at the end of test run. > **Note:** If you use `testRun` mode and using same test cases in different tests (yes i know it sounds funny), it will be overwritten with last test result. + - `existingTestRun` - Published test results to the existing test run, at the end of each test, parallel to test run. In this mode test results only added to the existing test run without its creation and completion. + > **Note:** If you use `existingTestRun` mode, `testRunId` should be specified. +- `testRunId` - Id of test run. Used only for `existingTestRun` publishing mode. diff --git a/src/playwright-azure-reporter.ts b/src/playwright-azure-reporter.ts index a3cedaf..5be50b2 100644 --- a/src/playwright-azure-reporter.ts +++ b/src/playwright-azure-reporter.ts @@ -31,7 +31,7 @@ const attachmentTypesArray = ['screenshot', 'video', 'trace'] as const; type TAttachmentType = Array<(typeof attachmentTypesArray)[number] | RegExp>; type TTestRunConfig = Omit | undefined; type TTestResultsToBePublished = { testCase: ITestCaseExtended; testResult: TestResult }; -type TPublishTestResults = 'testResult' | 'testRun'; +type TPublishTestResults = 'testResult' | 'testRun' | 'existingTestRun'; interface ITestCaseExtended extends TestCase { testAlias: string; @@ -52,6 +52,7 @@ export interface AzureReporterOptions { attachmentsType?: TAttachmentType | undefined; testRunConfig?: TTestRunConfig; testPointMapper?: (testCase: TestCase, testPoints: TestPoint[]) => Promise; + testRunId?: number; } interface TestResultsToTestRun { @@ -137,6 +138,7 @@ class AzureDevOpsReporter implements Reporter { maxRetries: 20, } as IRequestOptions; private _publishTestResultsMode: TPublishTestResults = 'testResult'; + private _testRunId = 0; public constructor(options: AzureReporterOptions) { this._runIdPromise = new Promise((resolve, reject) => { @@ -199,6 +201,14 @@ class AzureDevOpsReporter implements Reporter { this._isDisabled = true; return; } + if ( + options.publishTestResultsMode === 'existingTestRun' && + (!options?.testRunId || !Number.isInteger(options?.testRunId)) + ) { + this._warning("'testRunId' is not set for 'existingTestRun' mode. Reporting is disabled."); + this._isDisabled = true; + return; + } if (options?.uploadAttachments) { if (!options?.attachmentsType) { this._warning("'attachmentsType' is not set. Attachments Type will be set to 'screenshot' by default."); @@ -237,10 +247,15 @@ class AzureDevOpsReporter implements Reporter { if (this._logging) { debug.enable('azure'); } + this._testRunId = options.testRunId || 0; } async onBegin(): Promise { if (this._isDisabled) return; + if (this._publishTestResultsMode === 'existingTestRun') { + this._resolveRunId(this._testRunId); + this._log(chalk.green(`Using run ${this._testRunId} to publish test results`)); + } try { this._testApi = await this._connection.getTestApi(); @@ -271,7 +286,7 @@ class AzureDevOpsReporter implements Reporter { async onTestEnd(test: TestCase, testResult: TestResult): Promise { if (this._isDisabled) return; try { - if (this._publishTestResultsMode === 'testResult') { + if (this._publishTestResultsMode === 'testResult' || this._publishTestResultsMode === 'existingTestRun') { const runId = await this._runIdPromise; if (!runId) return; @@ -342,6 +357,7 @@ class AzureDevOpsReporter implements Reporter { return; } + if (this._publishTestResultsMode === 'existingTestRun') return; if (!this._testApi) this._testApi = await this._connection.getTestApi(); const runUpdatedResponse = await this._testApi.updateTestRun({ state: 'Completed' }, this._projectName, runId!); this._log(chalk.green(`Run ${runId} - ${runUpdatedResponse.state}`)); From 8ca1923fbebd1043773742f584f36ec2b1b4f100 Mon Sep 17 00:00:00 2001 From: Semen Syzov Date: Tue, 21 Nov 2023 15:39:26 +0200 Subject: [PATCH 2/4] change logic to enable publishing test results in both 'testRun' and 'testResult' modes --- README.md | 4 ++-- src/playwright-azure-reporter.ts | 26 +++++++++++++++----------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index eb517d4..f1d2888 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,6 @@ Reporter options (\* - required): - `testResult` - Published results of tests, at the end of each test, parallel to test run.. - `testRun` - Published test results to test run, at the end of test run. > **Note:** If you use `testRun` mode and using same test cases in different tests (yes i know it sounds funny), it will be overwritten with last test result. - - `existingTestRun` - Published test results to the existing test run, at the end of each test, parallel to test run. In this mode test results only added to the existing test run without its creation and completion. - > **Note:** If you use `existingTestRun` mode, `testRunId` should be specified. +- `isExistingTestRun` [true/false] - Published test results to the existing test run. In this mode test results only added to the existing test run without its creation and completion. + > **Note:** If you use `isExistingTestRun` mode, `testRunId` should be specified. - `testRunId` - Id of test run. Used only for `existingTestRun` publishing mode. diff --git a/src/playwright-azure-reporter.ts b/src/playwright-azure-reporter.ts index 5be50b2..c9b116f 100644 --- a/src/playwright-azure-reporter.ts +++ b/src/playwright-azure-reporter.ts @@ -31,7 +31,7 @@ const attachmentTypesArray = ['screenshot', 'video', 'trace'] as const; type TAttachmentType = Array<(typeof attachmentTypesArray)[number] | RegExp>; type TTestRunConfig = Omit | undefined; type TTestResultsToBePublished = { testCase: ITestCaseExtended; testResult: TestResult }; -type TPublishTestResults = 'testResult' | 'testRun' | 'existingTestRun'; +type TPublishTestResults = 'testResult' | 'testRun'; interface ITestCaseExtended extends TestCase { testAlias: string; @@ -52,6 +52,7 @@ export interface AzureReporterOptions { attachmentsType?: TAttachmentType | undefined; testRunConfig?: TTestRunConfig; testPointMapper?: (testCase: TestCase, testPoints: TestPoint[]) => Promise; + isExistingTestRun?: boolean; testRunId?: number; } @@ -139,6 +140,7 @@ class AzureDevOpsReporter implements Reporter { } as IRequestOptions; private _publishTestResultsMode: TPublishTestResults = 'testResult'; private _testRunId = 0; + private _isExistingTestRun = false; public constructor(options: AzureReporterOptions) { this._runIdPromise = new Promise((resolve, reject) => { @@ -201,11 +203,8 @@ class AzureDevOpsReporter implements Reporter { this._isDisabled = true; return; } - if ( - options.publishTestResultsMode === 'existingTestRun' && - (!options?.testRunId || !Number.isInteger(options?.testRunId)) - ) { - this._warning("'testRunId' is not set for 'existingTestRun' mode. Reporting is disabled."); + if (options?.isExistingTestRun && !Number.isInteger(options?.testRunId)) { + this._warning("'testRunId' is not set for 'isExistingTestRun'=true mode. Reporting is disabled."); this._isDisabled = true; return; } @@ -248,11 +247,12 @@ class AzureDevOpsReporter implements Reporter { debug.enable('azure'); } this._testRunId = options.testRunId || 0; + this._isExistingTestRun = options.isExistingTestRun || false; } async onBegin(): Promise { if (this._isDisabled) return; - if (this._publishTestResultsMode === 'existingTestRun') { + if (this._isExistingTestRun) { this._resolveRunId(this._testRunId); this._log(chalk.green(`Using run ${this._testRunId} to publish test results`)); } @@ -286,7 +286,7 @@ class AzureDevOpsReporter implements Reporter { async onTestEnd(test: TestCase, testResult: TestResult): Promise { if (this._isDisabled) return; try { - if (this._publishTestResultsMode === 'testResult' || this._publishTestResultsMode === 'existingTestRun') { + if (this._publishTestResultsMode === 'testResult') { const runId = await this._runIdPromise; if (!runId) return; @@ -337,8 +337,12 @@ class AzureDevOpsReporter implements Reporter { this._log(chalk.gray('No test results to publish')); return; } else { - const createRunResponse = await this._createRun(this._testRunTitle); - runId = createRunResponse?.id; + if (!this._isExistingTestRun) { + const createRunResponse = await this._createRun(this._testRunTitle); + runId = createRunResponse?.id; + } else { + runId = this._testRunId; + } if (runId) { this._resolveRunId(runId); this._log(chalk.green(`Using run ${runId} to publish test results`)); @@ -357,7 +361,7 @@ class AzureDevOpsReporter implements Reporter { return; } - if (this._publishTestResultsMode === 'existingTestRun') return; + if (this._isExistingTestRun) return; if (!this._testApi) this._testApi = await this._connection.getTestApi(); const runUpdatedResponse = await this._testApi.updateTestRun({ state: 'Completed' }, this._projectName, runId!); this._log(chalk.green(`Run ${runId} - ${runUpdatedResponse.state}`)); From 3c2bfb31fad8427dd1206202d934812a78290b27 Mon Sep 17 00:00:00 2001 From: Alex Neo Date: Tue, 21 Nov 2023 16:53:08 +0200 Subject: [PATCH 3/4] feat: set existing test run ID - added new option to set existing test run ID - added ability to set existing test run ID from environment variable - AZURE_TEST_RUN_ID --- README.md | 26 +- src/playwright-azure-reporter.ts | 22 +- .../reporter/reporter-existingTestRun.spec.ts | 586 ++++++++++++++++++ 3 files changed, 619 insertions(+), 15 deletions(-) create mode 100644 tests/reporter/reporter-existingTestRun.spec.ts diff --git a/README.md b/README.md index 2b737e0..c3279df 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,22 @@ # Playwright Azure Reporter + ![GitHub](https://img.shields.io/github/license/alexneo2003/playwright-azure-reporter) ![npm (scoped)](https://img.shields.io/npm/v/@alex_neo/playwright-azure-reporter) ![npm](https://img.shields.io/npm/dw/@alex_neo/playwright-azure-reporter) ![npm](https://img.shields.io/npm/dt/@alex_neo/playwright-azure-reporter) ## A must read! + **Since version 1.5.0 reporter allows using configurationIds to publish results for different configurations e.g. different browsers** **Necessarily defining `testRun.configurationIds` or/and `testPointMapper` function in reporter config, otherwise reporter will be publishing results for all configurations** - ## How to integrate + Install package ```bash npm install @alex_neo/playwright-azure-reporter ``` -or + +or + ```bash yarn add @alex_neo/playwright-azure-reporter ``` @@ -108,11 +112,11 @@ const config: PlaywrightTestConfig = { displayName: 'Alex Neo', }, comment: 'Playwright Test Run', - // the configuration ids of this test run, use + // the configuration ids of this test run, use // https://dev.azure.com/{organization}/{project}/_apis/test/configurations to get the ids of your project. - // if multiple configuration ids are used in one run a testPointMapper should be used to pick the correct one, + // if multiple configuration ids are used in one run a testPointMapper should be used to pick the correct one, // otherwise the results are pushed to all. - configurationIds: [ 1 ], + configurationIds: [1], }, } as AzureReporterOptions, ], @@ -146,6 +150,7 @@ Reporter options (\* - required): - `testRunTitle` - Title of test run using to create new test run. Default: `Playwright Test Run`. - `testRunConfig` - Extra data to pass when Test Run creating. Read [doc](https://learn.microsoft.com/en-us/rest/api/azure/devops/test/runs/create?view=azure-devops-rest-7.1&tabs=HTTP#request-body) from more information. Default: `empty`. - `testPointMapper` - A callback to map the test runs to test configurations, e.g. by browser + ``` testPointMapper: async (testCase: TestCase, testPoints: TestPoint[]) => { switch(testCase.parent.project()?.use.browserName) { @@ -160,13 +165,18 @@ Reporter options (\* - required): } } ``` + - `publishTestResultsMode` - Mode of publishing test results. Default: `'testResult'`. Available options: - `testResult` - Published results of tests, at the end of each test, parallel to test run.. - `testRun` - Published test results to test run, at the end of test run. > **Note:** If you use `testRun` mode and using same test cases in different tests (yes i know it sounds funny), it will be overwritten with last test result. -- `isExistingTestRun` [true/false] - Published test results to the existing test run. In this mode test results only added to the existing test run without its creation and completion. - > **Note:** If you use `isExistingTestRun` mode, `testRunId` should be specified. -- `testRunId` - Id of test run. Used only for `existingTestRun` publishing mode. +- `isExistingTestRun` [true/false] - Published test results to the existing test run. In this mode test results only added to the existing test run without its creation and completion. Default: `false`. + > **Note:** If you use `isExistingTestRun` mode, `testRunId` should be specified. +- `testRunId` [number] - Id of test run. Used only for `existingTestRun` publishing mode. Also can be set by `AZURE_PW_TEST_RUN_ID` environment variable. Default: `undefined`. + + > **Note:** If you set existing test run ID from reporter options and from environment variable - reporter options will be used + + > **Note:** If you use `isExistingTestRun` mode, test run doesn't complete automatically. You should complete it manually. ## Usefulness diff --git a/src/playwright-azure-reporter.ts b/src/playwright-azure-reporter.ts index 16b5b4c..22ca1d9 100644 --- a/src/playwright-azure-reporter.ts +++ b/src/playwright-azure-reporter.ts @@ -139,7 +139,7 @@ class AzureDevOpsReporter implements Reporter { maxRetries: 20, } as IRequestOptions; private _publishTestResultsMode: TPublishTestResults = 'testResult'; - private _testRunId = 0; + private _testRunId: number | undefined; private _isExistingTestRun = false; public constructor(options: AzureReporterOptions) { @@ -203,11 +203,17 @@ class AzureDevOpsReporter implements Reporter { this._isDisabled = true; return; } - if (options?.isExistingTestRun && !Number.isInteger(options?.testRunId)) { - this._warning("'testRunId' is not set for 'isExistingTestRun'=true mode. Reporting is disabled."); + this._testRunId = options.testRunId || Number(process.env.AZURE_PW_TEST_RUN_ID) || undefined; + if (options?.isExistingTestRun && !this._testRunId) { + this._warning( + "'testRunId' or AZURE_PW_TEST_RUN_ID is not set for 'isExistingTestRun'=true mode. Reporting is disabled." + ); this._isDisabled = true; return; } + if (this._testRunId) { + process.env.AZURE_PW_TEST_RUN_ID = String(this._testRunId); + } if (options?.uploadAttachments) { if (!options?.attachmentsType) { this._warning("'attachmentsType' is not set. Attachments Type will be set to 'screenshot' by default."); @@ -243,18 +249,20 @@ class AzureDevOpsReporter implements Reporter { if (options.testPointMapper) { this._testPointMapper = options.testPointMapper; } + this._isExistingTestRun = options.isExistingTestRun || false; + if (this._logging) { debug.enable('azure'); } - this._testRunId = options.testRunId || 0; - this._isExistingTestRun = options.isExistingTestRun || false; } async onBegin(): Promise { if (this._isDisabled) return; if (this._isExistingTestRun) { - this._resolveRunId(this._testRunId); - this._log(chalk.green(`Using run ${this._testRunId} to publish test results`)); + this._resolveRunId(this._testRunId!); + this._log(chalk.green(`Using existing run ${this._testRunId} to publish test results`)); + this._log(chalk.green(`AZURE_PW_TEST_RUN_ID: ${process.env.AZURE_PW_TEST_RUN_ID}`)); + return; } try { this._testApi = await this._connection.getTestApi(); diff --git a/tests/reporter/reporter-existingTestRun.spec.ts b/tests/reporter/reporter-existingTestRun.spec.ts new file mode 100644 index 0000000..b49f596 --- /dev/null +++ b/tests/reporter/reporter-existingTestRun.spec.ts @@ -0,0 +1,586 @@ +import path from 'path'; + +import { setHeaders } from '../config/utils'; +import azureAreas from './assets/azure-reporter/azureAreas'; +import headers from './assets/azure-reporter/azureHeaders'; +import location from './assets/azure-reporter/azureLocationOptionsResponse.json'; +import { reporterPath } from './reporterPath'; +import { expect, test } from './test-fixtures'; + +const TEST_OPTIONS_RESPONSE_PATH = path.join( + __dirname, + '.', + 'assets', + 'azure-reporter', + 'azureTestOptionsResponse.json' +); +const CORE_OPTIONS_RESPONSE_PATH = path.join( + __dirname, + '.', + 'assets', + 'azure-reporter', + 'azureCoreOptionsResponse.json' +); +const PROJECT_VALID_RESPONSE_PATH = path.join(__dirname, '.', 'assets', 'azure-reporter', 'projectValidResponse.json'); +const CREATE_RUN_VALID_RESPONSE_PATH = path.join( + __dirname, + '.', + 'assets', + 'azure-reporter', + 'createRunValidResponse.json' +); +const POINTS_3_VALID_RESPONSE_PATH = path.join(__dirname, '.', 'assets', 'azure-reporter', 'points3Response.json'); +const COMPLETE_RUN_VALID_RESPONSE_PATH = path.join( + __dirname, + '.', + 'assets', + 'azure-reporter', + 'completeRunValidResponse.json' +); +const TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH = path.join( + __dirname, + '.', + 'assets', + 'azure-reporter', + 'testRunResults3ValidResponse.json' +); + +test.describe('testRun - Publish results - isExistingTestRun', () => { + test('isExistingTestRun without testRunId', async ({ runInlineTest, server }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + reporter: [ + ['line'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + logging: true, + publishTestResultsMode: 'testRun', + isExistingTestRun: true, + }] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('[3] foobar', async () => { + expect(1).toBe(0); + }); + `, + }, + { reporter: '' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).not.toMatch(/azure: Using existing run (\d.*) to publish test results/); + expect(result.output).not.toContain('azure: AZURE_PW_TEST_RUN_ID: 150'); + expect(result.output).toContain( + "azure: 'testRunId' or AZURE_PW_TEST_RUN_ID is not set for 'isExistingTestRun'=true mode. Reporting is disabled." + ); + expect(result.output).not.toContain('azure: [3] foobar - failed'); + expect(result.output).not.toContain('azure: Start publishing test results for 1 test(s)'); + expect(result.output).not.toContain('azure: Left to publish: 0'); + expect(result.output).not.toContain('azure: Test results published for 1 test(s)'); + expect(result.output).not.toMatch(/azure: Run (\d.*) - Completed/); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); + + test('isExistingTestRun with testRunId', async ({ runInlineTest, server }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + reporter: [ + ['line'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + logging: true, + publishTestResultsMode: 'testRun', + isExistingTestRun: true, + testRunId: 150, + }] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('[3] foobar', async () => { + expect(1).toBe(0); + }); + `, + }, + { reporter: '' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).toMatch(/azure: Using existing run (\d.*) to publish test results/); + expect(result.output).toContain('azure: AZURE_PW_TEST_RUN_ID: 150'); + expect(result.output).not.toContain( + "azure: 'testRunId' or AZURE_PW_TEST_RUN_ID is not set for 'isExistingTestRun'=true mode. Reporting is disabled." + ); + expect(result.output).toContain('azure: [3] foobar - failed'); + expect(result.output).toContain('azure: Start publishing test results for 1 test(s)'); + expect(result.output).toContain('azure: Left to publish: 0'); + expect(result.output).toContain('azure: Test results published for 1 test(s)'); + expect(result.output).not.toMatch(/azure: Run (\d.*) - Completed/); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); + + test('isExistingTestRun with AZURE_PW_TEST_RUN_ID', async ({ runInlineTest, server }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + reporter: [ + ['line'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + logging: true, + publishTestResultsMode: 'testRun', + isExistingTestRun: true, + }] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('[3] foobar', async () => { + expect(1).toBe(0); + }); + `, + }, + { reporter: '' }, + { AZURE_PW_TEST_RUN_ID: '150' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).toMatch(/azure: Using existing run (\d.*) to publish test results/); + expect(result.output).toContain('azure: AZURE_PW_TEST_RUN_ID: 150'); + expect(result.output).not.toContain( + "azure: 'testRunId' or AZURE_PW_TEST_RUN_ID is not set for 'isExistingTestRun'=true mode. Reporting is disabled." + ); + expect(result.output).toContain('azure: [3] foobar - failed'); + expect(result.output).toContain('azure: Start publishing test results for 1 test(s)'); + expect(result.output).toContain('azure: Left to publish: 0'); + expect(result.output).toContain('azure: Test results published for 1 test(s)'); + expect(result.output).not.toMatch(/azure: Run (\d.*) - Completed/); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); +}); + +test.describe('testResult - Publish results - isExistingTestRun', () => { + test('isExistingTestRun without testRunId', async ({ runInlineTest, server }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + reporter: [ + ['line'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + logging: true, + publishTestResultsMode: 'testResult', + isExistingTestRun: true, + }] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('[3] foobar', async () => { + expect(1).toBe(0); + }); + `, + }, + { reporter: '' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).not.toMatch(/azure: Using existing run (\d.*) to publish test results/); + expect(result.output).not.toContain('azure: AZURE_PW_TEST_RUN_ID: 150'); + expect(result.output).toContain( + "azure: 'testRunId' or AZURE_PW_TEST_RUN_ID is not set for 'isExistingTestRun'=true mode. Reporting is disabled." + ); + expect(result.output).not.toContain('azure: [3] foobar - failed'); + expect(result.output).not.toContain('azure: Start publishing test results for 1 test(s)'); + expect(result.output).not.toContain('azure: Left to publish: 0'); + expect(result.output).not.toContain('azure: Test results published for 1 test(s)'); + expect(result.output).not.toMatch(/azure: Run (\d.*) - Completed/); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); + + test('isExistingTestRun with testRunId', async ({ runInlineTest, server }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + reporter: [ + ['line'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + logging: true, + publishTestResultsMode: 'testResult', + isExistingTestRun: true, + testRunId: 150, + }] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('[3] foobar', async () => { + expect(1).toBe(0); + }); + `, + }, + { reporter: '' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).toMatch(/azure: Using existing run (\d.*) to publish test results/); + expect(result.output).toContain('azure: AZURE_PW_TEST_RUN_ID: 150'); + expect(result.output).not.toContain( + "azure: 'testRunId' or AZURE_PW_TEST_RUN_ID is not set for 'isExistingTestRun'=true mode. Reporting is disabled." + ); + expect(result.output).toContain('azure: [3] foobar - failed'); + expect(result.output).toContain('azure: Start publishing: [3] foobar'); + expect(result.output).toContain('azure: Result published: [3] foobar'); + expect(result.output).not.toMatch(/azure: Run (\d.*) - Completed/); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); + + test('isExistingTestRun with AZURE_PW_TEST_RUN_ID', async ({ runInlineTest, server }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + reporter: [ + ['line'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + logging: true, + publishTestResultsMode: 'testResult', + isExistingTestRun: true, + }] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('[3] foobar', async () => { + expect(1).toBe(0); + }); + `, + }, + { reporter: '' }, + { AZURE_PW_TEST_RUN_ID: '150' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).toMatch(/azure: Using existing run (\d.*) to publish test results/); + expect(result.output).toContain('azure: AZURE_PW_TEST_RUN_ID: 150'); + expect(result.output).not.toContain( + "azure: 'testRunId' or AZURE_PW_TEST_RUN_ID is not set for 'isExistingTestRun'=true mode. Reporting is disabled." + ); + expect(result.output).toContain('azure: [3] foobar - failed'); + expect(result.output).toContain('azure: Start publishing: [3] foobar'); + expect(result.output).toContain('azure: Result published: [3] foobar'); + expect(result.output).not.toMatch(/azure: Run (\d.*) - Completed/); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); +}); From 6851b1069be208a4b5e5125a632c9679fcbfd3fb Mon Sep 17 00:00:00 2001 From: Alex Neo Date: Tue, 21 Nov 2023 16:54:13 +0200 Subject: [PATCH 4/4] 1.8.0 --- CHANGELOG.md | 9 +++++++++ package.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f99cb8b..bf75513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# [1.8.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.7.0...v1.8.0) (2023-11-21) + + +### Features + +* set existing test run ID ([3c2bfb3](https://github.com/alexneo2003/playwright-azure-reporter/commit/3c2bfb31fad8427dd1206202d934812a78290b27)) + + + # [1.7.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.7.0-beta.0...v1.7.0) (2023-11-21) diff --git a/package.json b/package.json index fe476a3..d106ff3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex_neo/playwright-azure-reporter", - "version": "1.7.0", + "version": "1.8.0", "description": "Playwright Azure Reporter", "main": "./dist/playwright-azure-reporter.js", "types": "./dist/playwright-azure-reporter.d.js",