Add ability to set Zephyr scale test ids via annotations

.env.example

@@ -0,0 +1 @@
+ZEPHYR_AUTHORIZATION_TOKEN=MySecretTokenValueHere

.gitignore

@@ -6,6 +6,11 @@ yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 
+# Dotenv environment variables file
+.env
+
+test-results/
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

README.md

@@ -77,6 +77,19 @@ test('[J79] basic test', async ({ page }) => {
 });
 ```
 
+You can also use annotations to set the test id, note this supports multiple test ids.
+```typescript
+test('An annotated test', {
+  annotation: [
+      { type: 'zephyrTestId', description: 'T2' },
+      { type: 'zephyrTestId', description: 'T3' }
+  ]
+}, async ({page}){
+  await page.goto('https://playwright.dev/');
+  const title = page.locator('.navbar__inner .navbar__title');
+  await expect(title).toHaveText('Playwright');
+});
+
 Then run your tests with `npx playwright test` command and you'll see the result in console:
 
 ```sh

package.json

@@ -37,15 +37,16 @@
   "homepage": "https://github.com/elaichenkov/playwright-zephyr#readme",
   "devDependencies": {
     "@playwright/test": "^1.36.2",
+    "dotenv": "^17.2.3",
     "prettier": "^3.0.1",
     "release-it": "^19.0.2",
     "typescript": "^5.1.6"
   },
   "dependencies": {
+    "@types/adm-zip": "^0.5.0",
+    "adm-zip": "^0.5.10",
     "axios": "^1.4.0",
     "picocolors": "^1.0.0",
-    "table": "^6.8.1",
-    "@types/adm-zip": "^0.5.0",
-    "adm-zip": "^0.5.10"
+    "table": "^6.8.1"
   }
 }

playwright.cloud.config.ts

@@ -0,0 +1,40 @@
+// playwright.config.ts
+import { PlaywrightTestConfig } from '@playwright/test';
+import * as dotenv from 'dotenv';
+
+dotenv.config({ path: '.env' });
+
+const config: PlaywrightTestConfig = {
+  reporter: [['list'], ['./src/cloud', { 
+    runName: `testrun-${new Date().getTime()}`,
+    authorizationToken: process.env.ZEPHYR_AUTHORIZATION_TOKEN,
+    projectKey: 'QE'
+  }]],
+  use: {
+      screenshot: 'only-on-failure'
+  },
+  projects: [
+    {
+      name: 'Chrome',
+      use: {
+        browserName: 'chromium',
+        channel: 'chrome',
+      },
+    },
+    {
+      name: 'Safari',
+      use: {
+        browserName: 'webkit',
+        viewport: { width: 1200, height: 750 },
+      }
+    },
+    {
+      name: 'Firefox',
+      use: {
+        browserName: 'firefox',
+        viewport: { width: 800, height: 600 },
+      }
+    },
+  ],
+};
+export default config;

playwright.config.ts

@@ -2,7 +2,7 @@
 import { PlaywrightTestConfig } from '@playwright/test';
 
 const config: PlaywrightTestConfig = {
-  reporter: [['list'], ['./src/zephyr.reporter.ts', { 
+  reporter: [['list'], ['./src', { 
     host: '',
     runName: '',
     user: '',

src/cloud.ts

@@ -13,7 +13,8 @@ export default class ZephyrReporter implements Reporter {
   private zephyrService!: ZephyrService;
   private testResults: ZephyrTestResult[] = [];
   private projectKey!: string;
-  private testCaseKeyPattern = /\[(.*?)\]/;
+  private readonly testCaseKeyPattern = /\[(.*?)\]/;
+  private readonly ansiRegex = /\x1b\[[0-9;]*m/g;
   private options: ZephyrOptions;
 
   constructor(options: ZephyrOptions) {
@@ -27,27 +28,39 @@ export default class ZephyrReporter implements Reporter {
   }
 
   onTestEnd(test: TestCase, result: TestResult) {
+    let testCaseIds: string[] | undefined;
     if (test.title.match(this.testCaseKeyPattern) && test.title.match(this.testCaseKeyPattern)!.length > 1) {
       const [, testCaseId] = test.title.match(this.testCaseKeyPattern)!;
-      const testCaseKey = `${this.projectKey}-${testCaseId}`;
-      const status = convertStatus(result.status);
-      const comment = result.error
-        ? `<b>❌ Error Message: </b> <br> <span style="color: rgb(226, 80, 65);">${result.error?.message?.replaceAll(
-            '\n',
-            '<br>',
-          )}</span> <br> <br> <b>🧱 Stack Trace:</b> <br> <span style="color: rgb(226, 80, 65);">${result.error?.stack?.replaceAll(
-            '\n',
-            '<br>',
-          )}</span>`
-        : undefined;
+      if(testCaseId) testCaseIds = [testCaseId];
+    } else if(test.annotations.some(annotation => annotation.type === 'zephyrTestId')){
+      testCaseIds = test.annotations
+        .filter(annotation => annotation.type === 'zephyrTestId')
+        .map(annotation => annotation.description || '');  
+    }
+
+    if(testCaseIds) {
+      for (const testCaseId of testCaseIds) {
+
+        const testCaseKey = `${this.projectKey}-${testCaseId}`;
+        const status = convertStatus(result.status);
+        const comment = result.error
+          ? `<b>❌ Error Message: </b> <br> <span style="color: rgb(226, 80, 65);">${this.stripAnsiCodes(result.error?.message)?.replaceAll(
+              '\n',
+              '<br>',
+            )}</span> <br> <br> <b>🧱 Stack Trace:</b> <br> <span style="color: rgb(226, 80, 65);">${this.stripAnsiCodes(result.error?.stack)?.replaceAll(
+              '\n',
+              '<br>',
+            )}</span>`
+          : undefined;
 
-      this.testResults.push({
-        result: status,
-        testCase: {
-          key: testCaseKey,
-          comment,
-        },
-      });
+        this.testResults.push({
+          result: status,
+          testCase: {
+            key: testCaseKey,
+            comment,
+          },
+        });
+      }
     }
   }
 
@@ -64,4 +77,9 @@ export default class ZephyrReporter implements Reporter {
       console.log(gray(`[zephyr reporter]: There's no Zephyr test case id in this spec file`));
     }
   }
+
+  private stripAnsiCodes(str: string | undefined): string | undefined {
+    return str?.replaceAll(this.ansiRegex, '');
+  }
+
 }

src/index.ts

@@ -32,17 +32,29 @@ class ZephyrReporter implements Reporter {
   }
 
   onTestEnd(test: TestCase, result: TestResult) {
+    let testCaseIds: string[] | undefined;
     if (test.title.match(this.testCaseKeyPattern) && test.title.match(this.testCaseKeyPattern)!.length > 1) {
-      const [, projectName] = test.titlePath();
       const [, testCaseId] = test.title.match(this.testCaseKeyPattern)!;
-      const testCaseKey = `${this.projectKey}-${testCaseId}`;
-      const status = convertPwStatusToZephyr(result.status);
-      this.testResults.push({
-        testCaseKey,
-        status,
-        environment: this.environment ?? projectName ?? 'Playwright',
-        executionDate: new Date().toISOString(),
-      });
+      if(testCaseId) testCaseIds = [testCaseId];
+    } else if(test.annotations.some(annotation => annotation.type === 'zephyrTestId')){
+      testCaseIds = test.annotations
+        .filter(annotation => annotation.type === 'zephyrTestId')
+        .map(annotation => annotation.description || '');  
+    }
+
+    if(testCaseIds) {
+      for (const testCaseId of testCaseIds) {
+        const [, projectName] = test.titlePath();
+        const testCaseKey = `${this.projectKey}-${testCaseId}`;
+        const status = convertPwStatusToZephyr(result.status);
+
+        this.testResults.push({
+          testCaseKey,
+          status,
+          environment: this.environment ?? projectName ?? 'Playwright',
+          executionDate: new Date().toISOString(),
+        });
+      }
     }
   }
 

src/server.ts

@@ -32,18 +32,29 @@ class ZephyrReporter implements Reporter {
   }
 
   onTestEnd(test: TestCase, result: TestResult) {
+    let testCaseIds: string[] | undefined;
     if (test.title.match(this.testCaseKeyPattern) && test.title.match(this.testCaseKeyPattern)!.length > 1) {
-      const [, projectName] = test.titlePath();
       const [, testCaseId] = test.title.match(this.testCaseKeyPattern)!;
-      const testCaseKey = `${this.projectKey}-${testCaseId}`;
-      const status = convertPwStatusToZephyr(result.status);
-
-      this.testResults.push({
-        testCaseKey,
-        status,
-        environment: this.environment ?? projectName ?? 'Playwright Test',
-        executionDate: new Date().toISOString(),
-      });
+      if(testCaseId) testCaseIds = [testCaseId];
+    } else if(test.annotations.some(annotation => annotation.type === 'zephyrTestId')){
+      testCaseIds = test.annotations
+        .filter(annotation => annotation.type === 'zephyrTestId')
+        .map(annotation => annotation.description || '');  
+    }
+
+    if(testCaseIds) {
+      for (const testCaseId of testCaseIds) {
+        const [, projectName] = test.titlePath();
+        const testCaseKey = `${this.projectKey}-${testCaseId}`;
+        const status = convertPwStatusToZephyr(result.status);
+
+        this.testResults.push({
+          testCaseKey,
+          status,
+          environment: this.environment ?? projectName ?? 'Playwright Test',
+          executionDate: new Date().toISOString(),
+        });
+      }
     }
   }
 

tests/annotated.test.ts

@@ -0,0 +1,29 @@
+import { test, expect } from "@playwright/test";
+
+test('annotated test', {
+    annotation: [{ type: 'zephyrTestId', description: 'T1' }]
+}, async ({page}) => {
+    await page.goto('https://playwright.dev/');
+    const title = page.locator('.navbar__inner .navbar__title');
+    await expect(title).toHaveText('Playwright');
+})
+
+test('Can annotate tests with multiple Zephyr IDs', {
+    annotation: [
+        { type: 'zephyrTestId', description: 'T2' },
+        { type: 'zephyrTestId', description: 'T3' }
+    ]
+}, async ({page}) => {
+        await page.goto('https://playwright.dev/');
+        const title = page.locator('.navbar__inner .navbar__title');
+        await expect(title).toHaveText('Playwright');
+    }
+)
+
+test.fail('Failed test with Zephyr ID', {
+    annotation: [{ type: 'zephyrTestId', description: 'T1' }]
+}, async ({page}) => {
+    await page.goto('https://playwright.dev/');
+    const title = page.locator('.navbar__inner .navbar__title');
+    await expect(title).toHaveText('Playright'); // Intentional typo to cause failure
+})