Merge pull request #9165 from nextcloud/backport/collectives-599/stable35

[stable35] Backport version comparison stack (#9047-#9050)

Author: blizzz

.github/workflows/playwright.yml

@@ -55,17 +55,21 @@ jobs:
           npm run build --if-present
 
       - name: Install Playwright Browsers
-        run: npx playwright install chromium --only-shell
+        run: npx playwright install --with-deps chromium webkit
 
       - name: Run Playwright tests
         run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        env:
+          TEXT_COMPARISON_E2E: '1'
 
       - name: Upload results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: ${{ !cancelled() }}
         with:
           name: playwright-report_shard${{ matrix.shardIndex }}
-          path: test-results/
+          path: |
+            test-results/
+            blob-report/
           retention-days: 7
 
   summary:

package-lock.json

@@ -79,6 +79,7 @@
     "@tiptap/vue-3": "^3.30.5",
     "@vueuse/shared": "^14.4.0",
     "debounce": "^3.0.0",
+    "diff": "^8.0.4",
     "escape-html": "^1.0.3",
     "highlight.js": "^11.12.0",
     "katex": "^0.18.4",
@@ -108,6 +109,7 @@
     "yjs": "^13.6.32"
   },
   "devDependencies": {
+    "@axe-core/playwright": "^4.13.0",
     "@nextcloud/babel-config": "^1.3.0",
     "@nextcloud/browserslist-config": "^3.1.2",
     "@nextcloud/e2e-test-server": "^0.5.1",

package.json

@@ -2,11 +2,57 @@
  * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
+/* eslint-disable jsdoc/require-jsdoc */
 
 import type { ReporterDescription } from '@playwright/test'
 
 import { defineConfig, devices } from '@playwright/test'
 
+const COMPARISON_E2E = process.env.TEXT_COMPARISON_E2E === '1'
+const COMPARISON_TESTS = /playwright\/comparison\/.*\.spec\.ts/
+const COMPARISON_BASE_URL = process.env.TEXT_COMPARISON_BASE_URL || process.env.baseURL || 'http://localhost:8089/index.php/'
+const EXTERNAL_COMPARISON_SERVER = Boolean(process.env.TEXT_COMPARISON_BASE_URL || process.env.baseURL)
+
+function comparisonProjects() {
+	if (!COMPARISON_E2E) {
+		return []
+	}
+	return [{
+		name: 'comparison-chromium',
+		testMatch: COMPARISON_TESTS,
+		grepInvert: /@memory/,
+		use: {
+			...devices['Desktop Chrome'],
+			baseURL: COMPARISON_BASE_URL,
+			ignoreHTTPSErrors: true,
+			screenshot: 'only-on-failure' as const,
+			trace: 'retain-on-failure' as const,
+		},
+	}, {
+		name: 'comparison-webkit',
+		testMatch: COMPARISON_TESTS,
+		grepInvert: /@memory/,
+		use: {
+			...devices['Desktop Safari'],
+			baseURL: COMPARISON_BASE_URL,
+			ignoreHTTPSErrors: true,
+			screenshot: 'only-on-failure' as const,
+			trace: 'retain-on-failure' as const,
+		},
+	}, {
+		name: 'comparison-chromium-memory',
+		testMatch: COMPARISON_TESTS,
+		grep: /@memory/,
+		use: {
+			...devices['Desktop Chrome'],
+			baseURL: COMPARISON_BASE_URL,
+			ignoreHTTPSErrors: true,
+			screenshot: 'only-on-failure' as const,
+			trace: 'retain-on-failure' as const,
+		},
+	}]
+}
+
 /**
  * Used locally - i.e. if `CI` is not set as an environment variable.
  */
@@ -24,55 +70,62 @@ const CI_CONFIG = {
 	// blob (so we can merge reports and download them for inspection),
 	// dot (so we have a quick overview in the logs while the tests are running)
 	// github (to have annotations in the PR)
-	reporter: [['blob'], ['line'], ['github']] as ReporterDescription[],
+	reporter: [
+		['blob'],
+		['json', { outputFile: 'test-results/results.json' }],
+		['line'],
+		['github'],
+	] as ReporterDescription[],
 	retries: 1,
 	timeout: 45_000,
 	// we shard to speed up the tests so no parallelism in workers
 	workers: 1,
 } as const
 
+function comparisonWebServer() {
+	if (EXTERNAL_COMPARISON_SERVER) {
+		return undefined
+	}
+	return {
+		command: 'npm run start:nextcloud',
+		gracefulShutdown: {
+			signal: 'SIGTERM' as const,
+			timeout: 10000,
+		},
+		reuseExistingServer: false,
+		stderr: 'pipe' as const,
+		stdout: 'pipe' as const,
+		timeout: 5 * 60 * 1000,
+		wait: {
+			stdout: /Nextcloud is now ready to use/,
+		},
+	}
+}
+
 /**
  * See https://playwright.dev/docs/test-configuration.
  */
 export default defineConfig({
 	testDir: './playwright',
 	...(process.env.CI ? CI_CONFIG : LOCAL_CONFIG),
+	workers: COMPARISON_E2E ? 1 : undefined,
 	use: {
 		// Base URL to use in actions like `await page.goto('./')`.
-		baseURL: process.env.baseURL ?? 'http://localhost:8089/index.php/',
+		baseURL: COMPARISON_BASE_URL,
 		// record traces but only keep them when the test fails
 		trace: 'on-first-retry',
 	},
 
 	projects: [
 		{
 			name: 'chromium',
+			testIgnore: COMPARISON_TESTS,
 			use: {
 				...devices['Desktop Chrome'],
 			},
 		},
+		...comparisonProjects(),
 	],
 
-	webServer: {
-		// Don't set `url` as it would take precedence over `wait.stdout` and tests start too early
-		// url: 'http://127.0.0.1:8089',
-		// Starts the Nextcloud docker container
-		command: 'npm run start:nextcloud',
-		// we use sigterm to notify the script to stop the container
-		// if it does not respond, we force kill it after 10 seconds
-		gracefulShutdown: {
-			signal: 'SIGTERM',
-			timeout: 10000,
-		},
-		// `start-nextcloud-server.mjs` only starts the server if not reachable yet.
-		reuseExistingServer: false,
-		stderr: 'pipe',
-		stdout: 'pipe',
-		// max. 5 minutes for creating the container
-		timeout: 5 * 60 * 1000,
-		wait: {
-			// we wait for this line to appear in the output of the webserver until consider it done
-			stdout: /Nextcloud is now ready to use/,
-		},
-	},
+	webServer: comparisonWebServer(),
 })