Add Playwright E2E tests

.github/workflows/CI.yml

@@ -189,3 +189,74 @@ jobs:
 
       - name: Package
         run: yarn package
+
+  # Tests that use a real VS Code through Playwright. They're separate from the
+  # tests above: see "End-to-End Tests" in DEVELOPMENT.md.
+  e2e:
+    name: "E2E Tests (OS: ${{ matrix.os }}, VS Code: ${{ matrix.vscode }})"
+    strategy:
+      matrix:
+        os: [macos-15, ubuntu-22.04, windows-2022]
+        vscode: ["stable"]
+        include:
+          - os: ubuntu-22.04
+            vscode: "1.90.0" # the oldest supported version of VS Code
+      fail-fast: false # don't immediately fail all other jobs if a single job fails
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    env:
+      DAFFODIL_TEST_VSCODE_VERSION: ${{ matrix.vscode }} # read by e2e/global.setup.ts
+    steps:
+      ############################################################
+      # Setup
+      ############################################################
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      # Daffodil 4.0.0 and newer need JDK 17 or newer
+      - name: Setup Java
+        uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
+        with:
+          distribution: temurin
+          java-version: 17
+
+      # some runner oses don't have sbt by default
+      - name: Setup sbt
+        uses: sbt/setup-sbt@ecad189dd245444b9ae0a0cd5922fd938b88906f # v1.5.12
+
+      # Playwright needs the latest patch release of a supported Node version
+      - name: Install Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+          cache: yarn
+          cache-dependency-path: yarn.lock
+
+      - name: Install node dependencies
+        run: yarn install --frozen-lockfile
+
+      ############################################################
+      # Test
+      ############################################################
+
+      # Linux runners have no screen, so xvfb-run gives VS Code a virtual one.
+      # -a makes it pick a free display number.
+      - name: Run E2E tests - Linux
+        run: xvfb-run -a yarn test:e2e
+        if: runner.os == 'Linux'
+
+      - name: Run E2E tests - Windows/Mac
+        run: yarn test:e2e
+        if: runner.os != 'Linux'
+
+      # The report has a trace of each test, for finding out why one failed. To
+      # read it, download it from the run's page and run:
+      # yarn playwright show-report <unzipped folder>
+      - name: Upload test report
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: failure()
+        with:
+          name: e2e-report-${{ matrix.os }}-${{ matrix.vscode }}
+          path: e2e/playwright-report/

.gitignore

@@ -35,6 +35,10 @@ src/version.ts
 # unknown
 .vscode-test/
 
+# playwright output
+e2e/test-results/
+e2e/playwright-report/
+
 # scala metals vscode extension compiled folder
 .metals
 .bloop

DEVELOPMENT.md

@@ -62,6 +62,8 @@
       - [Yarn Package](#yarn-package)
       - [Automated Testing Suite](#automated-testing-suite)
         - [Testing Against a Specific Version of VS Code](#testing-against-a-specific-version-of-vs-code)
+      - [End-to-End Tests](#end-to-end-tests)
+        - [How a Test Run Works](#how-a-test-run-works)
     - [Debugging the Extension](#debugging-the-extension)
       - [Changing the sampleWorkspace folder name and location](#changing-the-sampleworkspace-folder-name-and-location)
       - [Test a Local Version of Apache Daffodil Debugger](#test-a-local-version-of-apache-daffodil-debugger)
@@ -361,6 +363,38 @@ DAFFODIL_TEST_VSCODE_VERSION=1.74.3 yarn test
 
 Set `DAFFODIL_TEST_VSCODE_VERSION` to `stable` to use the latest stable release, or to `insiders` to use the latest (nightly) insiders build.
 
+#### End-to-End Tests
+
+The end-to-end (E2E) tests are separate from the test suite above, and everything for them is in the `e2e` folder. They use [Playwright](https://playwright.dev) to start a real VS Code and use the extension the way a person would, including the insides of webviews like the Data Editor. They need Node 22 or newer.
+
+| Command                | What it does                                                                         |
+| ---------------------- | ------------------------------------------------------------------------------------ |
+| `yarn test:e2e`        | Runs the tests                                                                       |
+| `yarn test:e2e:ui`     | Opens Playwright's interactive viewer, to pick tests, run them and inspect each step |
+| `yarn test:e2e:debug`  | Runs the tests paused in the Playwright Inspector, to step through them              |
+| `yarn test:e2e:report` | Opens the report of the last run, with a step-by-step trace of each test             |
+
+All of them except the report rebuild the extension first, and accept a test filter, such as `yarn test:e2e:ui dataEditor`. `DAFFODIL_TEST_VSCODE_VERSION` picks the version of VS Code, the same as for `yarn test`.
+
+In the interactive viewer, a test's timeline appears when the test finishes rather than while it runs, as Playwright can't trace VS Code live. The viewer can re-run tests when a spec changes. After changing the extension itself, run `yarn pretest:e2e` in another terminal to rebuild it.
+
+When adding tests, keep selectors for VS Code's own UI in `e2e/pages/workbench.ts`, as they can change between VS Code releases. Elements of our webviews that have no stable `id` can be given a `data-testid`.
+
+##### How a Test Run Works
+
+When you run `yarn test:e2e`:
+
+1. The `pretest:e2e` script type-checks the tests and rebuilds the extension, because the tests use the built copy in `dist`.
+2. Playwright reads its settings from `e2e/playwright.config.ts`.
+3. `e2e/global.setup.ts` gets a copy of VS Code. It's only downloaded once, and is then kept in `.vscode-test`.
+4. For each test in the `*.spec.ts` files:
+   1. `e2e/fixtures.ts` makes a temporary folder, copies `src/tests/data` into it, and starts VS Code on that copy with the extension loaded.
+   2. The test runs, and controls VS Code through the helpers in `e2e/pages`.
+   3. `e2e/fixtures.ts` saves a trace of the test, closes VS Code, stops the Data Editor's server and deletes the temporary folder.
+5. Playwright writes the report to `e2e/playwright-report`.
+
+To follow this in the code, start with a test in `e2e/dataEditor.spec.ts`, then read `e2e/fixtures.ts`.
+
 ### Debugging the Extension
 
 Create a `sampleWorkspace` folder in the folder one level higher than where `daffodil-vscode` currently resides. For example, if you have your `daffodil-vscode` folder stored in a folder called repos, then make a folder in repos called `sampleWorkspace`.

e2e/dataEditor.spec.ts

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs'
+import * as path from 'path'
+import { expect, test } from './fixtures'
+
+// `test` comes from ./fixtures, which provides the arguments that each test
+// asks for. Every test gets its own VS Code and its own copy of the test data,
+// so tests can't affect each other.
+//
+// VS Code is already running when a test's first line runs, and the fixtures
+// close it again after the last line.
+
+test('shows the bytes of the opened file', async ({
+  dataEditor,
+  workspaceDir,
+}) => {
+  // Open the test's copy of test.txt in the Data Editor
+  await dataEditor.open(path.join(workspaceDir, 'test.txt'))
+
+  // test.txt starts with "T", which is 54 in hex. expect() on a locator keeps
+  // checking until it passes or times out, so there's no need to wait for the
+  // file to load first.
+  await expect(dataEditor.byte(0)).toHaveText('54')
+})
+
+test('saves an overwritten byte to disk', async ({
+  dataEditor,
+  workspaceDir,
+}) => {
+  const file = path.join(workspaceDir, 'test.txt')
+
+  // The saved file should be the same, except that it starts with an "A" (0x41)
+  const expectedBytes = fs.readFileSync(file)
+  expectedBytes[0] = 0x41
+
+  // Open the file and change its first byte
+  await dataEditor.open(file)
+  await dataEditor.overwriteByte(0, '41')
+
+  // The new value should show on screen
+  await expect(dataEditor.byte(0)).toHaveText('41')
+
+  // Write the change to the file on disk
+  await dataEditor.save()
+
+  // Saving finishes some time after the click, so expect.poll keeps re-reading
+  // the file until it matches or times out
+  await expect.poll(() => fs.readFileSync(file)).toEqual(expectedBytes)
+})

e2e/debugger.spec.ts

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as path from 'path'
+import { expect, test } from './fixtures'
+
+// Debugs a schema through the UI, the way a person would. This test doesn't ask
+// for dataEditor, so it only sets up the fixtures that start VS Code.
+
+test('pauses on entry when debugging a schema', async ({
+  workbench,
+  workspaceDir,
+}) => {
+  // The "Debug File" command debugs the open file, so open the schema first
+  await workbench.openFile('test.dfdl.xsd')
+  await workbench.runCommand('Daffodil Debug: Debug File')
+
+  // The command asks for the data file to parse with the schema
+  await workbench.pickFile(path.join(workspaceDir, 'test.txt'))
+
+  // The extension now does several things, which this waits for:
+  //  1. Compiles the schema with the Daffodil CLI
+  //     (validateSchemaWithDaffodilCLI in src/daffodilDebugger/utils.ts)
+  //  2. Starts the debugger (runDebugger in the same file)
+  //  3. Connects to the debugger, and pauses on entry
+  // The Continue button appears once it has paused. All of this is slow, so it
+  // gets a longer timeout than the usual 30 seconds.
+  await expect(workbench.debugContinueButton()).toBeVisible({
+    timeout: 120_000,
+  })
+
+  // End the session before the fixtures close VS Code
+  await workbench.stopDebugging()
+})