research/perf

.github/workflows/review.yaml

@@ -151,22 +151,71 @@ jobs:
           ssh_key: ${{ secrets.SSH_PRIVATE_KEY }}
           env_file: .env
 
-  # load_test:
-  #   runs-on: self-hosted
-  #   needs: test_servers
-  #   timeout-minutes: 20
-  #   concurrency:
-  #     group: ${{ matrix.subdomain }}
-  #   strategy:
-  #     matrix:
-  #       subdomain: [aws, do, hetzner]
-  #       # gc, ionos
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: artilleryio/action-cli@v1
-  #       with:
-  #         command: run config/load/socket-io.yml ${{ matrix.subdomain }}.spaced.fun
-  #     # - run: ssh spaced@${{ matrix.subdomain }}.spaced.fun
+  load_test:
+    runs-on: self-hosted
+    needs: test_servers
+    timeout-minutes: 20
+    concurrency:
+      group: ${{ matrix.subdomain }}
+    strategy:
+      matrix:
+        subdomain: [do]
+        # subdomain: [aws, do] # ADD LAST MINIMUM
+        # subdomain: [aws, do, hetzner]
+        # gc, ionos
+    environment: manual_approval
+    env:
+      PLAYWRIGHT_BROWSERS_PATH: /data/runner_workspace/.cache/ms-playwright
+    steps:
+      - uses: actions/checkout@v4
+      - run: echo "TIMESTAMP=$(date +%s)" >> $GITHUB_ENV
+      - run: npm i && npx artillery
+      - run: |
+          npx artillery run config/load/playwright.yaml \
+            --tags env:review,host:${{ matrix.subdomain }},sha:${{ github.sha }} \
+            --record -o playwright_${{ env.TIMESTAMP }}.json -e breakpoint
+        continue-on-error: true
+        env:
+          ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY }}
+          SUBDOMAIN: ${{ matrix.subdomain }}
+      - run: |
+          jq '
+            .intermediate
+            | to_entries
+            | map(select(
+                .value.counters."vusers.failed" < 1 and
+                .value.summaries."browser.step.open".p99 < 1000 and
+                .value.summaries."browser.step.create".p99 < 1000 and
+                .value.summaries."browser.step.share".p99 < 1000 or
+                .value.summaries."browser.step.open".p99 < 1000 and
+                .value.summaries."browser.step.join".p99 < 1000 and
+                .value.summaries."browser.step.create_check_list".p99 < 2000
+              ))
+            | sort_by(.value.counters."vusers.created" | tonumber)
+            | last
+            | {
+                record: .key,
+                "vusers.created": .value.counters."vusers.created",
+                "vusers.session_length.p999": .value.summaries."vusers.session_length".p999,
+              }
+          ' playwright_${{ env.TIMESTAMP }}.json | tee | jq ."vusers.created" > NUM
+
+          echo SMOKE_RATE=$(cat NUM | jq '. * 0.10 | round') >> $GITHUB_ENV
+          echo AVG_RATE=$(cat NUM | jq '. * 0.50 | round') >> $GITHUB_ENV
+          echo STRESS_RATE=$(cat NUM | jq '. * 0.75 | round') >> $GITHUB_ENV
+          echo SPIKE_RATE=$(cat NUM) >> $GITHUB_ENV
+      - run: |
+          npx artillery run config/load/playwright.yaml \
+            --tags env:review,host:${{ matrix.subdomain }},sha:${{ github.sha }} \
+            --record -o playwright_${{ env.TIMESTAMP }}.json -e smoke_average_stress_spike
+        env:
+          ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY }}
+          SUBDOMAIN: ${{ matrix.subdomain }}
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: artillery-report
+          path: ./playwright_${{ env.TIMESTAMP }}.json
 
   # cleanup:
   #   runs-on: ubuntu-latest

.gitignore

@@ -4,6 +4,9 @@ srcs/e2e/playwright-report/
 srcs/e2e/blob-report/
 srcs/e2e/playwright/.cache/
 
+# Artillery
+config/load/*.json
+
 # Typescript
 tsconfig.tsbuildinfo
 srcs/slate-solid/lib

config/load/functions.cjs

@@ -0,0 +1,11 @@
+const { randomUUID } = require('node:crypto');
+
+function uuid(ctx, ee, next) {
+  ctx.vars['uuid'] = randomUUID();
+
+  return next();
+}
+
+module.exports = {
+  uuid,
+};

config/load/playwright.yaml

@@ -0,0 +1,67 @@
+# yaml-language-server: $schema=https://www.artillery.io/schema.json
+config:
+  processor: scenarios.mjs
+  engines:
+    playwright:
+      launchOptions:
+        headless: true
+  plugins:
+    ensure: {}
+  ensure:
+    maxErrorRate: 1
+    conditions:
+      - expression: 'browser.step.open.p99 < 1000 and browser.step.create.p99 < 1000 and browser.step.share.p99 < 1000 or browser.step.open.p99 < 1000 and browser.step.join.p99 < 1000 and browser.step.create_check_list.p99 < 2000'
+  variables:
+    uuid: 28cf22c3-77e8-49f5-b2ef-fa3eb212cce3
+  environments:
+    dev:
+      target: http://localhost:1420
+      engines:
+        playwright:
+          launchOptions:
+            headless: false
+          trace:
+            enabled: true
+    breakpoint:
+      target: http://{{ $env.SUBDOMAIN }}.spaced.fun
+      socketio:
+        query: EIO=4&transport=websocket
+        transports: [websocket]
+      phases:
+        # duration / 10 = reports
+        # rampTo / reports = concurrent users added per report (granularity)
+        - name: Find breakpoint
+          duration: 2m
+          arrivalRate: 1
+          rampTo: 100
+          # maxVusers: 500
+    smoke_average_stress_spike:
+      target: http://{{ $env.SUBDOMAIN }}.spaced.fun
+      socketio:
+        query: EIO=4&transport=websocket
+        transports: [websocket]
+      phases:
+        - name: smoke
+          duration: 10
+          arrivalRate: '{{ $env.SMOKE_RATE }}' # breakpoint * .10
+          maxVusers: '{{ $env.SMOKE_RATE }}' # breakpoint * .10
+        - name: average
+          duration: 60
+          arrivalRate: '{{ $env.AVG_RATE }}' # breakpoint * .50
+        - name: stress
+          duration: 120
+          arrivalRate: '{{ $env.STRESS_RATE }}' # breakpoint * .75
+        - name: pause
+          pause: 10
+        - name: spike
+          duration: 5
+          arrivalRate: '{{ $env.SPIKE_RATE }}' # breakpoint
+          maxVusers: 200
+
+scenarios:
+  - engine: playwright
+    testFunction: create_and_share_document
+    weight: 1
+  - engine: playwright
+    testFunction: join_and_update_document
+    weight: 6

config/load/scenarios.mjs

@@ -0,0 +1,180 @@
+export async function create_and_share_document(page, vuContext, events, test) {
+  const { step } = test;
+
+  await step('open', async () => {
+    await page.goto('/');
+  });
+
+  await step('create', async () => {
+    await page.getByTitle('Create Item').click();
+    const editor = page.locator('[data-slate-editor]');
+
+    await editor.dblclick();
+    await editor.pressSequentially(`Entering Title`);
+    await editor.press('Enter');
+    await editor.press('Enter');
+  });
+
+  await step('share', async () => {
+    await page.getByTitle('Share').click();
+
+    await page
+      .locator("input[name='username']")
+      .pressSequentially('RandomUser');
+
+    await page.evaluate(() => {
+      const getRandomHexColor = () => {
+        const randomColor = Math.floor(Math.random() * 16_777_215).toString(16);
+        return `#${randomColor.padStart(6, '0')}`;
+      };
+      const input = document.querySelector("input[name='color']");
+      if (input) {
+        input.value = getRandomHexColor();
+        input.dispatchEvent(new Event('change', { bubbles: true }));
+      }
+    });
+
+    await page.click("button[type='submit']");
+    const uuid = await page.evaluate(
+      () =>
+        (window.crypto.randomUUID = () => {
+          return 'mocked response';
+        }),
+    );
+    vuContext.vars.uuid = uuid;
+  });
+}
+
+export async function join_and_update_document(page, vuContext, events, test) {
+  const { step } = test;
+
+  await step('open', async () => {
+    await page.goto('/');
+  });
+
+  await step('join', async () => {
+    await page.getByPlaceholder('Enter code').fill(vuContext.vars.uuid);
+    await page.getByPlaceholder('Enter code').press('Enter');
+
+    await page
+      .locator("input[name='username']")
+      .pressSequentially('RandomUser');
+
+    await page.evaluate(() => {
+      const getRandomHexColor = () => {
+        const randomColor = Math.floor(Math.random() * 16_777_215).toString(16);
+        return `#${randomColor.padStart(6, '0')}`;
+      };
+      const input = document.querySelector("input[name='color']");
+      if (input) {
+        input.value = getRandomHexColor();
+        input.dispatchEvent(new Event('change', { bubbles: true }));
+      }
+    });
+  });
+
+  await step('create_check_list', async () => {
+    const editor = page.locator('[data-slate-editor]');
+    await editor.dblclick();
+    for (let index = 0; index < 6; index++) {
+      await editor.pressSequentially(
+        `-[] Implementing todo ${index + 1} for next week.`,
+      );
+      await editor.press('Enter');
+    }
+  });
+}
+
+// ###########
+
+// export async function document_connections(page, vuContext, events, test) {
+//   const { step } = test;
+
+//   await step('open', async () => {
+//     await page.goto('/');
+//   });
+
+//   await step('connect', async () => {
+//     await page.getByPlaceholder('Enter code').fill(vuContext.vars.code);
+//     await page.getByPlaceholder('Enter code').press('Enter');
+//   });
+
+//   await step('wait_for_connection', async () => {
+//     const editor = page.locator('[data-slate-editor]');
+//     await editor.waitFor();
+//   });
+// }
+
+// export async function document_updates(page, vuContext, events, test) {
+//   const { step } = test;
+
+//   await step('open', async () => {
+//     await page.goto('/');
+//   });
+
+//   await step('connect', async () => {
+//     // await page.getByTitle('Create Markdown').click();
+//     const uuid = await page.evaluate(
+//       () =>
+//         (window.crypto.randomUUID = () => {
+//           return 'mocked response';
+//         }),
+//     );
+//     await page.getByPlaceholder('Enter code').fill(uuid);
+//     await page.getByPlaceholder('Enter code').press('Enter');
+//   });
+
+//   await step('create_check_list', async () => {
+//     const editor = page.locator('[data-slate-editor]');
+//     await editor.dblclick();
+//     for (let index = 0; index < 6; index++) {
+//       await editor.pressSequentially(
+//         `-[] Implementing todo ${index + 1} for next week.`,
+//       );
+//       await editor.press('Enter');
+//     }
+//   });
+// }
+
+// export async function single_document_updates(page, vuContext, events, test) {
+//   const { step } = test;
+
+//   await step('open', async () => {
+//     await page.goto('/');
+//   });
+
+//   await step('connect', async () => {
+//     await page.getByPlaceholder('Enter code').fill(vuContext.vars.code);
+//     await page.getByPlaceholder('Enter code').press('Enter');
+//   });
+
+//   await step('create_check_list', async () => {
+//     const editor = page.locator('[data-slate-editor]');
+//     await editor.dblclick();
+//     for (let index = 0; index < 6; index++) {
+//       await editor.pressSequentially(
+//         `-[] Implementing todo ${index + 1} for next week.`,
+//       );
+//       await editor.press('Enter');
+//     }
+//   });
+// }
+
+// export async function load_documents(page, vuContext, events, test) {
+//   const { step } = test;
+
+//   await step('open', async () => {
+//     await page.goto('/');
+//   });
+
+//   await step('connect', async () => {
+//     const uuid = await page.evaluate(() => crypto.randomUUID());
+//     await page.getByPlaceholder('Enter code').fill(uuid);
+//     await page.getByPlaceholder('Enter code').press('Enter');
+//   });
+
+//   await step('wait_for_connection', async () => {
+//     const editor = page.locator('[data-slate-editor]');
+//     await editor.waitFor();
+//   });
+// }