From 04e271a906b32e579472e906d671c885eb0f163a Mon Sep 17 00:00:00 2001 From: Ekhorn Date: Fri, 15 Nov 2024 08:27:13 +0100 Subject: [PATCH 1/8] docs: initial load tests --- config/load_tests/playwright.yaml | 8 ++ config/load_tests/socketio.yaml | 17 ++++ config/load_tests/test.js | 21 +++++ .../archive/2024-2025/research-performance.md | 88 +++++++++++++++++++ .../archive/2024-2025/research-scalability.md | 80 +++++++++++++++++ 5 files changed, 214 insertions(+) create mode 100644 config/load_tests/playwright.yaml create mode 100644 config/load_tests/socketio.yaml create mode 100644 config/load_tests/test.js create mode 100644 docs/archive/2024-2025/research-performance.md create mode 100644 docs/archive/2024-2025/research-scalability.md diff --git a/config/load_tests/playwright.yaml b/config/load_tests/playwright.yaml new file mode 100644 index 00000000..23403deb --- /dev/null +++ b/config/load_tests/playwright.yaml @@ -0,0 +1,8 @@ +config: + target: http://localhost:1420 + engines: + playwright: {} + processor: './test.js' +scenarios: + - engine: playwright + testFunction: 'createItemAndWriteCheckList' diff --git a/config/load_tests/socketio.yaml b/config/load_tests/socketio.yaml new file mode 100644 index 00000000..6810a2e8 --- /dev/null +++ b/config/load_tests/socketio.yaml @@ -0,0 +1,17 @@ +config: + target: 'http://localhost:8080' + socketio: + query: 'EIO=4&transport=websocket' + transports: ['websocket'] + phases: + - duration: 60 + arrivalRate: 25 + +scenarios: + - name: initial test + engine: socketio + flow: + - namespace: '/yjs|test-room' + emit: + channel: 'echo' + data: '["awareness-update",{"_placeholder":true,"num":0}]' diff --git a/config/load_tests/test.js b/config/load_tests/test.js new file mode 100644 index 00000000..0b6b44bb --- /dev/null +++ b/config/load_tests/test.js @@ -0,0 +1,21 @@ +export async function createItemAndWriteCheckList(page, test) { + const { step } = test; + + await step('open', async () => { + await page.goto('http://localhost:1420'); + }); + + await step('create_item', async () => { + await page.getByTitle('Create Markdown').click(); + }); + + await step('create_check_list', async () => { + await page.click('[data-slate-editor]'); + await page.pressSequentially('-[] Implementing todo 1 for next week.'); + await page.pressSequentially('-[] Implementing todo 2 for next week.'); + await page.pressSequentially('-[] Implementing todo 3 for next week.'); + await page.pressSequentially('-[] Implementing todo 4 for next week.'); + await page.pressSequentially('-[] Implementing todo 5 for next week.'); + await page.pressSequentially('-[] Implementing todo 6 for next week.'); + }); +} diff --git a/docs/archive/2024-2025/research-performance.md b/docs/archive/2024-2025/research-performance.md new file mode 100644 index 00000000..cf4488ba --- /dev/null +++ b/docs/archive/2024-2025/research-performance.md @@ -0,0 +1,88 @@ + + +# Spaced + +## Blazingly Fast Knowledge Base + +![](../assets/spaced-2024-rich-text.png) + +| | | +| ------- | -------------------- | +| Date | 2024-11-15 | +| Version | 0.1 | +| State | initial draft | +| Author | Koen Schellingerhout | +| | | + +
+ +#### Version history + +| Version | Date | Author | Changes | State | +| ------- | ---------- | -------------------- | --------------- | ----- | +| 0.1 | 2024-11-15 | Koen Schellingerhout | initial version | draft | + + +#### Distribution + +| Version | Date | Receivers | +| ------- | ---------- | -------------------- | +| 0.1 | 2024-11-15 | Canvas: Erik Schriek | + + +
+ +### + +#### 1. Research questions + +**Primary question** +How to make creating and collaborating on documents in Spaced for students and experts as cheap as possible while maintaining a 99% reliable latency of less than 1 seconds? + +_\*Please read the remarks below on the "less than 1 second" and "threshold"._ + +| Vertical scalability | | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Preliminary** | 1. What actions are expected to be involved?
2. Which providers should be tested on their VPS hosting?
3. What is the expected cost of load testing on each hosting provider? | +| **Sub questions** | 1. How to generate realistic loads for the VPS instances?
2. How are the bottlenecks holding back the maxium threshold?
3. How can Spaced be optimized to increase the maximum threshold? | + +| Remark | Description | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Less than 1 second | The less than a second measurement **may** be taken on a local system instead of a hosting provider to reduce costs. It does mean these measurements should not be used to talk about "real world performance". | +| Threshold | The threshold is to be determined, but most likely involves the amount of clients connected and the traffic they are sending over a certain period of time. | + +
+ +### 2. Results + +#### 2.1 Preliminary questions + +##### 2.1.1. What actions are expected to be involved? + +Creating a document, editing, moving, sharing and deleting. + +##### 2.1.2. Which providers should be tested on their VPS hosting? + +- [Amazon Web Services - EC2](https://aws.amazon.com/ec2/) +- [Google Cloud](https://cloud.google.com/learn/what-is-a-virtual-private-server?hl=en), free 300 credit 90day trail +- [IONOS](https://www.ionos.com/servers/vps) +- [Fontys Netlab](https://fhict.topdesk.net) + +##### 2.1.3. What is the expected cost of load testing on each hosting provider? + +#### 2.2 Sub questions + +##### 2.2.1. How to generate realistic loads for the VPS instances? + +Consider [Artillery + Playwright](https://www.artillery.io/docs/reference/engines/playwright#why-load-test-with-headless-browsers) tests to determine realistic loads by calling APIs directly from the frontend. Other more difficult methods are using HAR files mimicking realistic traffic. + +Since document edits use CRDTS (Conflict-free replicated data types) combined with SocketIO that share binary data, using Playwright would be simpler compared to writing out specific requests manually. + +##### 2.2.2. How are the bottlenecks holding back the maxium threshold? +##### 2.2.3. How can Spaced be optimized to increase the maximum threshold? + +### 3. Conclusion diff --git a/docs/archive/2024-2025/research-scalability.md b/docs/archive/2024-2025/research-scalability.md new file mode 100644 index 00000000..5e3c3d35 --- /dev/null +++ b/docs/archive/2024-2025/research-scalability.md @@ -0,0 +1,80 @@ + + +# Spaced + +## Blazingly Fast Knowledge Base + +![](../assets/spaced-2024-rich-text.png) + +| | | +| ------- | -------------------- | +| Date | 2024-11-15 | +| Version | 0.1 | +| State | initial draft | +| Author | Koen Schellingerhout | +| | | + +
+ +#### Version history + +| Version | Date | Author | Changes | State | +| ------- | ---------- | -------------------- | --------------- | ----- | +| 0.1 | 2024-11-15 | Koen Schellingerhout | initial version | draft | + + +#### Distribution + +| Version | Date | Receivers | +| ------- | ---------- | -------------------- | +| 0.1 | 2024-11-15 | Canvas: Erik Schriek | + + +
+ +### + +#### 1. Research questions + +**Primary question** +How to make creating and collaborating on documents in Spaced for students and experts as cheap as possible while maintaining a 99% reliable latency of less than 1 seconds? + +_\*Please read the remarks below on the "less than 1 second" and "threshold"._ + +| Horizontal scalability | | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Preliminary** | 1. What actions are expected to be involved?
2. Which providers should be tested on their Kubernetes hosting?
3. What is the expected cost of load testing on each hosting provider? | +| **Sub questions** | 1. How to generate realistic loads for the Kubernetes instances?
2. How is Spaced failing to keep up while scaling past the maxium threshold?
3. How can Spaced be optimized to be more scalable? | + +| Remark | Description | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Less than 1 second | The less than a second measurement **may** be taken on a local system instead of a hosting provider to reduce costs. It does mean these measurements should not be used to talk about "real world performance". | +| Threshold | The threshold is to be determined, but most likely involves the amount of clients connected and the traffic they are sending over a certain period of time. | + +
+ +### 2. Results + +#### 2.1 Preliminary questions + +##### 2.1.1. What actions are expected to be involved? + +Creating a document, editing, moving, sharing and deleting. + +##### 2.1.2. What is the expected cost of load testing on each hosting provider? +##### 2.1.3. Which providers should be tested on their Kubernetes hosting? + +- [Amazon Web Services - EC2](https://aws.amazon.com/ec2/) +- [Google Cloud](https://cloud.google.com/learn/what-is-a-virtual-private-server?hl=en), free 300 credit 90day trail + +#### 2.2 Sub questions + +##### 2.2.1. How to generate realistic loads for the Kubernetes instances? +##### 2.2.2. How is Spaced failing to keep up while scaling past the maxium threshold? +##### 2.2.3. How can Spaced be optimized to be more scalable? + +### 3. Conclusion From 1e4aabc1f2b5f5f41de8d2eff1c115bc1bab19bf Mon Sep 17 00:00:00 2001 From: Ekhorn Date: Thu, 5 Dec 2024 08:40:51 +0100 Subject: [PATCH 2/8] test: wip --- .gitignore | 3 + config/load/document_connections.yaml | 61 +++++++++++++ config/load/document_updates.yaml | 59 +++++++++++++ config/load/load_documents.yaml | 59 +++++++++++++ config/load/single_document_updates.yaml | 61 +++++++++++++ config/{load_tests => load}/socketio.yaml | 0 config/load/test.mjs | 86 +++++++++++++++++++ config/load_tests/playwright.yaml | 8 -- config/load_tests/test.js | 21 ----- .../archive/2024-2025/research-performance.md | 74 ++++++++++++++++ package.json | 4 + 11 files changed, 407 insertions(+), 29 deletions(-) create mode 100644 config/load/document_connections.yaml create mode 100644 config/load/document_updates.yaml create mode 100644 config/load/load_documents.yaml create mode 100644 config/load/single_document_updates.yaml rename config/{load_tests => load}/socketio.yaml (100%) create mode 100644 config/load/test.mjs delete mode 100644 config/load_tests/playwright.yaml delete mode 100644 config/load_tests/test.js diff --git a/.gitignore b/.gitignore index dde00476..93f4aeda 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ srcs/e2e/playwright-report/ srcs/e2e/blob-report/ srcs/e2e/playwright/.cache/ +# Artillery +config/load/*.json + # Typescript tsconfig.tsbuildinfo srcs/slate-solid/lib diff --git a/config/load/document_connections.yaml b/config/load/document_connections.yaml new file mode 100644 index 00000000..b551bab1 --- /dev/null +++ b/config/load/document_connections.yaml @@ -0,0 +1,61 @@ +config: + ensure: + maxErrorRate: 1 + max: 500 + variables: + code: 28cf22c3-77e8-49f5-b2ef-fa3eb212cce3 + environments: + dev: + target: http://localhost:1420 + engines: + playwright: + launchOptions: + headless: false + ci: + target: http://localhost:7777 + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 50 + maxVusers: 50 + duration: 30 + prod: + target: https://spaced.fun + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 50 + maxVusers: 50 + duration: 30 + processor: test.mjs + engines: + playwright: + launchOptions: + headless: true +scenarios: + - engine: playwright + testFunction: document_connections diff --git a/config/load/document_updates.yaml b/config/load/document_updates.yaml new file mode 100644 index 00000000..0bf749c9 --- /dev/null +++ b/config/load/document_updates.yaml @@ -0,0 +1,59 @@ +config: + ensure: + maxErrorRate: 1 + max: 500 + environments: + dev: + target: http://localhost:1420 + engines: + playwright: + launchOptions: + headless: false + ci: + target: http://localhost:7777 + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 50 + maxVusers: 50 + duration: 30 + prod: + target: https://spaced.fun + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 75 + maxVusers: 75 + duration: 30 + processor: test.mjs + engines: + playwright: + launchOptions: + headless: true +scenarios: + - engine: playwright + testFunction: document_updates diff --git a/config/load/load_documents.yaml b/config/load/load_documents.yaml new file mode 100644 index 00000000..974fd959 --- /dev/null +++ b/config/load/load_documents.yaml @@ -0,0 +1,59 @@ +config: + ensure: + maxErrorRate: 1 + max: 500 + environments: + dev: + target: http://localhost:1420 + engines: + playwright: + launchOptions: + headless: false + ci: + target: http://localhost:7777 + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 50 + maxVusers: 50 + duration: 30 + prod: + target: https://spaced.fun + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 50 + maxVusers: 50 + duration: 30 + processor: test.mjs + engines: + playwright: + launchOptions: + headless: true +scenarios: + - engine: playwright + testFunction: load_documents diff --git a/config/load/single_document_updates.yaml b/config/load/single_document_updates.yaml new file mode 100644 index 00000000..21dcf5b4 --- /dev/null +++ b/config/load/single_document_updates.yaml @@ -0,0 +1,61 @@ +config: + ensure: + maxErrorRate: 1 + max: 500 + variables: + code: 253405a2-9a49-4106-b92f-231a91a59edc + environments: + dev: + target: http://localhost:1420 + engines: + playwright: + launchOptions: + headless: false + ci: + target: http://localhost:7777 + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 50 + maxVusers: 50 + duration: 30 + prod: + target: https://spaced.fun + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 30 + maxVusers: 30 + duration: 30 + processor: test.mjs + engines: + playwright: + launchOptions: + headless: true +scenarios: + - engine: playwright + testFunction: single_document_updates diff --git a/config/load_tests/socketio.yaml b/config/load/socketio.yaml similarity index 100% rename from config/load_tests/socketio.yaml rename to config/load/socketio.yaml diff --git a/config/load/test.mjs b/config/load/test.mjs new file mode 100644 index 00000000..626cf4b6 --- /dev/null +++ b/config/load/test.mjs @@ -0,0 +1,86 @@ +export async function document_connections(page, vuContext, events, test) { + const { step } = test; + + await step('open', async () => { + await page.goto('/'); + }); + + await step('connect', async () => { + await page.getByPlaceholder('Enter code').fill(vuContext.vars.code); + await page.getByPlaceholder('Enter code').press('Enter'); + }); + + await step('wait_for_connection', async () => { + const editor = page.locator('[data-slate-editor]'); + await editor.waitFor(); + }); +} + +export async function document_updates(page, vuContext, events, test) { + const { step } = test; + + await step('open', async () => { + await page.goto('/'); + }); + + await step('connect', async () => { + // await page.getByTitle('Create Markdown').click(); + const uuid = await page.evaluate(() => crypto.randomUUID()); + await page.getByPlaceholder('Enter code').fill(uuid); + await page.getByPlaceholder('Enter code').press('Enter'); + }); + + await step('create_check_list', async () => { + const editor = page.locator('[data-slate-editor]'); + await editor.dblclick(); + for (let index = 0; index < 6; index++) { + await editor.pressSequentially( + `-[] Implementing todo ${index + 1} for next week.`, + ); + await editor.press('Enter'); + } + }); +} + +export async function single_document_updates(page, vuContext, events, test) { + const { step } = test; + + await step('open', async () => { + await page.goto('/'); + }); + + await step('connect', async () => { + await page.getByPlaceholder('Enter code').fill(vuContext.vars.code); + await page.getByPlaceholder('Enter code').press('Enter'); + }); + + await step('create_check_list', async () => { + const editor = page.locator('[data-slate-editor]'); + await editor.dblclick(); + for (let index = 0; index < 6; index++) { + await editor.pressSequentially( + `-[] Implementing todo ${index + 1} for next week.`, + ); + await editor.press('Enter'); + } + }); +} + +export async function load_documents(page, vuContext, events, test) { + const { step } = test; + + await step('open', async () => { + await page.goto('/'); + }); + + await step('connect', async () => { + const uuid = await page.evaluate(() => crypto.randomUUID()); + await page.getByPlaceholder('Enter code').fill(uuid); + await page.getByPlaceholder('Enter code').press('Enter'); + }); + + await step('wait_for_connection', async () => { + const editor = page.locator('[data-slate-editor]'); + await editor.waitFor(); + }); +} diff --git a/config/load_tests/playwright.yaml b/config/load_tests/playwright.yaml deleted file mode 100644 index 23403deb..00000000 --- a/config/load_tests/playwright.yaml +++ /dev/null @@ -1,8 +0,0 @@ -config: - target: http://localhost:1420 - engines: - playwright: {} - processor: './test.js' -scenarios: - - engine: playwright - testFunction: 'createItemAndWriteCheckList' diff --git a/config/load_tests/test.js b/config/load_tests/test.js deleted file mode 100644 index 0b6b44bb..00000000 --- a/config/load_tests/test.js +++ /dev/null @@ -1,21 +0,0 @@ -export async function createItemAndWriteCheckList(page, test) { - const { step } = test; - - await step('open', async () => { - await page.goto('http://localhost:1420'); - }); - - await step('create_item', async () => { - await page.getByTitle('Create Markdown').click(); - }); - - await step('create_check_list', async () => { - await page.click('[data-slate-editor]'); - await page.pressSequentially('-[] Implementing todo 1 for next week.'); - await page.pressSequentially('-[] Implementing todo 2 for next week.'); - await page.pressSequentially('-[] Implementing todo 3 for next week.'); - await page.pressSequentially('-[] Implementing todo 4 for next week.'); - await page.pressSequentially('-[] Implementing todo 5 for next week.'); - await page.pressSequentially('-[] Implementing todo 6 for next week.'); - }); -} diff --git a/docs/archive/2024-2025/research-performance.md b/docs/archive/2024-2025/research-performance.md index cf4488ba..d2baf9d8 100644 --- a/docs/archive/2024-2025/research-performance.md +++ b/docs/archive/2024-2025/research-performance.md @@ -82,7 +82,81 @@ Consider [Artillery + Playwright](https://www.artillery.io/docs/reference/engine Since document edits use CRDTS (Conflict-free replicated data types) combined with SocketIO that share binary data, using Playwright would be simpler compared to writing out specific requests manually. + +--- + +understanding Artillery metrics + +FCP = [First Contentful Paint](https://web.dev/articles/fcp) +TTFB = [Time To First Byte](https://web.dev/articles/ttfb) +LCP = [Largest Contentful Paint](https://web.dev/articles/lcp) +FID = [First Input Delay](https://web.dev/articles/fid) + + +Web Vitals + +LCP (Largest Contentful Paint) +CLS (Cumulative Layout Shift) +INP (Interaction to Next Paint) +TTFB (Time To First Byte) +FCP (First Contentful Paint) +FID (First Input Delay) + +--- + +tests + +1. max single document connections +2. max document count +3. ... + +phases + +1. initial +2. 10min sustained expect load +3. stress test + +steps + +1. open page +2. + + ##### 2.2.2. How are the bottlenecks holding back the maxium threshold? + +How does a node implementation compare to the rust implementation? + +How do the different VPS instances hold up? + +1vCPU 1Gb RAM + +vs + +2vCPU 2Gb RAM + +How do different parts of the system add latencies? + +How does memory increase with and without metrics? + +How do default image install compare in resource usage, to nixos installs? + ##### 2.2.3. How can Spaced be optimized to increase the maximum threshold? +The rust implementation as seen already improves performance significantly compared to Node as no garbage collection is involved. + +Document garbage collection. + + // TODO: consider diff string for perf + // TODO: compare HashMap to DashMap + +MSGPack vs normal binary? vs Json + + +Limiting clients to debounce requests. + + +- Dive into websocket upgrades? +- Memory vs CPU bottleneck handling + + ### 3. Conclusion diff --git a/package.json b/package.json index bdb43a2c..ab66cea0 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,10 @@ "e2e": "npm --workspace e2e run e2e --", "fmt": "cargo fmt --check --all", "fmt:fix": "cargo fmt --all", + "load:document_connections": "npx artillery run config/load/document_connections.yaml -o config/load/document_connections_$(date +%s).json", + "load:document_updates": "npx artillery run config/load/document_updates.yaml -o config/load/document_updates_$(date +%s).json", + "load:load_documents": "npx artillery run config/load/load_documents.yaml -o config/load/load_documents_$(date +%s).json", + "load:single_document_updates": "npx artillery run config/load/single_document_updates.yaml -o config/load/single_document_updates_$(date +%s).json", "prepare": "husky", "preview": "vite preview", "services": "srcs/scripts/start.sh", From 527ffee216bddf6a6d5d3e51b608c2fdf5ce673e Mon Sep 17 00:00:00 2001 From: Ekhorn Date: Mon, 13 Jan 2025 13:15:17 +0100 Subject: [PATCH 3/8] ci: prepare load tests in pipeline The root namespace must be defined in the item_socket socket.io backend to allow artillery to run its initial "zero" task. --- .github/workflows/review.yaml | 40 +++++++++++++--------- config/load/socketio.yaml | 49 +++++++++++++++++++++------ srcs/services/item_socket/src/main.rs | 2 ++ 3 files changed, 65 insertions(+), 26 deletions(-) diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index ffb520a0..14361cdb 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -151,22 +151,30 @@ jobs: ssh_key: ${{ secrets.SSH_PRIVATE_KEY }} env_file: .env - # load_test: - # runs-on: self-hosted - # needs: test_servers - # timeout-minutes: 20 - # concurrency: - # group: ${{ matrix.subdomain }} - # strategy: - # matrix: - # subdomain: [aws, do, hetzner] - # # gc, ionos - # steps: - # - uses: actions/checkout@v4 - # - uses: artilleryio/action-cli@v1 - # with: - # command: run config/load/socket-io.yml ${{ matrix.subdomain }}.spaced.fun - # # - run: ssh spaced@${{ matrix.subdomain }}.spaced.fun + load_test: + runs-on: self-hosted + needs: test_servers + timeout-minutes: 20 + concurrency: + group: ${{ matrix.subdomain }} + strategy: + matrix: + subdomain: [aws, do, hetzner] + # gc, ionos + steps: + - uses: actions/checkout@v4 + - run: echo "TIMESTAMP=$(date +%s)" >> $GITHUB_ENV + - uses: artilleryio/action-cli@v1 + env: + ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY }} + SUBDOMAIN: ${{ matrix.subdomain }} + with: + command: run config/load/socketio.yaml --record -o socket-io_${{ env.TIMESTAMP }}.json -e review + - uses: actions/upload-artifact@v3 + if: always() + with: + name: artillery-report + path: ./socket-io_${{ env.TIMESTAMP }}.json # cleanup: # runs-on: ubuntu-latest diff --git a/config/load/socketio.yaml b/config/load/socketio.yaml index 6810a2e8..f3ff36fd 100644 --- a/config/load/socketio.yaml +++ b/config/load/socketio.yaml @@ -1,17 +1,46 @@ +# yaml-language-server: $schema=https://www.artillery.io/schema.json config: - target: 'http://localhost:8080' - socketio: - query: 'EIO=4&transport=websocket' - transports: ['websocket'] - phases: - - duration: 60 - arrivalRate: 25 + ensure: + maxErrorRate: 1 + max: 500 + environments: + local: + target: http://localhost:7777 + socketio: + query: EIO=4&transport=websocket + transports: ['websocket'] + phases: + - duration: 60 + arrivalRate: 50 + review: + target: http://{{ $env.SUBDOMAIN }}.spaced.fun + socketio: + query: EIO=4&transport=websocket + transports: ['websocket'] + phases: + # Create ramp up to 10 VUs per second + - name: initial + arrivalRate: 1 + rampTo: 10 + maxVusers: 10 + duration: 10 + # Creates 10 VUs every second + - name: sustained + arrivalRate: 1 + duration: 30 + - pause: 10 + # Creates 1 VU every second + - name: stress + arrivalRate: 5 + rampTo: 50 + maxVusers: 50 + duration: 30 scenarios: - name: initial test engine: socketio flow: - - namespace: '/yjs|test-room' + - namespace: /yjs|room-test emit: - channel: 'echo' - data: '["awareness-update",{"_placeholder":true,"num":0}]' + namespace: /yjs|room-test + data: '{}' diff --git a/srcs/services/item_socket/src/main.rs b/srcs/services/item_socket/src/main.rs index 3a92f32d..451a763d 100644 --- a/srcs/services/item_socket/src/main.rs +++ b/srcs/services/item_socket/src/main.rs @@ -94,6 +94,8 @@ async fn app() -> anyhow::Result { let io_clone = io.clone(); + io.ns("/", || {}); + io.dyn_ns( "/yjs|{doc_ns}", |socket: SocketRef, state: State, metrics: State| async move { From 7e80c5813d8a8b99a3b27cf5e2bfd36877733dfa Mon Sep 17 00:00:00 2001 From: Ekhorn Date: Wed, 15 Jan 2025 14:47:42 +0100 Subject: [PATCH 4/8] docs: finish perf/research doc --- .../archive/2024-2025/research-performance.md | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/archive/2024-2025/research-performance.md b/docs/archive/2024-2025/research-performance.md index d2baf9d8..8062b9c2 100644 --- a/docs/archive/2024-2025/research-performance.md +++ b/docs/archive/2024-2025/research-performance.md @@ -8,7 +8,7 @@ ## Blazingly Fast Knowledge Base -![](../assets/spaced-2024-rich-text.png) +![](../../assets/spaced-2024-rich-text.png) | | | | ------- | -------------------- | @@ -25,20 +25,44 @@ | Version | Date | Author | Changes | State | | ------- | ---------- | -------------------- | --------------- | ----- | | 0.1 | 2024-11-15 | Koen Schellingerhout | initial version | draft | - +| 1.0 | 2024-01-15 | Koen Schellingerhout | | final | #### Distribution | Version | Date | Receivers | | ------- | ---------- | -------------------- | | 0.1 | 2024-11-15 | Canvas: Erik Schriek | +| 1.0 | 2025-01-15 | Canvas: Erik Schriek |
-### +### Contents + + + +
+ +- [1. Research questions](#1-research-questions) +- [2. Results](#2-results) + - [2.1. Preliminary questions](#21-preliminary-questions) + - [2.1.1. What actions are expected to be involved?](#211-what-actions-are-expected-to-be-involved) + - [2.1.2. Which providers should be tested on their VPS hosting?](#212-which-providers-should-be-tested-on-their-vps-hosting) + - [2.1.3. What is the expected cost of load testing on each hosting provider?](#213-what-is-the-expected-cost-of-load-testing-on-each-hosting-provider) + - [2.2 Sub questions](#22-sub-questions) + - [2.2.1. How to generate realistic loads for the VPS instances?](#221-how-to-generate-realistic-loads-for-the-vps-instances) + - [2.2.2. How are the bottlenecks holding back the maxium threshold?](#222-how-are-the-bottlenecks-holding-back-the-maxium-threshold) + - [2.2.3. How can Spaced be optimized to increase the maximum threshold?](#223-how-can-spaced-be-optimized-to-increase-the-maximum-threshold) +- [3. Conclusion](#3-conclusion) + +
-#### 1. Research questions +### 1. Research questions **Primary question** How to make creating and collaborating on documents in Spaced for students and experts as cheap as possible while maintaining a 99% reliable latency of less than 1 seconds? @@ -59,7 +83,7 @@ _\*Please read the remarks below on the "less than 1 second" and "threshold"._ ### 2. Results -#### 2.1 Preliminary questions +#### 2.1. Preliminary questions ##### 2.1.1. What actions are expected to be involved? From 257dfb7bcda79b45fa75b5d00aebc9606ec84406 Mon Sep 17 00:00:00 2001 From: Ekhorn Date: Thu, 16 Jan 2025 09:57:33 +0100 Subject: [PATCH 5/8] wip --- .github/workflows/review.yaml | 51 ++++++- config/load/document_connections.yaml | 61 -------- config/load/document_updates.yaml | 59 -------- config/load/functions.cjs | 11 ++ config/load/load_documents.yaml | 59 -------- config/load/playwright.yaml | 67 +++++++++ config/load/scenarios.mjs | 180 +++++++++++++++++++++++ config/load/single_document_updates.yaml | 61 -------- config/load/socketio.yaml | 51 +++++-- config/load/test.mjs | 86 ----------- srcs/services/item_socket/src/y.rs | 2 +- 11 files changed, 346 insertions(+), 342 deletions(-) delete mode 100644 config/load/document_connections.yaml delete mode 100644 config/load/document_updates.yaml create mode 100644 config/load/functions.cjs delete mode 100644 config/load/load_documents.yaml create mode 100644 config/load/playwright.yaml create mode 100644 config/load/scenarios.mjs delete mode 100644 config/load/single_document_updates.yaml delete mode 100644 config/load/test.mjs diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index 14361cdb..01b91b96 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -159,22 +159,63 @@ jobs: group: ${{ matrix.subdomain }} strategy: matrix: - subdomain: [aws, do, hetzner] + subdomain: [do] + # subdomain: [aws, do] # ADD LAST MINIMUM + # subdomain: [aws, do, hetzner] # gc, ionos + environment: manual_approval + env: + PLAYWRIGHT_BROWSERS_PATH: /data/runner_workspace/.cache/ms-playwright steps: - uses: actions/checkout@v4 - run: echo "TIMESTAMP=$(date +%s)" >> $GITHUB_ENV - - uses: artilleryio/action-cli@v1 + - run: npm i && npx artillery + - run: | + npx artillery run config/load/playwright.yaml \ + --tags env:review,host:${{ matrix.subdomain }},sha:${{ github.sha }} \ + --record -o playwright_${{ env.TIMESTAMP }}.json -e breakpoint + continue-on-error: true + env: + ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY }} + SUBDOMAIN: ${{ matrix.subdomain }} + - run: | + jq ' + .intermediate + | to_entries + | map(select( + .value.counters."vusers.failed" < 1 and + .value.summaries."browser.step.open".p99 < 1000 and + .value.summaries."browser.step.create".p99 < 1000 and + .value.summaries."browser.step.share".p99 < 1000 or + .value.summaries."browser.step.open".p99 < 1000 and + .value.summaries."browser.step.join".p99 < 1000 and + .value.summaries."browser.step.create_check_list".p99 < 2000 + )) + | sort_by(.value.counters."vusers.created" | tonumber) + | last + | { + record: .key, + "vusers.created": .value.counters."vusers.created", + "vusers.session_length.p999": .value.summaries."vusers.session_length".p999, + } + ' playwright_${{ env.TIMESTAMP }}.json | tee | jq ."vusers.created" > NUM + + echo SMOKE_RATE=$(cat NUM | jq '. * 0.10 | round') >> $GITHUB_ENV + echo AVG_RATE=$(cat NUM | jq '. * 0.50 | round') >> $GITHUB_ENV + echo STRESS_RATE=$(cat NUM | jq '. * 0.75 | round') >> $GITHUB_ENV + echo SPIKE_RATE=$(cat NUM) >> $GITHUB_ENV + - run: | + npx artillery run config/load/playwright.yaml \ + --tags env:review,host:${{ matrix.subdomain }},sha:${{ github.sha }} \ + --record -o playwright_${{ env.TIMESTAMP }}.json -e smoke_average_stress_spike env: ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY }} SUBDOMAIN: ${{ matrix.subdomain }} - with: - command: run config/load/socketio.yaml --record -o socket-io_${{ env.TIMESTAMP }}.json -e review - uses: actions/upload-artifact@v3 if: always() with: name: artillery-report - path: ./socket-io_${{ env.TIMESTAMP }}.json + path: ./playwright_${{ env.TIMESTAMP }}.json # cleanup: # runs-on: ubuntu-latest diff --git a/config/load/document_connections.yaml b/config/load/document_connections.yaml deleted file mode 100644 index b551bab1..00000000 --- a/config/load/document_connections.yaml +++ /dev/null @@ -1,61 +0,0 @@ -config: - ensure: - maxErrorRate: 1 - max: 500 - variables: - code: 28cf22c3-77e8-49f5-b2ef-fa3eb212cce3 - environments: - dev: - target: http://localhost:1420 - engines: - playwright: - launchOptions: - headless: false - ci: - target: http://localhost:7777 - phases: - # Create ramp up to 10 VUs per second - - name: initial - arrivalRate: 1 - rampTo: 10 - maxVusers: 10 - duration: 10 - # Creates 10 VUs every second - - name: sustained - arrivalRate: 1 - duration: 30 - - pause: 10 - # Creates 1 VU every second - - name: stress - arrivalRate: 5 - rampTo: 50 - maxVusers: 50 - duration: 30 - prod: - target: https://spaced.fun - phases: - # Create ramp up to 10 VUs per second - - name: initial - arrivalRate: 1 - rampTo: 10 - maxVusers: 10 - duration: 10 - # Creates 10 VUs every second - - name: sustained - arrivalRate: 1 - duration: 30 - - pause: 10 - # Creates 1 VU every second - - name: stress - arrivalRate: 5 - rampTo: 50 - maxVusers: 50 - duration: 30 - processor: test.mjs - engines: - playwright: - launchOptions: - headless: true -scenarios: - - engine: playwright - testFunction: document_connections diff --git a/config/load/document_updates.yaml b/config/load/document_updates.yaml deleted file mode 100644 index 0bf749c9..00000000 --- a/config/load/document_updates.yaml +++ /dev/null @@ -1,59 +0,0 @@ -config: - ensure: - maxErrorRate: 1 - max: 500 - environments: - dev: - target: http://localhost:1420 - engines: - playwright: - launchOptions: - headless: false - ci: - target: http://localhost:7777 - phases: - # Create ramp up to 10 VUs per second - - name: initial - arrivalRate: 1 - rampTo: 10 - maxVusers: 10 - duration: 10 - # Creates 10 VUs every second - - name: sustained - arrivalRate: 1 - duration: 30 - - pause: 10 - # Creates 1 VU every second - - name: stress - arrivalRate: 5 - rampTo: 50 - maxVusers: 50 - duration: 30 - prod: - target: https://spaced.fun - phases: - # Create ramp up to 10 VUs per second - - name: initial - arrivalRate: 1 - rampTo: 10 - maxVusers: 10 - duration: 10 - # Creates 10 VUs every second - - name: sustained - arrivalRate: 1 - duration: 30 - - pause: 10 - # Creates 1 VU every second - - name: stress - arrivalRate: 5 - rampTo: 75 - maxVusers: 75 - duration: 30 - processor: test.mjs - engines: - playwright: - launchOptions: - headless: true -scenarios: - - engine: playwright - testFunction: document_updates diff --git a/config/load/functions.cjs b/config/load/functions.cjs new file mode 100644 index 00000000..4d908a22 --- /dev/null +++ b/config/load/functions.cjs @@ -0,0 +1,11 @@ +const { randomUUID } = require('node:crypto'); + +function uuid(ctx, ee, next) { + ctx.vars['uuid'] = randomUUID(); + + return next(); +} + +module.exports = { + uuid, +}; diff --git a/config/load/load_documents.yaml b/config/load/load_documents.yaml deleted file mode 100644 index 974fd959..00000000 --- a/config/load/load_documents.yaml +++ /dev/null @@ -1,59 +0,0 @@ -config: - ensure: - maxErrorRate: 1 - max: 500 - environments: - dev: - target: http://localhost:1420 - engines: - playwright: - launchOptions: - headless: false - ci: - target: http://localhost:7777 - phases: - # Create ramp up to 10 VUs per second - - name: initial - arrivalRate: 1 - rampTo: 10 - maxVusers: 10 - duration: 10 - # Creates 10 VUs every second - - name: sustained - arrivalRate: 1 - duration: 30 - - pause: 10 - # Creates 1 VU every second - - name: stress - arrivalRate: 5 - rampTo: 50 - maxVusers: 50 - duration: 30 - prod: - target: https://spaced.fun - phases: - # Create ramp up to 10 VUs per second - - name: initial - arrivalRate: 1 - rampTo: 10 - maxVusers: 10 - duration: 10 - # Creates 10 VUs every second - - name: sustained - arrivalRate: 1 - duration: 30 - - pause: 10 - # Creates 1 VU every second - - name: stress - arrivalRate: 5 - rampTo: 50 - maxVusers: 50 - duration: 30 - processor: test.mjs - engines: - playwright: - launchOptions: - headless: true -scenarios: - - engine: playwright - testFunction: load_documents diff --git a/config/load/playwright.yaml b/config/load/playwright.yaml new file mode 100644 index 00000000..1a7833ca --- /dev/null +++ b/config/load/playwright.yaml @@ -0,0 +1,67 @@ +# yaml-language-server: $schema=https://www.artillery.io/schema.json +config: + processor: scenarios.mjs + engines: + playwright: + launchOptions: + headless: true + plugins: + ensure: {} + ensure: + maxErrorRate: 1 + conditions: + - expression: 'browser.step.open.p99 < 1000 and browser.step.create.p99 < 1000 and browser.step.share.p99 < 1000 or browser.step.open.p99 < 1000 and browser.step.join.p99 < 1000 and browser.step.create_check_list.p99 < 2000' + variables: + uuid: 28cf22c3-77e8-49f5-b2ef-fa3eb212cce3 + environments: + dev: + target: http://localhost:1420 + engines: + playwright: + launchOptions: + headless: false + trace: + enabled: true + breakpoint: + target: http://{{ $env.SUBDOMAIN }}.spaced.fun + socketio: + query: EIO=4&transport=websocket + transports: [websocket] + phases: + # duration / 10 = reports + # rampTo / reports = concurrent users added per report (granularity) + - name: Find breakpoint + duration: 2m + arrivalRate: 1 + rampTo: 100 + # maxVusers: 500 + smoke_average_stress_spike: + target: http://{{ $env.SUBDOMAIN }}.spaced.fun + socketio: + query: EIO=4&transport=websocket + transports: [websocket] + phases: + - name: smoke + duration: 10 + arrivalRate: '{{ $env.SMOKE_RATE }}' # breakpoint * .10 + maxVusers: '{{ $env.SMOKE_RATE }}' # breakpoint * .10 + - name: average + duration: 60 + arrivalRate: '{{ $env.AVG_RATE }}' # breakpoint * .50 + - name: stress + duration: 120 + arrivalRate: '{{ $env.STRESS_RATE }}' # breakpoint * .75 + - name: pause + pause: 10 + - name: spike + duration: 5 + arrivalRate: '{{ $env.SPIKE_RATE }}' # breakpoint + maxVusers: 200 + +scenarios: + - engine: playwright + testFunction: create_and_share_document + weight: 1 + - engine: playwright + testFunction: join_and_update_document + weight: 6 diff --git a/config/load/scenarios.mjs b/config/load/scenarios.mjs new file mode 100644 index 00000000..04f661cc --- /dev/null +++ b/config/load/scenarios.mjs @@ -0,0 +1,180 @@ +export async function create_and_share_document(page, vuContext, events, test) { + const { step } = test; + + await step('open', async () => { + await page.goto('/'); + }); + + await step('create', async () => { + await page.getByTitle('Create Item').click(); + const editor = page.locator('[data-slate-editor]'); + + await editor.dblclick(); + await editor.pressSequentially(`Entering Title`); + await editor.press('Enter'); + await editor.press('Enter'); + }); + + await step('share', async () => { + await page.getByTitle('Share').click(); + + await page + .locator("input[name='username']") + .pressSequentially('RandomUser'); + + await page.evaluate(() => { + const getRandomHexColor = () => { + const randomColor = Math.floor(Math.random() * 16_777_215).toString(16); + return `#${randomColor.padStart(6, '0')}`; + }; + const input = document.querySelector("input[name='color']"); + if (input) { + input.value = getRandomHexColor(); + input.dispatchEvent(new Event('change', { bubbles: true })); + } + }); + + await page.click("button[type='submit']"); + const uuid = await page.evaluate( + () => + (window.crypto.randomUUID = () => { + return 'mocked response'; + }), + ); + vuContext.vars.uuid = uuid; + }); +} + +export async function join_and_update_document(page, vuContext, events, test) { + const { step } = test; + + await step('open', async () => { + await page.goto('/'); + }); + + await step('join', async () => { + await page.getByPlaceholder('Enter code').fill(vuContext.vars.uuid); + await page.getByPlaceholder('Enter code').press('Enter'); + + await page + .locator("input[name='username']") + .pressSequentially('RandomUser'); + + await page.evaluate(() => { + const getRandomHexColor = () => { + const randomColor = Math.floor(Math.random() * 16_777_215).toString(16); + return `#${randomColor.padStart(6, '0')}`; + }; + const input = document.querySelector("input[name='color']"); + if (input) { + input.value = getRandomHexColor(); + input.dispatchEvent(new Event('change', { bubbles: true })); + } + }); + }); + + await step('create_check_list', async () => { + const editor = page.locator('[data-slate-editor]'); + await editor.dblclick(); + for (let index = 0; index < 6; index++) { + await editor.pressSequentially( + `-[] Implementing todo ${index + 1} for next week.`, + ); + await editor.press('Enter'); + } + }); +} + +// ########### + +// export async function document_connections(page, vuContext, events, test) { +// const { step } = test; + +// await step('open', async () => { +// await page.goto('/'); +// }); + +// await step('connect', async () => { +// await page.getByPlaceholder('Enter code').fill(vuContext.vars.code); +// await page.getByPlaceholder('Enter code').press('Enter'); +// }); + +// await step('wait_for_connection', async () => { +// const editor = page.locator('[data-slate-editor]'); +// await editor.waitFor(); +// }); +// } + +// export async function document_updates(page, vuContext, events, test) { +// const { step } = test; + +// await step('open', async () => { +// await page.goto('/'); +// }); + +// await step('connect', async () => { +// // await page.getByTitle('Create Markdown').click(); +// const uuid = await page.evaluate( +// () => +// (window.crypto.randomUUID = () => { +// return 'mocked response'; +// }), +// ); +// await page.getByPlaceholder('Enter code').fill(uuid); +// await page.getByPlaceholder('Enter code').press('Enter'); +// }); + +// await step('create_check_list', async () => { +// const editor = page.locator('[data-slate-editor]'); +// await editor.dblclick(); +// for (let index = 0; index < 6; index++) { +// await editor.pressSequentially( +// `-[] Implementing todo ${index + 1} for next week.`, +// ); +// await editor.press('Enter'); +// } +// }); +// } + +// export async function single_document_updates(page, vuContext, events, test) { +// const { step } = test; + +// await step('open', async () => { +// await page.goto('/'); +// }); + +// await step('connect', async () => { +// await page.getByPlaceholder('Enter code').fill(vuContext.vars.code); +// await page.getByPlaceholder('Enter code').press('Enter'); +// }); + +// await step('create_check_list', async () => { +// const editor = page.locator('[data-slate-editor]'); +// await editor.dblclick(); +// for (let index = 0; index < 6; index++) { +// await editor.pressSequentially( +// `-[] Implementing todo ${index + 1} for next week.`, +// ); +// await editor.press('Enter'); +// } +// }); +// } + +// export async function load_documents(page, vuContext, events, test) { +// const { step } = test; + +// await step('open', async () => { +// await page.goto('/'); +// }); + +// await step('connect', async () => { +// const uuid = await page.evaluate(() => crypto.randomUUID()); +// await page.getByPlaceholder('Enter code').fill(uuid); +// await page.getByPlaceholder('Enter code').press('Enter'); +// }); + +// await step('wait_for_connection', async () => { +// const editor = page.locator('[data-slate-editor]'); +// await editor.waitFor(); +// }); +// } diff --git a/config/load/single_document_updates.yaml b/config/load/single_document_updates.yaml deleted file mode 100644 index 21dcf5b4..00000000 --- a/config/load/single_document_updates.yaml +++ /dev/null @@ -1,61 +0,0 @@ -config: - ensure: - maxErrorRate: 1 - max: 500 - variables: - code: 253405a2-9a49-4106-b92f-231a91a59edc - environments: - dev: - target: http://localhost:1420 - engines: - playwright: - launchOptions: - headless: false - ci: - target: http://localhost:7777 - phases: - # Create ramp up to 10 VUs per second - - name: initial - arrivalRate: 1 - rampTo: 10 - maxVusers: 10 - duration: 10 - # Creates 10 VUs every second - - name: sustained - arrivalRate: 1 - duration: 30 - - pause: 10 - # Creates 1 VU every second - - name: stress - arrivalRate: 5 - rampTo: 50 - maxVusers: 50 - duration: 30 - prod: - target: https://spaced.fun - phases: - # Create ramp up to 10 VUs per second - - name: initial - arrivalRate: 1 - rampTo: 10 - maxVusers: 10 - duration: 10 - # Creates 10 VUs every second - - name: sustained - arrivalRate: 1 - duration: 30 - - pause: 10 - # Creates 1 VU every second - - name: stress - arrivalRate: 5 - rampTo: 30 - maxVusers: 30 - duration: 30 - processor: test.mjs - engines: - playwright: - launchOptions: - headless: true -scenarios: - - engine: playwright - testFunction: single_document_updates diff --git a/config/load/socketio.yaml b/config/load/socketio.yaml index f3ff36fd..2051c3de 100644 --- a/config/load/socketio.yaml +++ b/config/load/socketio.yaml @@ -1,22 +1,41 @@ # yaml-language-server: $schema=https://www.artillery.io/schema.json config: + processor: ./functions.cjs + plugins: + ensure: {} ensure: maxErrorRate: 1 - max: 500 + thresholds: + - 'vusers.session_length.p999': 1000 environments: local: - target: http://localhost:7777 + target: http://localhost:8081 socketio: query: EIO=4&transport=websocket - transports: ['websocket'] + transports: [websocket] phases: - - duration: 60 - arrivalRate: 50 - review: + - name: Find breakpoint + duration: 10 + arrivalCount: 1 + maxVusers: 1 + breakpoint: target: http://{{ $env.SUBDOMAIN }}.spaced.fun socketio: query: EIO=4&transport=websocket - transports: ['websocket'] + transports: [websocket] + phases: + # duration / 10 = reports + # rampTo / reports = concurrent users added per report (granularity) + - name: Find breakpoint + duration: 10m + arrivalRate: 1 + rampTo: 30 + # maxVusers: 500 + smoke_average_stress_spike: + target: http://{{ $env.SUBDOMAIN }}.spaced.fun + socketio: + query: EIO=4&transport=websocket + transports: [websocket] phases: # Create ramp up to 10 VUs per second - name: initial @@ -36,11 +55,23 @@ config: maxVusers: 50 duration: 30 + # - name: Breakpoint: this test type might be how I determine the average load the VPS can take on - so the initial load test. + # - name: Smoke: I could include as initial phase + # - name: Average load: Not sure how to conduct this for different systems + # - name: Stress: When load exceeds expected average - not sure either - maybe through initial tests, and hardcoded values + # - name: Spike: I should can do this + scenarios: - - name: initial test + - name: open_new_document engine: socketio flow: - - namespace: /yjs|room-test + - function: uuid + - namespace: /yjs|{{ uuid }} + emit: + data: '{}' + - namespace: /yjs|{{ uuid }} emit: - namespace: /yjs|room-test + channel: sync-step-1 data: '{}' + acknowledge: + data: '0["sync-step-1",{"_placeholder":true,"num":0}]' diff --git a/config/load/test.mjs b/config/load/test.mjs deleted file mode 100644 index 626cf4b6..00000000 --- a/config/load/test.mjs +++ /dev/null @@ -1,86 +0,0 @@ -export async function document_connections(page, vuContext, events, test) { - const { step } = test; - - await step('open', async () => { - await page.goto('/'); - }); - - await step('connect', async () => { - await page.getByPlaceholder('Enter code').fill(vuContext.vars.code); - await page.getByPlaceholder('Enter code').press('Enter'); - }); - - await step('wait_for_connection', async () => { - const editor = page.locator('[data-slate-editor]'); - await editor.waitFor(); - }); -} - -export async function document_updates(page, vuContext, events, test) { - const { step } = test; - - await step('open', async () => { - await page.goto('/'); - }); - - await step('connect', async () => { - // await page.getByTitle('Create Markdown').click(); - const uuid = await page.evaluate(() => crypto.randomUUID()); - await page.getByPlaceholder('Enter code').fill(uuid); - await page.getByPlaceholder('Enter code').press('Enter'); - }); - - await step('create_check_list', async () => { - const editor = page.locator('[data-slate-editor]'); - await editor.dblclick(); - for (let index = 0; index < 6; index++) { - await editor.pressSequentially( - `-[] Implementing todo ${index + 1} for next week.`, - ); - await editor.press('Enter'); - } - }); -} - -export async function single_document_updates(page, vuContext, events, test) { - const { step } = test; - - await step('open', async () => { - await page.goto('/'); - }); - - await step('connect', async () => { - await page.getByPlaceholder('Enter code').fill(vuContext.vars.code); - await page.getByPlaceholder('Enter code').press('Enter'); - }); - - await step('create_check_list', async () => { - const editor = page.locator('[data-slate-editor]'); - await editor.dblclick(); - for (let index = 0; index < 6; index++) { - await editor.pressSequentially( - `-[] Implementing todo ${index + 1} for next week.`, - ); - await editor.press('Enter'); - } - }); -} - -export async function load_documents(page, vuContext, events, test) { - const { step } = test; - - await step('open', async () => { - await page.goto('/'); - }); - - await step('connect', async () => { - const uuid = await page.evaluate(() => crypto.randomUUID()); - await page.getByPlaceholder('Enter code').fill(uuid); - await page.getByPlaceholder('Enter code').press('Enter'); - }); - - await step('wait_for_connection', async () => { - const editor = page.locator('[data-slate-editor]'); - await editor.waitFor(); - }); -} diff --git a/srcs/services/item_socket/src/y.rs b/srcs/services/item_socket/src/y.rs index 2b08d877..0d8e28d7 100644 --- a/srcs/services/item_socket/src/y.rs +++ b/srcs/services/item_socket/src/y.rs @@ -132,7 +132,7 @@ pub async fn start_synchronization( let update = Update::decode_v1(binary).unwrap(); awareness.doc().transact_mut().apply_update(update).unwrap(); } - Err(err) => println!("Ack error {:?}", err), + Err(err) => println!("Ack error {:?}", err), // TODO: Exit if timeout } let data = awareness.update().unwrap().encode_v1(); From 9cfe63d073248ca999e5986ef7dda8473c1b625b Mon Sep 17 00:00:00 2001 From: Ekhorn Date: Fri, 17 Jan 2025 13:10:08 +0100 Subject: [PATCH 6/8] docs: add final research report --- .../archive/2024-2025/research-performance.md | 191 +++++++++++------- docs/assets/artillery_dashboard.png | Bin 0 -> 206716 bytes docs/assets/item_socket_leakish_memory.png | Bin 0 -> 44378 bytes docs/assets/proxy_resource_usage.png | Bin 0 -> 46068 bytes 4 files changed, 114 insertions(+), 77 deletions(-) create mode 100644 docs/assets/artillery_dashboard.png create mode 100644 docs/assets/item_socket_leakish_memory.png create mode 100644 docs/assets/proxy_resource_usage.png diff --git a/docs/archive/2024-2025/research-performance.md b/docs/archive/2024-2025/research-performance.md index 8062b9c2..c4e6b5a2 100644 --- a/docs/archive/2024-2025/research-performance.md +++ b/docs/archive/2024-2025/research-performance.md @@ -1,9 +1,3 @@ - - # Spaced ## Blazingly Fast Knowledge Base @@ -12,9 +6,9 @@ | | | | ------- | -------------------- | -| Date | 2024-11-15 | -| Version | 0.1 | -| State | initial draft | +| Date | 2024-11-17 | +| Version | 1.0 | +| State | final | | Author | Koen Schellingerhout | | | | @@ -25,19 +19,20 @@ | Version | Date | Author | Changes | State | | ------- | ---------- | -------------------- | --------------- | ----- | | 0.1 | 2024-11-15 | Koen Schellingerhout | initial version | draft | -| 1.0 | 2024-01-15 | Koen Schellingerhout | | final | +| 1.0 | 2024-01-15 | Koen Schellingerhout | | draft | +| 1.0 | 2024-01-17 | Koen Schellingerhout | | final | #### Distribution | Version | Date | Receivers | | ------- | ---------- | -------------------- | | 0.1 | 2024-11-15 | Canvas: Erik Schriek | -| 1.0 | 2025-01-15 | Canvas: Erik Schriek | +| 1.0 | 2025-01-17 | Canvas: Erik Schriek |
-### Contents +## Contents