feat(graphql): backoff for dropped connection after ACK

packages/javascript-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qminder-api",
-  "version": "17.0.4",
+  "version": "17.0.5",
   "description": "Qminder Javascript API. Makes it easy to leverage Qminder capabilities in your system.",
   "scripts": {
     "test": "jest",

packages/javascript-api/src/lib/services/graphql/__tests__/graphql-reconnect-resilience.spec.ts

@@ -1,11 +1,13 @@
 import { WebSocket } from 'mock-socket';
 
+import { ConnectionStatus } from '../../../model/connection-status';
 import { GraphQLSubscriptionsFixture } from '../__fixtures__/graphql-subscriptions-fixture';
 import * as backoff from '../../../util/randomized-exponential-backoff/randomized-exponential-backoff';
+import * as sleep from '../../../util/sleep-ms/sleep-ms';
 
 jest.mock('isomorphic-ws', () => WebSocket);
 jest.mock('../../../util/sleep-ms/sleep-ms', () => ({
-  sleepMs: () => new Promise((resolve) => setTimeout(resolve, 4)),
+  sleepMs: jest.fn(() => new Promise((resolve) => setTimeout(resolve, 4))),
 }));
 
 describe('GraphQL reconnect resilience', () => {
@@ -41,6 +43,88 @@ describe('GraphQL reconnect resilience', () => {
     sub.unsubscribe();
   });
 
+  it('should back off using the attempt counter when a connection drops after ACK', async () => {
+    const service = fixture.graphqlService as any;
+    service.connectionStatus = ConnectionStatus.CONNECTED;
+    service.connectionAttemptsCount = 2;
+
+    const openSocketSpy = jest
+      .spyOn(service, 'openSocket')
+      .mockResolvedValue(undefined);
+
+    await service.handleConnectionDrop();
+
+    expect(backoffSpy).toHaveBeenCalledWith(2);
+    expect(service.connectionAttemptsCount).toBe(3);
+    expect(openSocketSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('should defer the reconnect until the backoff delay elapses', async () => {
+    const service = fixture.graphqlService as any;
+    service.connectionStatus = ConnectionStatus.CONNECTED;
+
+    const sleepMs = sleep.sleepMs as jest.Mock;
+    sleepMs.mockClear();
+    let resolveSleep: () => void;
+    sleepMs.mockReturnValueOnce(
+      new Promise<void>((resolve) => {
+        resolveSleep = resolve;
+      }),
+    );
+
+    const openSocketSpy = jest
+      .spyOn(service, 'openSocket')
+      .mockResolvedValue(undefined);
+
+    const dropPromise = service.handleConnectionDrop();
+
+    // The backoff is armed, but the reconnect must wait for it to elapse.
+    expect(sleepMs).toHaveBeenCalledTimes(1);
+    expect(openSocketSpy).not.toHaveBeenCalled();
+
+    resolveSleep();
+    await dropPromise;
+
+    expect(openSocketSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('should tear down the old socket before the backoff sleep so its onclose cannot reconnect', async () => {
+    const service = fixture.graphqlService as any;
+    service.connectionStatus = ConnectionStatus.CONNECTED;
+
+    const closeSpy = jest.fn();
+    const staleSocket: Partial<WebSocket> = {
+      onclose: () => {},
+      onmessage: () => {},
+      onopen: () => {},
+      onerror: () => {},
+      close: closeSpy,
+    };
+    service.socket = staleSocket;
+
+    const sleepMs = sleep.sleepMs as jest.Mock;
+    sleepMs.mockClear();
+    let resolveSleep: () => void;
+    sleepMs.mockReturnValueOnce(
+      new Promise<void>((resolve) => {
+        resolveSleep = resolve;
+      }),
+    );
+
+    jest.spyOn(service, 'openSocket').mockResolvedValue(undefined);
+
+    const dropPromise = service.handleConnectionDrop();
+
+    // The socket is detached and closed before the backoff window begins, so a
+    // late onclose can't start a second, parallel reconnect.
+    expect(closeSpy).toHaveBeenCalledTimes(1);
+    expect(staleSocket.onclose).toBeNull();
+    expect(service.socket).toBeNull();
+
+    resolveSleep();
+    await dropPromise;
+  });
+
   it('should not leak ping intervals when the connection monitor restarts', async () => {
     const setIntervalSpy = jest.spyOn(global, 'setInterval');
     const clearIntervalSpy = jest.spyOn(global, 'clearInterval');

packages/javascript-api/src/lib/services/graphql/graphql.service.ts

@@ -429,14 +429,7 @@ export class GraphqlService {
   }
 
   private createSocketConnection(temporaryApiKey: string): void {
-    if (this.socket) {
-      this.socket.onclose = null;
-      this.socket.onmessage = null;
-      this.socket.onopen = null;
-      this.socket.onerror = null;
-      this.socket.close();
-      this.socket = null;
-    }
+    this.tearDownSocket();
 
     this.socket = new WebSocket(this.getServerUrl(temporaryApiKey));
 
@@ -774,9 +767,33 @@ export class GraphqlService {
 
     this.setConnectionStatus(ConnectionStatus.DISCONNECTED);
     this.clearPingMonitoring();
+
+    // Tear down the old socket before sleeping so its onclose handler can't fire
+    // a second, parallel reconnect during the backoff window.
+    this.tearDownSocket();
+
+    const timer = calculateRandomizedExponentialBackoffTime(
+      this.connectionAttemptsCount,
+    );
+
+    this.connectionAttemptsCount++;
+    this.logger.info(`Reconnect socket after drop in ${timer.toFixed(0)}ms`);
+
+    await sleepMs(timer);
     await this.openSocket();
   }
 
+  private tearDownSocket(): void {
+    if (this.socket) {
+      this.socket.onclose = null;
+      this.socket.onmessage = null;
+      this.socket.onopen = null;
+      this.socket.onerror = null;
+      this.socket.close();
+      this.socket = null;
+    }
+  }
+
   private clearPingMonitoring(): void {
     clearTimeout(this.pongTimeout);
     clearInterval(this.pingPongInterval);