Merge branch '301012-improve-actioncable-reconnection' into 'master'

Improve ActionCable reconnection logic

See merge request gitlab-org/gitlab!53392

app/assets/javascripts/actioncable_connection_monitor.js

+/* eslint-disable no-restricted-globals */
+
+import { logger } from '@rails/actioncable';
+
+// This is based on https://github.com/rails/rails/blob/5a477890c809d4a17dc0dede43c6b8cef81d8175/actioncable/app/javascript/action_cable/connection_monitor.js
+// so that we can take advantage of the improved reconnection logic. We can remove this once we upgrade @rails/actioncable to a version that includes this.
+
+// Responsible for ensuring the cable connection is in good health by validating the heartbeat pings sent from the server, and attempting
+// revival reconnections if things go astray. Internal class, not intended for direct user manipulation.
+
+const now = () => new Date().getTime();
+
+const secondsSince = (time) => (now() - time) / 1000;
+class ConnectionMonitor {
+  constructor(connection) {
+    this.visibilityDidChange = this.visibilityDidChange.bind(this);
+    this.connection = connection;
+    this.reconnectAttempts = 0;
+  }
+
+  start() {
+    if (!this.isRunning()) {
+      this.startedAt = now();
+      delete this.stoppedAt;
+      this.startPolling();
+      addEventListener('visibilitychange', this.visibilityDidChange);
+      logger.log(
+        `ConnectionMonitor started. stale threshold = ${this.constructor.staleThreshold} s`,
+      );
+    }
+  }
+
+  stop() {
+    if (this.isRunning()) {
+      this.stoppedAt = now();
+      this.stopPolling();
+      removeEventListener('visibilitychange', this.visibilityDidChange);
+      logger.log('ConnectionMonitor stopped');
+    }
+  }
+
+  isRunning() {
+    return this.startedAt && !this.stoppedAt;
+  }
+
+  recordPing() {
+    this.pingedAt = now();
+  }
+
+  recordConnect() {
+    this.reconnectAttempts = 0;
+    this.recordPing();
+    delete this.disconnectedAt;
+    logger.log('ConnectionMonitor recorded connect');
+  }
+
+  recordDisconnect() {
+    this.disconnectedAt = now();
+    logger.log('ConnectionMonitor recorded disconnect');
+  }
+
+  // Private
+
+  startPolling() {
+    this.stopPolling();
+    this.poll();
+  }
+
+  stopPolling() {
+    clearTimeout(this.pollTimeout);
+  }
+
+  poll() {
+    this.pollTimeout = setTimeout(() => {
+      this.reconnectIfStale();
+      this.poll();
+    }, this.getPollInterval());
+  }
+
+  getPollInterval() {
+    const { staleThreshold, reconnectionBackoffRate } = this.constructor;
+    const backoff = (1 + reconnectionBackoffRate) ** Math.min(this.reconnectAttempts, 10);
+    const jitterMax = this.reconnectAttempts === 0 ? 1.0 : reconnectionBackoffRate;
+    const jitter = jitterMax * Math.random();
+    return staleThreshold * 1000 * backoff * (1 + jitter);
+  }
+
+  reconnectIfStale() {
+    if (this.connectionIsStale()) {
+      logger.log(
+        `ConnectionMonitor detected stale connection. reconnectAttempts = ${
+          this.reconnectAttempts
+        }, time stale = ${secondsSince(this.refreshedAt)} s, stale threshold = ${
+          this.constructor.staleThreshold
+        } s`,
+      );
+      this.reconnectAttempts += 1;
+      if (this.disconnectedRecently()) {
+        logger.log(
+          `ConnectionMonitor skipping reopening recent disconnect. time disconnected = ${secondsSince(
+            this.disconnectedAt,
+          )} s`,
+        );
+      } else {
+        logger.log('ConnectionMonitor reopening');
+        this.connection.reopen();
+      }
+    }
+  }
+
+  get refreshedAt() {
+    return this.pingedAt ? this.pingedAt : this.startedAt;
+  }
+
+  connectionIsStale() {
+    return secondsSince(this.refreshedAt) > this.constructor.staleThreshold;
+  }
+
+  disconnectedRecently() {
+    return (
+      this.disconnectedAt && secondsSince(this.disconnectedAt) < this.constructor.staleThreshold
+    );
+  }
+
+  visibilityDidChange() {
+    if (document.visibilityState === 'visible') {
+      setTimeout(() => {
+        if (this.connectionIsStale() || !this.connection.isOpen()) {
+          logger.log(
+            `ConnectionMonitor reopening stale connection on visibilitychange. visibilityState = ${document.visibilityState}`,
+          );
+          this.connection.reopen();
+        }
+      }, 200);
+    }
+  }
+}
+
+ConnectionMonitor.staleThreshold = 6; // Server::Connections::BEAT_INTERVAL * 2 (missed two pings)
+ConnectionMonitor.reconnectionBackoffRate = 0.15;
+
+export default ConnectionMonitor;

app/assets/javascripts/actioncable_consumer.js

 import { createConsumer } from '@rails/actioncable';
+import ConnectionMonitor from './actioncable_connection_monitor';
 
-export default createConsumer();
+const consumer = createConsumer();
+
+if (consumer.connection) {
+  consumer.connection.monitor = new ConnectionMonitor(consumer.connection);
+}
+
+export default consumer;

spec/frontend/actioncable_connection_monitor_spec.js

+import ConnectionMonitor from '~/actioncable_connection_monitor';
+
+describe('ConnectionMonitor', () => {
+  let monitor;
+
+  beforeEach(() => {
+    monitor = new ConnectionMonitor({});
+  });
+
+  describe('#getPollInterval', () => {
+    beforeEach(() => {
+      Math.originalRandom = Math.random;
+    });
+    afterEach(() => {
+      Math.random = Math.originalRandom;
+    });
+
+    const { staleThreshold, reconnectionBackoffRate } = ConnectionMonitor;
+    const backoffFactor = 1 + reconnectionBackoffRate;
+    const ms = 1000;
+
+    it('uses exponential backoff', () => {
+      Math.random = () => 0;
+
+      monitor.reconnectAttempts = 0;
+      expect(monitor.getPollInterval()).toEqual(staleThreshold * ms);
+
+      monitor.reconnectAttempts = 1;
+      expect(monitor.getPollInterval()).toEqual(staleThreshold * backoffFactor * ms);
+
+      monitor.reconnectAttempts = 2;
+      expect(monitor.getPollInterval()).toEqual(
+        staleThreshold * backoffFactor * backoffFactor * ms,
+      );
+    });
+
+    it('caps exponential backoff after some number of reconnection attempts', () => {
+      Math.random = () => 0;
+      monitor.reconnectAttempts = 42;
+      const cappedPollInterval = monitor.getPollInterval();
+
+      monitor.reconnectAttempts = 9001;
+      expect(monitor.getPollInterval()).toEqual(cappedPollInterval);
+    });
+
+    it('uses 100% jitter when 0 reconnection attempts', () => {
+      Math.random = () => 0;
+      expect(monitor.getPollInterval()).toEqual(staleThreshold * ms);
+
+      Math.random = () => 0.5;
+      expect(monitor.getPollInterval()).toEqual(staleThreshold * 1.5 * ms);
+    });
+
+    it('uses reconnectionBackoffRate for jitter when >0 reconnection attempts', () => {
+      monitor.reconnectAttempts = 1;
+
+      Math.random = () => 0.25;
+      expect(monitor.getPollInterval()).toEqual(
+        staleThreshold * backoffFactor * (1 + reconnectionBackoffRate * 0.25) * ms,
+      );
+
+      Math.random = () => 0.5;
+      expect(monitor.getPollInterval()).toEqual(
+        staleThreshold * backoffFactor * (1 + reconnectionBackoffRate * 0.5) * ms,
+      );
+    });
+
+    it('applies jitter after capped exponential backoff', () => {
+      monitor.reconnectAttempts = 9001;
+
+      Math.random = () => 0;
+      const withoutJitter = monitor.getPollInterval();
+      Math.random = () => 0.5;
+      const withJitter = monitor.getPollInterval();
+
+      expect(withJitter).toBeGreaterThan(withoutJitter);
+    });
+  });
+});