feat: Node.js CircuitBreaker with exponential OPEN cooldown — GH#4775

onboarding-enabler-nodejs/src/CircuitBreaker.js

@@ -0,0 +1,197 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+/* eslint-disable no-underscore-dangle */
+import { EventEmitter } from 'events';
+
+const STATES = {
+  CLOSED: 'CLOSED',
+  OPEN: 'OPEN',
+  HALF_OPEN: 'HALF_OPEN',
+};
+
+/**
+ * Circuit breaker state machine for the Node.js onboarding enabler.
+ *
+ * Three states: CLOSED (normal), OPEN (failing, no requests), HALF_OPEN (probing).
+ * Transitions:
+ *   CLOSED → OPEN when failureCount >= maxFailures
+ *   OPEN  → HALF_OPEN when cooldown expires (checked via allowRequest())
+ *   HALF_OPEN → CLOSED on recordSuccess()
+ *   HALF_OPEN → OPEN  on recordFailure()
+ *
+ * OPEN cooldown uses exponential backoff: cooldownTime × 2^(openCycleCount-1),
+ * capped at backoffMax. CLOSED backoff uses cooldownTime × 2^(failureCount-1).
+ *
+ * Emits events: 'circuitOpen', 'circuitHalfOpen', 'circuitClose'
+ */
+export default class CircuitBreaker extends EventEmitter {
+  /**
+   * @param {Object} options
+   * @param {number} [options.maxFailures=5] Consecutive failures before circuit opens
+   * @param {number} [options.cooldownTime=60000] Base cooldown time in ms (used for
+   *   both CLOSED exponential backoff and OPEN exponential cooldown base)
+   * @param {number} [options.backoffMax=300000] Maximum backoff cap in ms
+   */
+  constructor({
+    maxFailures = 5,
+    cooldownTime = 60000,
+    backoffMax = 300000,
+  } = {}) {
+    super();
+    this.maxFailures = maxFailures;
+    this.cooldownTime = cooldownTime;
+    this.backoffMax = backoffMax;
+
+    this._state = STATES.CLOSED;
+    this.failureCount = 0;
+    this._openedAt = null;
+    this._openCycleCount = 0;
+  }
+
+  /** @returns {string} Current state: CLOSED, OPEN, or HALF_OPEN */
+  get state() {
+    return this._state;
+  }
+
+  /** @returns {boolean} True if circuit is OPEN (not accepting requests) */
+  isOpen() {
+    return this._state === STATES.OPEN;
+  }
+
+  /**
+   * Check whether a request should be allowed.
+   * If circuit is OPEN but cooldown has expired, transitions to HALF_OPEN and returns true.
+   *
+   * @returns {boolean} True if a request may proceed
+   */
+  allowRequest() {
+    if (this._state === STATES.CLOSED) {
+      return true;
+    }
+    if (this._state === STATES.OPEN) {
+      if (this._cooldownExpired()) {
+        this._transitionTo(STATES.HALF_OPEN);
+        return true;
+      }
+      return false;
+    }
+    // HALF_OPEN: allow probe request
+    return true;
+  }
+
+  /**
+   * Record a successful request. Resets failureCount.
+   * If transitioning from HALF_OPEN to CLOSED, resets openCycleCount and emits 'circuitClose'.
+   *
+   * @returns {{ transition: string|null }} Transition if state changed
+   */
+  recordSuccess() {
+    const prevState = this._state;
+    this.failureCount = 0;
+    if (prevState === STATES.HALF_OPEN) {
+      this._openCycleCount = 0;
+      this._transitionTo(STATES.CLOSED);
+      return { transition: STATES.CLOSED };
+    }
+    return { transition: null };
+  }
+
+  /**
+   * Record a failed request. Increments failureCount.
+   * May transition to OPEN if threshold reached.
+   * OPEN delay returned is the exponential cooldown .
+   *
+   * @returns {{ transition: string|null, delay: number }}
+   *   transition — state change if any; delay — suggested wait before next attempt (ms)
+   */
+  recordFailure() {
+    this.failureCount += 1;
+    const prevState = this._state;
+
+    // HALF_OPEN probe failure → immediately re-open with exponential delay
+    if (prevState === STATES.HALF_OPEN) {
+      this._transitionTo(STATES.OPEN);
+      return { transition: STATES.OPEN, delay: this._computeOpenCooldown() };
+    }
+
+    // CLOSED + threshold reached → open circuit with exponential delay
+    if (prevState === STATES.CLOSED && this.failureCount >= this.maxFailures) {
+      this._transitionTo(STATES.OPEN);
+      return { transition: STATES.OPEN, delay: this._computeOpenCooldown() };
+    }
+
+    // Still CLOSED, below threshold — return exponential backoff delay
+    return { transition: null, delay: this.getNextCooldown() };
+  }
+
+  /**
+   * Compute the cooldown/delay for the next scheduling cycle.
+   * OPEN state: exponential cooldown based on openCycleCount.
+   * CLOSED state: exponential backoff based on failureCount.
+   * Both use cooldownTime as base, capped at backoffMax.
+   *
+   * @returns {number} Delay in milliseconds
+   */
+  getNextCooldown() {
+    if (this._state === STATES.OPEN) {
+      return this._computeOpenCooldown();
+    }
+    if (this.failureCount === 0) {
+      return this.cooldownTime;
+    }
+    const backoff = this.cooldownTime * (2 ** (this.failureCount - 1));
+    return Math.min(backoff, this.backoffMax);
+  }
+
+  /** Reset breaker to CLOSED, zero failures and open cycle count. */
+  reset() {
+    this._state = STATES.CLOSED;
+    this.failureCount = 0;
+    this._openedAt = null;
+    this._openCycleCount = 0;
+  }
+
+  // ---- internal ----
+
+  /**
+   * Compute the OPEN cooldown with exponential backoff.
+   * cooldownTime × 2^(openCycleCount-1), capped at backoffMax.
+   * @returns {number} Cooldown in milliseconds
+   */
+  _computeOpenCooldown() {
+    if (this._openCycleCount === 0) return this.cooldownTime;
+    const backoff = this.cooldownTime * (2 ** (this._openCycleCount - 1));
+    return Math.min(backoff, this.backoffMax);
+  }
+
+  _transitionTo(newState) {
+    const oldState = this._state;
+    this._state = newState;
+
+    if (newState === STATES.OPEN) {
+      this._openedAt = Date.now();
+      this._openCycleCount += 1;
+      this.emit('circuitOpen', { from: oldState, to: newState });
+    } else if (newState === STATES.HALF_OPEN) {
+      this.emit('circuitHalfOpen', { from: oldState, to: newState });
+    } else if (newState === STATES.CLOSED) {
+      this._openedAt = null;
+      this.emit('circuitClose', { from: oldState, to: newState });
+    }
+  }
+
+  _cooldownExpired() {
+    if (this._openedAt === null) {
+      return false;
+    }
+    return (Date.now() - this._openedAt) >= this._computeOpenCooldown();
+  }
+}

onboarding-enabler-nodejs/src/EurekaClient.js

@@ -48,6 +48,8 @@
  * SOFTWARE.
  */
 
+
+/* eslint-disable no-underscore-dangle */
 import fs from 'fs';
 import yaml from 'js-yaml';
 import lodash from 'lodash';
@@ -58,6 +60,7 @@ import { series, waterfall } from 'async';
 import { EventEmitter } from 'events';
 
 import AwsMetadata from './AwsMetadata.js';
+import CircuitBreaker from './CircuitBreaker.js';
 import ConfigClusterResolver from './ConfigClusterResolver.js';
 import DnsClusterResolver from './DnsClusterResolver.js';
 import Logger from './Logger.js';
@@ -138,6 +141,28 @@ export default class Eureka extends EventEmitter {
       app: {},
       vip: {},
     };
+
+    // Initialize circuit breaker
+    const cbConfig = this.config.eureka.circuitBreaker || {};
+    this.circuitBreaker = new CircuitBreaker({
+      maxFailures: cbConfig.maxFailures,
+      cooldownTime: cbConfig.cooldownTime,
+      backoffMax: cbConfig.backoffMax,
+    });
+
+    this.circuitBreaker.on('circuitOpen', ({ from, to }) => {
+      this.logger.warn(`Circuit breaker transition: ${from} → ${to}. Eureka requests suspended.`);
+    });
+    this.circuitBreaker.on('circuitHalfOpen', ({ from, to }) => {
+      this.logger.info(`Circuit breaker transition: ${from} → ${to}. Probing Eureka server.`);
+    });
+    this.circuitBreaker.on('circuitClose', ({ from, to }) => {
+      this.logger.info(`Circuit breaker transition: ${from} → ${to}. Eureka requests resumed.`);
+    });
+
+    // Timeout tracking for setTimeout-based scheduling
+    this._heartbeatTimeout = null;
+    this._registryFetchTimeout = null;
   }
 
   /*
@@ -217,6 +242,11 @@ export default class Eureka extends EventEmitter {
     De-registers instance with Eureka, stops heartbeats / registry fetches.
   */
   stop(callback = noop) {
+    clearTimeout(this._registryFetchTimeout);
+    this._registryFetchTimeout = null;
+    clearTimeout(this._heartbeatTimeout);
+    this._heartbeatTimeout = null;
+    // Clear legacy interval IDs for backward compatibility
     clearInterval(this.registryFetch);
     if (this.config.eureka.registerWithEureka) {
       clearInterval(this.heartbeat);
@@ -307,26 +337,62 @@ export default class Eureka extends EventEmitter {
   }
 
   /*
-    Sets up heartbeats on interval for the life of the application.
+    Sets up heartbeats using setTimeout-based scheduling with circuit breaker.
     Heartbeat interval by setting configuration property: eureka.heartbeatInterval
+    When circuit breaker is enabled, each tick checks allowRequest() before
+    making the HTTP call and adjusts scheduling based on success/failure.
   */
   startHeartbeats() {
-    this.heartbeat = setInterval(() => {
-      this.renew();
-    }, this.config.eureka.heartbeatInterval);
+    const cbEnabled = this.config.eureka.circuitBreaker &&
+      this.config.eureka.circuitBreaker.enabled !== false;
+
+    if (!cbEnabled) {
+      // Fallback: legacy setInterval behavior
+      this.heartbeat = setInterval(() => {
+        this.renew();
+      }, this.config.eureka.heartbeatInterval);
+      return;
+    }
+
+    const schedule = () => {
+      if (!this.circuitBreaker.allowRequest()) {
+        // Circuit is OPEN — skip HTTP call, wait for cooldown
+        this._heartbeatTimeout = setTimeout(schedule, this.circuitBreaker.getNextCooldown());
+        return;
+      }
+
+      this.renew((err) => {
+        if (err) {
+          const result = this.circuitBreaker.recordFailure();
+          this._heartbeatTimeout = setTimeout(schedule,
+            result.delay || this.config.eureka.heartbeatInterval);
+        } else {
+          this.circuitBreaker.recordSuccess();
+          this._heartbeatTimeout = setTimeout(schedule, this.config.eureka.heartbeatInterval);
+        }
+      });
+    };
+
+    this._heartbeatTimeout = setTimeout(schedule, this.config.eureka.heartbeatInterval);
   }
 
-  renew() {
+  /*
+    Sends a heartbeat renewal to Eureka. Optionally accepts a callback for
+    success/failure notification (used by circuit breaker scheduling).
+  */
+  renew(callback = noop) {
     this.eurekaRequest({
       method: 'PUT',
       uri: `${this.config.instance.app}/${this.instanceId}`,
     }, (error, response, body) => {
       if (!error && response.statusCode === 200) {
         this.logger.debug('eureka heartbeat success');
         this.emit('heartbeat');
+        callback(null);
       } else if (!error && response.statusCode === 404) {
         this.logger.warn('eureka heartbeat FAILED, Re-registering app');
         this.register();
+        callback(null);
       } else {
         if (error) {
           this.logger.error('An error in the request occured.', error);
@@ -336,20 +402,54 @@ export default class Eureka extends EventEmitter {
           `statusCode: ${response ? response.statusCode : 'unknown'}` +
           `body: ${body} ${error | ''} `
         );
+        callback(error || new Error(
+          `Heartbeat failed: status ${response ? response.statusCode : 'unknown'}`
+        ));
       }
     });
   }
 
   /*
-    Sets up registry fetches on interval for the life of the application.
+    Sets up registry fetches using setTimeout-based scheduling with circuit breaker.
     Registry fetch interval setting configuration property: eureka.registryFetchInterval
   */
   startRegistryFetches() {
-    this.registryFetch = setInterval(() => {
-      this.fetchRegistry(err => {
-        if (err) this.logger.warn('Error fetching registry', err);
+    const cbEnabled = this.config.eureka.circuitBreaker &&
+      this.config.eureka.circuitBreaker.enabled !== false;
+
+    if (!cbEnabled) {
+      // Fallback: legacy setInterval behavior
+      this.registryFetch = setInterval(() => {
+        this.fetchRegistry(err => {
+          if (err) this.logger.warn('Error fetching registry', err);
+        });
+      }, this.config.eureka.registryFetchInterval);
+      return;
+    }
+
+    const schedule = () => {
+      if (!this.circuitBreaker.allowRequest()) {
+        // Circuit is OPEN — skip HTTP call, wait for cooldown
+        this._registryFetchTimeout = setTimeout(schedule, this.circuitBreaker.getNextCooldown());
+        return;
+      }
+
+      this.fetchRegistry((err) => {
+        if (err) {
+          const result = this.circuitBreaker.recordFailure();
+          this._registryFetchTimeout = setTimeout(schedule,
+            result.delay || this.config.eureka.registryFetchInterval);
+        } else {
+          this.circuitBreaker.recordSuccess();
+          this._registryFetchTimeout = setTimeout(
+            schedule,
+            this.config.eureka.registryFetchInterval
+          );
+        }
       });
-    }, this.config.eureka.registryFetchInterval);
+    };
+
+    this._registryFetchTimeout = setTimeout(schedule, this.config.eureka.registryFetchInterval);
   }
 
   /*

onboarding-enabler-nodejs/src/defaultConfig.js

@@ -68,6 +68,12 @@ export default {
     registerWithEureka: true,
     useLocalMetadata: false,
     preferIpAddress: false,
+    circuitBreaker: {
+      enabled: true,
+      maxFailures: 5,
+      cooldownTime: 60000,
+      backoffMax: 300000,
+    },
   },
   instance: {},
 };