diff --git a/onboarding-enabler-nodejs/src/CircuitBreaker.js b/onboarding-enabler-nodejs/src/CircuitBreaker.js new file mode 100644 index 0000000000..da689dcd27 --- /dev/null +++ b/onboarding-enabler-nodejs/src/CircuitBreaker.js @@ -0,0 +1,197 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +/* eslint-disable no-underscore-dangle */ +import { EventEmitter } from 'events'; + +const STATES = { + CLOSED: 'CLOSED', + OPEN: 'OPEN', + HALF_OPEN: 'HALF_OPEN', +}; + +/** + * Circuit breaker state machine for the Node.js onboarding enabler. + * + * Three states: CLOSED (normal), OPEN (failing, no requests), HALF_OPEN (probing). + * Transitions: + * CLOSED → OPEN when failureCount >= maxFailures + * OPEN → HALF_OPEN when cooldown expires (checked via allowRequest()) + * HALF_OPEN → CLOSED on recordSuccess() + * HALF_OPEN → OPEN on recordFailure() + * + * OPEN cooldown uses exponential backoff: cooldownTime × 2^(openCycleCount-1), + * capped at backoffMax. CLOSED backoff uses cooldownTime × 2^(failureCount-1). + * + * Emits events: 'circuitOpen', 'circuitHalfOpen', 'circuitClose' + */ +export default class CircuitBreaker extends EventEmitter { + /** + * @param {Object} options + * @param {number} [options.maxFailures=5] Consecutive failures before circuit opens + * @param {number} [options.cooldownTime=60000] Base cooldown time in ms (used for + * both CLOSED exponential backoff and OPEN exponential cooldown base) + * @param {number} [options.backoffMax=300000] Maximum backoff cap in ms + */ + constructor({ + maxFailures = 5, + cooldownTime = 60000, + backoffMax = 300000, + } = {}) { + super(); + this.maxFailures = maxFailures; + this.cooldownTime = cooldownTime; + this.backoffMax = backoffMax; + + this._state = STATES.CLOSED; + this.failureCount = 0; + this._openedAt = null; + this._openCycleCount = 0; + } + + /** @returns {string} Current state: CLOSED, OPEN, or HALF_OPEN */ + get state() { + return this._state; + } + + /** @returns {boolean} True if circuit is OPEN (not accepting requests) */ + isOpen() { + return this._state === STATES.OPEN; + } + + /** + * Check whether a request should be allowed. + * If circuit is OPEN but cooldown has expired, transitions to HALF_OPEN and returns true. + * + * @returns {boolean} True if a request may proceed + */ + allowRequest() { + if (this._state === STATES.CLOSED) { + return true; + } + if (this._state === STATES.OPEN) { + if (this._cooldownExpired()) { + this._transitionTo(STATES.HALF_OPEN); + return true; + } + return false; + } + // HALF_OPEN: allow probe request + return true; + } + + /** + * Record a successful request. Resets failureCount. + * If transitioning from HALF_OPEN to CLOSED, resets openCycleCount and emits 'circuitClose'. + * + * @returns {{ transition: string|null }} Transition if state changed + */ + recordSuccess() { + const prevState = this._state; + this.failureCount = 0; + if (prevState === STATES.HALF_OPEN) { + this._openCycleCount = 0; + this._transitionTo(STATES.CLOSED); + return { transition: STATES.CLOSED }; + } + return { transition: null }; + } + + /** + * Record a failed request. Increments failureCount. + * May transition to OPEN if threshold reached. + * OPEN delay returned is the exponential cooldown . + * + * @returns {{ transition: string|null, delay: number }} + * transition — state change if any; delay — suggested wait before next attempt (ms) + */ + recordFailure() { + this.failureCount += 1; + const prevState = this._state; + + // HALF_OPEN probe failure → immediately re-open with exponential delay + if (prevState === STATES.HALF_OPEN) { + this._transitionTo(STATES.OPEN); + return { transition: STATES.OPEN, delay: this._computeOpenCooldown() }; + } + + // CLOSED + threshold reached → open circuit with exponential delay + if (prevState === STATES.CLOSED && this.failureCount >= this.maxFailures) { + this._transitionTo(STATES.OPEN); + return { transition: STATES.OPEN, delay: this._computeOpenCooldown() }; + } + + // Still CLOSED, below threshold — return exponential backoff delay + return { transition: null, delay: this.getNextCooldown() }; + } + + /** + * Compute the cooldown/delay for the next scheduling cycle. + * OPEN state: exponential cooldown based on openCycleCount. + * CLOSED state: exponential backoff based on failureCount. + * Both use cooldownTime as base, capped at backoffMax. + * + * @returns {number} Delay in milliseconds + */ + getNextCooldown() { + if (this._state === STATES.OPEN) { + return this._computeOpenCooldown(); + } + if (this.failureCount === 0) { + return this.cooldownTime; + } + const backoff = this.cooldownTime * (2 ** (this.failureCount - 1)); + return Math.min(backoff, this.backoffMax); + } + + /** Reset breaker to CLOSED, zero failures and open cycle count. */ + reset() { + this._state = STATES.CLOSED; + this.failureCount = 0; + this._openedAt = null; + this._openCycleCount = 0; + } + + // ---- internal ---- + + /** + * Compute the OPEN cooldown with exponential backoff. + * cooldownTime × 2^(openCycleCount-1), capped at backoffMax. + * @returns {number} Cooldown in milliseconds + */ + _computeOpenCooldown() { + if (this._openCycleCount === 0) return this.cooldownTime; + const backoff = this.cooldownTime * (2 ** (this._openCycleCount - 1)); + return Math.min(backoff, this.backoffMax); + } + + _transitionTo(newState) { + const oldState = this._state; + this._state = newState; + + if (newState === STATES.OPEN) { + this._openedAt = Date.now(); + this._openCycleCount += 1; + this.emit('circuitOpen', { from: oldState, to: newState }); + } else if (newState === STATES.HALF_OPEN) { + this.emit('circuitHalfOpen', { from: oldState, to: newState }); + } else if (newState === STATES.CLOSED) { + this._openedAt = null; + this.emit('circuitClose', { from: oldState, to: newState }); + } + } + + _cooldownExpired() { + if (this._openedAt === null) { + return false; + } + return (Date.now() - this._openedAt) >= this._computeOpenCooldown(); + } +} diff --git a/onboarding-enabler-nodejs/src/EurekaClient.js b/onboarding-enabler-nodejs/src/EurekaClient.js index 122d59c794..d0d9d88026 100644 --- a/onboarding-enabler-nodejs/src/EurekaClient.js +++ b/onboarding-enabler-nodejs/src/EurekaClient.js @@ -48,6 +48,8 @@ * SOFTWARE. */ + +/* eslint-disable no-underscore-dangle */ import fs from 'fs'; import yaml from 'js-yaml'; import lodash from 'lodash'; @@ -58,6 +60,7 @@ import { series, waterfall } from 'async'; import { EventEmitter } from 'events'; import AwsMetadata from './AwsMetadata.js'; +import CircuitBreaker from './CircuitBreaker.js'; import ConfigClusterResolver from './ConfigClusterResolver.js'; import DnsClusterResolver from './DnsClusterResolver.js'; import Logger from './Logger.js'; @@ -138,6 +141,28 @@ export default class Eureka extends EventEmitter { app: {}, vip: {}, }; + + // Initialize circuit breaker + const cbConfig = this.config.eureka.circuitBreaker || {}; + this.circuitBreaker = new CircuitBreaker({ + maxFailures: cbConfig.maxFailures, + cooldownTime: cbConfig.cooldownTime, + backoffMax: cbConfig.backoffMax, + }); + + this.circuitBreaker.on('circuitOpen', ({ from, to }) => { + this.logger.warn(`Circuit breaker transition: ${from} → ${to}. Eureka requests suspended.`); + }); + this.circuitBreaker.on('circuitHalfOpen', ({ from, to }) => { + this.logger.info(`Circuit breaker transition: ${from} → ${to}. Probing Eureka server.`); + }); + this.circuitBreaker.on('circuitClose', ({ from, to }) => { + this.logger.info(`Circuit breaker transition: ${from} → ${to}. Eureka requests resumed.`); + }); + + // Timeout tracking for setTimeout-based scheduling + this._heartbeatTimeout = null; + this._registryFetchTimeout = null; } /* @@ -217,6 +242,11 @@ export default class Eureka extends EventEmitter { De-registers instance with Eureka, stops heartbeats / registry fetches. */ stop(callback = noop) { + clearTimeout(this._registryFetchTimeout); + this._registryFetchTimeout = null; + clearTimeout(this._heartbeatTimeout); + this._heartbeatTimeout = null; + // Clear legacy interval IDs for backward compatibility clearInterval(this.registryFetch); if (this.config.eureka.registerWithEureka) { clearInterval(this.heartbeat); @@ -307,16 +337,50 @@ export default class Eureka extends EventEmitter { } /* - Sets up heartbeats on interval for the life of the application. + Sets up heartbeats using setTimeout-based scheduling with circuit breaker. Heartbeat interval by setting configuration property: eureka.heartbeatInterval + When circuit breaker is enabled, each tick checks allowRequest() before + making the HTTP call and adjusts scheduling based on success/failure. */ startHeartbeats() { - this.heartbeat = setInterval(() => { - this.renew(); - }, this.config.eureka.heartbeatInterval); + const cbEnabled = this.config.eureka.circuitBreaker && + this.config.eureka.circuitBreaker.enabled !== false; + + if (!cbEnabled) { + // Fallback: legacy setInterval behavior + this.heartbeat = setInterval(() => { + this.renew(); + }, this.config.eureka.heartbeatInterval); + return; + } + + const schedule = () => { + if (!this.circuitBreaker.allowRequest()) { + // Circuit is OPEN — skip HTTP call, wait for cooldown + this._heartbeatTimeout = setTimeout(schedule, this.circuitBreaker.getNextCooldown()); + return; + } + + this.renew((err) => { + if (err) { + const result = this.circuitBreaker.recordFailure(); + this._heartbeatTimeout = setTimeout(schedule, + result.delay || this.config.eureka.heartbeatInterval); + } else { + this.circuitBreaker.recordSuccess(); + this._heartbeatTimeout = setTimeout(schedule, this.config.eureka.heartbeatInterval); + } + }); + }; + + this._heartbeatTimeout = setTimeout(schedule, this.config.eureka.heartbeatInterval); } - renew() { + /* + Sends a heartbeat renewal to Eureka. Optionally accepts a callback for + success/failure notification (used by circuit breaker scheduling). + */ + renew(callback = noop) { this.eurekaRequest({ method: 'PUT', uri: `${this.config.instance.app}/${this.instanceId}`, @@ -324,9 +388,11 @@ export default class Eureka extends EventEmitter { if (!error && response.statusCode === 200) { this.logger.debug('eureka heartbeat success'); this.emit('heartbeat'); + callback(null); } else if (!error && response.statusCode === 404) { this.logger.warn('eureka heartbeat FAILED, Re-registering app'); this.register(); + callback(null); } else { if (error) { this.logger.error('An error in the request occured.', error); @@ -336,20 +402,54 @@ export default class Eureka extends EventEmitter { `statusCode: ${response ? response.statusCode : 'unknown'}` + `body: ${body} ${error | ''} ` ); + callback(error || new Error( + `Heartbeat failed: status ${response ? response.statusCode : 'unknown'}` + )); } }); } /* - Sets up registry fetches on interval for the life of the application. + Sets up registry fetches using setTimeout-based scheduling with circuit breaker. Registry fetch interval setting configuration property: eureka.registryFetchInterval */ startRegistryFetches() { - this.registryFetch = setInterval(() => { - this.fetchRegistry(err => { - if (err) this.logger.warn('Error fetching registry', err); + const cbEnabled = this.config.eureka.circuitBreaker && + this.config.eureka.circuitBreaker.enabled !== false; + + if (!cbEnabled) { + // Fallback: legacy setInterval behavior + this.registryFetch = setInterval(() => { + this.fetchRegistry(err => { + if (err) this.logger.warn('Error fetching registry', err); + }); + }, this.config.eureka.registryFetchInterval); + return; + } + + const schedule = () => { + if (!this.circuitBreaker.allowRequest()) { + // Circuit is OPEN — skip HTTP call, wait for cooldown + this._registryFetchTimeout = setTimeout(schedule, this.circuitBreaker.getNextCooldown()); + return; + } + + this.fetchRegistry((err) => { + if (err) { + const result = this.circuitBreaker.recordFailure(); + this._registryFetchTimeout = setTimeout(schedule, + result.delay || this.config.eureka.registryFetchInterval); + } else { + this.circuitBreaker.recordSuccess(); + this._registryFetchTimeout = setTimeout( + schedule, + this.config.eureka.registryFetchInterval + ); + } }); - }, this.config.eureka.registryFetchInterval); + }; + + this._registryFetchTimeout = setTimeout(schedule, this.config.eureka.registryFetchInterval); } /* diff --git a/onboarding-enabler-nodejs/src/defaultConfig.js b/onboarding-enabler-nodejs/src/defaultConfig.js index fbb77f26eb..492748bd9c 100644 --- a/onboarding-enabler-nodejs/src/defaultConfig.js +++ b/onboarding-enabler-nodejs/src/defaultConfig.js @@ -68,6 +68,12 @@ export default { registerWithEureka: true, useLocalMetadata: false, preferIpAddress: false, + circuitBreaker: { + enabled: true, + maxFailures: 5, + cooldownTime: 60000, + backoffMax: 300000, + }, }, instance: {}, }; diff --git a/onboarding-enabler-nodejs/src/index.js b/onboarding-enabler-nodejs/src/index.js index 96ff4de4ee..7ae46ad5f8 100644 --- a/onboarding-enabler-nodejs/src/index.js +++ b/onboarding-enabler-nodejs/src/index.js @@ -75,7 +75,7 @@ function init() { }; } else { throw new Error( - 'Invalid TLS configuration: provide either p12File or certificate + keystore' + 'Invalid TLS configuration: provide either p12File or certificate + keystore' ); } diff --git a/onboarding-enabler-nodejs/test/CircuitBreaker.test.js b/onboarding-enabler-nodejs/test/CircuitBreaker.test.js new file mode 100644 index 0000000000..efaf1fdb7b --- /dev/null +++ b/onboarding-enabler-nodejs/test/CircuitBreaker.test.js @@ -0,0 +1,495 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +/* eslint-disable no-unused-expressions, max-len, no-underscore-dangle */ +import sinon from 'sinon'; +import * as chai from 'chai'; + +const { expect } = chai; +import sinonChai from 'sinon-chai'; +import { EventEmitter } from 'events'; + +import CircuitBreaker from '../src/CircuitBreaker.js'; + +chai.use(sinonChai); + +describe('CircuitBreaker', () => { + let breaker; + let clock; + + beforeEach(() => { + clock = sinon.useFakeTimers(); + breaker = new CircuitBreaker({ + maxFailures: 3, + cooldownTime: 10000, + backoffMax: 60000, + }); + }); + + afterEach(() => { + clock.restore(); + }); + + describe('construction and initial state', () => { + it('should extend EventEmitter', () => { + expect(breaker).to.be.instanceof(EventEmitter); + }); + + it('should start in CLOSED state', () => { + expect(breaker.state).to.equal('CLOSED'); + }); + + it('should have failureCount of 0', () => { + expect(breaker.failureCount).to.equal(0); + }); + + it('should have openCycleCount of 0', () => { + expect(breaker._openCycleCount).to.equal(0); + }); + + it('should use default values when none provided', () => { + const defaultBreaker = new CircuitBreaker(); + expect(defaultBreaker.maxFailures).to.equal(5); + expect(defaultBreaker.cooldownTime).to.equal(60000); + expect(defaultBreaker.backoffMax).to.equal(300000); + }); + }); + + describe('isOpen()', () => { + it('should return false when CLOSED', () => { + expect(breaker.isOpen()).to.be.false; + }); + + it('should return true when OPEN', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker.isOpen()).to.be.true; + }); + + it('should return false when HALF_OPEN', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); // transitions to HALF_OPEN + expect(breaker.isOpen()).to.be.false; + }); + }); + + describe('allowRequest()', () => { + it('should return true when CLOSED', () => { + expect(breaker.allowRequest()).to.be.true; + }); + + it('should return false when OPEN and cooldown has not expired', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker.state).to.equal('OPEN'); + expect(breaker.allowRequest()).to.be.false; + }); + + it('should transition OPEN → HALF_OPEN when cooldown has expired', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker.state).to.equal('OPEN'); + clock.tick(10001); + expect(breaker.allowRequest()).to.be.true; + expect(breaker.state).to.equal('HALF_OPEN'); + }); + + it('should return true while HALF_OPEN (probe allowed)', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); // transitions to HALF_OPEN + expect(breaker.allowRequest()).to.be.true; + expect(breaker.state).to.equal('HALF_OPEN'); + }); + + it('should not transition OPEN → HALF_OPEN before cooldown expires', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(9999); // 1ms short of cooldown + expect(breaker.allowRequest()).to.be.false; + expect(breaker.state).to.equal('OPEN'); + }); + }); + + describe('recordSuccess()', () => { + it('should reset failureCount to 0 in CLOSED', () => { + breaker.recordFailure(); + breaker.recordFailure(); + expect(breaker.failureCount).to.equal(2); + breaker.recordSuccess(); + expect(breaker.failureCount).to.equal(0); + }); + + it('should transition HALF_OPEN → CLOSED and reset openCycleCount', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker._openCycleCount).to.equal(1); + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + const result = breaker.recordSuccess(); + expect(result.transition).to.equal('CLOSED'); + expect(breaker.state).to.equal('CLOSED'); + expect(breaker._openCycleCount).to.equal(0); + }); + + it('should not transition when already CLOSED', () => { + const result = breaker.recordSuccess(); + expect(result.transition).to.be.null; + expect(breaker.state).to.equal('CLOSED'); + }); + + it('should return transition=null when CLOSED and no state change needed', () => { + breaker.recordFailure(); + expect(breaker.failureCount).to.equal(1); + const result = breaker.recordSuccess(); + expect(result.transition).to.be.null; + expect(breaker.failureCount).to.equal(0); + }); + }); + + describe('recordFailure()', () => { + it('should increment failureCount', () => { + breaker.recordFailure(); + expect(breaker.failureCount).to.equal(1); + breaker.recordFailure(); + expect(breaker.failureCount).to.equal(2); + }); + + it('should transition CLOSED → OPEN when failureCount reaches maxFailures', () => { + breaker.recordFailure(); // 1 + expect(breaker.state).to.equal('CLOSED'); + breaker.recordFailure(); // 2 + expect(breaker.state).to.equal('CLOSED'); + const result = breaker.recordFailure(); // 3 → OPEN + expect(result.transition).to.equal('OPEN'); + expect(result.delay).to.equal(10000); // cooldownTime (first open cycle) + expect(breaker.state).to.equal('OPEN'); + }); + + it('should return exponential backoff delay when below threshold', () => { + const result = breaker.recordFailure(); // failureCount=1 + expect(result.transition).to.be.null; + expect(result.delay).to.equal(10000); // cooldownTime × 2^(1-1) = 10000 + }); + + it('should transition HALF_OPEN → OPEN on probe failure with exponential delay', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + // First open: openCycleCount=1, cooldown=10000 + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + const result = breaker.recordFailure(); + expect(result.transition).to.equal('OPEN'); + // Probe failure starts a NEW open cycle (openCycleCount=2), + // so delay = cooldownTime * 2^(2-1) = 20000 + expect(result.delay).to.equal(20000); + expect(breaker.state).to.equal('OPEN'); + expect(breaker._openCycleCount).to.equal(2); + }); + }); + + describe('getNextCooldown()', () => { + it('should return cooldownTime when failureCount is 0', () => { + expect(breaker.getNextCooldown()).to.equal(10000); + }); + + it('should double with each failure (exponential backoff)', () => { + breaker.recordFailure(); // 1 failure → 10000 × 2^0 = 10000 + expect(breaker.getNextCooldown()).to.equal(10000); + + breaker.recordFailure(); // 2 failures → 10000 × 2^1 = 20000 + expect(breaker.getNextCooldown()).to.equal(20000); + + // 3rd failure would open circuit, test with maxFailures=10 instead + const bigBreaker = new CircuitBreaker({ maxFailures: 10, cooldownTime: 1000, backoffMax: 60000 }); + for (let i = 0; i < 5; i += 1) bigBreaker.recordFailure(); + // 5 failures → 1000 × 2^(5-1) = 1000 × 16 = 16000 + expect(bigBreaker.getNextCooldown()).to.equal(16000); + + bigBreaker.recordFailure(); // 6 failures → 1000 × 2^5 = 32000 + expect(bigBreaker.getNextCooldown()).to.equal(32000); + }); + + it('should cap at backoffMax', () => { + const cappedBreaker = new CircuitBreaker({ + maxFailures: 20, + cooldownTime: 1000, + backoffMax: 5000, + }); + for (let i = 0; i < 10; i += 1) cappedBreaker.recordFailure(); + expect(cappedBreaker.getNextCooldown()).to.equal(5000); // capped + }); + + it('should return exponential cooldown when OPEN (AC6)', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker.state).to.equal('OPEN'); + expect(breaker._openCycleCount).to.equal(1); + expect(breaker.getNextCooldown()).to.equal(10000); // cooldownTime × 2^0 + + // Simulate a second open cycle + breaker._openCycleCount = 2; + expect(breaker.getNextCooldown()).to.equal(20000); // cooldownTime × 2^1 + + breaker._openCycleCount = 3; + expect(breaker.getNextCooldown()).to.equal(40000); // cooldownTime × 2^2 + }); + + it('should cap OPEN cooldown at backoffMax', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + breaker._openCycleCount = 10; // 10000 × 2^9 = 5120000 > backoffMax(60000) + expect(breaker.getNextCooldown()).to.equal(60000); // capped + }); + + it('should return cooldownTime after success resets failureCount', () => { + breaker.recordFailure(); + breaker.recordFailure(); + expect(breaker.getNextCooldown()).to.be.above(10000); + breaker.recordSuccess(); + expect(breaker.getNextCooldown()).to.equal(10000); + }); + }); + + describe('OPEN cooldown — exponential backoff (AC5+AC6)', () => { + it('should use cooldownTime for first OPEN cycle', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker._openCycleCount).to.equal(1); + expect(breaker.getNextCooldown()).to.equal(10000); + }); + + it('should double cooldown on second OPEN cycle', () => { + // First OPEN + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + breaker.recordFailure(); // → OPEN again, openCycleCount=2 + expect(breaker._openCycleCount).to.equal(2); + expect(breaker.getNextCooldown()).to.equal(20000); + }); + + it('should double cooldown again on third OPEN cycle', () => { + // First OPEN + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); + breaker.recordFailure(); // openCycleCount=2 + clock.tick(20001); + breaker.allowRequest(); + breaker.recordFailure(); // openCycleCount=3 + expect(breaker._openCycleCount).to.equal(3); + expect(breaker.getNextCooldown()).to.equal(40000); + }); + + it('should reset openCycleCount on successful HALF_OPEN probe', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker._openCycleCount).to.equal(1); + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + breaker.recordSuccess(); // → CLOSED + expect(breaker._openCycleCount).to.equal(0); + }); + + it('should use longer cooldown after multiple OPEN cycles', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + // openCycleCount=1, cooldown=10000 + expect(breaker._cooldownExpired()).to.be.false; + clock.tick(5000); + expect(breaker._cooldownExpired()).to.be.false; + clock.tick(5001); // total 10001ms > 10000 + expect(breaker._cooldownExpired()).to.be.true; + }); + }); + + describe('state transitions — events', () => { + it('should emit "circuitOpen" on CLOSED → OPEN', () => { + const spy = sinon.spy(); + breaker.on('circuitOpen', spy); + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(spy).to.have.been.calledOnce; + expect(spy).to.have.been.calledWith({ from: 'CLOSED', to: 'OPEN' }); + }); + + it('should emit "circuitHalfOpen" on OPEN → HALF_OPEN', () => { + const spy = sinon.spy(); + breaker.on('circuitHalfOpen', spy); + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); + expect(spy).to.have.been.calledOnce; + expect(spy).to.have.been.calledWith({ from: 'OPEN', to: 'HALF_OPEN' }); + }); + + it('should emit "circuitClose" on HALF_OPEN → CLOSED', () => { + const spy = sinon.spy(); + breaker.on('circuitClose', spy); + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + breaker.recordSuccess(); // → CLOSED + expect(spy).to.have.been.calledOnce; + expect(spy).to.have.been.calledWith({ from: 'HALF_OPEN', to: 'CLOSED' }); + }); + + it('should emit "circuitOpen" on HALF_OPEN → OPEN (probe fail)', () => { + const spy = sinon.spy(); + breaker.on('circuitOpen', spy); + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + breaker.recordFailure(); // → OPEN + // Already called once for the first OPEN, and now a second time + expect(spy).to.have.been.calledTwice; + expect(spy.secondCall).to.have.been.calledWith({ from: 'HALF_OPEN', to: 'OPEN' }); + }); + }); + + describe('reset()', () => { + it('should reset to CLOSED with failureCount=0 and openCycleCount=0', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker.state).to.equal('OPEN'); + expect(breaker.failureCount).to.equal(3); + expect(breaker._openCycleCount).to.equal(1); + breaker.reset(); + expect(breaker.state).to.equal('CLOSED'); + expect(breaker.failureCount).to.equal(0); + expect(breaker._openCycleCount).to.equal(0); + }); + + it('should reset from HALF_OPEN to CLOSED', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + breaker.reset(); + expect(breaker.state).to.equal('CLOSED'); + expect(breaker.failureCount).to.equal(0); + expect(breaker._openCycleCount).to.equal(0); + }); + }); + + describe('complete lifecycle', () => { + it('should cycle through CLOSED → OPEN → HALF_OPEN → CLOSED', () => { + // CLOSED → OPEN (3 failures) + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker.state).to.equal('OPEN'); + + // OPEN → HALF_OPEN (cooldown passes) + clock.tick(10001); + breaker.allowRequest(); + expect(breaker.state).to.equal('HALF_OPEN'); + + // HALF_OPEN → CLOSED (success) + breaker.recordSuccess(); + expect(breaker.state).to.equal('CLOSED'); + expect(breaker.failureCount).to.equal(0); + }); + + it('should cycle through CLOSED → OPEN → HALF_OPEN → OPEN (probe fails)', () => { + // CLOSED → OPEN + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker.state).to.equal('OPEN'); + + // OPEN → HALF_OPEN + clock.tick(10001); + breaker.allowRequest(); + expect(breaker.state).to.equal('HALF_OPEN'); + + // HALF_OPEN → OPEN (probe fails) + breaker.recordFailure(); + expect(breaker.state).to.equal('OPEN'); + + // Another cooldown cycle with exponential (20000ms now) + clock.tick(20001); + breaker.allowRequest(); + expect(breaker.state).to.equal('HALF_OPEN'); + }); + }); + + describe('edge cases', () => { + it('should handle zero failures gracefully', () => { + expect(breaker.failureCount).to.equal(0); + expect(breaker.allowRequest()).to.be.true; + expect(breaker.state).to.equal('CLOSED'); + }); + + it('should handle rapid success after many failures', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); // OPEN + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + const result = breaker.recordSuccess(); // CLOSED + expect(result.transition).to.equal('CLOSED'); + expect(breaker.failureCount).to.equal(0); + expect(breaker.allowRequest()).to.be.true; + }); + + it('should handle single failure without opening circuit', () => { + const result = breaker.recordFailure(); + expect(result.transition).to.be.null; + expect(breaker.state).to.equal('CLOSED'); + expect(breaker.failureCount).to.equal(1); + }); + + it('should resume normal interval after success resets failures', () => { + breaker.recordFailure(); + breaker.recordFailure(); + expect(breaker.getNextCooldown()).to.equal(20000); // 10000 × 2^1 + breaker.recordSuccess(); + expect(breaker.getNextCooldown()).to.equal(10000); // back to base + }); + + it('should keep same OPEN timestamp on repeated failures while OPEN', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + const firstOpenAt = breaker._openedAt; + + // Additional recordFailure while OPEN shouldn't reset the clock + breaker.recordFailure(); + breaker.recordFailure(); + expect(breaker._openedAt).to.equal(firstOpenAt); + }); + + it('should reset failureCount in HALF_OPEN after success', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN, failureCount still 3 + expect(breaker.failureCount).to.equal(3); + breaker.recordSuccess(); // → CLOSED + expect(breaker.failureCount).to.equal(0); + }); + + it('should allow new failure count to accumulate after circuit closes', () => { + // Open circuit + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); // HALF_OPEN + breaker.recordSuccess(); // CLOSED, failureCount=0 + + // New failures should count from 0 again + breaker.recordFailure(); + expect(breaker.failureCount).to.equal(1); + breaker.recordFailure(); + expect(breaker.failureCount).to.equal(2); + expect(breaker.state).to.equal('CLOSED'); // still below threshold + }); + + it('should increment openCycleCount only once per CLOSED→OPEN transition', () => { + for (let i = 0; i < 3; i += 1) breaker.recordFailure(); + expect(breaker._openCycleCount).to.equal(1); + + // HALF_OPEN→OPEN should increment again + clock.tick(10001); + breaker.allowRequest(); + breaker.recordFailure(); + expect(breaker._openCycleCount).to.equal(2); + }); + + it('should handle many rapid OPEN cycles without overflow', () => { + for (let i = 0; i < 50; i += 1) { + breaker.recordFailure(); + clock.tick(10001); + breaker.allowRequest(); + } + // Should not crash, and cooldown should be capped + expect(breaker.getNextCooldown()).to.be.at.most(breaker.backoffMax); + }); + }); +}); diff --git a/onboarding-enabler-nodejs/test/EurekaClient.test.js b/onboarding-enabler-nodejs/test/EurekaClient.test.js index 701eec773e..0b6bd9465c 100644 --- a/onboarding-enabler-nodejs/test/EurekaClient.test.js +++ b/onboarding-enabler-nodejs/test/EurekaClient.test.js @@ -48,7 +48,7 @@ * SOFTWARE. */ -/* eslint-disable no-unused-expressions, max-len */ +/* eslint-disable no-unused-expressions, max-len, no-underscore-dangle */ import sinon from 'sinon'; import * as chai from 'chai'; const expect = chai.expect; @@ -342,7 +342,11 @@ describe('Eureka client', () => { let renewSpy; let clock; before(() => { - config = makeConfig(); + config = makeConfig({ + eureka: { + circuitBreaker: { enabled: false }, + }, + }); client = new Eureka(config); renewSpy = sinon.stub(client, 'renew'); clock = sinon.useFakeTimers(); @@ -368,7 +372,11 @@ describe('Eureka client', () => { let fetchRegistrySpy; let clock; before(() => { - config = makeConfig(); + config = makeConfig({ + eureka: { + circuitBreaker: { enabled: false }, + }, + }); client = new Eureka(config); fetchRegistrySpy = sinon.stub(client, 'fetchRegistry'); clock = sinon.useFakeTimers(); @@ -1420,4 +1428,272 @@ describe('Eureka client', () => { expect(client.cache.app.THEAPP).to.have.length(0); }); }); + + describe('CircuitBreaker integration', () => { + let config; + let client; + let clock; + + beforeEach(() => { + config = makeConfig({ + eureka: { + host: '127.0.0.1', + port: 9999, + heartbeatInterval: 5000, + registryFetchInterval: 5000, + circuitBreaker: { + enabled: true, + maxFailures: 3, + cooldownTime: 10000, + backoffMax: 300000, + }, + }, + }); + client = new Eureka(config); + clock = sinon.useFakeTimers(); + }); + + afterEach(() => { + clock.restore(); + if (client._heartbeatTimeout) clearTimeout(client._heartbeatTimeout); + if (client._registryFetchTimeout) clearTimeout(client._registryFetchTimeout); + }); + + // AC1: Normal interval behavior preserved when Eureka reachable + it('should call renew at heartbeat interval (AC1)', () => { + const renewSpy = sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(null); }); + + client.startHeartbeats(); + clock.tick(5000); + expect(renewSpy).to.have.been.calledOnce; + + clock.tick(5000); + expect(renewSpy).to.have.been.calledTwice; + + renewSpy.restore(); + }); + + // AC2: Circuit opens after N consecutive failures, WARN logged + it('should open circuit after maxFailures and log WARN (AC2)', () => { + const warnSpy = sinon.spy(client.logger, 'warn'); + sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(new Error('fail')); }); + + client.startHeartbeats(); + + // Exponential backoff with cooldownTime=10000 base: + // Failure 1 after 5000ms initial, failure 2 after 10000ms backoff, + // failure 3 after 20000ms backoff → circuit opens. + clock.tick(5000); // failure 1 + clock.tick(10000); // failure 2 + clock.tick(20000); // failure 3 → OPEN + + expect(warnSpy).to.have.been.calledWithMatch(/Circuit breaker transition.*CLOSED.*OPEN/); + expect(client.circuitBreaker.isOpen()).to.be.true; + + warnSpy.restore(); + }); + + // AC3: No HTTP calls while circuit is OPEN + it('should not make HTTP calls while circuit is OPEN (AC3)', () => { + const renewSpy = sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(new Error('fail')); }); + + // Open the circuit by causing failures + client.startHeartbeats(); + clock.tick(5000); // failure 1 + clock.tick(10000); // failure 2 + clock.tick(20000); // failure 3 → OPEN + + renewSpy.resetHistory(); + + // Tick just under cooldown (9999ms < 10000ms cooldownTime). + // The setTimeout from recordFailure won't fire yet, renew stays uncalled. + clock.tick(9999); + expect(renewSpy).to.not.have.been.called; + + renewSpy.restore(); + }); + + // AC4: HALF_OPEN after cooldown, probe sent + it('should enter HALF_OPEN after cooldown and send probe (AC4)', () => { + const halfOpenSpy = sinon.spy(); + client.circuitBreaker.on('circuitHalfOpen', halfOpenSpy); + sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(new Error('fail')); }); + + // Open the circuit + client.startHeartbeats(); + clock.tick(5000); // failure 1 + clock.tick(10000); // failure 2 + clock.tick(20000); // failure 3 → OPEN + + expect(client.circuitBreaker.isOpen()).to.be.true; + + // Advance past cooldown → allowRequest transitions to HALF_OPEN, + // 'circuitHalfOpen' event emitted, probe sent via renew + clock.tick(client.config.eureka.circuitBreaker.cooldownTime); + + // Verify probe was sent and HALF_OPEN was reached + expect(halfOpenSpy).to.have.been.calledOnce; + // Probe fails → back to OPEN (verified by AC6) + + client.renew.restore(); + }); + + // AC5: Probe success → CLOSED, 'circuitClose' emitted + it('should close circuit on probe success and emit circuitClose (AC5)', () => { + // Open the circuit first with failures + sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(new Error('fail')); }); + client.startHeartbeats(); + clock.tick(5000); + clock.tick(10000); + clock.tick(20000); // OPEN + client.renew.restore(); + + expect(client.circuitBreaker.isOpen()).to.be.true; + + // Advance past cooldown — allowRequest will set HALF_OPEN, renew called + const closeSpy = sinon.spy(); + client.circuitBreaker.on('circuitClose', closeSpy); + + // Stub renew to succeed this time + sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(null); }); + + clock.tick(client.config.eureka.circuitBreaker.cooldownTime); + + expect(client.circuitBreaker.state).to.equal('CLOSED'); + expect(closeSpy).to.have.been.calledOnce; + + client.renew.restore(); + }); + + // AC6: Probe failure → re-OPEN, cooldown restarts + it('should re-open circuit on probe failure (AC6)', () => { + // Open the circuit first + sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(new Error('fail')); }); + client.startHeartbeats(); + clock.tick(5000); + clock.tick(10000); + clock.tick(20000); // OPEN + + expect(client.circuitBreaker.isOpen()).to.be.true; + + // Advance past cooldown → HALF_OPEN, probe sent, fail → re-OPEN + clock.tick(client.config.eureka.circuitBreaker.cooldownTime); + + expect(client.circuitBreaker.isOpen()).to.be.true; + + client.renew.restore(); + }); + + // AC7: Exponential backoff in CLOSED state, capped at backoffMax + it('should use exponential backoff capped at backoffMax (AC7)', () => { + sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(new Error('fail')); }); + client.startHeartbeats(); + + // After 1 failure: backoff = cooldownTime * 2^0 = 10000 + clock.tick(5000); + expect(client.circuitBreaker.failureCount).to.equal(1); + expect(client.circuitBreaker.getNextCooldown()).to.equal(10000); + + // After 2 failures: backoff = 10000 * 2^1 = 20000 + clock.tick(10000); + expect(client.circuitBreaker.failureCount).to.equal(2); + expect(client.circuitBreaker.getNextCooldown()).to.equal(20000); + + // After 3 failures: circuit opens, openCycleCount=1, cooldown = 10000 + clock.tick(20000); + expect(client.circuitBreaker.isOpen()).to.be.true; + expect(client.circuitBreaker.getNextCooldown()).to.equal(client.config.eureka.circuitBreaker.cooldownTime); + + // Backoff cap test: manually set failure count high + client.circuitBreaker.reset(); + client.circuitBreaker.failureCount = 20; + expect(client.circuitBreaker.getNextCooldown()).to.be.at.most(client.config.eureka.circuitBreaker.backoffMax); + + client.renew.restore(); + }); + + // AC8: Success resets failure counter + it('should reset failure counter on success (AC8)', () => { + sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(new Error('fail')); }); + client.startHeartbeats(); + + // Cause a failure + clock.tick(5000); + expect(client.circuitBreaker.failureCount).to.equal(1); + + client.renew.restore(); + + // Now succeed + sinon.stub(client, 'renew').callsFake((cb) => { if (cb) cb(null); }); + clock.tick(10000); // after backoff (cooldownTime * 2^0), renew succeeds + + expect(client.circuitBreaker.failureCount).to.equal(0); + expect(client.circuitBreaker.state).to.equal('CLOSED'); + + client.renew.restore(); + }); + + // AC9: Config merges correctly from defaultConfig.js + it('should merge circuitBreaker config from defaults (AC9)', () => { + // Client was created with explicit circuitBreaker config + expect(client.config.eureka.circuitBreaker).to.exist; + expect(client.config.eureka.circuitBreaker.enabled).to.equal(true); + expect(client.config.eureka.circuitBreaker.maxFailures).to.equal(3); + expect(client.config.eureka.circuitBreaker.cooldownTime).to.equal(10000); + expect(client.config.eureka.circuitBreaker.backoffMax).to.equal(300000); + + // Test that defaults fill in when values are not provided + const defaultClient = new Eureka(makeConfig({ + eureka: { + host: '127.0.0.1', + port: 9999, + heartbeatInterval: 5000, + }, + })); + expect(defaultClient.config.eureka.circuitBreaker).to.exist; + expect(defaultClient.config.eureka.circuitBreaker.maxFailures).to.equal(5); // from defaultConfig + expect(defaultClient.config.eureka.circuitBreaker.cooldownTime).to.equal(60000); // from defaultConfig + }); + + // Test circuit-breaker-disabled falls back to setInterval + it('should use setInterval when circuitBreaker is disabled', () => { + const disabledClient = new Eureka(makeConfig({ + eureka: { + host: '127.0.0.1', + port: 9999, + heartbeatInterval: 30000, + circuitBreaker: { enabled: false }, + }, + })); + const renewSpy = sinon.stub(disabledClient, 'renew'); + + disabledClient.startHeartbeats(); + clock.tick(30000); + expect(renewSpy).to.have.been.calledOnce; + clock.tick(30000); + expect(renewSpy).to.have.been.calledTwice; + + renewSpy.restore(); + clearInterval(disabledClient.heartbeat); + }); + + // Test registry fetch with circuit breaker + it('should skip registry fetch when circuit is OPEN', () => { + const fetchSpy = sinon.stub(client, 'fetchRegistry').callsFake((cb) => { if (cb) cb(null); }); + + // Open the circuit first + client.circuitBreaker.failureCount = 3; + client.circuitBreaker._transitionTo('OPEN'); + + expect(client.circuitBreaker.isOpen()).to.be.true; + + client.startRegistryFetches(); + clock.tick(5000); + + // Circuit is OPEN, allowRequest returns false, fetchRegistry NOT called + expect(fetchSpy).to.not.have.been.called; + + fetchSpy.restore(); + }); + }); });