From 98a8f65f72175ac79f3c4ab5133a5866301ae636 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 26 Jun 2024 18:56:15 +0100 Subject: [PATCH 1/2] feat(metrics): calculate remaining capacity in pg connection pool --- packages/server/logging/knexMonitoring.js | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/server/logging/knexMonitoring.js b/packages/server/logging/knexMonitoring.js index 90892458fe0..500310d8179 100644 --- a/packages/server/logging/knexMonitoring.js +++ b/packages/server/logging/knexMonitoring.js @@ -7,11 +7,15 @@ const prometheusClient = require('prom-client') let metricFree = null let metricUsed = null +let metricPendingCreates = null +let metricRemainingCapacity = null let metricPendingAquires = null let metricQueryDuration = null let metricQueryErrors = null const queryStartTime = {} +const postgresMaxConnections = + parseInt(process.env.POSTGRES_MAX_CONNECTIONS_SERVER) || 4 module.exports = { initKnexPrometheusMetrics() { @@ -39,6 +43,32 @@ module.exports = { } }) + metricPendingCreates = new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_creates', + help: 'Number of pending DB connection creates', + collect() { + this.set(knex.client.pool.numPendingCreates()) + } + }) + + metricRemainingCapacity = new prometheusClient.Gauge({ + name: 'speckle_server_knex_remaining_capacity', + help: 'Remaining capacity of the DB connection pool', + collect() { + const postgresMaxConnections = + parseInt(process.env.POSTGRES_MAX_CONNECTIONS_SERVER) || 4 + const demand = + knex.client.pool.numUsed() + + knex.client.pool.numPendingCreates() + + knex.client.pool.numPendingAcquires() + + //the higher value of zero or the difference between the postgresMaxConnections and the demand + const remainingCapacity = + postgresMaxConnections <= demand ? 0 : postgresMaxConnections - demand + this.set(remainingCapacity) + } + }) + metricQueryDuration = new prometheusClient.Summary({ name: 'speckle_server_knex_query_duration', help: 'Summary of the DB query durations in seconds' From 8bf8afdf04a042be8fd8aa06d1495fc288850141 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 26 Jun 2024 20:20:26 +0100 Subject: [PATCH 2/2] Apply metric to all relevant services --- packages/fileimport-service/knex.js | 5 +++- .../src/prometheusMetrics.js | 28 +++++++++++++++++++ .../bg_service/prometheusMetrics.js | 28 +++++++++++++++++++ packages/preview-service/knex.js | 5 +++- packages/webhook-service/src/knex.js | 5 +++- .../src/observability/prometheusMetrics.js | 28 +++++++++++++++++++ 6 files changed, 96 insertions(+), 3 deletions(-) diff --git a/packages/fileimport-service/knex.js b/packages/fileimport-service/knex.js index 37b663b9cbc..7e85a7ae257 100644 --- a/packages/fileimport-service/knex.js +++ b/packages/fileimport-service/knex.js @@ -8,6 +8,9 @@ module.exports = require('knex')({ connectionString: process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@127.0.0.1/speckle' }, - pool: { min: 0, max: 1 } + pool: { + min: 0, + max: parseInt(process.env.POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE) || 1 + } // migrations are in managed in the server package }) diff --git a/packages/fileimport-service/src/prometheusMetrics.js b/packages/fileimport-service/src/prometheusMetrics.js index e8ad631692a..621dcd23539 100644 --- a/packages/fileimport-service/src/prometheusMetrics.js +++ b/packages/fileimport-service/src/prometheusMetrics.js @@ -8,6 +8,8 @@ const knex = require('../knex') let metricFree = null let metricUsed = null let metricPendingAquires = null +let metricPendingCreates = null +let metricRemainingCapacity = null let metricQueryDuration = null let metricQueryErrors = null @@ -46,6 +48,32 @@ function initKnexPrometheusMetrics() { } }) + metricPendingCreates = new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_creates', + help: 'Number of pending DB connection creates', + collect() { + this.set(knex.client.pool.numPendingCreates()) + } + }) + + metricRemainingCapacity = new prometheusClient.Gauge({ + name: 'speckle_server_knex_remaining_capacity', + help: 'Remaining capacity of the DB connection pool', + collect() { + const postgresMaxConnections = + parseInt(process.env.POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE) || 1 + const demand = + knex.client.pool.numUsed() + + knex.client.pool.numPendingCreates() + + knex.client.pool.numPendingAcquires() + + //the higher value of zero or the difference between the postgresMaxConnections and the demand + const remainingCapacity = + postgresMaxConnections <= demand ? 0 : postgresMaxConnections - demand + this.set(remainingCapacity) + } + }) + metricQueryDuration = new prometheusClient.Summary({ name: 'speckle_server_knex_query_duration', help: 'Summary of the DB query durations in seconds' diff --git a/packages/preview-service/bg_service/prometheusMetrics.js b/packages/preview-service/bg_service/prometheusMetrics.js index 4e3c84f73e2..8882a284fbc 100644 --- a/packages/preview-service/bg_service/prometheusMetrics.js +++ b/packages/preview-service/bg_service/prometheusMetrics.js @@ -8,6 +8,8 @@ const knex = require('../knex') let metricFree = null let metricUsed = null let metricPendingAquires = null +let metricPendingCreates = null +let metricRemainingCapacity = null let metricQueryDuration = null let metricQueryErrors = null @@ -46,6 +48,32 @@ function initKnexPrometheusMetrics() { } }) + metricPendingCreates = new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_creates', + help: 'Number of pending DB connection creates', + collect() { + this.set(knex.client.pool.numPendingCreates()) + } + }) + + metricRemainingCapacity = new prometheusClient.Gauge({ + name: 'speckle_server_knex_remaining_capacity', + help: 'Remaining capacity of the DB connection pool', + collect() { + const postgresMaxConnections = + parseInt(process.env.POSTGRES_MAX_CONNECTIONS_PREVIEW_SERVICE) || 2 + const demand = + knex.client.pool.numUsed() + + knex.client.pool.numPendingCreates() + + knex.client.pool.numPendingAcquires() + + //the higher value of zero or the difference between the postgresMaxConnections and the demand + const remainingCapacity = + postgresMaxConnections <= demand ? 0 : postgresMaxConnections - demand + this.set(remainingCapacity) + } + }) + metricQueryDuration = new prometheusClient.Summary({ name: 'speckle_server_knex_query_duration', help: 'Summary of the DB query durations in seconds' diff --git a/packages/preview-service/knex.js b/packages/preview-service/knex.js index ac84a971218..3a7d7d7cddf 100644 --- a/packages/preview-service/knex.js +++ b/packages/preview-service/knex.js @@ -8,6 +8,9 @@ module.exports = require('knex')({ connectionString: process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@127.0.0.1/speckle' }, - pool: { min: 0, max: 2 } + pool: { + min: 0, + max: parseInt(process.env.POSTGRES_MAX_CONNECTIONS_PREVIEW_SERVICE) || 2 + } // migrations are in managed in the server package }) diff --git a/packages/webhook-service/src/knex.js b/packages/webhook-service/src/knex.js index 9ec1673e09c..7c002a8c7a7 100644 --- a/packages/webhook-service/src/knex.js +++ b/packages/webhook-service/src/knex.js @@ -8,6 +8,9 @@ module.exports = require('knex')({ connectionString: process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@127.0.0.1/speckle' }, - pool: { min: 0, max: 1 } + pool: { + min: 0, + max: parseInt(process.env.POSTGRES_MAX_CONNECTIONS_WEBHOOK_SERVICE) || 1 + } // migrations are in managed in the server package }) diff --git a/packages/webhook-service/src/observability/prometheusMetrics.js b/packages/webhook-service/src/observability/prometheusMetrics.js index 553540e2ab7..8d9a745c7ce 100644 --- a/packages/webhook-service/src/observability/prometheusMetrics.js +++ b/packages/webhook-service/src/observability/prometheusMetrics.js @@ -8,6 +8,8 @@ const knex = require('../knex') let metricFree = null let metricUsed = null let metricPendingAquires = null +let metricPendingCreates = null +let metricRemainingCapacity = null let metricQueryDuration = null let metricQueryErrors = null @@ -46,6 +48,32 @@ function initKnexPrometheusMetrics() { } }) + metricPendingCreates = new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_creates', + help: 'Number of pending DB connection creates', + collect() { + this.set(knex.client.pool.numPendingCreates()) + } + }) + + metricRemainingCapacity = new prometheusClient.Gauge({ + name: 'speckle_server_knex_remaining_capacity', + help: 'Remaining capacity of the DB connection pool', + collect() { + const postgresMaxConnections = + parseInt(process.env.POSTGRES_MAX_CONNECTIONS_WEBHOOK_SERVICE) || 1 + const demand = + knex.client.pool.numUsed() + + knex.client.pool.numPendingCreates() + + knex.client.pool.numPendingAcquires() + + //the higher value of zero or the difference between the postgresMaxConnections and the demand + const remainingCapacity = + postgresMaxConnections <= demand ? 0 : postgresMaxConnections - demand + this.set(remainingCapacity) + } + }) + metricQueryDuration = new prometheusClient.Summary({ name: 'speckle_server_knex_query_duration', help: 'Summary of the DB query durations in seconds'