feat(metrics): calculate remaining capacity in pg connection pool

packages/fileimport-service/knex.js

@@ -8,6 +8,9 @@ module.exports = require('knex')({
     connectionString:
       process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@127.0.0.1/speckle'
   },
-  pool: { min: 0, max: 1 }
+  pool: {
+    min: 0,
+    max: parseInt(process.env.POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE) || 1
+  }
   // migrations are in managed in the server package
 })

packages/fileimport-service/src/prometheusMetrics.js

@@ -8,6 +8,8 @@ const knex = require('../knex')
 let metricFree = null
 let metricUsed = null
 let metricPendingAquires = null
+let metricPendingCreates = null
+let metricRemainingCapacity = null
 let metricQueryDuration = null
 let metricQueryErrors = null
 
@@ -46,6 +48,32 @@ function initKnexPrometheusMetrics() {
     }
   })
 
+  metricPendingCreates = new prometheusClient.Gauge({
+    name: 'speckle_server_knex_pending_creates',
+    help: 'Number of pending DB connection creates',
+    collect() {
+      this.set(knex.client.pool.numPendingCreates())
+    }
+  })
+
+  metricRemainingCapacity = new prometheusClient.Gauge({
+    name: 'speckle_server_knex_remaining_capacity',
+    help: 'Remaining capacity of the DB connection pool',
+    collect() {
+      const postgresMaxConnections =
+        parseInt(process.env.POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE) || 1
+      const demand =
+        knex.client.pool.numUsed() +
+        knex.client.pool.numPendingCreates() +
+        knex.client.pool.numPendingAcquires()
+
+      //the higher value of zero or the difference between the postgresMaxConnections and the demand
+      const remainingCapacity =
+        postgresMaxConnections <= demand ? 0 : postgresMaxConnections - demand
+      this.set(remainingCapacity)
+    }
+  })
+
   metricQueryDuration = new prometheusClient.Summary({
     name: 'speckle_server_knex_query_duration',
     help: 'Summary of the DB query durations in seconds'

packages/preview-service/bg_service/prometheusMetrics.js

@@ -8,6 +8,8 @@ const knex = require('../knex')
 let metricFree = null
 let metricUsed = null
 let metricPendingAquires = null
+let metricPendingCreates = null
+let metricRemainingCapacity = null
 let metricQueryDuration = null
 let metricQueryErrors = null
 
@@ -46,6 +48,32 @@ function initKnexPrometheusMetrics() {
     }
   })
 
+  metricPendingCreates = new prometheusClient.Gauge({
+    name: 'speckle_server_knex_pending_creates',
+    help: 'Number of pending DB connection creates',
+    collect() {
+      this.set(knex.client.pool.numPendingCreates())
+    }
+  })
+
+  metricRemainingCapacity = new prometheusClient.Gauge({
+    name: 'speckle_server_knex_remaining_capacity',
+    help: 'Remaining capacity of the DB connection pool',
+    collect() {
+      const postgresMaxConnections =
+        parseInt(process.env.POSTGRES_MAX_CONNECTIONS_PREVIEW_SERVICE) || 2
+      const demand =
+        knex.client.pool.numUsed() +
+        knex.client.pool.numPendingCreates() +
+        knex.client.pool.numPendingAcquires()
+
+      //the higher value of zero or the difference between the postgresMaxConnections and the demand
+      const remainingCapacity =
+        postgresMaxConnections <= demand ? 0 : postgresMaxConnections - demand
+      this.set(remainingCapacity)
+    }
+  })
+
   metricQueryDuration = new prometheusClient.Summary({
     name: 'speckle_server_knex_query_duration',
     help: 'Summary of the DB query durations in seconds'

packages/preview-service/knex.js

@@ -8,6 +8,9 @@ module.exports = require('knex')({
     connectionString:
       process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@127.0.0.1/speckle'
   },
-  pool: { min: 0, max: 2 }
+  pool: {
+    min: 0,
+    max: parseInt(process.env.POSTGRES_MAX_CONNECTIONS_PREVIEW_SERVICE) || 2
+  }
   // migrations are in managed in the server package
 })

packages/server/logging/knexMonitoring.js

@@ -7,11 +7,15 @@ const prometheusClient = require('prom-client')
 
 let metricFree = null
 let metricUsed = null
+let metricPendingCreates = null
+let metricRemainingCapacity = null
 let metricPendingAquires = null
 let metricQueryDuration = null
 let metricQueryErrors = null
 
 const queryStartTime = {}
+const postgresMaxConnections =
+  parseInt(process.env.POSTGRES_MAX_CONNECTIONS_SERVER) || 4
 
 module.exports = {
   initKnexPrometheusMetrics() {
@@ -39,6 +43,32 @@ module.exports = {
       }
     })
 
+    metricPendingCreates = new prometheusClient.Gauge({
+      name: 'speckle_server_knex_pending_creates',
+      help: 'Number of pending DB connection creates',
+      collect() {
+        this.set(knex.client.pool.numPendingCreates())
+      }
+    })
+
+    metricRemainingCapacity = new prometheusClient.Gauge({
+      name: 'speckle_server_knex_remaining_capacity',
+      help: 'Remaining capacity of the DB connection pool',
+      collect() {
+        const postgresMaxConnections =
+          parseInt(process.env.POSTGRES_MAX_CONNECTIONS_SERVER) || 4
+        const demand =
+          knex.client.pool.numUsed() +
+          knex.client.pool.numPendingCreates() +
+          knex.client.pool.numPendingAcquires()
+
+        //the higher value of zero or the difference between the postgresMaxConnections and the demand
+        const remainingCapacity =
+          postgresMaxConnections <= demand ? 0 : postgresMaxConnections - demand
+        this.set(remainingCapacity)
+      }
+    })
+
     metricQueryDuration = new prometheusClient.Summary({
       name: 'speckle_server_knex_query_duration',
       help: 'Summary of the DB query durations in seconds'

packages/webhook-service/src/knex.js

@@ -8,6 +8,9 @@ module.exports = require('knex')({
     connectionString:
       process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@127.0.0.1/speckle'
   },
-  pool: { min: 0, max: 1 }
+  pool: {
+    min: 0,
+    max: parseInt(process.env.POSTGRES_MAX_CONNECTIONS_WEBHOOK_SERVICE) || 1
+  }
   // migrations are in managed in the server package
 })

packages/webhook-service/src/observability/prometheusMetrics.js

@@ -8,6 +8,8 @@ const knex = require('../knex')
 let metricFree = null
 let metricUsed = null
 let metricPendingAquires = null
+let metricPendingCreates = null
+let metricRemainingCapacity = null
 let metricQueryDuration = null
 let metricQueryErrors = null
 
@@ -46,6 +48,32 @@ function initKnexPrometheusMetrics() {
     }
   })
 
+  metricPendingCreates = new prometheusClient.Gauge({
+    name: 'speckle_server_knex_pending_creates',
+    help: 'Number of pending DB connection creates',
+    collect() {
+      this.set(knex.client.pool.numPendingCreates())
+    }
+  })
+
+  metricRemainingCapacity = new prometheusClient.Gauge({
+    name: 'speckle_server_knex_remaining_capacity',
+    help: 'Remaining capacity of the DB connection pool',
+    collect() {
+      const postgresMaxConnections =
+        parseInt(process.env.POSTGRES_MAX_CONNECTIONS_WEBHOOK_SERVICE) || 1
+      const demand =
+        knex.client.pool.numUsed() +
+        knex.client.pool.numPendingCreates() +
+        knex.client.pool.numPendingAcquires()
+
+      //the higher value of zero or the difference between the postgresMaxConnections and the demand
+      const remainingCapacity =
+        postgresMaxConnections <= demand ? 0 : postgresMaxConnections - demand
+      this.set(remainingCapacity)
+    }
+  })
+
   metricQueryDuration = new prometheusClient.Summary({
     name: 'speckle_server_knex_query_duration',
     help: 'Summary of the DB query durations in seconds'