VPR-141 feat(healthchecks): add /health endpoints and UI dashboard

JenkinsFile

@@ -109,6 +109,27 @@ pipeline {
 				networkShares()
 				filecopy copyOfflineOperations, 'test', env.WORKSPACE
 				filecopy copyOperations, 'test', env.WORKSPACE
+				powershell label: 'Verify /health returns 200', script: '''
+					$url = "https://secure-test.vetmed.ucdavis.edu/2/health"
+					$maxAttempts = 15
+					for ($i = 1; $i -le $maxAttempts; $i++) {
+						# 2s for the first few attempts then 4s - absorbs IIS app-pool warm-up
+						$delay = if ($i -le 5) { 2 } else { 4 }
+						try {
+							$response = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 10
+							if ($response.StatusCode -eq 200) {
+								Write-Host "Attempt ${i}: $url returned 200 OK"
+								exit 0
+							}
+							Write-Host "Attempt ${i}: $url returned $($response.StatusCode)"
+						} catch {
+							Write-Host "Attempt ${i}: $($_.Exception.Message)"
+						}
+						if ($i -lt $maxAttempts) { Start-Sleep -Seconds $delay }
+					}
+					Write-Error "Health check at $url failed after $maxAttempts attempts"
+					exit 1
+				'''
 			}
 		}
 		stage('Deploy to prod') {
@@ -122,6 +143,26 @@ pipeline {
 				networkShares()
 				filecopy copyOfflineOperations, 'prod', env.WORKSPACE
 				filecopy copyOperations, 'prod', env.WORKSPACE
+				powershell label: 'Verify /health returns 200', script: '''
+					$url = "https://secure.vetmed.ucdavis.edu/2/health"
+					$maxAttempts = 15
+					for ($i = 1; $i -le $maxAttempts; $i++) {
+						$delay = if ($i -le 5) { 2 } else { 4 }
+						try {
+							$response = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 10
+							if ($response.StatusCode -eq 200) {
+								Write-Host "Attempt ${i}: $url returned 200 OK"
+								exit 0
+							}
+							Write-Host "Attempt ${i}: $url returned $($response.StatusCode)"
+						} catch {
+							Write-Host "Attempt ${i}: $($_.Exception.Message)"
+						}
+						if ($i -lt $maxAttempts) { Start-Sleep -Seconds $delay }
+					}
+					Write-Error "Health check at $url failed after $maxAttempts attempts"
+					exit 1
+				'''
 			}
 		}
     }

web/Classes/CloudflareNetworks.cs

@@ -0,0 +1,59 @@
+namespace Viper.Classes
+{
+    /// <summary>
+    /// Cloudflare's published IPv4/IPv6 networks, used to mark CF as a known
+    /// proxy in ForwardedHeadersOptions. Fetched from cloudflare.com at startup
+    /// so we automatically pick up rotations; falls back to a hardcoded snapshot
+    /// when the fetch fails (CF outage during deploy, sandboxed network, etc).
+    /// </summary>
+    public static class CloudflareNetworks
+    {
+        // Snapshot of https://www.cloudflare.com/ips/ - only used when the
+        // runtime fetch fails. Refresh occasionally if logs show this falling
+        // through and current CF IPs aren't in the list.
+        private static readonly string[] HardcodedFallback =
+        [
+            "173.245.48.0/20",
+            "103.21.244.0/22",
+            "103.22.200.0/22",
+            "103.31.4.0/22",
+            "141.101.64.0/18",
+            "108.162.192.0/18",
+            "190.93.240.0/20",
+            "188.114.96.0/20",
+            "197.234.240.0/22",
+            "198.41.128.0/17",
+            "162.158.0.0/15",
+            "104.16.0.0/13",
+            "104.24.0.0/14",
+            "172.64.0.0/13",
+            "131.0.72.0/22",
+            "2400:cb00::/32",
+            "2606:4700::/32",
+            "2803:f800::/32",
+            "2405:b500::/32",
+            "2405:8100::/32",
+            "2a06:98c0::/29",
+            "2c0f:f248::/32",
+        ];
+
+        public static IReadOnlyList<string> FetchOrFallback(NLog.Logger logger)
+        {
+            try
+            {
+                using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(5) };
+                var v4 = http.GetStringAsync("https://www.cloudflare.com/ips-v4/").GetAwaiter().GetResult();
+                var v6 = http.GetStringAsync("https://www.cloudflare.com/ips-v6/").GetAwaiter().GetResult();
+                var cidrs = (v4 + "\n" + v6)
+                    .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+                logger.Info("Fetched {Count} Cloudflare networks from cloudflare.com", cidrs.Length);
+                return cidrs;
+            }
+            catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
+            {
+                logger.Warn(ex, "Failed to fetch Cloudflare IP ranges; using hardcoded fallback ({Count} entries)", HardcodedFallback.Length);
+                return HardcodedFallback;
+            }
+        }
+    }
+}

web/Classes/HealthChecks/AdaptivePollingHealthCheck.cs

@@ -0,0 +1,81 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace Viper.Classes.HealthChecks
+{
+    /// <summary>
+    /// Throttles an inner health check by caching its last result for a
+    /// status-dependent duration: Healthy results are reused for longer, while
+    /// Unhealthy/Degraded results refresh on a tighter cycle so recovery is
+    /// noticed quickly. When a cached result is returned, the original probe
+    /// timestamp is appended to the description so operators can tell how
+    /// stale the reading is.
+    /// </summary>
+    public class AdaptivePollingHealthCheck : IHealthCheck
+    {
+        private readonly IHealthCheck _inner;
+        private readonly TimeSpan _healthyCacheDuration;
+        private readonly TimeSpan _unhealthyCacheDuration;
+        private readonly SemaphoreSlim _semaphore = new(1, 1);
+        private DateTime _lastCheckTime;
+        private HealthCheckResult? _lastResult;
+
+        public AdaptivePollingHealthCheck(
+            IHealthCheck inner,
+            TimeSpan healthyCacheDuration,
+            TimeSpan unhealthyCacheDuration)
+        {
+            _inner = inner;
+            _healthyCacheDuration = healthyCacheDuration;
+            _unhealthyCacheDuration = unhealthyCacheDuration;
+        }
+
+        public async Task<HealthCheckResult> CheckHealthAsync(
+            HealthCheckContext context,
+            CancellationToken cancellationToken = default)
+        {
+            await _semaphore.WaitAsync(cancellationToken);
+            try
+            {
+                if (_lastResult.HasValue)
+                {
+                    // S6561: DateTime.Now used for elapsed-time calc. Accepted
+                    // because VIPER convention is DateTimeKind.Local and a
+                    // sub-hour DST skew only shifts one cache window.
+#pragma warning disable S6561
+                    var elapsed = DateTime.Now - _lastCheckTime;
+#pragma warning restore S6561
+                    var cacheDuration = _lastResult.Value.Status == HealthStatus.Healthy
+                        ? _healthyCacheDuration
+                        : _unhealthyCacheDuration;
+
+                    if (elapsed < cacheDuration)
+                    {
+                        return AppendTimestamp(_lastResult.Value, _lastCheckTime);
+                    }
+                }
+
+                var result = await _inner.CheckHealthAsync(context, cancellationToken);
+                _lastResult = result;
+                _lastCheckTime = DateTime.Now;
+                return result;
+            }
+            finally
+            {
+                _semaphore.Release();
+            }
+        }
+
+        private static HealthCheckResult AppendTimestamp(HealthCheckResult result, DateTime lastCheckedAt)
+        {
+            var stamp = $"Last checked: {lastCheckedAt:MMM d, h:mm tt}";
+            var description = string.IsNullOrWhiteSpace(result.Description)
+                ? stamp
+                : $"{result.Description}\n{stamp}";
+            return new HealthCheckResult(
+                result.Status,
+                description,
+                result.Exception,
+                result.Data);
+        }
+    }
+}

web/Classes/HealthChecks/AwsSsmHealthCheck.cs

@@ -0,0 +1,56 @@
+using Amazon;
+using Amazon.Runtime;
+using Amazon.SimpleSystemsManagement;
+using Amazon.SimpleSystemsManagement.Model;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace Viper.Classes.HealthChecks
+{
+    /// <summary>
+    /// Verifies AWS SSM Parameter Store is reachable with the app's credentials.
+    /// Uses a lightweight DescribeParameters probe (MaxResults=1) so the check
+    /// does not actually fetch any parameter values.
+    /// </summary>
+    public class AwsSsmHealthCheck : IHealthCheck
+    {
+        private readonly RegionEndpoint _region;
+        private readonly bool _healthyWhenMissing;
+
+        /// <param name="healthyWhenMissing">
+        /// If true, missing credentials or client-side SDK errors return Healthy
+        /// with a "skipped" description. Use for Development where local machines
+        /// may not have AWS credentials configured.
+        /// </param>
+        public AwsSsmHealthCheck(RegionEndpoint? region = null, bool healthyWhenMissing = false)
+        {
+            _region = region ?? RegionEndpoint.USWest1;
+            _healthyWhenMissing = healthyWhenMissing;
+        }
+
+        public async Task<HealthCheckResult> CheckHealthAsync(
+            HealthCheckContext context,
+            CancellationToken cancellationToken = default)
+        {
+            try
+            {
+                using var client = new AmazonSimpleSystemsManagementClient(_region);
+                await client.DescribeParametersAsync(
+                    new DescribeParametersRequest { MaxResults = 1 },
+                    cancellationToken);
+                return HealthCheckResult.Healthy("AWS SSM reachable.");
+            }
+            catch (AmazonServiceException ex)
+            {
+                return _healthyWhenMissing
+                    ? HealthCheckResult.Healthy("AWS SSM not configured (skipped).")
+                    : HealthCheckResult.Unhealthy($"AWS SSM unreachable: {ex.ErrorCode}.");
+            }
+            catch (AmazonClientException)
+            {
+                return _healthyWhenMissing
+                    ? HealthCheckResult.Healthy("AWS SSM not configured (skipped).")
+                    : HealthCheckResult.Unhealthy("AWS SSM client error (credentials or network).");
+            }
+        }
+    }
+}