Introduce exclude cidrs list from rate-limiting feature

acceptance-tests/rate_limit_test.go

@@ -3,6 +3,7 @@ package acceptance_tests
 import (
 	"fmt"
 	"net/http"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -11,32 +12,15 @@ import (
 var _ = Describe("Rate-Limiting", func() {
 	It("Connections/Requests aren't blocked when block config isn't set", func() {
 		rateLimit := 5
-		opsfileConnectionsRateLimit := fmt.Sprintf(`---
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit?/requests
-  value: %d
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/window_size?
-  value: 10s
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/table_size?
-  value: 1k
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections
-  value: %d
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size?
-  value: 10s
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size?
-  value: 1k
-`, rateLimit, rateLimit)
 		haproxyBackendPort := 12000
 		haproxyInfo, _ := deployHAProxy(baseManifestVars{
 			haproxyBackendPort:    haproxyBackendPort,
 			haproxyBackendServers: []string{"127.0.0.1"},
 			deploymentName:        deploymentNameForTestNode(),
-		}, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true)
+		}, []string{
+			requestsRateLimitOps(rateLimit, "10s", "1k", false),
+			connectionsRateLimitOps(rateLimit, "10s", "1k", false, nil),
+		}, map[string]interface{}{}, true)
 
 		closeLocalServer, localPort := startDefaultTestServer()
 		defer closeLocalServer()
@@ -60,27 +44,13 @@ var _ = Describe("Rate-Limiting", func() {
 
 	It("Request Based Limiting Works", func() {
 		requestLimit := 5
-		opsfileRequestRateLimit := fmt.Sprintf(`---
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit?/requests
-  value: %d
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/window_size?
-  value: 10s
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/table_size?
-  value: 100
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/block?
-  value: true
-`, requestLimit)
 
 		haproxyBackendPort := 12000
 		haproxyInfo, _ := deployHAProxy(baseManifestVars{
 			haproxyBackendPort:    haproxyBackendPort,
 			haproxyBackendServers: []string{"127.0.0.1"},
 			deploymentName:        deploymentNameForTestNode(),
-		}, []string{opsfileRequestRateLimit}, map[string]interface{}{}, true)
+		}, []string{requestsRateLimitOps(requestLimit, "10s", "100", true)}, map[string]interface{}{}, true)
 
 		closeLocalServer, localPort := startDefaultTestServer()
 		defer closeLocalServer()
@@ -112,26 +82,12 @@ var _ = Describe("Rate-Limiting", func() {
 
 	It("Connection Based Limiting Works", func() {
 		connLimit := 5
-		opsfileConnectionsRateLimit := fmt.Sprintf(`---
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections
-  value: %d
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size?
-  value: 10s
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size?
-  value: 1k
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/block?
-  value: true
-`, connLimit)
 		haproxyBackendPort := 12000
 		haproxyInfo, _ := deployHAProxy(baseManifestVars{
 			haproxyBackendPort:    haproxyBackendPort,
 			haproxyBackendServers: []string{"127.0.0.1"},
 			deploymentName:        deploymentNameForTestNode(),
-		}, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true)
+		}, []string{connectionsRateLimitOps(connLimit, "10s", "1k", true, nil)}, map[string]interface{}{}, true)
 
 		closeLocalServer, localPort := startDefaultTestServer()
 		defer closeLocalServer()
@@ -165,33 +121,53 @@ var _ = Describe("Rate-Limiting", func() {
 		Expect(successfulRequestCount).To(Equal(connLimit))
 	})
 
+	It("Excluded CIDRs are never connection rate-limited", func() {
+		connLimit := 5
+		// exclude_cidrs covers all IPv4 sources so the test runner's egress IP is
+		// guaranteed to match, proving the negated reject rule lets excluded sources through
+		haproxyBackendPort := 12000
+		haproxyInfo, _ := deployHAProxy(baseManifestVars{
+			haproxyBackendPort:    haproxyBackendPort,
+			haproxyBackendServers: []string{"127.0.0.1"},
+			deploymentName:        deploymentNameForTestNode(),
+		}, []string{connectionsRateLimitOps(connLimit, "100s", "100", true, []string{"0.0.0.0/0"})}, map[string]interface{}{}, true)
+
+		closeLocalServer, localPort := startDefaultTestServer()
+		defer closeLocalServer()
+
+		closeTunnel := setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort)
+		defer closeTunnel()
+
+		By("Sending more connections than the limit, expecting none to be blocked because the source is excluded")
+		testRequestCount := connLimit * 3
+		for i := 0; i < testRequestCount; i++ {
+			rt := &http.Transport{
+				DisableKeepAlives: true,
+			}
+			client := &http.Client{Transport: rt}
+			resp, err := client.Get(fmt.Sprintf("http://%s/foo", haproxyInfo.PublicIP))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		}
+	})
+
 	It("Connection Based Limiting Works with Proxy Protocol enabled", func() {
 		connLimit := 5
-		opsfileConnRateLimitWithProxyProtocol := fmt.Sprintf(`---
-# Enable Proxy Protocol
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/accept_proxy?
-  value: true
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections
-  value: %d
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size?
-  value: 100s
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size?
-  value: 100
+		opsfileAcceptProxy := fmt.Sprintf(`---
 - type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/block?
+  path: %s/accept_proxy?
   value: true
-`, connLimit)
+`, rateLimitPropsPath)
 
 		haproxyBackendPort := 12000
 		haproxyInfo, _ := deployHAProxy(baseManifestVars{
 			haproxyBackendPort:    haproxyBackendPort,
 			haproxyBackendServers: []string{"127.0.0.1"},
 			deploymentName:        deploymentNameForTestNode(),
-		}, []string{opsfileConnRateLimitWithProxyProtocol}, map[string]interface{}{}, true)
+		}, []string{
+			opsfileAcceptProxy,
+			connectionsRateLimitOps(connLimit, "100s", "100", true, nil),
+		}, map[string]interface{}{}, true)
 
 		closeLocalServer, localPort := startDefaultTestServer()
 		defer closeLocalServer()
@@ -224,38 +200,15 @@ var _ = Describe("Rate-Limiting", func() {
 		requestLimit := 5
 		connLimit := 6 // needs to be higher than request limit for this test
 		// connection based rate-limiting has priority over request based rate-limiting so we expect some sucesses, then one status 429 response, then no response at all
-		opsfileConnectionsRateLimit := fmt.Sprintf(`---
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit?/requests
-  value: %d
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/window_size?
-  value: 10s
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/table_size?
-  value: 100
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/requests_rate_limit/block?
-  value: true
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections
-  value: %d
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size?
-  value: 100s
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size?
-  value: 100
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/block?
-  value: true
-`, requestLimit, connLimit)
 		haproxyBackendPort := 12000
 		haproxyInfo, _ := deployHAProxy(baseManifestVars{
 			haproxyBackendPort:    haproxyBackendPort,
 			haproxyBackendServers: []string{"127.0.0.1"},
 			deploymentName:        deploymentNameForTestNode(),
-		}, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true)
+		}, []string{
+			requestsRateLimitOps(requestLimit, "10s", "100", true),
+			connectionsRateLimitOps(connLimit, "100s", "100", true, nil),
+		}, map[string]interface{}{}, true)
 
 		closeLocalServer, localPort := startDefaultTestServer()
 		defer closeLocalServer()
@@ -288,26 +241,12 @@ var _ = Describe("Rate-Limiting", func() {
 
 	It("Connection Based Limiting works via manifest and can be overridden at runtime via socket", func() {
 		connLimit := 5
-		opsfileConnectionsRateLimit := fmt.Sprintf(`---
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit?/connections
-  value: %d
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/window_size?
-  value: 10s
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/table_size?
-  value: 100
-- type: replace
-  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/connections_rate_limit/block?
-  value: true
-`, connLimit)
 		haproxyBackendPort := 12000
 		haproxyInfo, _ := deployHAProxy(baseManifestVars{
 			haproxyBackendPort:    haproxyBackendPort,
 			haproxyBackendServers: []string{"127.0.0.1"},
 			deploymentName:        deploymentNameForTestNode(),
-		}, []string{opsfileConnectionsRateLimit}, map[string]interface{}{}, true)
+		}, []string{connectionsRateLimitOps(connLimit, "10s", "100", true, nil)}, map[string]interface{}{}, true)
 
 		closeLocalServer, localPort := startDefaultTestServer()
 		defer closeLocalServer()
@@ -381,3 +320,48 @@ var _ = Describe("Rate-Limiting", func() {
 		Expect(successfulRequestCount).To(Equal(newLimit))
 	})
 })
+
+const rateLimitPropsPath = "/instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy"
+
+// requestsRateLimitOps builds an opsfile fragment configuring the requests_rate_limit properties.
+func requestsRateLimitOps(requests int, windowSize, tableSize string, block bool) string {
+	return fmt.Sprintf(`---
+- type: replace
+  path: %[1]s/requests_rate_limit?/requests
+  value: %[2]d
+- type: replace
+  path: %[1]s/requests_rate_limit/window_size?
+  value: %[3]s
+- type: replace
+  path: %[1]s/requests_rate_limit/table_size?
+  value: %[4]s
+- type: replace
+  path: %[1]s/requests_rate_limit/block?
+  value: %[5]t
+`, rateLimitPropsPath, requests, windowSize, tableSize, block)
+}
+
+// connectionsRateLimitOps builds an opsfile fragment configuring the connections_rate_limit properties.
+func connectionsRateLimitOps(connections int, windowSize, tableSize string, block bool, excludeCIDRs []string) string {
+	quotedCIDRs := make([]string, len(excludeCIDRs))
+	for i, cidr := range excludeCIDRs {
+		quotedCIDRs[i] = fmt.Sprintf("%q", cidr)
+	}
+	return fmt.Sprintf(`---
+- type: replace
+  path: %[1]s/connections_rate_limit?/connections
+  value: %[2]d
+- type: replace
+  path: %[1]s/connections_rate_limit/window_size?
+  value: %[3]s
+- type: replace
+  path: %[1]s/connections_rate_limit/table_size?
+  value: %[4]s
+- type: replace
+  path: %[1]s/connections_rate_limit/block?
+  value: %[5]t
+- type: replace
+  path: %[1]s/connections_rate_limit/exclude_cidrs?
+  value: [%[6]s]
+`, rateLimitPropsPath, connections, windowSize, tableSize, block, strings.Join(quotedCIDRs, ", "))
+}

docs/rate_limiting.md

@@ -30,6 +30,40 @@ This will not result in a log statement on the HAProxy side, which can make trac
 > If both rate-limits are reached simultaneously (e.g. if they are configured identically and every incoming HTTP request uses a new TCP connection), connection based rate-limiting will come into effect first, resulting in a dropped TCP connection.
 
 
+## Excluding CIDRs from Connection Rate Limiting
+Some sources should never be rate-limited on connections — for example internal/NAT ranges, health checkers, or trusted peers whose traffic all appears to originate from a small set of IPs. `connections_rate_limit.exclude_cidrs` takes a list of CIDRs that are exempt from connection based rate limiting.
+
+Excluded sources are **still tracked** in the `st_tcp_conn_rate` stick-table (so they remain visible via `show table st_tcp_conn_rate`), but they are **never rejected**, regardless of the configured threshold or the runtime `block` setting.
+
+The list is rendered to `/var/vcap/jobs/haproxy/config/rate_limit_exclusion_cidrs.txt` and referenced by an ACL that negates the reject rule on both the `http-in` and `https-in` frontends.
+
+#### Configuration
+```yml
+config:
+    # [...]
+    connections_rate_limit:
+        connections: 10
+        window_size: 10s
+        table_size: 1m
+        block: true
+        exclude_cidrs:
+        - 10.0.0.0/8
+        - 192.168.0.0/16
+        - 2001:db8::/32
+```
+
+The value may also be provided as a single base64-encoded, gzipped string (useful for very long lists), matching the format accepted by `cidr_whitelist` and `cidr_blocklist_tcp`.
+
+#### Resulting `haproxy.config`
+```ini
+frontend http-in
+    # [...]
+    acl rate_limit_exclude src -f /var/vcap/jobs/haproxy/config/rate_limit_exclusion_cidrs.txt
+    tcp-request connection track-sc0 src table st_tcp_conn_rate
+    tcp-request connection reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 } !rate_limit_exclude
+```
+
+
 ## Configuration Examples
 > Note:
 > The following examples assume only an `http-in` frontend is configured; an `https-in` frontend would behave identically.

jobs/haproxy/spec

@@ -26,6 +26,7 @@ templates:
   whitelist_cidrs.txt.erb:      config/whitelist_cidrs.txt
   expect_proxy_cidrs.txt.erb:   config/expect_proxy_cidrs.txt
   trusted_domain_cidrs.txt.erb: config/trusted_domain_cidrs.txt
+  rate_limit_exclusion_cidrs.txt.erb: config/rate_limit_exclusion_cidrs.txt
 
 consumes:
   - name: http_backend
@@ -827,3 +828,12 @@ properties:
   ha_proxy.connections_rate_limit.block:
     description: Whether or not to block connections. See docs/rate_limiting.md
     default: false
+  ha_proxy.connections_rate_limit.exclude_cidrs:
+    description: "List of CIDRs (e.g. private/NAT ranges) to exclude from connection based rate-limiting. Excluded sources are still tracked in the stick-table but are never rejected. Format is string array of CIDRs or single string of base64 encoded gzip. See docs/rate_limiting.md"
+    default: ~
+    example:
+      connections_rate_limit:
+        exclude_cidrs:
+        - 10.0.0.0/8
+        - 192.168.0.0/16
+        - 2001:db8::/32

jobs/haproxy/templates/haproxy.config.erb

@@ -451,9 +451,10 @@ frontend http-in
     acl layer4_block src -f /var/vcap/jobs/haproxy/config/blocklist_cidrs_tcp.txt
     tcp-request <%= tcp_request_phase %> reject if layer4_block
   <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%>
+    acl rate_limit_exclude src -f /var/vcap/jobs/haproxy/config/rate_limit_exclusion_cidrs.txt
     tcp-request <%= tcp_request_phase %> track-sc0 src table st_tcp_conn_rate
     # use sub() converter as variable references are only accepted as arguments to converters
-    tcp-request <%= tcp_request_phase %> reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }
+    tcp-request <%= tcp_request_phase %> reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 } !rate_limit_exclude
   <%- end -%>
   <%- if_p("ha_proxy.requests_rate_limit.table_size", "ha_proxy.requests_rate_limit.window_size") do -%>
     http-request track-sc1 src table st_http_req_rate
@@ -582,9 +583,10 @@ frontend https-in
     acl layer4_block src -f /var/vcap/jobs/haproxy/config/blocklist_cidrs_tcp.txt
     tcp-request <%= tcp_request_phase %> reject if layer4_block
   <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%>
+    acl rate_limit_exclude src -f /var/vcap/jobs/haproxy/config/rate_limit_exclusion_cidrs.txt
     tcp-request <%= tcp_request_phase %> track-sc0 src table st_tcp_conn_rate
     # use sub() converter as variable references are only accepted as arguments to converters
-    tcp-request <%= tcp_request_phase %> reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 }
+    tcp-request <%= tcp_request_phase %> reject if { var(proc.connections_rate_limit_block) -m bool } { var(proc.connections_rate_limit_connections) -m int gt 0 } { sc_conn_rate(0),sub(proc.connections_rate_limit_connections) gt 0 } !rate_limit_exclude
   <%- end -%>
   <%- if_p("ha_proxy.requests_rate_limit.table_size", "ha_proxy.requests_rate_limit.window_size") do -%>
     http-request track-sc1 src table st_http_req_rate