Attach lock context to overloaded exceptions for better metrics

lib/berater.rb

@@ -12,7 +12,14 @@ module Berater
   include Meddleware
   extend self
 
-  class Overloaded < StandardError; end
+  class Overloaded < StandardError
+    attr_reader :lock
+
+    def initialize(lock = nil)
+      @lock = lock
+      super()
+    end
+  end
 
   attr_accessor :redis
 

lib/berater/concurrency_limiter.rb

@@ -77,7 +77,7 @@ def timeout
         [ capacity, ts, @timeout, cost ]
       )
 
-      raise Overloaded if lock_ids.empty?
+      raise Overloaded.new(Lock.new(capacity, count)) if lock_ids.empty?
 
       release_fn = if cost > 0
         proc { redis.zrem(cache_key, lock_ids) }

lib/berater/inhibitor.rb

@@ -10,7 +10,7 @@ def to_s
     end
 
     protected def acquire_lock(*)
-      raise Overloaded
+      raise Overloaded.new(Lock.new(capacity, capacity))
     end
 
   end

lib/berater/middleware/statsd.rb

@@ -13,6 +13,10 @@ def call(limiter, **opts)
         # capture exception for reporting, then propagate
         raise
       ensure
+        if !lock && error.is_a?(Berater::Overloaded)
+          lock = error.lock
+        end
+
         duration += Process.clock_gettime(Process::CLOCK_MONOTONIC)
         duration = (duration * 1_000).round(2) # milliseconds
 

lib/berater/middleware/trace.rb

@@ -15,6 +15,10 @@ def call(limiter, **)
             # capture exception for reporting, then propagate
             raise
           ensure
+            if !lock && error.is_a?(Berater::Overloaded)
+              lock = error.lock
+            end
+
             span.set_tag('capacity', limiter.capacity)
             span.set_tag('contention', lock.contention) if lock
             span.set_tag('key', limiter.key)

lib/berater/rate_limiter.rb

@@ -82,7 +82,7 @@ def interval
 
       count = count.include?('.') ? count.to_f : count.to_i
 
-      raise Overloaded unless allowed
+      raise Overloaded.new(Lock.new(capacity, count)) unless allowed
 
       Lock.new(capacity, count)
     end
@@ -103,4 +103,3 @@ def to_s
 
   end
 end
-

lib/berater/static_limiter.rb

@@ -32,7 +32,7 @@ class StaticLimiter < Limiter
       # Redis returns Floats as strings to maintain precision
       count = count.include?('.') ? count.to_f : count.to_i
 
-      raise Overloaded unless allowed
+      raise Overloaded.new(Lock.new(capacity, count)) unless allowed
 
       release_fn = if cost > 0
         proc { redis.incrbyfloat(cache_key, -cost) }

spec/middleware/statsd_spec.rb

@@ -243,10 +243,16 @@ def expect_tags_to(matcher)
         )
       end
 
-      it 'does not track lock-based stats' do
-        expect(client).not_to receive(:gauge).with(
-          /berater.lock/,
-          any_args,
+      it 'tracks lock-based stats from the overloaded error lock' do
+        expect(client).to receive(:gauge).with(
+          'berater.lock.capacity',
+          limiter.capacity,
+          Hash,
+        )
+        expect(client).to receive(:gauge).with(
+          'berater.lock.contention',
+          limiter.capacity,
+          Hash,
         )
       end
 

spec/middleware/trace_spec.rb

@@ -96,6 +96,7 @@
 
       it 'tags the span as overloaded and raises' do
         expect(span).to receive(:set_tag).with('overloaded', true)
+        expect(span).to receive(:set_tag).with('contention', 0)
 
         expect {
           limiter.limit