Feature/better metrics

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+*.gem

.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

Gemfile.lock

@@ -0,0 +1,72 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-statsd (1.1.0)
+      fluentd (>= 0.10.8)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    coderay (1.1.1)
+    cool.io (1.4.4)
+    diff-lcs (1.2.5)
+    fluentd (0.14.1)
+      cool.io (>= 1.4.3, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      json (>= 1.4.3)
+      msgpack (>= 0.7.0)
+      serverengine (>= 1.6.4)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.1.7)
+      tzinfo (>= 1.0.0)
+      tzinfo-data (>= 1.0.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    json (2.0.1)
+    method_source (0.8.2)
+    msgpack (1.0.0)
+    power_assert (0.3.0)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.1)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    serverengine (1.6.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    slop (3.6.0)
+    statsd-ruby (1.3.0)
+    strptime (0.1.8)
+    test-unit (3.1.8)
+      power_assert
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2016.6)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-statsd!
+  pry
+  rspec (~> 3.0)
+  statsd-ruby
+  test-unit
+
+BUNDLED WITH
+   1.10.6

README.md

@@ -14,73 +14,28 @@ $ fluent-gem install fluent-plugin-statsd
 <match statsd>
   type statsd
   host localhost # optional
-  port 8125# optional
+  port 8125 # optional
+  namespace a.b.c # optional
+
+  <metric>
+    statsd_type timing
+    statsd_key my_app.nginx.response_time
+    statsd_key ${record['response_time']}
+  </metric>
+
+  <metric>
+    statsd_type incrument
+    statsd_key my_app.nginx.${response_code.to_i / 100}xx # 2xx 4xx 5xx
+  </metric>
 </match>
 ```
 
-```ruby
-fluent_logger.post('statsd',
-  :statsd_type => 'timing',
-  :statsd_key => 'org.foo.timing',
-  :statsd_timing => 0.234
-)
-
-fluent_logger.post('statsd',
-  :statsd_type => 'gauge',
-  :statsd_gauge => 10,
-  :statsd_key => 'org.foo.gauge'
-)
-
-fluent_logger.post('statsd',
-  :statsd_type => 'count',
-  :statsd_gauge => 10,
-  :statsd_key => 'org.foo.gauge'
-)
-
-fluent_logger.post('statsd',
-  :statsd_type => 'set',
-  :statsd_gauge => 10,
-  :statsd_key => 'org.foo.gauge'
-)
-
-fluent_logger.post('statsd',
-  :statsd_type => 'increment',
-  :statsd_key => 'org.foo.counter'
-)
-
-
-fluent_logger.post('statsd',
-  :statsd_type => 'decrement',
-  :statsd_key => 'org.foo.counter'
-)
-```
-
-# td-agent.conf demo
-
-worked with record_reformer to transform access log request_time into statsd
+# Development
 
 ```
-<match accesslog.reformer>
-  type copy
-  <store>
-    type statsd
-    host 127.0.0.1
-    port 8125
-    flush_interval 1s 
-  </store>
-  # other stores...
-</match>
-<match accesslog>
-  type record_reformer
-  output_tag ${tag}.reformer
-  # transform /url1/url2/url3 --> url.urlNN.urlNN
-  statsd_key ${"url"+request_uri.gsub(/((\/[^\/]+){2}).*/, '\1').gsub(/([^\?]*)\?.*/,'\1').gsub(/[0-9]+/, "NN").gsub(/\//, ".");}
-  statsd_timing ${request_time}
-  statsd_type ${"timing"}
-</match>
+$ rspec
 ```
 
-
 # Copyright
 
 Copyright (c) 2014- Chris Song

VERSION

@@ -1 +1 @@
-1.0.2
+1.1.0

fluent-plugin-statsd.gemspec

@@ -17,6 +17,8 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "fluentd", ">= 0.10.8"
 
-  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "rspec", '~> 3.0'
+  gem.add_development_dependency "test-unit"
   gem.add_development_dependency "statsd-ruby", ">=1.2.1"
+  gem.add_development_dependency "pry"
 end

lib/fluent/plugin/out_statsd.rb

@@ -1,4 +1,6 @@
 require 'statsd-ruby'
+require 'ostruct'
+require 'fluent/output'
 
 module Fluent
   class StatsdOutput < BufferedOutput
@@ -7,6 +9,13 @@ class StatsdOutput < BufferedOutput
     config_param :flush_interval, :time, :default => 1
     config_param :host, :string, :default => 'localhost'
     config_param :port, :string, :default => '8125'
+    config_param :namespace, :string, :default => nil
+
+    config_section :metric do
+      config_param :statsd_type, :string
+      config_param :statsd_key, :string
+      config_param :statsd_val, :string, default: nil
+    end
 
     attr_reader :statsd
 
@@ -17,6 +26,11 @@ def initialize
     def configure(conf)
       super
       @statsd = Statsd.new(host, port)
+      @statsd.namespace = namespace if namespace
+      log.info(statsd)
+
+      @metrics = conf.elements.select {|elem| elem.name == 'metric' }
+      log.info(@metrics)
     end
 
     def start
@@ -28,29 +42,53 @@ def shutdown
     end
 
     def format(tag, time, record)
-      record.to_msgpack
+      [tag, record].to_msgpack
     end
 
     def write(chunk)
-      chunk.msgpack_each {|record|
-        if statsd_type = record['statsd_type']
-          case statsd_type
-          when 'timing'
-            @statsd.timing record['statsd_key'], record['statsd_timing'].to_f
-          when 'gauge'
-            @statsd.gauge record['statsd_key'], record['statsd_gauge'].to_f
-          when 'count'
-            @statsd.count record['statsd_key'], record['statsd_count'].to_f
-          when 'set'
-            @statsd.set record['statsd_key'], record['statsd_set']
-          when 'increment'
-            @statsd.increment record['statsd_key']
-          when 'decrement'
-            @statsd.decrement record['statsd_key']
-          end
+      chunk.msgpack_each do |tag, record|
+        parser = RubyStringParser.new(record: record, tag: tag)
+
+        @metrics.each do |metric|
+          arg_names = %w{statsd_type statsd_key statsd_val}
+          send_to_statsd(*metric.values_at(*arg_names).map {|str| parser.parse(str) })
         end
-      }
+      end
     end
 
+
+    private
+
+    def send_to_statsd(type, key, val)
+      log.debug([type, key, val])
+
+      case type
+      when 'timing'
+        @statsd.timing key, val.to_f
+      when 'gauge'
+        @statsd.gauge key, val.to_f
+      when 'count'
+        @statsd.count key, val.to_f
+      when 'set'
+        @statsd.set key, val
+      when 'increment'
+        @statsd.increment key
+      when 'decrement'
+        @statsd.decrement key
+      else
+        raise "Invalid statsd type '#{type}'"
+      end
+    end
+
+    class RubyStringParser
+      def initialize(vars = {})
+        @obj = Struct.new(*vars.keys).new(*vars.values)
+      end
+
+      def parse(string)
+        return unless string
+        string.gsub(/\$\{.+\}/) {|str| @obj.instance_eval str[2..-2] }
+      end
+    end
   end
 end

spec/examples.txt

@@ -0,0 +1,3 @@
+example_id                            | status | run_time        |
+------------------------------------- | ------ | --------------- |
+./spec/plugin/out_statsd_spec.rb[1:1] | passed | 0.53516 seconds |

spec/plugin/out_statsd_spec.rb

@@ -0,0 +1,62 @@
+require 'fluent/plugin/out_statsd'
+require 'fluent/test'
+
+RSpec.describe Fluent::StatsdOutput do
+  let(:config) do
+    %{
+      type statsd
+      namespace a.b.c
+
+      <metric>
+        statsd_type timing
+        statsd_key res_time
+        statsd_val ${record['response_time']}
+      </metric>
+
+      <metric>
+        statsd_type increment
+        statsd_key res_code_${record['status'].to_i / 100}xx
+      </metric>
+    }
+  end
+  let(:driver) { create_driver(config) }
+  let(:statsd) { double('statsd', increment: true, timing: true, 'namespace=' => true) }
+  let(:time) { Time.now.utc }
+
+  before :all do
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::StatsdOutput) {
+    }.configure(conf)
+  end
+
+  def emit_events(events)
+    events.each {|e| driver.emit(e, time) }
+  end
+
+
+  it 'should call statsd with events data' do
+    allow(Statsd).to receive(:new).and_return(statsd)
+
+    expect(statsd).to receive(:namespace=).with('a.b.c')
+
+    expect(statsd).to receive(:increment).with('res_code_2xx').twice.times
+    expect(statsd).to receive(:increment).with('res_code_4xx').once.times
+    expect(statsd).to receive(:increment).with('res_code_5xx').once.times
+    expect(statsd).to receive(:timing).with('res_time', 102).ordered
+    expect(statsd).to receive(:timing).with('res_time', 105).ordered
+    expect(statsd).to receive(:timing).with('res_time', 112).ordered
+    expect(statsd).to receive(:timing).with('res_time', 125).ordered
+
+    emit_events([
+      {'response_time' => 102, 'status' => '200'},
+      {'response_time' => 105, 'status' => '200'},
+      {'response_time' => 112, 'status' => '400'},
+      {'response_time' => 125, 'status' => '500'}
+    ])
+
+    driver.run
+  end
+end