Support encoding to custom json object

lib/logstash/codecs/json.rb

@@ -29,6 +29,18 @@ class LogStash::Codecs::JSON < LogStash::Codecs::Base
   # For nxlog users, you may to set this to "CP1252".
   config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
 
+  # When specified, build a json object based on the mapping structure,
+  # taking necessary information from event.
+  # This setting allows you to build specific JSON object inside an output configuration that support codecs.
+  #
+  # * JSON key can either be literal string  `"abc"` or interpolated string `"%{[event_accessor][subfield]}"`
+  # * JSON value can either be literal scalar,array,object or interpolated string `"%{[field]}"`, or part of the event `"[field][nested_field]"` (array,hash or scalar)
+  #
+  # The difference between using interpolated string `"%{[field]}"` or only the accessor `"[field]"` is the resulting element type.
+  # Using Interpolated string will always return a string value, while accessor will directly inject the event field as-is if it is an integer,boolean
+  #
+  config :encode_mapping, :validate => :hash
+
   def register
     @converter = LogStash::Util::Charset.new(@charset)
     @converter.logger = @logger
@@ -38,8 +50,40 @@ def decode(data, &block)
     parse(@converter.convert(data), &block)
   end
 
+  def build_mapping(event, mapping)
+    map = Hash.new
+    mapping.each do |key, value|
+      k = event.sprintf(key)
+      v = build_mapping_value(event,value)
+      map[k] = v
+    end
+    return map
+  end
+
+  def build_mapping_value(event,value)
+    if value.is_a?(Hash)
+      v = build_mapping(event, value)
+    elsif value.is_a?(Array)
+      v = value.map do |val|
+        build_mapping_value(event, val)
+      end
+    else
+      if /^(?:\[[^\[\]]+\])+$/.match(value)
+        v = event.get(value)
+      else
+        v = event.sprintf(value)
+      end
+    end
+    return v
+  end
+
   def encode(event)
-    @on_event.call(event, event.to_json)
+    if @encode_mapping
+      encode_result = LogStash::Json.dump(build_mapping(event,@encode_mapping))
+    else
+      encode_result = event.to_json
+    end
+    @on_event.call(event, encode_result)
   end
 
   private

spec/codecs/json_spec.rb

@@ -149,6 +149,105 @@
       end
     end
 
+    context "#build_mapping" do
+
+      context "event sprintf on key" do
+        it "should interpolate mapping key with sprintf syntax" do
+          mapping = { "%{my_key}" => "%{int}", "prefixed_%{my_key}" => "%{int}"}
+          data = { "my_key" => "abc", "int" => 123 }
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "abc" => "123", "prefixed_abc" => "123" }
+        end
+      end
+
+      context "event sprintf on value" do
+
+        it "should interpolate event fields that are scalar" do
+          mapping = { "int" => "%{int}", "double" => "%{double}" ,"bar" => "%{string}", "baz" => "%{bool}"}
+          data = { "int" => 123, "double" => 123.4 ,"string" => "string content", "bool" => false}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "int" => "123", "double" => "123.4","bar" => "string content", "baz" => "false" }
+        end
+
+        it "should interpolate event fields that are array" do
+          mapping = { "foo" => "%{[baz][bah]}"}
+          data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "foo" => "a,b,c" }
+        end
+
+        #Documenting the current behaviour, not expected to be used
+        it "should interpolate event fields that are hash" do
+          mapping = { "foo" => "%{[baz]}"}
+          data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "foo" => "{\"bah\":[\"a\",\"b\",\"c\"]}" }
+        end
+      end
+
+      context "event accessor on value" do
+
+        it "should include event fields that are scalar" do
+          mapping = { "int" => "[int]", "double" => "[double]" ,"bar" => "[string]", "baz" => "[bool]"}
+          data = { "int" => 123, "double" => 123.4 ,"string" => "string content", "bool" => false}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "int" => 123, "double" => 123.4,"bar" => "string content", "baz" => false }
+        end
+
+        #keep object type (hash/number/boolean)
+        it "should include event field from accessor string" do
+          mapping = { "foo" => "[baz][bah]"}
+          data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "foo" => ["a","b","c"] }
+        end
+      end
+      context "with mapping nested_hash" do
+        it "should support hash nested in hash" do
+          mapping = { "int" => "[int]", "nested_hash" => { "nested_hash-2" => {"foo" => "[bool]"}, "bar" => "%{bool}"} }
+          data = { "int" => 123, "double" => 123.4 ,"string" => "string content", "bool" => false}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "int" => 123, "nested_hash" => { "nested_hash-2" => {"foo" => false}, "bar" => "false"} }
+        end
+      end
+
+      context "with mapping array" do
+        it "should create array from event field accessor string" do
+          mapping = { "array" => [ "[int]", "[double]", "[string]", "[bool]" ] }
+          data = { "int" => 123, "double" => 123.4 ,"string" => "string content", "bool" => false}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "array" => [123, 123.4, "string content", false] }
+        end
+      end
+
+      context "with complex mapping array of hash && hash of array" do
+        it "should support array nested in array" do
+          mapping = { "array" => [ "[int]", "[double]", "[string]", "nested_array" => [ "[int]", "[double]" ] ] }
+          data = { "int" => 123, "double" => 123.4 ,"string" => "string content", "bool" => false}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "array" => [123, 123.4, "string content", "nested_array" => [123,123.4]] }
+        end
+
+        it "should support hash nested in array" do
+          mapping = { "array" => [ "[int]", "[double]", "[string]", "nested_hash" => { "foo" => "[bool]", "bar" => "%{bool}"} ] }
+          data = { "int" => 123, "double" => 123.4 ,"string" => "string content", "bool" => false}
+          event = LogStash::Event.new(data)
+          result = subject.build_mapping(event,mapping)
+          insist { result } == { "array" => [123, 123.4, "string content", "nested_hash" => { "foo" => false, "bar" => "false"}] }
+        end
+
+      end
+    end
+
     context "#encode" do
       it "should return json data" do
         data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
@@ -167,6 +266,23 @@
     end
   end
 
+  context "#encode with mapping" do
+    subject do
+      LogStash::Codecs::JSON.new("encode_mapping" => {"prefixed_%{foo}" => "[baz][bah]"})
+    end
+    it "should return mapped json data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      event = LogStash::Event.new(data)
+      got_event = false
+      subject.on_event do |e, d|
+        insist { LogStash::Json.load(d) } == { "prefixed_bar" => ["a","b","c"]}
+        got_event = true
+      end
+      subject.encode(event)
+      insist { got_event }
+    end
+  end
+
   context "forcing legacy parsing" do
     it_behaves_like :codec do
       before(:each) do
@@ -187,5 +303,4 @@
       # do nothing
     end
   end
-
 end