Kouhei Sutou
null+****@clear*****
Mon Oct 20 14:26:50 JST 2014
Kouhei Sutou 2014-10-20 14:26:50 +0900 (Mon, 20 Oct 2014) New Revision: 17e2e90cc99bea3ac5f02fc483ae7baf429cc01b https://github.com/groonga/fluent-plugin-groonga/commit/17e2e90cc99bea3ac5f02fc483ae7baf429cc01b Message: Reduce the number of load calls to pack multiple values into one load Modified files: lib/fluent/plugin/out_groonga.rb test/test_output.rb Modified: lib/fluent/plugin/out_groonga.rb (+20 -13) =================================================================== --- lib/fluent/plugin/out_groonga.rb 2014-10-15 18:29:36 +0900 (cebdf0a) +++ lib/fluent/plugin/out_groonga.rb 2014-10-20 14:26:50 +0900 (c57dc94) @@ -17,6 +17,8 @@ require "fileutils" +require "yajl" + require "groonga/client" module Fluent @@ -60,10 +62,7 @@ module Fluent end def write(chunk) - chunk.msgpack_each do |message| - tag, _, record = message - @emitter.emit(tag, record) - end + @emitter.emit(chunk) end private @@ -82,13 +81,22 @@ module Fluent @table = table end - def emit(tag, record) - if /\Agroonga\.command\./ =~ tag - name = $POSTMATCH - send_command(name, record) - else - store_chunk(data) + def emit(chunk) + values = [] + chunk.msgpack_each do |message| + tag, _, record = message + if /\Agroonga\.command\./ =~ tag + name = $POSTMATCH + unless values.empty? + store_values(values) + values.clear + end + send_command(name, record) + else + values << record + end end + store_values(values) unless values.empty? end private @@ -98,13 +106,12 @@ module Fluent @client.send(command) end - def store_chunk(value) + def store_values(values) return if****@table*****? - values = [value] arguments = { "table" => @table, - "values" => Yajl::Enocder.encode(values), + "values" => Yajl::Encoder.encode(values), } send_command("load", arguments) end Modified: test/test_output.rb (+39 -0) =================================================================== --- test/test_output.rb 2014-10-15 18:29:36 +0900 (202d37b) +++ test/test_output.rb 2014-10-20 14:26:50 +0900 (9c477cf) @@ -104,6 +104,45 @@ EOC @request_parser.request_url) end end + + class StoreTest < self + def configuration + <<-CONFIGURATION + #{super} + table Logs + CONFIGURATION + end + + def test_one_message + @response_body = JSON.generate([[0, 0.0, 0.0], [1]]) + driver = create_driver("log") + time = Time.parse("2012-10-26T08:45:42Z").to_i + driver.run do + driver.emit({"message" => "1st message"}, time) + end + assert_equal("/d/load?table=Logs", + @request_parser.request_url) + assert_equal([{"message" => "1st message"}], + JSON.parse(@request_body)) + end + + def test_multiple_messages + @response_body = JSON.generate([[0, 0.0, 0.0], [2]]) + driver = create_driver("log") + time = Time.parse("2012-10-26T08:45:42Z").to_i + driver.run do + driver.emit({"message" => "1st message"}, time) + driver.emit({"message" => "2nd message"}, time + 1) + end + assert_equal("/d/load?table=Logs", + @request_parser.request_url) + assert_equal([ + {"message" => "1st message"}, + {"message" => "2nd message"}, + ], + JSON.parse(@request_body)) + end + end end class CommandLineTest < self -------------- next part -------------- HTML����������������������������...下载
