Kouhei Sutou
null+****@clear*****
Mon Dec 16 18:44:56 JST 2013
Kouhei Sutou 2013-12-16 18:44:56 +0900 (Mon, 16 Dec 2013) New Revision: cc1dd4aa70c4acb49cc9094a0f9248fb85ab28eb https://github.com/groonga/groonga-query-log/commit/cc1dd4aa70c4acb49cc9094a0f9248fb85ab28eb Message: Add groonga-query-log-verify-server command It verifies that two Groonga servers returns the same response for the same request. Request list is created from query log. Added files: lib/groonga/query-log/command/verify-server.rb lib/groonga/query-log/server-verifier.rb Copied files: bin/groonga-query-log-verify-server (from lib/groonga/query-log.rb) Modified files: lib/groonga/query-log.rb Copied: bin/groonga-query-log-verify-server (+6 -6) 76% Mode: 100644 -> 100755 =================================================================== --- lib/groonga/query-log.rb 2013-12-16 16:32:05 +0900 (3524c0a) +++ bin/groonga-query-log-verify-server 2013-12-16 18:44:56 +0900 (89150ae) @@ -1,6 +1,7 @@ +#!/usr/bin/env ruby # -*- coding: utf-8 -*- # -# Copyright (C) 2012 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2013 Kouhei Sutou <kou �� clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,8 +17,7 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -require "groonga/query-log/version" -require "groonga/query-log/analyzer" -require "groonga/query-log/extractor" -require "groonga/query-log/parser" -require "groonga/query-log/replayer" +require "groonga/query-log/command/verify-server" + +verify_server_command = Groonga::QueryLog::Command::VerifyServer.new +verify_server_command.run(*ARGV) Modified: lib/groonga/query-log.rb (+1 -0) =================================================================== --- lib/groonga/query-log.rb 2013-12-16 16:32:05 +0900 (3524c0a) +++ lib/groonga/query-log.rb 2013-12-16 18:44:56 +0900 (8af3ad4) @@ -21,3 +21,4 @@ require "groonga/query-log/analyzer" require "groonga/query-log/extractor" require "groonga/query-log/parser" require "groonga/query-log/replayer" +require "groonga/query-log/server-verifier" Added: lib/groonga/query-log/command/verify-server.rb (+131 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/command/verify-server.rb 2013-12-16 18:44:56 +0900 (deaa929) @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2013 Kouhei Sutou <kou �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +require "optparse" + +require "groonga/query-log" + +module Groonga + module QueryLog + module Command + class VerifyServer + def initialize + @options = ServerVerifier::Options.new + end + + def run(*command_line) + input_paths = create_parser.parse(*command_line) + verifier = ServerVerifier.new(@options) + input_paths.each do |input_path| + File.open(input_path) do |input| + verifier.verify(input) + end + end + end + + private + def create_parser + parser = OptionParser.new + parser.version = VERSION + parser.banner += " QUERY_LOG1 QUERY_LOG2 ..." + + parser.separator("") + parser.separator("Options:") + + available_protocols = [:gqtp, :http] + available_protocols_label = "[#{available_protocols.join(', ')}]" + + parser.on("--groonga1-host=HOST", + "Host name or IP address of Groonga server 1", + "[#{@options.groonga1.host}]") do |host| + @options.groonga1.host = host + end + + parser.on("--groonga1-port=PORT", Integer, + "Port number of Groonga server 1", + "[#{@options.groonga1.port}]") do |port| + @options.groonga1.port = port + end + + parser.on("--groonga1-protocol=PROTOCOL", available_protocols, + "Protocol of Groonga server 1", + available_protocols_label) do |protocol| + @options.groonga1.protocol = protocol + end + + parser.on("--groonga2-host=HOST", + "Host name or IP address of Groonga server 2", + "[#{@options.groonga2.host}]") do |host| + @options.groonga2.host = host + end + + parser.on("--groonga2-port=PORT", Integer, + "Port number of Groonga server 2", + "[#{@options.groonga2.port}]") do |port| + @options.groonga2.port = port + end + + parser.on("--groonga2-protocol=PROTOCOL", available_protocols, + "Protocol of Groonga server 2", + available_protocols_label) do |protocol| + @options.groonga2.protocol = protocol + end + + parser.on("--n-clients=N", Integer, + "The max number of concurrency", + "[#{@options.n_clients}]") do |n_clients| + @options.n_clients = n_clients + end + + parser.on("--request-queue-size=SIZE", Integer, + "The size of request queue", + "[auto]") do |size| + @options.request_queue_size = size + end + + parser.on("--disable-cache", + "Add 'cache=no' parameter to request", + "[#{@options.disable_cache?}]") do + @options.disable_cache = true + end + + parser.on("--target-command-name=NAME", + "Add NAME to target command names", + "You can specify this option zero or more times", + "See also --target-command-names") do |name| + @options.target_command_names << name + end + + target_command_names_label =****@optio*****_command_names.join(", ") + parser.on("--target-command-names=NAME1,NAME2,...", Array, + "Replay only NAME1,NAME2,... commands", + "You can use glob to choose command name", + "[#{target_command_names_label}]") do |names| + @options.target_command_names = names + end + + parser.on("--output=PATH", + "Output results to PATH", + "[stdout]") do |path| + @options.output_path = path + end + end + end + end + end +end Added: lib/groonga/query-log/server-verifier.rb (+188 -0) 100644 =================================================================== --- /dev/null +++ lib/groonga/query-log/server-verifier.rb 2013-12-16 18:44:56 +0900 (a70e3c0) @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2013 Kouhei Sutou <kou �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +require "time" +require "thread" + +require "groonga/client" + +require "groonga/query-log/parser" + +module Groonga + module QueryLog + class ServerVerifier + def initialize(options) + @options = options + @queue = SizedQueue.new(@options.request_queue_size) + @different_results = Queue.new + end + + def verify(input) + producer = run_producer(input) + consumers = run_consumers + reporter = run_reporter + producer.join + consumers.each(&:join) + @different_results.push(nil) + reporter.join + end + + private + def run_producer(input) + Thread.new do + parser = Parser.new + parser.parse(input) do |statistic| + next unless target_command?(statistic.command) + @queue.push(statistic) + end + @options.n_clients.times do + @queue.push(nil) + end + end + end + + def run_consumers + @options.n_clients.times.collect do + Thread.new do + loop do + break if run_consumer + end + end + end + end + + def run_consumer + @options.groonga1.create_client do |groonga1_client| + @options.groonga2.create_client do |groonga2_client| + loop do + statistic =****@queue***** + return true if statistic.nil? + begin + verify_command(groonga1_client, groonga2_client, + statistic.command) + rescue Groonga::Client::Connection::Error + # TODO: add error log mechanism + $stderr.puts(Time.now.iso8601) + $stderr.puts(statistic.command.original_source) + $stderr.puts($!.raw_error.message) + $stderr.puts($!.raw_error.backtrace) + return false + end + end + end + end + end + + def run_reporter + Thread.new do + @options.create_output do |output| + loop do + result = @different_results.pop + break if result.nil? + report_result(output, result) + end + end + end + end + + def target_command?(command) + @options.target_command_name?(command.name) + end + + def verify_command(groonga1_client, groonga2_client, command) + command["cache"] = "no" if****@optio*****_cache? + response1 = groonga1_client.execute(command) + response2 = groonga2_client.execute(command) + unless same_response?(response1, response2) + @different_results.push([command, response1, response2]) + end + end + + def same_response?(response1, response2) + response1.body == response2.body + end + + def report_result(output, result) + command, response1, response2 = result + output.puts("command: #{command.original_format}") + output.puts("response1: #{response1.body}") + output.puts("response2: #{response2.body}") + end + + class Options + attr_reader :groonga1 + attr_reader :groonga2 + attr_accessor :n_clients + attr_writer :request_queue_size + attr_accessor :target_command_names + attr_accessor :output_path + def initialize + @groonga1 = GroongaOptions.new + @groonga2 = GroongaOptions.new + @n_clients = 8 + @request_queue_size = nil + @disable_cache = false + @output_path = nil + @target_command_names = ["select"] + end + + def request_queue_size + @request_queue_size || @n_clients * 3 + end + + def disable_cache? + @disable_cache + end + + def target_command_name?(name) + @target_command_names.any? do |name_pattern| + flags = 0 + flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB) + File.fnmatch(name_pattern, name, flags) + end + end + + def create_output(&block) + if @output_path + File.open(@output_path, "w", &block) + else + yield($stdout) + end + end + end + + class GroongaOptions + attr_accessor :host + attr_accessor :port + attr_accessor :protocol + def initialize + @host = "127.0.0.1" + @port = 10041 + @protocol = :gqtp + end + + def create_client(&block) + Groonga::Client.open(:host => @host, + :port => @port, + :protocol => @protocol, + &block) + end + end + end + end +end -------------- next part -------------- HTML����������������������������...下载
