Kouhei Sutou
null+****@clear*****
Tue Apr 10 11:33:04 JST 2018
Kouhei Sutou 2018-04-10 11:33:04 +0900 (Tue, 10 Apr 2018) New Revision: b6605ed70706c42a50090f117d55f1cf9673be6c https://github.com/groonga/groonga/commit/b6605ed70706c42a50090f117d55f1cf9673be6c Message: logical_range_filter: add sort_keys option TODO: Document Added files: test/command/suite/sharding/logical_range_filter/sort_keys/ascending.expected test/command/suite/sharding/logical_range_filter/sort_keys/ascending.test test/command/suite/sharding/logical_range_filter/sort_keys/descending.expected test/command/suite/sharding/logical_range_filter/sort_keys/descending.test test/command/suite/sharding/logical_range_filter/sort_keys/multiple.expected test/command/suite/sharding/logical_range_filter/sort_keys/multiple.test Modified files: plugins/sharding/logical_range_filter.rb Modified: plugins/sharding/logical_range_filter.rb (+33 -9) =================================================================== --- plugins/sharding/logical_range_filter.rb 2018-04-10 11:19:28 +0900 (b070bf592) +++ plugins/sharding/logical_range_filter.rb 2018-04-10 11:33:04 +0900 (2bbec6f0a) @@ -16,6 +16,7 @@ module Groonga "output_columns", "use_range_index", "post_filter", + "sort_keys", ]) def run_body(input) @@ -73,18 +74,23 @@ module Groonga key << "#{input[:output_columns]}\0" key << "#{input[:use_range_index]}\0" key << "#{input[:post_filter]}\0" + key << "#{input[:sort_keys]}\0" dynamic_columns = DynamicColumns.parse(input) key << dynamic_columns.cache_key key end class ExecuteContext + include KeysParsable + attr_reader :use_range_index attr_reader :enumerator attr_reader :order attr_reader :filter attr_reader :offset attr_reader :limit + attr_reader :post_filter + attr_reader :sort_keys attr_reader :dynamic_columns attr_accessor :current_offset attr_accessor :current_limit @@ -92,7 +98,6 @@ module Groonga attr_reader :unsorted_result_sets attr_reader :temporary_tables attr_reader :threshold - attr_reader :post_filter def initialize(input) @input = input @use_range_index = parse_use_range_index(@input[:use_range_index]) @@ -101,6 +106,9 @@ module Groonga @filter = @input[:filter] @offset = (@input[:offset] || 0).to_i @limit = (@input[:limit] || 10).to_i + @post_filter = @input[:post_filter] + @sort_keys = parse_keys(@input[:sort_keys]) + @dynamic_columns = DynamicColumns.parse(@input) @current_offset = @offset @@ -112,8 +120,6 @@ module Groonga @temporary_tables = [] @threshold = compute_threshold - - @post_filter = @input[:post_filter] end def close @@ -560,6 +566,11 @@ module Groonga return decide_use_range_index(false, reason, __LINE__, __method__) end + unles****@conte*****_keys.empty? + reason = "sort_keys is used" + return decide_use_range_index(false, reason, __LINE__, __method__) + end + current_limit =****@conte*****_limit if current_limit < 0 reason = "limit is negative: <#{current_limit}>" @@ -934,12 +945,25 @@ module Groonga end @unsorted_result_sets << result_set if result_set.temporary? - sort_keys = [ - { - :key => @context.enumerator.shard_key_name, - :order => @context.order, - }, - ] + if****@conte*****_keys.empty? + sort_keys = [ + { + :key => @context.enumerator.shard_key_name, + :order => @context.order, + }, + ] + else + sort_keys =****@conte*****_keys.collect do |sort_key| + if sort_key.start_with?("-") + key = sort_key[1..-1] + order = :descending + else + key = sort_key + order = :ascending + end + {:key => key, :order => order} + end + end if****@conte*****_limit > 0 limit =****@conte*****_limit else Added: test/command/suite/sharding/logical_range_filter/sort_keys/ascending.expected (+112 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/sort_keys/ascending.expected 2018-04-10 11:33:04 +0900 (59844716e) @@ -0,0 +1,112 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 300}, +{"timestamp": "2017/03/15 01:00:00", "price": 1000}, +{"timestamp": "2017/03/15 02:00:00", "price": 900} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 110}, +{"timestamp": "2017/03/16 12:00:00", "price": 520} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 400}, +{"timestamp": "2017/03/17 21:00:00", "price": 800}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --sort_keys 'price' --output_columns _id,price +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "_id", + "UInt32" + ], + [ + "price", + "UInt32" + ] + ], + [ + 1, + 300 + ], + [ + 3, + 900 + ], + [ + 2, + 1000 + ], + [ + 2, + 110 + ], + [ + 3, + 520 + ], + [ + 1, + 530 + ], + [ + 3, + 300 + ], + [ + 1, + 400 + ], + [ + 2, + 800 + ] + ] +] +#|d| [logical_range_filter][select] <Logs_20170315>: sort_keys is used +#|d| [logical_range_filter][select] <Logs_20170316>: sort_keys is used +#|d| [logical_range_filter][select] <Logs_20170317>: sort_keys is used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/sort_keys/ascending.test (+50 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/sort_keys/ascending.test 2018-04-10 11:33:04 +0900 (3791acba8) @@ -0,0 +1,50 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 300}, +{"timestamp": "2017/03/15 01:00:00", "price": 1000}, +{"timestamp": "2017/03/15 02:00:00", "price": 900} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 110}, +{"timestamp": "2017/03/16 12:00:00", "price": 520} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 400}, +{"timestamp": "2017/03/17 21:00:00", "price": 800}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --sort_keys 'price' \ + --output_columns _id,price +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/sort_keys/descending.expected (+112 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/sort_keys/descending.expected 2018-04-10 11:33:04 +0900 (71cc4570a) @@ -0,0 +1,112 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 300}, +{"timestamp": "2017/03/15 01:00:00", "price": 1000}, +{"timestamp": "2017/03/15 02:00:00", "price": 900} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 110}, +{"timestamp": "2017/03/16 12:00:00", "price": 520} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 400}, +{"timestamp": "2017/03/17 21:00:00", "price": 800}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --sort_keys '-price' --output_columns _id,price +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "_id", + "UInt32" + ], + [ + "price", + "UInt32" + ] + ], + [ + 2, + 1000 + ], + [ + 3, + 900 + ], + [ + 1, + 300 + ], + [ + 1, + 530 + ], + [ + 3, + 520 + ], + [ + 2, + 110 + ], + [ + 2, + 800 + ], + [ + 1, + 400 + ], + [ + 3, + 300 + ] + ] +] +#|d| [logical_range_filter][select] <Logs_20170315>: sort_keys is used +#|d| [logical_range_filter][select] <Logs_20170316>: sort_keys is used +#|d| [logical_range_filter][select] <Logs_20170317>: sort_keys is used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/sort_keys/descending.test (+50 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/sort_keys/descending.test 2018-04-10 11:33:04 +0900 (6a64626a8) @@ -0,0 +1,50 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 300}, +{"timestamp": "2017/03/15 01:00:00", "price": 1000}, +{"timestamp": "2017/03/15 02:00:00", "price": 900} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 110}, +{"timestamp": "2017/03/16 12:00:00", "price": 520} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 400}, +{"timestamp": "2017/03/17 21:00:00", "price": 800}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --sort_keys '-price' \ + --output_columns _id,price +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/sort_keys/multiple.expected (+127 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/sort_keys/multiple.expected 2018-04-10 11:33:04 +0900 (9d1d3cd1c) @@ -0,0 +1,127 @@ +plugin_register sharding +[[0,0.0,0.0],true] +plugin_register functions/time +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 300}, +{"timestamp": "2017/03/15 01:00:00", "price": 1000}, +{"timestamp": "2017/03/15 02:00:00", "price": 900} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 110}, +{"timestamp": "2017/03/16 12:00:00", "price": 520} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 400}, +{"timestamp": "2017/03/17 21:00:00", "price": 800}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[hour2].stage initial --columns[hour2].type Time --columns[hour2].flags COLUMN_SCALAR --columns[hour2].value 'time_classify_hour(timestamp, 2)' --sort_keys 'hour2,-price' --output_columns _id,hour2,price +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "_id", + "UInt32" + ], + [ + "hour2", + "Time" + ], + [ + "price", + "UInt32" + ] + ], + [ + 2, + 1489503600.0, + 1000 + ], + [ + 1, + 1489503600.0, + 300 + ], + [ + 3, + 1489510800.0, + 900 + ], + [ + 1, + 1489626000.0, + 530 + ], + [ + 2, + 1489626000.0, + 110 + ], + [ + 3, + 1489633200.0, + 520 + ], + [ + 2, + 1489748400.0, + 800 + ], + [ + 1, + 1489748400.0, + 400 + ], + [ + 3, + 1489755600.0, + 300 + ] + ] +] +#|d| [logical_range_filter][select] <Logs_20170315>: dynamic columns are used +#|d| [logical_range_filter][select] <Logs_20170316>: dynamic columns are used +#|d| [logical_range_filter][select] <Logs_20170317>: dynamic columns are used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/sort_keys/multiple.test (+56 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/sort_keys/multiple.test 2018-04-10 11:33:04 +0900 (a7b1b6de2) @@ -0,0 +1,56 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +plugin_register functions/time + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 300}, +{"timestamp": "2017/03/15 01:00:00", "price": 1000}, +{"timestamp": "2017/03/15 02:00:00", "price": 900} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 110}, +{"timestamp": "2017/03/16 12:00:00", "price": 520} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 400}, +{"timestamp": "2017/03/17 21:00:00", "price": 800}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[hour2].stage initial \ + --columns[hour2].type Time \ + --columns[hour2].flags COLUMN_SCALAR \ + --columns[hour2].value 'time_classify_hour(timestamp, 2)' \ + --sort_keys 'hour2,-price' \ + --output_columns _id,hour2,price +log_level --level notice +#@remove-important-log-levels debug -------------- next part -------------- HTML����������������������������... URL: https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20180410/824ac1ba/attachment-0001.htm
