Skip to content

Commit

Permalink
generate-series: skip parsing generated load command
Browse files Browse the repository at this point in the history
It improves performance.
  • Loading branch information
kou committed Apr 12, 2024
1 parent a8f0310 commit f4382a6
Showing 1 changed file with 33 additions and 18 deletions.
51 changes: 33 additions & 18 deletions lib/grntest/executors/base-executor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -308,28 +308,43 @@ def execute_directive_collect_query_log(line, content, options)
@context.collect_query_log = (options[0] == "true")
end

def each_generated_series_chunk(evaluator, start, stop)
max_chunk_size = 1 * 1024 * 1024 # 1MiB
chunk_size = 0
records = []
(Integer(start)..Integer(stop)).each do |i|
record = evaluator.evaluate(i: i).to_json
records << record
chunk_size += record.bytesize
if chunk_size > max_chunk_size
yield(records)
records.clear
chunk_size = 0
end
end
yield(records) unless records.empty?
end

def execute_directive_generate_series(parser, line, content, options)
start, stop, table, template, = options
evaluator = TemplateEvaluator.new(template.force_encoding("UTF-8"))
(Integer(start)..Integer(stop)).each_slice(1000) do |range|
parser << "load --table #{table}\n"
parser << "["
first_record = true
range.each do |i|
record = ""
if first_record
first_record = false
else
record << ","
end
record << "\n"
record << evaluator.evaluate(i: i).to_json
before = Time.now
parser << record
elapsed = Time.now - before
Thread.pass if elapsed > 1.0
each_generated_series_chunk(evaluator,
Integer(start),
Integer(stop)) do |records|
source = "load --table #{table}\n"
values_part_start_position = source.size
source << "["
records.each_with_index do |record, i|
source << "," unless i.zero?
source << "\n"
source << record
end
parser << "\n]\n"
source << "\n]"
values = source[values_part_start_position..-1]
command = Groonga::Command::Load.new(table: table, values: values)
command.original_source = source
execute_command(command)
Thread.pass
end
end

Expand Down

0 comments on commit f4382a6

Please sign in to comment.