Class: NoSE::CLI::NoSECLI

Inherits:
Thor
  • Object
show all
Defined in:
lib/nose_cli.rb,
lib/nose_cli/why.rb,
lib/nose_cli/dump.rb,
lib/nose_cli/list.rb,
lib/nose_cli/load.rb,
lib/nose_cli/repl.rb,
lib/nose_cli/graph.rb,
lib/nose_cli/proxy.rb,
lib/nose_cli/create.rb,
lib/nose_cli/export.rb,
lib/nose_cli/recost.rb,
lib/nose_cli/search.rb,
lib/nose_cli/texify.rb,
lib/nose_cli/analyze.rb,
lib/nose_cli/console.rb,
lib/nose_cli/execute.rb,
lib/nose_cli/reformat.rb,
lib/nose_cli/benchmark.rb,
lib/nose_cli/diff_plans.rb,
lib/nose_cli/search_all.rb,
lib/nose_cli/genworkload.rb,
lib/nose_cli/plan_schema.rb,
lib/nose_cli/random_plans.rb,
lib/nose_cli/search_bench.rb,
lib/nose_cli/shared_options.rb,
lib/nose_cli/collect_results.rb

Overview

Add a command to generate a graphic of the schema from a workload

Constant Summary collapse

CONFIG_FILE_NAME =

The path to the configuration file in the working directory

'nose.yml'
AVAILABLE_TYPES = 
%w(backend cost).freeze

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(_options, local_options, config) ⇒ NoSECLI

Returns a new instance of NoSECLI.



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/nose_cli.rb', line 31

def initialize(_options, local_options, config)
  super

  # Set up a logger for this command
  cmd_name = config[:current_command].name
  @logger = Logging.logger["nose::#{cmd_name}"]

  # Peek ahead into the options and prompt the user to create a config
  check_config_file interactive?(local_options)

  force_colour(options[:colour]) unless options[:colour].nil?

  # Disable parallel processing if desired
  Parallel.instance_variable_set(:@processor_count, 0) \
    unless options[:parallel]
end

Class Method Details

.share_option(name, options = {}) ⇒ Object

Add a new option to those which can be potentially shared



8
9
10
11
 
# File 'lib/nose_cli/shared_options.rb', line 8

def self.share_option(name, options = {})
  @options ||= {}
  @options[name] = options
end

.shared_option(name) ⇒ void

This method returns an undefined value.

Use a shared option for the current command



15
16
17
 
# File 'lib/nose_cli/shared_options.rb', line 15

def self.shared_option(name)
  method_option name, @options[name]
end

Instance Method Details

#analyze(output_file, *csv_files) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/nose_cli/analyze.rb', line 24

def analyze(output_file, *csv_files)
  # Load data from the files
  data = load_data csv_files, options[:total]

  # Set graph properties
  g = Gruff::Bar.new '2000x800'
  g.title = 'NoSE Schema Performance'
  g.x_axis_label = '\nWorkload group'
  g.y_axis_label = 'Weighted execution time (s)'
  g.title_font_size = 20
  g.legend_font_size = 10
  g.marker_font_size = 10
  g.label_stagger_height = 15
  g.legend_box_size = 10
  g.bar_spacing = 0.5

  # Add each data element to the graph
  data.each do |datum|
    g.data datum.first['label'], datum.map { |row| row['mean'] }
  end
  g.labels = Hash[data.first.map.with_index do |row, n|
    [n, row['group']]
  end]

  g.write output_file
end

#benchmark(plan_file) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/nose_cli/benchmark.rb', line 38

def benchmark(plan_file)
  label = File.basename plan_file, '.*'
  result = load_results plan_file, options[:mix]

  backend = get_backend(options, result)

  index_values = index_values result.indexes, backend,
                              options[:num_iterations],
                              options[:fail_on_empty]

  group_tables = Hash.new { |h, k| h[k] = [] }
  group_totals = Hash.new { |h, k| h[k] = 0 }
  result.plans.each do |plan|
    query = plan.query
    weight = result.workload.statement_weights[query]
    next if query.is_a?(SupportQuery) || !weight
    @logger.debug { "Executing #{query.text}" }

    next unless options[:group].nil? || plan.group == options[:group]

    indexes = plan.select do |step|
      step.is_a? Plans::IndexLookupPlanStep
    end.map(&:index)

    measurement = bench_query backend, indexes, plan, index_values,
                              options[:num_iterations], options[:repeat],
                              weight: weight
    next if measurement.empty?

    measurement.estimate = plan.cost
    group_totals[plan.group] += measurement.mean
    group_tables[plan.group] << measurement
  end

  result.workload.updates.each do |update|
    weight = result.workload.statement_weights[update]
    next unless weight

    plans = (result.update_plans || []).select do |possible_plan|
      possible_plan.statement == update
    end
    next if plans.empty?

    @logger.debug { "Executing #{update.text}" }

    plans.each do |plan|
      next unless options[:group].nil? || plan.group == options[:group]

      # Get all indexes used by support queries
      indexes = plan.query_plans.flat_map(&:indexes) << plan.index

      measurement = bench_update backend, indexes, plan, index_values,
                                 options[:num_iterations],
                                 options[:repeat], weight: weight
      next if measurement.empty?

      measurement.estimate = plan.cost
      group_totals[plan.group] += measurement.mean
      group_tables[plan.group] << measurement
    end
  end

  total = 0
  table = []
  group_totals.each do |group, group_total|
    total += group_total
    total_measurement = Measurements::Measurement.new nil, 'TOTAL'
    group_table = group_tables[group]
    total_measurement << group_table.map(&:weighted_mean) \
                         .inject(0, &:+)
    group_table << total_measurement if options[:totals]
    table << OpenStruct.new(label: label, group: group,
                            measurements: group_table)
  end

  if options[:totals]
    total_measurement = Measurements::Measurement.new nil, 'TOTAL'
    total_measurement << table.map do |group|
      group.measurements.find { |m| m.name == 'TOTAL' }.mean
    end.inject(0, &:+)
    table << OpenStruct.new(label: label, group: 'TOTAL',
                            measurements: [total_measurement])
  end

  case options[:format]
  when 'txt'
    output_table table
  else
    output_csv table
  end
end

#collect_results(*csv_files) ⇒ Object 



20
21
22
23
24
25
26
27
28
 
# File 'lib/nose_cli/collect_results.rb', line 20

def collect_results(*csv_files)
  # Load the data and output the header
  data = load_data csv_files, options[:total]
  labels = data.map { |datum| datum.first['label'] }
  puts((['Group'] + labels).join("\t"))

  # Output the mean for each schema
  group_data(data).each { |group| collect_group_data group, data }
end

#console(plan_file) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/nose_cli/console.rb', line 19

def console(plan_file)
  # Load the results from the plan file and define each as a variable
  result = load_results plan_file
  expose_result result

  # Also extract the model as a variable
  TOPLEVEL_BINDING.local_variable_set :model, result.workload.model

  # Load the options and backend as variables
  TOPLEVEL_BINDING.local_variable_set :options, options
  TOPLEVEL_BINDING.local_variable_set :backend,
                                      get_backend(options, result)

  TOPLEVEL_BINDING.pry
end

#create(*plan_files) ⇒ Object 



23
24
25
26
27
28
29
30
31
32
 
# File 'lib/nose_cli/create.rb', line 23

def create(*plan_files)
  plan_files.each do |plan_file|
    _, backend = load_plans plan_file, options

    # Produce the DDL and execute unless the dry run option was given
    backend.indexes_ddl(!options[:dry_run], options[:skip_existing],
                        options[:drop_existing]) \
           .each { |ddl| puts ddl }
  end
end

#diff_plans(plan1, plan2) ⇒ Object 



15
16
17
18
19
20
21
 
# File 'lib/nose_cli/diff_plans.rb', line 15

def diff_plans(plan1, plan2)
  result1 = load_results plan1
  result2 = load_results plan2

  output_diff plan1, result1, result2
  output_diff plan2, result2, result1
end

#dump(plan_name) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/nose_cli/dump.rb', line 19

def dump(plan_name)
  plans = Plans::ExecutionPlans.load plan_name
  plans.mix = options[:mix].to_sym \
    unless options[:mix] == 'default' && plans.mix != :default

  # Set the cost of each plan
  cost_model = get_class_from_config options, 'cost', :cost_model
  plans.calculate_cost cost_model

  results = OpenStruct.new
  results.workload = Workload.new plans.schema.model
  results.workload.mix = plans.mix
  results.model = results.workload.model
  results.indexes = plans.schema.indexes.values
  results.enumerated_indexes = []

  results.plans = []
  results.update_plans = []

  # Store all the query and update plans
  plans.groups.values.flatten(1).each do |plan|
    if plan.update_steps.empty?
      results.plans << plan
    else
      # XXX: Hack to build a valid update plan
      statement = OpenStruct.new group: plan.group
      update_plan = Plans::UpdatePlan.new statement, plan.index, nil,
                                          plan.update_steps, cost_model
      update_plan.instance_variable_set :@group, plan.group
      update_plan.instance_variable_set :@query_plans, plan.query_plans
      results.update_plans << update_plan
    end
  end

  results.cost_model = cost_model
  results.weights = Hash[plans.weights.map { |g, w| [g, w[plans.mix]] }]
  results.total_size = results.indexes.sum_by(&:size)
  results.total_cost = plans.groups.values.flatten(1).sum_by do |plan|
    next 0 if plan.weight.nil?

    plan.cost * plan.weight
  end

  # Output the results in the specified format
  send(('output_' + options[:format]).to_sym, results)
end

#execute(plans_name) ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/nose_cli/execute.rb', line 37

def execute(plans_name)
  # Load the execution plans
  plans = Plans::ExecutionPlans.load plans_name

  # Construct an instance of the backend
  result = OpenStruct.new
  result.workload = Workload.new plans.schema.model
  result.workload.mix = options[:mix].to_sym \
    unless options[:mix] == 'default' && result.workload.mix != :default
  result.model = result.workload.model
  result.indexes = plans.schema.indexes.values
  backend = get_backend(options, result)

  # Get sample index values to use in queries
  index_values = index_values plans.schema.indexes.values, backend,
                              options[:num_iterations],
                              options[:fail_on_empty]

  table = []
  total = 0
  plans.groups.each do |group, group_plans|
    next if options[:group] && group != options[:group]

    group_table = []
    group_total = 0
    group_weight = plans.weights[group][result.workload.mix]
    next unless group_weight

    group_plans.each do |plan|
      next if options[:plan] && plan.name != options[:plan]

      update = !plan.steps.last.is_a?(Plans::IndexLookupPlanStep)
      method = update ? :bench_update : :bench_query
      measurement = send method, backend, plans.schema.indexes.values,
                         plan, index_values,
                         options[:num_iterations],
                         options[:repeat], weight: group_weight

      # Run the query and get the total time
      group_total += measurement.mean
      group_table << measurement
    end

    if options[:totals]
      total_measurement = Measurements::Measurement.new nil, 'TOTAL'
      total_measurement << group_table.map(&:weighted_mean) \
                           .inject(0, &:+)
      group_table << total_measurement
    end

    table << OpenStruct.new(label: plans_name, group: group,
                            measurements: group_table)
    group_total *= group_weight
    total += group_total
  end

  if options[:totals]
    total_measurement = Measurements::Measurement.new nil, 'TOTAL'
    total_measurement << table.map do |group|
      group.measurements.find { |m| m.name == 'TOTAL' }.mean
    end.inject(0, &:+)
    table << OpenStruct.new(label: plans_name, group: 'TOTAL',
                            measurements: [total_measurement])
  end

  case options[:format]
  when 'txt'
    output_table table
  else
    output_csv table
  end
end

#export ⇒ Object 



14
15
16
 
# File 'lib/nose_cli/export.rb', line 14

def export
  export_value [], options
end

#genworkload(name) ⇒ Object 



15
16
17
18
19
20
21
 
# File 'lib/nose_cli/genworkload.rb', line 15

def genworkload(name)
  loader_class = get_class 'loader', options
  workload = loader_class.new.workload options[:loader]
  File.open("./workloads/#{name}.rb", 'w') do |file|
    file.write workload.source_code
  end
end

#graph(workload_name, filename) ⇒ Object 



17
18
19
20
21
 
# File 'lib/nose_cli/graph.rb', line 17

def graph(workload_name, filename)
  workload = Workload.load workload_name
  type = filename.split('.').last.to_sym
  workload.model.output type, filename, options[:include_fields]
end

#list(type) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/nose_cli/list.rb', line 18

def list(type)
  case type
  when 'backend'
    cls = Backend::Backend
  when 'cost'
    cls = Cost::Cost
  else
    fail Thor::UnknownArgumentError,
         "Invalid type. Available types are #{AVAILABLE_TYPES.join ', '}"
  end

  cls.subclasses.each_value { |c| puts c.subtype_name }
end

#load(*plan_files) ⇒ Object 



26
27
28
 
# File 'lib/nose_cli/load.rb', line 26

def load(*plan_files)
  plan_files.each { |plan_file| load_plan plan_file, options }
end

#plan_schema(workload_name, schema_name) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/nose_cli/plan_schema.rb', line 21

def plan_schema(workload_name, schema_name)
  workload = Workload.load workload_name
  workload.mix = options[:mix].to_sym \
    unless options[:mix] == 'default' && workload.mix != :default
  schema = Schema.load schema_name
  indexes = schema.indexes.values

  # Build the statement plans
  cost_model = get_class_from_config options, 'cost', :cost_model
  planner = Plans::QueryPlanner.new workload, indexes, cost_model
  trees = workload.queries.map { |q| planner.find_plans_for_query q }
  plans = trees.map(&:min)

  update_plans = build_update_plans workload.statements, indexes,
                                    workload.model, trees, cost_model

  # Construct a result set
  results = plan_schema_results workload, indexes, plans, update_plans,
                                cost_model

  # Output the results in the specified format
  send(('output_' + options[:format]).to_sym, results)
end

#proxy(plan_file) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/nose_cli/proxy.rb', line 16

def proxy(plan_file)
  result = load_results plan_file
  backend = get_backend(options, result)

  # Create a new instance of the proxy class
  proxy_class = get_class 'proxy', options
  proxy = proxy_class.new options[:proxy], result, backend

  # Start the proxy server
  trap 'INT' do
    proxy.stop
  end
  proxy.start
end

#random_plans(workload_name, tag) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/nose_cli/random_plans.rb', line 25

def random_plans(workload_name, tag)
  # Find the statement with the given tag
  workload = Workload.load workload_name
  statement = workload.find_with_tag tag

  # Generate a random set of plans
  indexes = IndexEnumerator.new(workload).indexes_for_workload
  cost_model = get_class('cost', options[:cost_model][:name]) \
               .new(**options[:cost_model])
  plans = find_random_plans statement, workload, indexes, cost_model,
                            options
  results = random_plan_results workload, indexes, plans, cost_model
  output_random_plans results, options[:output], options[:format]
end

#recost(plan_file, cost_model) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/nose_cli/recost.rb', line 17

def recost(plan_file, cost_model)
  result = load_results plan_file

  # Get the new cost model and recost all the queries
  new_cost_model = get_class('cost', cost_model).new
  result.plans.each { |plan| plan.cost_model = new_cost_model }

  # Update the cost values
  result.cost_model = new_cost_model
  result.total_cost = total_cost result.workload, result.plans

  output_json result
end

#reformat(plan_file) ⇒ Object 



13
14
15
16
17
18
19
 
# File 'lib/nose_cli/reformat.rb', line 13

def reformat(plan_file)
  result = load_results plan_file, options[:mix]
  result.recalculate_cost

  # Output the results in the specified format
  send(('output_' + options[:format]).to_sym, result)
end

#repl(plan_file) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/nose_cli/repl.rb', line 24

def repl(plan_file)
  result = load_results plan_file
  backend = get_backend(options, result)

  load_history
  loop do
    begin
      line = read_line
    rescue Interrupt
      line = nil
    end
    break if line.nil?

    next if line.empty?

    execute_statement line, result, backend
  end
end

#search(name) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/nose_cli/search.rb', line 38

def search(name)
  # Get the workload from file or name
  if File.exist? name
    result = load_results name, options[:mix]
    workload = result.workload
  else
    workload = Workload.load name
  end

  # Prepare the workload and the cost model
  workload.mix = options[:mix].to_sym \
    unless options[:mix] == 'default' && workload.mix != :default
  workload.remove_updates if options[:read_only]
  cost_model = get_class_from_config options, 'cost', :cost_model

  # Execute the advisor
  objective = Search::Objective.const_get options[:objective].upcase
  result = search_result workload, cost_model, options[:max_space],
                         objective, options[:by_id_graph]
  output_search_result result, options unless result.nil?
end

#search_all(name, directory) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/nose_cli/search_all.rb', line 30

def search_all(name, directory)
  # Load the workload and cost model and create the output directory
  workload = Workload.load name
  workload.mix = options[:mix].to_sym \
    unless options[:mix] == 'default' && workload.mix != :default
  workload.remove_updates if options[:read_only]
  cost_model = get_class_from_config options, 'cost', :cost_model
  FileUtils.mkdir_p(directory) unless Dir.exist?(directory)

  # Run the search and output the results
  results = search_results workload, cost_model, options[:max_results]
  output_results results, directory, options
end

#search_bench(name) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/nose_cli/search_bench.rb', line 16

def search_bench(name)
  # Open a tempfile which will be used for advisor output
  filename = Tempfile.new('workload').path

  # Set some default options for various commands
  opts = options.to_h
  opts[:output] = filename
  opts[:format] = 'json'
  opts[:skip_existing] = true

  o = filter_command_options opts, 'search'
  $stderr.puts "Running advisor #{o}..."
  invoke self.class, :search, [name], o

  invoke self.class, :reformat, [filename], {}

  o = filter_command_options opts, 'create'
  $stderr.puts "Creating indexes #{o}..."
  invoke self.class, :create, [filename], o

  o = filter_command_options opts, 'load'
  $stderr.puts "Loading data #{o}..."
  invoke self.class, :load, [filename], o

  o = filter_command_options opts, 'benchmark'
  $stderr.puts "Running benchmark #{o}..."
  invoke self.class, :benchmark, [filename], o
end

#texify(plan_file) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/nose_cli/texify.rb', line 17

def texify(plan_file)
  # Load the indexes from the file
  result, = load_plans plan_file, options

  # If these are manually generated plans, load them separately
  if result.plans.nil?
    plans = Plans::ExecutionPlans.load(plan_file) \
            .groups.values.flatten(1)
    result.plans = plans.select { |p| p.update_steps.empty? }
    result.update_plans = plans.reject { |p| p.update_steps.empty? }
  end

  # Print document header
  puts "\\documentclass{article}\n\\begin{document}\n\\begin{flushleft}"

  # Print the LaTeX for all indexes and plans
  texify_indexes result.indexes
  texify_plans result.plans + result.update_plans

  # End the document
  puts "\\end{flushleft}\n\\end{document}"
end

#why(plan_file) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/nose_cli/why.rb', line 16

def why(plan_file)
  result = load_results plan_file
  indexes_usage = Hash.new { |h, k| h[k] = [] }

  # Count the indexes used in queries
  query_count = Set.new
  update_index_usage result.plans, indexes_usage, query_count

  # Count the indexes used in support queries
  # (ignoring those used in queries)
  support_count = Set.new
  result.update_plans.each do |plan|
    update_index_usage plan.query_plans, indexes_usage,
                       support_count, query_count
  end

  # Produce the final output of index usage
  print_index_usage indexes_usage, query_count, support_count
end