Class: Benchmark::Sweet::Job

Inherits:
Object
  • Object
show all
Includes:
IPS, Memory, Queries
Defined in:
lib/benchmark/sweet/job.rb

Overview

abstract notion of a job

Constant Summary collapse

IPS_METRICS =

metrics calculated by the ips test suite

%w(ips).freeze
MEMORY_METRICS =

metrics calculated by the memory test suite

%w(memsize memsize_retained objects objects_retained strings strings_retained).freeze
DATABASE_METRICS =

metrics calculated by the database test suite

%w(rows queries ignored ignored cached).freeze
ALL_METRICS = 
(IPS_METRICS + MEMORY_METRICS + DATABASE_METRICS).freeze
HIGHER_BETTER = 
%w(ips).freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Queries

#run_queries

Methods included from Memory

#run_memory

Methods included from IPS

#run_ips

Constructor Details

#initialize(options = {}) ⇒ Job

Returns a new instance of Job.



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/benchmark/sweet/job.rb', line 38

def initialize(options = {})
  @options = options
  @options[:metrics] ||= IPS_METRICS + %w()
  validate_metrics(@options[:metrics])
  @items = []
  @entries = {}
  @symbolize_keys = false
  # load / save
  @filename = nil
  # display
  @grouping = nil
  @report_options = {}
  @report_block = nil
  # current item metadata
  @meta = {}
end

Instance Attribute Details

#entries ⇒ Object (readonly)

Returns the value of attribute entries.



21
22
23
 
# File 'lib/benchmark/sweet/job.rb', line 21

def entries
  @entries
end

#grouping ⇒ Nil|Lambda (readonly)

lambda used to group metrics that should be compared The lambda takes the label as an argument and returns a unique object per comparison group NOTE: This lambda takes a label hash as an argument

While other lambdas in this api take a comparison object

a symbol is assumed to refer to the label

Returns:

  • (Nil|Lambda) —

    lambda for grouping



36
37
38
 
# File 'lib/benchmark/sweet/job.rb', line 36

def grouping
  @grouping
end

#items ⇒ Object (readonly)

Returns the value of attribute items.



19
20
21
 
# File 'lib/benchmark/sweet/job.rb', line 19

def items
  @items
end

#options ⇒ Object (readonly)

TODO: :confidence



28
29
30
 
# File 'lib/benchmark/sweet/job.rb', line 28

def options
  @options
end

Instance Method Details

#add_entry(label, metric, stat) ⇒ Object

report results



131
132
133
 
# File 'lib/benchmark/sweet/job.rb', line 131

def add_entry(label, metric, stat)
  (@entries[metric] ||= {})[label] = stat.respond_to?(:central_tendency) ? stat : create_stats(stat)
end

#compare_by(*symbol, &block) ⇒ Object

&block - a lambda that accepts a label and a stats object returns a unique object for each set of metrics that should be compared with each other

unfortunatly, this currently has a different signature than all other lambdas at this time, there are no comparisons created yet. so it is hard to pass one in example:

x.compare_by { |label, value| label[:data] }
x.compare_by :data



104
105
106
 
# File 'lib/benchmark/sweet/job.rb', line 104

def compare_by(*symbol, &block)
  @grouping = symbol.empty? ? block : Proc.new { |label, value| symbol.map { |s| label[s] } }
end

#comparison_values ⇒ Object

of note, this groups with @grouping (defined by group_by) but then all data continues to the next step this allows you to make comparisons across rows / columns / grouping



212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/benchmark/sweet/job.rb', line 212

def comparison_values
  relevant_entries.flat_map do |metric_name, metric_entries|
    # TODO: map these to Comparison(metric_name, label, stats) So we only have 1 type of lambda
    partitioned_metrics = grouping ? metric_entries.group_by(&grouping) : {nil => metric_entries}
    partitioned_metrics.flat_map do |grouping_name, grouped_metrics|
      sorted = grouped_metrics.sort_by { |n, e| e.central_tendency }
      sorted.reverse! if HIGHER_BETTER.include?(metric_name)

      _best_label, best_stats = sorted.first
      total = sorted.count

      # TODO: fix ranking. i / total doesn't work as well when there is only 1 entry or some entries are the same
      sorted.each_with_index.map { |(label, stats), i| Comparison.new(metric_name, label, stats, i, total, best_stats) }
    end
  end
end

#configure(options) ⇒ Object 



55
56
57
 
# File 'lib/benchmark/sweet/job.rb', line 55

def configure(options)
  @options.merge!(options)
end

#database? ⇒ Boolean

Returns:

  • (Boolean) 


64
 
# File 'lib/benchmark/sweet/job.rb', line 64

def database? ; !(relevant_metric_names & DATABASE_METRICS).empty? ; end

#display_report(comparisons) ⇒ Object 



201
202
203
204
205
206
207
 
# File 'lib/benchmark/sweet/job.rb', line 201

def display_report(comparisons)
  if !@report_block || @report_block.arity == 2
    Benchmark::Sweet.table(comparisons, **@report_options, &@report_block)
  else
    @report_block.call(comparisons)
  end
end

#entry_stat(label, metric) ⇒ Object 



135
136
137
 
# File 'lib/benchmark/sweet/job.rb', line 135

def entry_stat(label, metric)
  @entries.dig(metric, label)
end

#force? ⇒ Boolean

Returns:

  • (Boolean) 


70
 
# File 'lib/benchmark/sweet/job.rb', line 70

def force? ; options[:force] ; end

#ips? ⇒ Boolean

Returns:

  • (Boolean) 


60
 
# File 'lib/benchmark/sweet/job.rb', line 60

def ips? ; !(relevant_metric_names & IPS_METRICS).empty? ; end

#item(label, action = nil, &block) ⇒ Object Also known as: report

items to run (typical benchmark/benchmark-ips use case)



76
77
78
79
80
 
# File 'lib/benchmark/sweet/job.rb', line 76

def item(label, action = nil, &block)
  # could use Benchmark::IPS::Job::Entry
  current_meta = label.kind_of?(Hash) ? @meta.merge(label) : @meta.merge(method: label)
  @items << Item.new(current_meta, action || block)
end

#labels_have_symbols! ⇒ Object

if we are using symbols as keys for our labels



127
128
 
# File 'lib/benchmark/sweet/job.rb', line 127

def labels_have_symbols!
end

#load_entries(filename = @filename) ⇒ Object

serialization



144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/benchmark/sweet/job.rb', line 144

def load_entries(filename = @filename)
  # ? have ips save / load their own data?
  return unless filename && File.exist?(filename)
  require "json"

  JSON.load(IO.read(filename)).each do |v|
    n = v["name"]
    n.symbolize_keys!
    add_entry n, v["metric"], v["samples"]
  end

end

#memory? ⇒ Boolean

Returns:

  • (Boolean) 


62
 
# File 'lib/benchmark/sweet/job.rb', line 62

def memory? ; !(relevant_metric_names & MEMORY_METRICS).empty? ; end

#metadata(options) ⇒ Object 



83
84
85
86
87
88
89
 
# File 'lib/benchmark/sweet/job.rb', line 83

def metadata(options)
  @old_meta = @meta
  @meta = @meta.merge(options)
  return unless block_given?
  yield
  @meta = @old_meta
end

#quiet? ⇒ Boolean

Returns:

  • (Boolean) 


67
 
# File 'lib/benchmark/sweet/job.rb', line 67

def quiet? ; options[:quiet] ; end

#relevant_entries ⇒ Object 



139
140
141
 
# File 'lib/benchmark/sweet/job.rb', line 139

def relevant_entries
  relevant_metric_names.map { |n| [n, @entries[n] ] }
end

#relevant_metric_names ⇒ Object 



73
 
# File 'lib/benchmark/sweet/job.rb', line 73

def relevant_metric_names ; options[:metrics] ; end

#report_with(args = {}, &block) ⇒ Object

Setup the testing framework TODO: would be easier to debug if these were part of run_report



117
118
119
120
121
122
123
124
 
# File 'lib/benchmark/sweet/job.rb', line 117

def report_with(args = {}, &block)
  @report_options = args
  @report_block = block
  # Assume the display grouping is the same as comparison grouping unless an explicit value was provided
  if !args.key?(:grouping) && @grouping
    args[:grouping] = @grouping.respond_to?(:call) ? -> v { @grouping.call(v.label, v.stats) } : @grouping
  end
end

#run ⇒ Object 



181
182
183
184
185
186
187
188
189
190
 
# File 'lib/benchmark/sweet/job.rb', line 181

def run
  # run metrics if they are requested and haven't run yet
  # only run the suites that provide the data the user needs.
  # if the first node has the data, assumes all do
  #
  # TODO: may want to override these values
  run_ips     if ips?      && (force? || !@entries.dig(IPS_METRICS.first, items.first.label))
  run_memory  if memory?   && (force? || !@entries.dig(MEMORY_METRICS.first, items.first.label))
  run_queries if database? && (force? || !@entries.dig(DATABASE_METRICS.first, items.first.label))
end

#run_report ⇒ Object

? metric => label(:version, :method) => stats ? label(:metric, :version, :method) => stats



195
196
197
198
199
 
# File 'lib/benchmark/sweet/job.rb', line 195

def run_report
  comparison_values.tap do |results|
    display_report(results)
  end
end

#save_entries(filename = @filename) ⇒ Object 



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/benchmark/sweet/job.rb', line 157

def save_entries(filename = @filename)
  return unless filename
  require "json"

  # sanity checking
  symbol_value = false

  data = @entries.flat_map do |metric_name, metric_values|
    metric_values.map do |label, stat|
      # warnings
      symbol_values ||= label.kind_of?(Hash) && label.values.detect { |v| v.nil? || v.kind_of?(Symbol) }
      {
        'name'    => label,
        'metric'  => metric_name,
        'samples' => stat.samples,
        # extra data like measured_us, iter, and others?
      }
    end
  end

  puts "", "Warning: Please use strings or numbers for label hash values (not nils or symbols). Symbols are not JSON friendly." if symbol_value
  IO.write(filename, JSON.pretty_generate(data) << "\n")
end

#save_file(filename) ⇒ Object 



91
92
93
 
# File 'lib/benchmark/sweet/job.rb', line 91

def save_file(filename)
  @filename = filename
end