Class: Benchmark::IPS::Job

Inherits:
Object
  • Object
show all
Defined in:
lib/benchmark/ips/job.rb,
lib/benchmark/ips/job/entry.rb,
lib/benchmark/ips/job/multi_report.rb,
lib/benchmark/ips/job/stream_report.rb

Overview

Benchmark jobs.

Defined Under Namespace

Classes: Entry, MultiReport, StreamReport

Constant Summary collapse

MICROSECONDS_PER_100MS =

Microseconds per 100 millisecond.

100_000
MICROSECONDS_PER_SECOND =

Microseconds per second.

Timing::MICROSECONDS_PER_SECOND
MAX_TIME_SKEW =

The percentage of the expected runtime to allow before reporting a weird runtime

0.05
POW_2_30 = 
1 << 30

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ Job

Instantiate the Benchmark::IPS::Job.



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/benchmark/ips/job.rb', line 67

def initialize opts={}
  @list = []
  @run_single = false
  @json_path = false
  @compare = false
  @compare_order = :fastest
  @held_path = nil
  @held_results = nil

  @timing = Hash.new 1 # default to 1 in case warmup isn't run
  @full_report = Report.new

  # Default warmup and calculation time in seconds.
  @warmup = 2
  @time = 5
  @iterations = 1

  # Default statistical model
  @stats = :sd
  @confidence = 95

  @out = MultiReport.new(StreamReport.new)
end

Instance Attribute Details

#compare ⇒ Boolean (readonly)

Determining whether to run comparison utility.

Returns:

  • (Boolean) —

    true if needs to run compare.



20
21
22
 
# File 'lib/benchmark/ips/job.rb', line 20

def compare
  @compare
end

#confidence ⇒ Integer

Confidence.

Returns:

  • (Integer) 


52
53
54
 
# File 'lib/benchmark/ips/job.rb', line 52

def confidence
  @confidence
end

#full_report ⇒ Report (readonly)

Report object containing information about the run.

Returns:

  • (Report) —

    the report object.



28
29
30
 
# File 'lib/benchmark/ips/job.rb', line 28

def full_report
  @full_report
end

#hold ⇒ Boolean

Determining whether to hold results between Ruby invocations

Returns:

  • (Boolean) 


24
25
26
 
# File 'lib/benchmark/ips/job.rb', line 24

def hold
  @hold
end

#iterations ⇒ Integer

Warmup and calculation iterations.

Returns:

  • (Integer) 


44
45
46
 
# File 'lib/benchmark/ips/job.rb', line 44

def iterations
  @iterations
end

#list ⇒ Array<Entry> (readonly)

Two-element arrays, consisting of label and block pairs.

Returns:

  • (Array<Entry>) —

    list of entries



16
17
18
 
# File 'lib/benchmark/ips/job.rb', line 16

def list
  @list
end

#stats ⇒ Object

Statistics model.

Returns:

  • (Object) 


48
49
50
 
# File 'lib/benchmark/ips/job.rb', line 48

def stats
  @stats
end

#time ⇒ Integer

Calculation time setter and getter (in seconds).

Returns:

  • (Integer) 


40
41
42
 
# File 'lib/benchmark/ips/job.rb', line 40

def time
  @time
end

#timing ⇒ Hash (readonly)

Storing Iterations in time period.

Returns:

  • (Hash) 


32
33
34
 
# File 'lib/benchmark/ips/job.rb', line 32

def timing
  @timing
end

#warmup ⇒ Integer

Warmup time setter and getter (in seconds).

Returns:

  • (Integer) 


36
37
38
 
# File 'lib/benchmark/ips/job.rb', line 36

def warmup
  @warmup
end

Instance Method Details

#all_results_have_been_run? ⇒ Boolean

Returns:

  • (Boolean) 


241
242
243
 
# File 'lib/benchmark/ips/job.rb', line 241

def all_results_have_been_run?
  @full_report.entries.size == @list.size
end

#clear_held_results ⇒ Object 



245
246
247
 
# File 'lib/benchmark/ips/job.rb', line 245

def clear_held_results
  File.delete @held_path if File.exist?(@held_path)
end

#compare!(order: :fastest) ⇒ Object

Run comparison utility.



124
125
126
127
 
# File 'lib/benchmark/ips/job.rb', line 124

def compare!(order: :fastest)
  @compare = true
  @compare_order = order
end

#compare? ⇒ Boolean

Return true if job needs to be compared.

Returns:

  • (Boolean) —

    Need to compare?



119
120
121
 
# File 'lib/benchmark/ips/job.rb', line 119

def compare?
  @compare
end

#config(opts) ⇒ Object

Job configuration options, set @warmup and @time.

Parameters:

  • opts (Hash) —

    a customizable set of options

  • iterations (Hash) —

    a customizable set of options

Options Hash (opts):

  • :warmup (Integer) —

    Warmup time.

  • :time (Integer) —

    Calculation time.



95
96
97
98
99
100
101
102
103
 
# File 'lib/benchmark/ips/job.rb', line 95

def config opts
  @warmup = opts[:warmup] if opts[:warmup]
  @time = opts[:time] if opts[:time]
  @iterations = opts[:iterations] if opts[:iterations]
  @stats = opts[:stats] if opts[:stats]
  @confidence = opts[:confidence] if opts[:confidence]
  self.quiet = opts[:quiet] if opts.key?(:quiet)
  self.suite = opts[:suite] if opts[:suite]
end

#create_report(label, measured_us, iter, samples, cycles) ⇒ Report::Entry

Create report by add entry to @full_report.

Parameters:

  • label (String) —

    Report item label.

  • measured_us (Integer) —

    Measured time in microsecond.

  • iter (Integer) —

    Iterations.

  • samples (Array<Float>) —

    Sampled iterations per second.

  • cycles (Integer) —

    Number of Cycles.

Returns:



392
393
394
 
# File 'lib/benchmark/ips/job.rb', line 392

def create_report(label, measured_us, iter, samples, cycles)
  @full_report.add_entry label, measured_us, iter, samples, cycles
end

#create_stats(samples) ⇒ Object 



364
365
366
367
368
369
370
371
372
373
 
# File 'lib/benchmark/ips/job.rb', line 364

def create_stats(samples)
  case @stats
    when :sd
      Stats::SD.new(samples)
    when :bootstrap
      Stats::Bootstrap.new(samples, @confidence)
    else
      raise "unknown stats #{@stats}"
  end
end

#cycles_per_100ms(time_msec, iters) ⇒ Integer

Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time.

Parameters:

  • time_msec (Float) —

    Each iteration’s time in ms.

  • iters (Integer) —

    Iterations.

Returns:

  • (Integer) —

    Cycles per 100ms.



193
194
195
196
 
# File 'lib/benchmark/ips/job.rb', line 193

def cycles_per_100ms time_msec, iters
  cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
  cycles <= 0 ? 1 : cycles
end

#generate_json ⇒ Object

Generate json from @full_report.



381
382
383
 
# File 'lib/benchmark/ips/job.rb', line 381

def generate_json
  @full_report.generate_json @json_path if json?
end

#hold!(held_path) ⇒ Object

Hold after each iteration.

Parameters:

  • held_path (String) —

    File name to store hold file.



137
138
139
140
 
# File 'lib/benchmark/ips/job.rb', line 137

def hold!(held_path)
  @held_path = held_path
  @run_single = true
end

#hold? ⇒ Boolean

Return true if results are held while multiple Ruby invocations

Returns:

  • (Boolean) —

    Need to hold results between multiple Ruby invocations?



131
132
133
 
# File 'lib/benchmark/ips/job.rb', line 131

def hold?
  !!@held_path
end

#item(label = "", str = nil, &blk) ⇒ Object Also known as: report

Registers the given label and block pair in the job list.

Parameters:

  • label (String) (defaults to: "") —

    Label of benchmarked code.

  • str (String) (defaults to: nil) —

    Code to be benchmarked.

  • blk (Proc) —

    Code to be benchmarked.

Raises:

  • (ArgumentError) —

    Raises if str and blk are both present.

  • (ArgumentError) —

    Raises if str and blk are both absent.



175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/benchmark/ips/job.rb', line 175

def item(label="", str=nil, &blk) # :yield:
  if blk and str
    raise ArgumentError, "specify a block and a str, but not both"
  end

  action = str || blk
  raise ArgumentError, "no block or string" unless action

  @list.push Entry.new(label, action)
  self
end

#iterations_per_sec(cycles, time_us) ⇒ Float

Calculate the interations per second given the number of cycles run and the time in microseconds that elapsed.

Parameters:

  • cycles (Integer) —

    Cycles.

  • time_us (Integer) —

    Time in microsecond.

Returns:

  • (Float) —

    Iteration per second.



211
212
213
 
# File 'lib/benchmark/ips/job.rb', line 211

def iterations_per_sec cycles, time_us
  MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
end

#json!(path = "data.json") ⇒ Object

Generate json to given path, defaults to “data.json”.



165
166
167
 
# File 'lib/benchmark/ips/job.rb', line 165

def json!(path="data.json")
  @json_path = path
end

#json? ⇒ Boolean

Return true if job needs to generate json.

Returns:

  • (Boolean) —

    Need to generate json?



160
161
162
 
# File 'lib/benchmark/ips/job.rb', line 160

def json?
  !!@json_path
end

#load_held_results ⇒ Object 



215
216
217
218
219
220
221
222
223
224
 
# File 'lib/benchmark/ips/job.rb', line 215

def load_held_results
  return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
  require "json"
  @held_results = {}
  JSON.load(IO.read(@held_path)).each do |result|
    @held_results[result['item']] = result
    create_report(result['item'], result['measured_us'], result['iter'],
                  create_stats(result['samples']), result['cycles'])
  end
end

#quiet ⇒ Boolean

Silence output

Returns:

  • (Boolean) 


56
57
58
 
# File 'lib/benchmark/ips/job.rb', line 56

def quiet
  @out.quiet?
end

#quiet=(val) ⇒ Object 



105
106
107
108
109
110
111
 
# File 'lib/benchmark/ips/job.rb', line 105

def quiet=(val)
  if val # remove instances of StreamReport
    @out.quiet!
  else # ensure there is an instance of StreamReport
    @out << StreamReport.new if @out.quiet?
  end
end

#run ⇒ Object 



249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
 
# File 'lib/benchmark/ips/job.rb', line 249

def run
  if @warmup && @warmup != 0 then
    @out.start_warming
    @iterations.times do
      run_warmup
    end
  end

  @out.start_running

  @iterations.times do |n|
    run_benchmark
  end

  @out.footer
end

#run_benchmark ⇒ Object

Run calculation.



312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
 
# File 'lib/benchmark/ips/job.rb', line 312

def run_benchmark
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @out.running item.label, @time

    Timing.clean_env

    iter = 0

    measurements_us = []

    # Running this number of cycles should take around 100ms.
    cycles = @timing[item]

    target = Timing.add_second Timing.now, @time

    begin
      before = Timing.now
      item.call_times cycles
      after = Timing.now

      # If for some reason the timing said this took no time (O_o)
      # then ignore the iteration entirely and start another.
      iter_us = Timing.time_us before, after
      next if iter_us <= 0.0

      iter += cycles

      measurements_us << iter_us
    end while Timing.now < target

    final_time = before

    measured_us = measurements_us.inject(:+)

    samples = measurements_us.map { |time_us|
      iterations_per_sec cycles, time_us
    }

    rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)

    if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
      rep.show_total_time!
    end

    @out.add_report rep, caller(1).first

    break if run_single?
  end
end

#run_comparison ⇒ Object

Run comparison of entries in @full_report.



376
377
378
 
# File 'lib/benchmark/ips/job.rb', line 376

def run_comparison
  @full_report.run_comparison(@compare_order) if compare?
end

#run_single? ⇒ Boolean

Return true if items are to be run one at a time. For the traditional hold, this is true

Returns:

  • (Boolean) —

    Run just a single item?



154
155
156
 
# File 'lib/benchmark/ips/job.rb', line 154

def run_single?
  @run_single
end

#run_warmup ⇒ Object

Run warmup.



267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
 
# File 'lib/benchmark/ips/job.rb', line 267

def run_warmup
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @out.warming item.label, @warmup

    Timing.clean_env

    # Run for up to half of the configured warmup time with an increasing
    # number of cycles to reduce overhead and improve accuracy.
    # This also avoids running with a constant number of cycles, which a
    # JIT might speculate on and then have to recompile in #run_benchmark.
    before = Timing.now
    target = Timing.add_second before, @warmup / 2.0

    cycles = 1
    begin
      t0 = Timing.now
      item.call_times cycles
      t1 = Timing.now
      warmup_iter = cycles
      warmup_time_us = Timing.time_us(t0, t1)

      # If the number of cycles would go outside the 32-bit signed integers range
      # then exit the loop to avoid overflows and start the 100ms warmup runs
      break if cycles >= POW_2_30
      cycles *= 2
    end while Timing.now + warmup_time_us * 2 < target

    cycles = cycles_per_100ms warmup_time_us, warmup_iter
    @timing[item] = cycles

    # Run for the remaining of warmup in a similar way as #run_benchmark.
    target = Timing.add_second before, @warmup
    while Timing.now + MICROSECONDS_PER_100MS < target
      item.call_times cycles
    end

    @out.warmup_stats warmup_time_us, @timing[item]

    break if run_single?
  end
end

#save!(held_path) ⇒ Object

Save interim results. Similar to hold, but all reports are run The report label must change for each invocation. One way to achieve this is to include the version in the label.

Parameters:

  • held_path (String) —

    File name to store hold file.



146
147
148
149
 
# File 'lib/benchmark/ips/job.rb', line 146

def save!(held_path)
  @held_path = held_path
  @run_single = false
end

#save_held_results ⇒ Object 



226
227
228
229
230
231
232
233
234
235
236
237
238
239
 
# File 'lib/benchmark/ips/job.rb', line 226

def save_held_results
  return unless @held_path
  require "json"
  data = full_report.entries.map { |e|
    {
      'item' => e.label,
      'measured_us' => e.microseconds,
      'iter' => e.iterations,
      'samples' => e.samples,
      'cycles' => e.measurement_cycle
    }
  }
  IO.write(@held_path, JSON.generate(data) << "\n")
end

#suite ⇒ Benchmark::IPS::MultiReport

Suite

Returns:

  • (Benchmark::IPS::MultiReport) 


62
63
64
 
# File 'lib/benchmark/ips/job.rb', line 62

def suite
  @out
end

#suite=(suite) ⇒ Object 



113
114
115
 
# File 'lib/benchmark/ips/job.rb', line 113

def suite=(suite)
  @out << suite
end

#time_us(before, after) ⇒ Float

Calculate the time difference of before and after in microseconds.

Parameters:

  • before (Time) —

    time.

  • after (Time) —

    time.

Returns:

  • (Float) —

    Time difference of before and after.



202
203
204
 
# File 'lib/benchmark/ips/job.rb', line 202

def time_us before, after
  (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
end