Class: Vanity::Experiment::AbTest

Inherits:
Base show all
Defined in:
lib/vanity/experiment/ab_test.rb

Overview

The meat.

Instance Attribute Summary

Attributes inherited from Base

#completed_at, #id, #name, #playground

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Base

#active?, #complete_if, #created_at, #description, #identify, load, #type, type

Constructor Details

#initialize(*args) ⇒ AbTest

Returns a new instance of AbTest.



113
114
115
 
# File 'lib/vanity/experiment/ab_test.rb', line 113

def initialize(*args)
  super
end

Class Method Details

.friendly_name ⇒ Object 



107
108
109
 
# File 'lib/vanity/experiment/ab_test.rb', line 107

def friendly_name
  "A/B Test" 
end

.probability(score) ⇒ Object

Convert z-score to probability.



101
102
103
104
105
 
# File 'lib/vanity/experiment/ab_test.rb', line 101

def probability(score)
  score = score.abs
  probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
  probability ? probability.last : 0
end

Instance Method Details

#alternative(key) ⇒ Object

Returns an Alternative with the specified key.

Examples:

alternative(:red) == alternatives[0]
alternative(:blue) == alternatives[2]


191
192
193
 
# File 'lib/vanity/experiment/ab_test.rb', line 191

def alternative(key)
  alternatives.find { |alt| alt.key == key }
end

#alternatives(*args) ⇒ Object

Call this method once to set alternative values for this experiment (requires at least two values). Call without arguments to obtain current list of alternatives.

Examples:

Define A/B test with three alternatives

ab_test "Background color" do
  metrics :coolness
  alternatives "red", "blue", "orange"
end

Find out which alternatives this test uses

alts = experiment(:background_color).alternatives
puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"


150
151
152
153
154
155
156
157
158
159
 
# File 'lib/vanity/experiment/ab_test.rb', line 150

def alternatives(*args)
  @alternatives = args.empty? ? [true, false] : args.clone
  if @control && !@alternatives.include?(@control)
    @alternatives.push(@control) # add to end
  end
  class << self
    define_method :alternatives, instance_method(:_alternatives)
  end
  nil
end

#assign_on(event) ⇒ Object 



317
318
319
 
# File 'lib/vanity/experiment/ab_test.rb', line 317

def assign_on(event)
  @assign_event = event
end

#assign_on?(event) ⇒ Boolean

Returns:

  • (Boolean) 


321
322
323
 
# File 'lib/vanity/experiment/ab_test.rb', line 321

def assign_on?(event)
  @assign_event == event
end

#choose ⇒ Object

Chooses a value for this experiment. You probably want to use the Rails helper method ab_test instead.

This method picks an alternative for the current identity and returns the alternative’s value. It will consistently choose the same alternative for the same identity, and randomly split alternatives between different identities.

Examples:

color = experiment(:which_blue).choose


220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/vanity/experiment/ab_test.rb', line 220

def choose
  if @playground.collecting?
    if active?
      identity = identity()
      index = connection.ab_showing(@id, identity)
      unless index
        index = alternative_for(identity)
        if !@playground.using_js?
          connection.ab_add_participant @id, index, identity
          check_completion!
        end
      end
    else
      index = connection.ab_get_outcome(@id) || alternative_for(identity)
    end
  else
    identity = identity()
    @showing ||= {}
    @showing[identity] ||= alternative_for(identity)
    index = @showing[identity]
  end
  alternatives[index.to_i]
end

#chooses(key) ⇒ Object

Forces this experiment to use a particular alternative. You’ll want to use this from your test cases to test for the different alternatives.

Examples:

Setup test to red button

setup do
  experiment(:button_color).select(:red)
end

def test_shows_red_button
  . . .
end

Use nil to clear selection

teardown do
  experiment(:green_button).select(nil)
end


270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
 
# File 'lib/vanity/experiment/ab_test.rb', line 270

def chooses(key)
  if @playground.collecting?
    if key.nil?
      connection.ab_not_showing @id, identity
    else
      index = @alternatives.index(key)
      #add them to the experiment unless they are already in it
      unless index == connection.ab_showing(@id, identity)
        connection.ab_add_participant @id, index, identity
        check_completion!
      end
      raise ArgumentError, "No alternative #{key.inspect} for #{name}" unless index
      if (connection.ab_showing(@id, identity) && connection.ab_showing(@id, identity) != index) || 
   alternative_for(identity) != index
        connection.ab_show @id, identity, index
      end
    end
  else
    @showing ||= {}
    @showing[identity] = key.nil? ? nil : @alternatives.index(key)
  end
  self
end

#chosen? ⇒ Boolean

True if this experiment has been selected for the current id (see #chooses).

Returns:

  • (Boolean) 


306
307
308
309
310
311
312
313
314
315
 
# File 'lib/vanity/experiment/ab_test.rb', line 306

def chosen?
  # True if experiment is active and a value has been chosen for current identity
  !!if @playground.collecting? # return a boolean value
    active? && (connection.ab_showing(@id, identity()) ||
                connection.ab_chosen(@id, identity()))
            # TODO: implement ab_chosen on all vanity adapters!!
  else
    @showing && @showing[identity()]
  end
end

#complete! ⇒ Object 



461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
 
# File 'lib/vanity/experiment/ab_test.rb', line 461

def complete!
  return unless @playground.collecting? && active?
  super
  if @outcome_is
    begin
      result = @outcome_is.call
      outcome = result.id if Alternative === result && result.experiment == self
    rescue 
      warn "Error in AbTest#complete!: #{$!}"
    end
  else
    best = score.best
    outcome = best.id if best
  end
  # TODO: logging
  connection.ab_set_outcome @id, outcome || 0
end

#conclusion(score = score) ⇒ Object

Use the result of #score to derive a conclusion. Returns an array of claims.



391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
 
# File 'lib/vanity/experiment/ab_test.rb', line 391

def conclusion(score = score)
  claims = []
  participants = score.alts.inject(0) { |t,alt| t + alt.participants }
  claims << case participants
    when 0 ; "There are no participants in this experiment yet."
    when 1 ; "There is one participant in this experiment."
    else ; "There are #{participants} participants in this experiment."
  end
  # only interested in sorted alternatives with conversion
  sorted = score.alts.select { |alt| alt.measure > 0.0 }.sort_by(&:measure).reverse
  if sorted.size > 1
    # start with alternatives that have conversion, from best to worst,
    # then alternatives with no conversion.
    sorted |= score.alts
    # we want a result that's clearly better than 2nd best.
    best, second = sorted[0], sorted[1]
    if best.measure > second.measure
      diff = ((best.measure - second.measure) / second.measure * 100).round
      better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
      claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
      if best.probability >= 90
        claims << "With %d%% probability this result is statistically significant." % score.best.probability
      else
        claims << "This result is not statistically significant, suggest you continue this experiment."
      end
      sorted.delete best
    end
    sorted.each do |alt|
      if alt.measure > 0.0
        claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.measure * 100]
      else
        claims << "%s did not convert." % alt.name.gsub(/^o/, "O")
      end
    end
  else
    claims << "This experiment did not run long enough to find a clear winner."
  end
  claims << "#{score.choice.name.gsub(/^o/, "O")} selected as the best alternative." if score.choice
  claims
end

#control_percent(pct) ⇒ Object

Raises:

  • (RuntimeError) 


330
331
332
333
 
# File 'lib/vanity/experiment/ab_test.rb', line 330

def control_percent(pct)
  raise RuntimeError, "Control percent must be an integer" unless pct.kind_of?(Integer)
  test_percent(100-pct)
end

#control_value(value) ⇒ Object 



335
336
337
338
339
340
 
# File 'lib/vanity/experiment/ab_test.rb', line 335

def control_value(value)
  @control = value
  if @alternatives && !@alternatives.include?(@control)
    @alternatives.push(@control) # add to end
  end
end

#destroy ⇒ Object

– Store/validate –



482
483
484
485
 
# File 'lib/vanity/experiment/ab_test.rb', line 482

def destroy
  connection.destroy_experiment @id
  super
end

#false_true ⇒ Object Also known as: true_false

Defines an A/B test with two alternatives: false and true. This is the default pair of alternatives, so just syntactic sugar for those who love being explicit.

Examples:

ab_test "More bacon" do
  metrics :yummyness 
  false_true
end


205
206
207
 
# File 'lib/vanity/experiment/ab_test.rb', line 205

def false_true
  alternatives false, true
end

#fingerprint(alternative) ⇒ Object

Returns fingerprint (hash) for given alternative. Can be used to lookup alternative for experiment without revealing what values are available (e.g. choosing alternative from HTTP query parameter).



247
248
249
 
# File 'lib/vanity/experiment/ab_test.rb', line 247

def fingerprint(alternative)
  Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
end

#metrics(*args) ⇒ Object

Tells A/B test which metric we’re measuring, or returns metric in use.

Examples:

Define A/B test against coolness metric

ab_test "Background color" do
  metrics :coolness
  alternatives "red", "blue", "orange"
end

Find metric for A/B test

puts "Measures: " + experiment(:background_color).metrics.map(&:name)


129
130
131
132
 
# File 'lib/vanity/experiment/ab_test.rb', line 129

def metrics(*args)
  @metrics = args.map { |id| @playground.metric(id) } unless args.empty?
  @metrics
end

#outcome ⇒ Object

Alternative chosen when this experiment completed.



455
456
457
458
459
 
# File 'lib/vanity/experiment/ab_test.rb', line 455

def outcome
  return unless @playground.collecting?
  outcome = connection.ab_get_outcome(@id)
  outcome && _alternatives[outcome]
end

#outcome_is(&block) ⇒ Object

Defines how the experiment can choose the optimal outcome on completion.

By default, Vanity will take the best alternative (highest conversion rate) and use that as the outcome. You experiment may have different needs, maybe you want the least performing alternative, or factor cost in the equation?

The default implementation reads like this:

outcome_is do
  a, b = alternatives
  # a is expensive, only choose a if it performs 2x better than b
  a.measure > b.measure * 2 ? a : b
end

Raises:

  • (ArgumentError) 


448
449
450
451
452
 
# File 'lib/vanity/experiment/ab_test.rb', line 448

def outcome_is(&block)
  raise ArgumentError, "Missing block" unless block
  raise "outcome_is already called on this experiment" if @outcome_is
  @outcome_is = block
end

#save ⇒ Object 



487
488
489
490
491
492
493
494
495
496
497
498
499
 
# File 'lib/vanity/experiment/ab_test.rb', line 487

def save
  true_false unless @alternatives
  fail "Experiment #{name} needs at least two alternatives" unless @alternatives.size >= 2
  super
  if @metrics.nil? || @metrics.empty?
    warn "Please use metrics method to explicitly state which metric you are measuring against."
    metric = @playground.metrics[id] ||= Vanity::Metric.new(@playground, name)
    @metrics = [metric]
  end
  @metrics.each do |metric|
    metric.hook &method(:track!)
  end
end

#score(probability = 90) ⇒ Object

Scores alternatives based on the current tracking data. This method returns a structure with the following attributes:

:alts

Ordered list of alternatives, populated with scoring info.

:base

Second best performing alternative.

:least

Least performing alternative (but more than zero conversion).

:choice

Choice alterntive, either the outcome or best alternative.

Alternatives returned by this method are populated with the following attributes:

:z_score

Z-score (relative to the base alternative).

:probability

Probability (z-score mapped to 0, 90, 95, 99 or 99.9%).

:difference

Difference from the least performant altenative.

The choice alternative is set only if its probability is higher or equal to the specified probability (default is 90%).



360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
 
# File 'lib/vanity/experiment/ab_test.rb', line 360

def score(probability = 90)
  alts = alternatives
  # sort by conversion rate to find second best and 2nd best
  sorted = alts.sort_by(&:measure)
  base = sorted[-2]
  # calculate z-score
  pc = base.measure
  nc = base.participants
  alts.each do |alt|
    p = alt.measure
    n = alt.participants
    alt.z_score = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
    alt.probability = AbTest.probability(alt.z_score)
  end
  # difference is measured from least performant
  if least = sorted.find { |alt| alt.measure > 0 }
    alts.each do |alt|
      if alt.measure > least.measure
        alt.difference = (alt.measure - least.measure) / least.measure * 100
      end
    end
  end
  # best alternative is one with highest conversion rate (best shot).
  # choice alternative can only pick best if we have high probability (>90%).
  best = sorted.last if sorted.last.measure > 0.0
  choice = outcome ? alts[outcome.id] : (best && best.probability >= probability ? best : nil)
  Struct.new(:alts, :best, :base, :least, :choice).new(alts, best, base, least, choice)
end

#showing?(alternative) ⇒ Boolean

True if this alternative is currently showing (see #chooses).

Returns:

  • (Boolean) 


295
296
297
298
299
300
301
302
303
 
# File 'lib/vanity/experiment/ab_test.rb', line 295

def showing?(alternative)
  identity = identity()
  if @playground.collecting?
    (connection.ab_showing(@id, identity) || alternative_for(identity)) == alternative.id
  else
    @showing ||= {}
    @showing[identity] == alternative.id
  end
end

#test_percent(pct) ⇒ Object

Raises:

  • (RuntimeError) 


325
326
327
328
 
# File 'lib/vanity/experiment/ab_test.rb', line 325

def test_percent(pct)
  raise RuntimeError, "Test percent must be an integer" unless pct.kind_of?(Integer)
  @test_pct = pct
end

#track!(metric_id, timestamp, count, *args) ⇒ Object

Called when tracking associated metric.



502
503
504
505
506
507
508
509
510
511
 
# File 'lib/vanity/experiment/ab_test.rb', line 502

def track!(metric_id, timestamp, count, *args)
  return unless active?
  identity = identity() rescue nil
  if identity
    return if connection.ab_showing(@id, identity)
    index = alternative_for(identity)
    connection.ab_add_conversion @id, index, identity, count
    check_completion!
  end
end

#values(data) ⇒ Object

Call this method to add complex values for alternatives, and use alternative values as keys to access them.

Examples:

Define A/B test with complex values

ab_test "Option type" do
  metrics :coolness
  alternatives "a", "b", "c"
  values "a" => {:some => :data},
    "b" => {:more => :data}, "c" => {:other => :data}
end

Raises:

  • (ArgumentError) 


171
172
173
174
 
# File 'lib/vanity/experiment/ab_test.rb', line 171

def values(data)
  raise ArgumentError, "values should respond to []" unless data.respond_to?(:[])
  @values = data.clone
end