Class: Split::Experiment
- Inherits:
-
Object
- Object
- Split::Experiment
- Defined in:
- lib/split/experiment.rb
Constant Summary collapse
- DEFAULT_OPTIONS =
{ :resettable => true }
Instance Attribute Summary collapse
- #algorithm ⇒ Object
-
#alternative_probabilities ⇒ Object
Returns the value of attribute alternative_probabilities.
-
#alternatives ⇒ Object
Returns the value of attribute alternatives.
-
#goals ⇒ Object
Returns the value of attribute goals.
-
#metadata ⇒ Object
Returns the value of attribute metadata.
-
#name ⇒ Object
Returns the value of attribute name.
-
#resettable ⇒ Object
Returns the value of attribute resettable.
Class Method Summary collapse
Instance Method Summary collapse
- #==(obj) ⇒ Object
- #[](name) ⇒ Object
- #calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object
- #calc_beta_params(goal = nil) ⇒ Object
- #calc_simulated_conversion_rates(beta_params) ⇒ Object
- #calc_time ⇒ Object
- #calc_time=(time) ⇒ Object
- #calc_winning_alternatives ⇒ Object
- #control ⇒ Object
- #count_simulated_wins(winning_alternatives) ⇒ Object
- #delete ⇒ Object
- #delete_metadata ⇒ Object
- #estimate_winning_alternative(goal = nil) ⇒ Object
- #extract_alternatives_from_options(options) ⇒ Object
- #find_simulated_winner(simulated_cr_hash) ⇒ Object
- #finished_key ⇒ Object
- #goals_key ⇒ Object
- #has_winner? ⇒ Boolean
- #increment_version ⇒ Object
-
#initialize(name, options = {}) ⇒ Experiment
constructor
A new instance of Experiment.
- #jstring(goal = nil) ⇒ Object
- #key ⇒ Object
- #load_from_redis ⇒ Object
- #metadata_key ⇒ Object
- #new_record? ⇒ Boolean
- #next_alternative ⇒ Object
- #participant_count ⇒ Object
- #random_alternative ⇒ Object
- #reset ⇒ Object
- #reset_winner ⇒ Object
- #resettable? ⇒ Boolean
- #save ⇒ Object
- #set_alternatives_and_options(options) ⇒ Object
- #start ⇒ Object
- #start_time ⇒ Object
- #validate! ⇒ Object
- #version ⇒ Object
- #winner ⇒ Object
- #winner=(winner_name) ⇒ Object
- #write_to_alternatives(goal = nil) ⇒ Object
Constructor Details
#initialize(name, options = {}) ⇒ Experiment
Returns a new instance of Experiment.
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
# File 'lib/split/experiment.rb', line 16 def initialize(name, = {}) = DEFAULT_OPTIONS.merge() @name = name.to_s alternatives = () if alternatives.empty? && (exp_config = Split.configuration.experiment_for(name)) = { alternatives: load_alternatives_from_configuration, goals: Split::GoalsCollection.new(@name).load_from_configuration, metadata: , resettable: exp_config[:resettable], algorithm: exp_config[:algorithm] } else [:alternatives] = alternatives end () end |
Instance Attribute Details
#algorithm ⇒ Object
115 116 117 |
# File 'lib/split/experiment.rb', line 115 def algorithm @algorithm ||= Split.configuration.algorithm end |
#alternative_probabilities ⇒ Object
Returns the value of attribute alternative_probabilities.
9 10 11 |
# File 'lib/split/experiment.rb', line 9 def alternative_probabilities @alternative_probabilities end |
#alternatives ⇒ Object
Returns the value of attribute alternatives.
8 9 10 |
# File 'lib/split/experiment.rb', line 8 def alternatives @alternatives end |
#goals ⇒ Object
Returns the value of attribute goals.
7 8 9 |
# File 'lib/split/experiment.rb', line 7 def goals @goals end |
#metadata ⇒ Object
Returns the value of attribute metadata.
10 11 12 |
# File 'lib/split/experiment.rb', line 10 def @metadata end |
#name ⇒ Object
Returns the value of attribute name.
4 5 6 |
# File 'lib/split/experiment.rb', line 4 def name @name end |
#resettable ⇒ Object
Returns the value of attribute resettable.
6 7 8 |
# File 'lib/split/experiment.rb', line 6 def resettable @resettable end |
Class Method Details
.finished_key(key) ⇒ Object
38 39 40 |
# File 'lib/split/experiment.rb', line 38 def self.finished_key(key) "#{key}:finished" end |
Instance Method Details
#==(obj) ⇒ Object
107 108 109 |
# File 'lib/split/experiment.rb', line 107 def ==(obj) self.name == obj.name end |
#[](name) ⇒ Object
111 112 113 |
# File 'lib/split/experiment.rb', line 111 def [](name) alternatives.find{|a| a.name == name} end |
#calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object
314 315 316 317 318 319 320 |
# File 'lib/split/experiment.rb', line 314 def calc_alternative_probabilities(winning_counts, number_of_simulations) alternative_probabilities = {} winning_counts.each do |alternative, wins| alternative_probabilities[alternative] = wins / number_of_simulations.to_f end return alternative_probabilities end |
#calc_beta_params(goal = nil) ⇒ Object
365 366 367 368 369 370 371 372 373 374 375 376 377 |
# File 'lib/split/experiment.rb', line 365 def calc_beta_params(goal = nil) beta_params = {} alternatives.each do |alternative| conversions = goal.nil? ? alternative.completed_count : alternative.completed_count(goal) alpha = 1 + conversions beta = 1 + alternative.participant_count - conversions params = [alpha, beta] beta_params[alternative] = params end return beta_params end |
#calc_simulated_conversion_rates(beta_params) ⇒ Object
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 |
# File 'lib/split/experiment.rb', line 347 def calc_simulated_conversion_rates(beta_params) # initialize a random variable (from which to simulate conversion rates ~beta-distributed) rand = SimpleRandom.new rand.set_seed simulated_cr_hash = {} # create a hash which has the conversion rate pulled from each alternative's beta distribution beta_params.each do |alternative, params| alpha = params[0] beta = params[1] simulated_conversion_rate = rand.beta(alpha, beta) simulated_cr_hash[alternative] = simulated_conversion_rate end return simulated_cr_hash end |
#calc_time ⇒ Object
383 384 385 |
# File 'lib/split/experiment.rb', line 383 def calc_time redis.hget(experiment_config_key, :calc_time).to_i end |
#calc_time=(time) ⇒ Object
379 380 381 |
# File 'lib/split/experiment.rb', line 379 def calc_time=(time) redis.hset(experiment_config_key, :calc_time, time) end |
#calc_winning_alternatives ⇒ Object
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 |
# File 'lib/split/experiment.rb', line 264 def calc_winning_alternatives # Super simple cache so that we only recalculate winning alternatives once per day days_since_epoch = Time.now.utc.to_i / 86400 if self.calc_time != days_since_epoch if goals.empty? self.estimate_winning_alternative else goals.each do |goal| self.estimate_winning_alternative(goal) end end self.calc_time = days_since_epoch self.save end end |
#control ⇒ Object
158 159 160 |
# File 'lib/split/experiment.rb', line 158 def control alternatives.first end |
#count_simulated_wins(winning_alternatives) ⇒ Object
322 323 324 325 326 327 328 329 330 331 332 333 |
# File 'lib/split/experiment.rb', line 322 def count_simulated_wins(winning_alternatives) # initialize a hash to keep track of winning alternative in simulations winning_counts = {} alternatives.each do |alternative| winning_counts[alternative] = 0 end # count number of times each alternative won, calculate probabilities, place in hash winning_alternatives.each do |alternative| winning_counts[alternative] += 1 end return winning_counts end |
#delete ⇒ Object
234 235 236 237 238 239 240 241 242 243 244 |
# File 'lib/split/experiment.rb', line 234 def delete Split.configuration.on_before_experiment_delete.call(self) if Split.configuration.start_manually redis.hdel(:experiment_start_times, @name) end reset_winner redis.srem(:experiments, name) remove_experiment_configuration Split.configuration.on_experiment_delete.call(self) increment_version end |
#delete_metadata ⇒ Object
246 247 248 |
# File 'lib/split/experiment.rb', line 246 def redis.del() end |
#estimate_winning_alternative(goal = nil) ⇒ Object
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 |
# File 'lib/split/experiment.rb', line 283 def estimate_winning_alternative(goal = nil) # initialize a hash of beta distributions based on the alternatives' conversion rates beta_params = calc_beta_params(goal) winning_alternatives = [] Split.configuration.beta_probability_simulations.times do # calculate simulated conversion rates from the beta distributions simulated_cr_hash = calc_simulated_conversion_rates(beta_params) winning_alternative = find_simulated_winner(simulated_cr_hash) # push the winning pair to the winning_alternatives array winning_alternatives.push(winning_alternative) end winning_counts = count_simulated_wins(winning_alternatives) @alternative_probabilities = calc_alternative_probabilities(winning_counts, Split.configuration.beta_probability_simulations) write_to_alternatives(goal) self.save end |
#extract_alternatives_from_options(options) ⇒ Object
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
# File 'lib/split/experiment.rb', line 50 def () alts = [:alternatives] || [] if alts.length == 1 if alts[0].is_a? Hash alts = alts[0].map{|k,v| {k => v} } end end if alts.empty? exp_config = Split.configuration.experiment_for(name) if exp_config alts = load_alternatives_from_configuration [:goals] = Split::GoalsCollection.new(@name).load_from_configuration [:metadata] = [:resettable] = exp_config[:resettable] [:algorithm] = exp_config[:algorithm] end end [:alternatives] = alts () # calculate probability that each alternative is the winner @alternative_probabilities = {} alts end |
#find_simulated_winner(simulated_cr_hash) ⇒ Object
335 336 337 338 339 340 341 342 343 344 345 |
# File 'lib/split/experiment.rb', line 335 def find_simulated_winner(simulated_cr_hash) # figure out which alternative had the highest simulated conversion rate winning_pair = ["",0.0] simulated_cr_hash.each do |alternative, rate| if rate > winning_pair[1] winning_pair = [alternative, rate] end end winner = winning_pair[0] return winner end |
#finished_key ⇒ Object
214 215 216 |
# File 'lib/split/experiment.rb', line 214 def finished_key self.class.finished_key(key) end |
#goals_key ⇒ Object
210 211 212 |
# File 'lib/split/experiment.rb', line 210 def goals_key "#{name}:goals" end |
#has_winner? ⇒ Boolean
146 147 148 |
# File 'lib/split/experiment.rb', line 146 def has_winner? !winner.nil? end |
#increment_version ⇒ Object
198 199 200 |
# File 'lib/split/experiment.rb', line 198 def increment_version @version = redis.incr("#{name}:version") end |
#jstring(goal = nil) ⇒ Object
387 388 389 390 391 392 393 394 |
# File 'lib/split/experiment.rb', line 387 def jstring(goal = nil) js_id = if goal.nil? name else name + "-" + goal end js_id.gsub('/', '--') end |
#key ⇒ Object
202 203 204 205 206 207 208 |
# File 'lib/split/experiment.rb', line 202 def key if version.to_i > 0 "#{name}:#{version}" else name end end |
#load_from_redis ⇒ Object
250 251 252 253 254 255 256 257 258 259 260 261 262 |
# File 'lib/split/experiment.rb', line 250 def load_from_redis exp_config = redis.hgetall(experiment_config_key) = { resettable: exp_config['resettable'], algorithm: exp_config['algorithm'], alternatives: load_alternatives_from_redis, goals: Split::GoalsCollection.new(@name).load_from_redis, metadata: } () end |
#metadata_key ⇒ Object
218 219 220 |
# File 'lib/split/experiment.rb', line 218 def "#{name}:metadata" end |
#new_record? ⇒ Boolean
103 104 105 |
# File 'lib/split/experiment.rb', line 103 def new_record? !redis.exists(name) end |
#next_alternative ⇒ Object
182 183 184 |
# File 'lib/split/experiment.rb', line 182 def next_alternative winner || random_alternative end |
#participant_count ⇒ Object
154 155 156 |
# File 'lib/split/experiment.rb', line 154 def participant_count alternatives.inject(0){|sum,a| sum + a.participant_count} end |
#random_alternative ⇒ Object
186 187 188 189 190 191 192 |
# File 'lib/split/experiment.rb', line 186 def random_alternative if alternatives.length > 1 algorithm.choose_alternative(self) else alternatives.first end end |
#reset ⇒ Object
226 227 228 229 230 231 232 |
# File 'lib/split/experiment.rb', line 226 def reset Split.configuration.on_before_experiment_reset.call(self) alternatives.each(&:reset) reset_winner Split.configuration.on_experiment_reset.call(self) increment_version end |
#reset_winner ⇒ Object
162 163 164 |
# File 'lib/split/experiment.rb', line 162 def reset_winner redis.hdel(:experiment_winner, name) end |
#resettable? ⇒ Boolean
222 223 224 |
# File 'lib/split/experiment.rb', line 222 def resettable? resettable end |
#save ⇒ Object
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
# File 'lib/split/experiment.rb', line 79 def save validate! if new_record? start unless Split.configuration.start_manually elsif experiment_configuration_has_changed? reset unless Split.configuration.reset_manually end persist_experiment_configuration if new_record? || experiment_configuration_has_changed? redis.hset(experiment_config_key, :resettable, resettable) redis.hset(experiment_config_key, :algorithm, algorithm.to_s) self end |
#set_alternatives_and_options(options) ⇒ Object
42 43 44 45 46 47 48 |
# File 'lib/split/experiment.rb', line 42 def () self.alternatives = [:alternatives] self.goals = [:goals] self.resettable = [:resettable] self.algorithm = [:algorithm] self. = [:metadata] end |
#start ⇒ Object
166 167 168 |
# File 'lib/split/experiment.rb', line 166 def start redis.hset(:experiment_start_times, @name, Time.now.to_i) end |
#start_time ⇒ Object
170 171 172 173 174 175 176 177 178 179 180 |
# File 'lib/split/experiment.rb', line 170 def start_time t = redis.hget(:experiment_start_times, @name) if t # Check if stored time is an integer if t =~ /^[-+]?[0-9]+$/ Time.at(t.to_i) else Time.parse(t) end end end |
#validate! ⇒ Object
95 96 97 98 99 100 101 |
# File 'lib/split/experiment.rb', line 95 def validate! if @alternatives.empty? && Split.configuration.experiment_for(@name).nil? raise ExperimentNotFound.new("Experiment #{@name} not found") end @alternatives.each {|a| a.validate! } goals_collection.validate! end |
#version ⇒ Object
194 195 196 |
# File 'lib/split/experiment.rb', line 194 def version @version ||= (redis.get("#{name}:version").to_i || 0) end |
#winner ⇒ Object
137 138 139 140 141 142 143 144 |
# File 'lib/split/experiment.rb', line 137 def winner experiment_winner = redis.hget(:experiment_winner, name) if experiment_winner Split::Alternative.new(experiment_winner, name) else nil end end |
#winner=(winner_name) ⇒ Object
150 151 152 |
# File 'lib/split/experiment.rb', line 150 def winner=(winner_name) redis.hset(:experiment_winner, name, winner_name.to_s) end |
#write_to_alternatives(goal = nil) ⇒ Object
308 309 310 311 312 |
# File 'lib/split/experiment.rb', line 308 def write_to_alternatives(goal = nil) alternatives.each do |alternative| alternative.set_p_winner(@alternative_probabilities[alternative], goal) end end |