Class: CorrectHorseBatteryStaple::Corpus::Redis2

Inherits:
Redis show all
Defined in:
lib/correct_horse_battery_staple/corpus/redis2.rb

Constant Summary collapse

MAX_ITERATIONS = 
1000

Instance Attribute Summary

Attributes inherited from Redis

#dest, #options

Attributes inherited from Base

#frequency_mean, #frequency_stddev, #original_size, #probability_mean, #probability_stddev, #weighted_size

Instance Method Summary collapse

Methods inherited from Redis

#close, #corpus_length_range, #count_all, #discontiguous_range_map, #each, #entries, #get_word_ids_in_zset, #get_words_for_ids, #initialize, #intersection, #pick_drange, #pick_random_words, #pick_standard, #range_cover?, read, #sorted_entries, #zcount

Methods included from Backend::Redis

included

Methods inherited from Base

#candidates, #compose_filters, #count, #count_by_options, #count_candidates, #each, #entropy_per_word, #entropy_per_word_by_filter, #filter, #filter_for_options, #frequencies, #initialize, #inspect, #load_stats_from_hash, #precache, read, #recalculate, #reset, #result, #sorted_entries, #stats, #words

Methods included from Memoize

included

Methods included from CorrectHorseBatteryStaple::Common

#array_sample, #logger, #random_in_range, #random_number, #set_sample

Methods inherited from CorrectHorseBatteryStaple::Corpus

format_for, read

Constructor Details

This class inherits a constructor from CorrectHorseBatteryStaple::Corpus::Redis

Instance Method Details

#make_subset(spec) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
 
# File 'lib/correct_horse_battery_staple/corpus/redis2.rb', line 55

def make_subset(spec)
  key = gensym_temp
  source_key = spec.shift
  db.zunionstore(key, [source_key])
  db.expire(key, 180)
  spec.each do |(min, max)|
    db.zremrangebyscore(key, min, max)
  end
  key
end

#make_subset_spec(key, range) ⇒ Object 



51
52
53
 
# File 'lib/correct_horse_battery_staple/corpus/redis2.rb', line 51

def make_subset_spec(key, range)
  [key, ["-inf", "(#{range.begin}"], ["(#{range.end}", "inf"]]
end

#pick(count, options = {}) ⇒ Object

our own collection operations optimized pick implementations - they do NOT support :filter, though



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/correct_horse_battery_staple/corpus/redis2.rb', line 17

def pick(count, options = {})
  percentile_range = options[:percentile]
  length_range     = options[:word_length]
  tempkey = nil

  if percentile_range && percentile_range.begin == 0 && percentile_range.end == 100
    percentile_range = nil
  end

  pick_sset_random(@words_key, 4)

  if (!percentile_range && !length_range)
    get_words_for_ids(pick_random_words(count))
  else
    sets = []
    sets << make_subset_spec(@percentile_key, percentile_range) if percentile_range

    # this isn't correct because lenprod_key will have values in the range 18...19
    # sets << make_subset_spec(@lenprod_key, length_range)         if length_range
    if length_range
      sets << [@lenprod_key, ["-inf", "(#{length_range.begin}"],
                             ["#{length_range.end.floor + 1}", "inf"]]
    end

    # returns union set key
    tempkey = subset_and_union(sets)
    # STDERR.puts "result count in #{tempkey} is #{db.zcard(tempkey)}"

    get_words_for_ids(pick_sset_random(tempkey, count))
  end
ensure
  db.del tempkey if tempkey
end

#pick_sset_random(key, count) ⇒ Object 



79
80
81
82
83
84
85
86
87
 
# File 'lib/correct_horse_battery_staple/corpus/redis2.rb', line 79

def pick_sset_random(key, count)
  max = db.zcard(key)
  db.multi do
    count.times.map do
      rnd = random_number(max)
      db.zrange(key, rnd, rnd)
    end
  end.flatten
end

#size ⇒ Object 



8
9
10
 
# File 'lib/correct_horse_battery_staple/corpus/redis2.rb', line 8

def size
  @size ||= db.zcard(@percentile_key)
end

#subset_and_union(specs) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/correct_horse_battery_staple/corpus/redis2.rb', line 66

def subset_and_union(specs)
  result_key = gensym_temp
  db.multi do
    keys = specs.map do |spec|
      make_subset(spec)
    end
    db.zinterstore(result_key, keys)
    db.del(*keys)
  end
  db.expire(result_key, 1800)
  result_key
end