Class: LSH::Storage::RedisBackend

Inherits:
Object
  • Object
show all
Defined in:
lib/lsh/storage/redis_backend.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 }, :data_dir => 'data' }) ⇒ RedisBackend

Returns a new instance of RedisBackend.



28
29
30
31
32
33
34
35
 
# File 'lib/lsh/storage/redis_backend.rb', line 28

def initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 }, :data_dir => 'data' })
  @redis = Redis.new(params[:redis])
  @data_dir = params[:data_dir]
  unless File.exists?(@data_dir)
    Dir.mkdir(@data_dir)
    Dir.mkdir(File.join(@data_dir, 'projections'))
  end
end

Instance Attribute Details

#data_dir ⇒ Object (readonly)

Returns the value of attribute data_dir.



26
27
28
 
# File 'lib/lsh/storage/redis_backend.rb', line 26

def data_dir
  @data_dir
end

#redis ⇒ Object (readonly)

Returns the value of attribute redis.



26
27
28
 
# File 'lib/lsh/storage/redis_backend.rb', line 26

def redis
  @redis
end

Instance Method Details

#add_vector_id(vector, id) ⇒ Object 



132
133
134
135
136
 
# File 'lib/lsh/storage/redis_backend.rb', line 132

def add_vector_id(vector, id)
  save_vector(vector) # Writing vector to disk if not already there
  @redis.set "lsh:vector_to_id:#{vector.hash}", id
  @redis.set "lsh:id_to_vector:#{id}", vector.hash.to_s
end

#add_vector_to_bucket(bucket, hash, vector) ⇒ Object 



127
128
129
130
 
# File 'lib/lsh/storage/redis_backend.rb', line 127

def add_vector_to_bucket(bucket, hash, vector)
  save_vector(vector) # Writing vector to disk if not already there
  @redis.sadd "#{bucket}:#{hash}", vector.hash.to_s # Only storing vector's hash in Redis
end

#clear_data! ⇒ Object 



42
43
44
45
46
 
# File 'lib/lsh/storage/redis_backend.rb', line 42

def clear_data!
  keys = @redis.keys("lsh:bucket:*")
  @redis.del(keys) unless keys.empty?
  delete_dat_files_in_dir(@data_dir)
end

#clear_projections! ⇒ Object 



48
49
50
51
52
 
# File 'lib/lsh/storage/redis_backend.rb', line 48

def clear_projections!
  @redis.del("lsh:parameters")
  @redis.del("lsh:buckets")
  delete_dat_files_in_dir(File.join(@data_dir, 'projections'))
end

#create_new_bucket ⇒ Object 



112
113
114
 
# File 'lib/lsh/storage/redis_backend.rb', line 112

def create_new_bucket
  @redis.incr "lsh:buckets"
end

#delete_dat_files_in_dir(dir) ⇒ Object 



54
55
56
 
# File 'lib/lsh/storage/redis_backend.rb', line 54

def delete_dat_files_in_dir(dir)
  Dir.foreach(dir) {|f| File.delete(File.join(dir, f)) if f != '.' and f != '..' and f.end_with?('.dat')}
end

#find_bucket(i) ⇒ Object 



147
148
149
 
# File 'lib/lsh/storage/redis_backend.rb', line 147

def find_bucket(i)
  "lsh:bucket:#{i}"
end

#has_index? ⇒ Boolean

Returns:

  • (Boolean) 


58
59
60
 
# File 'lib/lsh/storage/redis_backend.rb', line 58

def has_index?
  parameters and projections and number_of_buckets > 0
end

#id_to_vector(id) ⇒ Object 



142
143
144
145
 
# File 'lib/lsh/storage/redis_backend.rb', line 142

def id_to_vector(id)
  vector_hash = @redis.get "lsh:id_to_vector:#{id}"
  load_vector(vector_hash)
end

#load_vector(hash) ⇒ Object 



121
122
123
124
125
 
# File 'lib/lsh/storage/redis_backend.rb', line 121

def load_vector(hash)
  vector = MathUtil.zeros(parameters[:dim])
  vector.load(File.join(@data_dir, hash+'.dat'))
  vector
end

#number_of_buckets ⇒ Object 



62
63
64
 
# File 'lib/lsh/storage/redis_backend.rb', line 62

def number_of_buckets
  @redis.get("lsh:buckets").to_i || 0
end

#parameters ⇒ Object 



99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/lsh/storage/redis_backend.rb', line 99

def parameters
  begin
    @parms ||= (
      parms = JSON.parse(@redis.get "lsh:parameters")
      parms.keys.each { |k| parms[k.to_sym] = parms[k]; parms.delete(k) }
      parms[:window] = Float::INFINITY if parms[:window] == 'Infinity'
      parms
    )
  rescue TypeError
    nil
  end 
end

#parameters=(parms) ⇒ Object 



94
95
96
97
 
# File 'lib/lsh/storage/redis_backend.rb', line 94

def parameters=(parms)
  parms[:window] = 'Infinity' if parms[:window] == Float::INFINITY
  @redis.set "lsh:parameters", parms.to_json
end

#projections ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/lsh/storage/redis_backend.rb', line 77

def projections
  return unless parameters
  @projections ||= (
    projections = []
    parameters[:number_of_independent_projections].times do |i|
      vectors = []
      parameters[:number_of_random_vectors].times do |j|
        v = MathUtil.zeros(parameters[:dim])
        v.load(File.join(@data_dir, 'projections', "vector_#{i}_#{j}.dat"))
        vectors << v
      end
      projections << vectors
    end
    projections
  )
end

#projections=(projections) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
 
# File 'lib/lsh/storage/redis_backend.rb', line 66

def projections=(projections)
  # Saving the projections to disk
  # (too slow to serialize and store in Redis for
  # large number of dimensions/projections)
  projections.each_with_index do |projection, i|
    projection.each_with_index do |vector, j|
      vector.save(File.join(@data_dir, 'projections', "vector_#{i}_#{j}.dat"))
    end
  end
end

#query_buckets(hashes) ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/lsh/storage/redis_backend.rb', line 151

def query_buckets(hashes)
  vector_hashes = []
  hashes.each_with_index do |hash, i|
    bucket = find_bucket(i)
    results = @redis.smembers("#{bucket}:#{hash}")
    vector_hashes += results if results
  end
  # Making sure we don't load the same vectors twice if they match
  # in different random projections
  vector_hashes.uniq!
  results = []
  vector_hashes.each do |vector_hash|
    vector = load_vector(vector_hash)
    results << vector
  end
  results
end

#reset! ⇒ Object 



37
38
39
40
 
# File 'lib/lsh/storage/redis_backend.rb', line 37

def reset!
  clear_data!
  clear_projections!
end

#save_vector(vector) ⇒ Object 



116
117
118
119
 
# File 'lib/lsh/storage/redis_backend.rb', line 116

def save_vector(vector)
  path = File.join(@data_dir, vector.hash.to_s+'.dat')
  vector.save(path) unless File.exists?(path)
end

#vector_to_id(vector) ⇒ Object 



138
139
140
 
# File 'lib/lsh/storage/redis_backend.rb', line 138

def vector_to_id(vector)
  @redis.get "lsh:vector_to_id:#{vector.hash}"
end