Module: Mongoid::FTS::Util
Instance Method Summary collapse
- #boolean_and(*strings) ⇒ Object
- #boolean_or(*strings) ⇒ Object
- #chars(string) ⇒ Object
- #connect! ⇒ Object
- #create_indexes ⇒ Object
- #destroy_all ⇒ Object
- #enable!(*args) ⇒ Object
- #find_in_batches(queries = {}) ⇒ Object
- #find_or_create(finder, creator) ⇒ Object
- #fts_models ⇒ Object
- #fuzzy(*args) ⇒ Object (also: #fuzzy_for)
- #index(*args, &block) ⇒ Object
- #index!(*args, &block) ⇒ Object
- #list_of_strings(*args) ⇒ Object
- #literals_for(*args) ⇒ Object
- #models ⇒ Object
- #ngrams_for(*args) ⇒ Object
- #normalized_array(*array) ⇒ Object
- #reset! ⇒ Object
- #session ⇒ Object
- #session=(session) ⇒ Object
- #setup!(*args) ⇒ Object
- #stems_for(*args, &block) ⇒ Object
- #stopword?(word) ⇒ Boolean
- #strip(word) ⇒ Object
- #terms_for(*args, &block) ⇒ Object
- #unidecode(string) ⇒ Object
- #unindex(*args, &block) ⇒ Object
- #unindex!(*args, &block) ⇒ Object
- #utf8ify(string) ⇒ Object
- #words_for(*args, &block) ⇒ Object
Instance Method Details
#boolean_and(*strings) ⇒ Object
307 308 309 310 |
# File 'lib/mongoid-fts/util.rb', line 307 def boolean_and(*strings) strings = Coerce.list_of_strings(*strings) strings.map{|s| '"%s"' % s.gsub('"', '')}.join(' ') end |
#boolean_or(*strings) ⇒ Object
312 313 314 315 |
# File 'lib/mongoid-fts/util.rb', line 312 def boolean_or(*strings) strings = Coerce.list_of_strings(*strings) strings.join(' ') end |
#chars(string) ⇒ Object
209 210 211 212 213 |
# File 'lib/mongoid-fts/util.rb', line 209 def chars(string) chars = [] UnicodeUtils.each_grapheme(string.to_s){|g| chars.push(g)} chars end |
#connect! ⇒ Object
301 302 303 304 305 |
# File 'lib/mongoid-fts/util.rb', line 301 def connect! Mongoid.configure do |config| config.connect_to('mongoid-fts') end end |
#create_indexes ⇒ Object
34 35 36 |
# File 'lib/mongoid-fts/util.rb', line 34 def create_indexes fts_models.each{|model| model.create_indexes} end |
#destroy_all ⇒ Object
38 39 40 |
# File 'lib/mongoid-fts/util.rb', line 38 def destroy_all fts_models.map{|model| model.destroy_all} end |
#enable!(*args) ⇒ Object
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 |
# File 'lib/mongoid-fts/util.rb', line 279 def enable!(*args) = Map.(args) unless .has_key?(:warn) [:warn] = true end begin session = Mongoid::Sessions.default session.with(database: :admin).command({ setParameter: 1, textSearchEnabled: true }) rescue Object => e unless e.is_a?(Mongoid::Errors::NoSessionsConfig) warn "failed to enable search with #{ e.class }(#{ e. })" end end end |
#find_in_batches(queries = {}) ⇒ Object
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
# File 'lib/mongoid-fts/util.rb', line 43 def find_in_batches(queries = {}) models = queries.map do |model_class, model_ids| unless model_class.is_a?(Class) model_class = eval(model_class.to_s) end model_ids = Array(model_ids) begin model_class.find(model_ids) rescue Mongoid::Errors::DocumentNotFound model_ids.map do |model_id| begin model_class.find(model_id) rescue Mongoid::Errors::DocumentNotFound nil end end end end models.flatten! models.compact! models end |
#find_or_create(finder, creator) ⇒ Object
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
# File 'lib/mongoid-fts/util.rb', line 70 def find_or_create(finder, creator) doc = finder.call() return doc if doc n, max = 0, 2 begin creator.call() rescue Object => e n += 1 raise if n > max sleep(rand(0.1)) finder.call() or retry end end |
#fts_models ⇒ Object
5 6 7 8 9 |
# File 'lib/mongoid-fts/util.rb', line 5 def fts_models [ Mongoid::FTS::Index ] end |
#fuzzy(*args) ⇒ Object Also known as: fuzzy_for
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 |
# File 'lib/mongoid-fts/util.rb', line 166 def fuzzy(*args) strings = Coerce.list_of_strings(args).map{|string| utf8ify(string)} list = [] strings.each do |string| list.push(*ngrams_for(string)) decoded = unidecode(string) unless decoded == string list.push(*ngrams_for(decoded)) end end list.uniq end |
#index(*args, &block) ⇒ Object
238 239 240 241 242 243 244 245 246 247 248 249 250 251 |
# File 'lib/mongoid-fts/util.rb', line 238 def index(*args, &block) if args.empty? and block.nil? Index else args.each do |arg| case arg when Class arg.all.each{|model| Index.add(model)} else Index.add(arg, &block) end end end end |
#index!(*args, &block) ⇒ Object
257 258 259 |
# File 'lib/mongoid-fts/util.rb', line 257 def index!(*args, &block) Index.add!(*args, &block) end |
#list_of_strings(*args) ⇒ Object
233 234 235 |
# File 'lib/mongoid-fts/util.rb', line 233 def list_of_strings(*args) args.flatten.compact.map{|arg| arg.to_s}.select{|arg| !arg.empty?}.uniq end |
#literals_for(*args) ⇒ Object
148 149 150 151 152 |
# File 'lib/mongoid-fts/util.rb', line 148 def literals_for(*args) words = FTS.normalized_array(args) return words.map{|word| "__#{ Digest::MD5.hexdigest(word) }__"} end |
#models ⇒ Object
266 267 268 |
# File 'lib/mongoid-fts/util.rb', line 266 def models @models ||= [] end |
#ngrams_for(*args) ⇒ Object
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 |
# File 'lib/mongoid-fts/util.rb', line 185 def ngrams_for(*args) = Map.(args) strings = Coerce.list_of_strings(args).map{|string| utf8ify(string)} list = [] sizes = [:sizes] || [2,3] strings.each do |string| chars = Util.chars('_' + string + '_') sizes.each do |size| (chars.size - (size - 1)).times do |i| ngram = chars[i, size].join list.push(ngram) end end end list end |
#normalized_array(*array) ⇒ Object
229 230 231 |
# File 'lib/mongoid-fts/util.rb', line 229 def normalized_array(*array) array.flatten.map{|_| _.to_s.strip}.select{|_| !_.empty?}.uniq end |
#reset! ⇒ Object
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
# File 'lib/mongoid-fts/util.rb', line 11 def reset! Mongoid::FTS.setup!(:warn => true) fts_models.each do |model| model.destroy_all begin model.collection.indexes.drop rescue Object => e end begin model.collection.drop rescue Object => e end begin model.create_indexes rescue Object => e end end end |
#session ⇒ Object
271 272 273 |
# File 'lib/mongoid-fts/util.rb', line 271 def session @session ||= Mongoid::Sessions.default end |
#session=(session) ⇒ Object
275 276 277 |
# File 'lib/mongoid-fts/util.rb', line 275 def session=(session) @session = session end |
#setup!(*args) ⇒ Object
296 297 298 299 |
# File 'lib/mongoid-fts/util.rb', line 296 def setup!(*args) enable!(*args) Index.setup! end |
#stems_for(*args, &block) ⇒ Object
140 141 142 143 144 145 146 |
# File 'lib/mongoid-fts/util.rb', line 140 def stems_for(*args, &block) = Map.(args) words = Coerce.list_of_strings(*args).map{|word| utf8ify(word)} Stemming.stem(*words) end |
#stopword?(word) ⇒ Boolean
154 155 156 157 |
# File 'lib/mongoid-fts/util.rb', line 154 def stopword?(word) word = utf8ify(word) word.empty? or Stemming::Stopwords.stopword?(word) end |
#strip(word) ⇒ Object
159 160 161 162 163 164 |
# File 'lib/mongoid-fts/util.rb', line 159 def strip(word) word = utf8ify(word) word.gsub!(/\A(?:[^\w]|_|\s)+/, '') # leading punctuation/spaces word.gsub!(/(?:[^\w]|_|\s+)+\Z/, '') # trailing punctuation/spaces word end |
#terms_for(*args, &block) ⇒ Object
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 |
# File 'lib/mongoid-fts/util.rb', line 87 def terms_for(*args, &block) = Map.(args) words = words_for(*args) list = [:list] || [] words.each do |word| word = word.downcase next if stopword?(word) stems = stems_for(word) stems.each do |stem| [stem, unidecode(stem)].uniq.each do |stem| next if stopword?(stem) block ? block.call(stem) : list.push(stem) substems = stem.split(/_/) if [:subterms] and substems.size > 1 substems.each do |substem| terms_for(substem.gsub(/_+/, '-'), :list => list) end end end end end list.uniq! block ? nil : list end |
#unidecode(string) ⇒ Object
215 216 217 |
# File 'lib/mongoid-fts/util.rb', line 215 def unidecode(string) Stringex::Unidecoder.decode(utf8ify(string.to_s)) end |
#unindex(*args, &block) ⇒ Object
253 254 255 |
# File 'lib/mongoid-fts/util.rb', line 253 def unindex(*args, &block) Index.remove(*args, &block) end |
#unindex!(*args, &block) ⇒ Object
261 262 263 |
# File 'lib/mongoid-fts/util.rb', line 261 def unindex!(*args, &block) Index.remove!(*args, &block) end |
#utf8ify(string) ⇒ Object
219 220 221 222 223 224 225 226 227 |
# File 'lib/mongoid-fts/util.rb', line 219 def utf8ify(string) UnicodeUtils.nfkd( begin string.force_encoding('UTF-8') rescue string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') end ) end |
#words_for(*args, &block) ⇒ Object
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
# File 'lib/mongoid-fts/util.rb', line 122 def words_for(*args, &block) = Map.(args) string = args.join(' ') list = [] UnicodeUtils.each_word(string) do |word| word = strip(utf8ify(word)) next if word.empty? block ? block.call(word) : list.push(word) end block ? nil : list end |