Class: SolrBatch

Inherits:
Object
  • Object
show all
Defined in:
lib/solr_batch.rb

Class Method Summary collapse

Class Method Details

.flag_record(model_name, model_id) ⇒ Object

Flags an individual record as pending synchronization with Solr


24
25
26
27
 
# File 'lib/solr_batch.rb', line 24

def SolrBatch.flag_record(model_name, model_id)
  sql = 'insert into solr_sync_records (model, model_id) values (?, ?)'
  ActiveRecord::Base.connection.execute(sanitize_sql_array([sql, model_name, model_id]))
end

.process_all ⇒ Object

This procedure should be run in the background with whatever job scheduler you are comfortable with


5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
 
# File 'lib/solr_batch.rb', line 5

def SolrBatch.process_all
  models = ActiveRecord::Base.connection.select_values("select distinct model from solr_sync_records")
  if models.size == 0
    RAILS_DEFAULT_LOGGER.debug("SolrBatch.process_all: No records pending synchronization")
  else
    first_klass = nil
    models.each do |model|
      klass = eval(model)
      first_klass = klass unless first_klass
      SolrBatch.sync_pending_records(klass, { :do_commit => false, :do_optimize => false })
    end

    ## Commit/optimize after ALL records have been processed
    first_klass.send(:solr_commit)
    first_klass.send(:solr_optimize)
  end
end

.requeue_all(ar_model) ⇒ Object

Marks all records for this model as pending synchronization with Solr


30
31
32
33
34
35
36
37
38
39
 
# File 'lib/solr_batch.rb', line 30

def SolrBatch.requeue_all(ar_model)
  sql = <<-endsql
      insert into solr_sync_records 
      (model, model_id) 
      select ?, id from #{ar_model.send(:table_name)} 
      EXCEPT 
      select ?, model_id from solr_sync_records where model = ?
  endsql
  ActiveRecord::Base.connection.execute(sanitize_sql_array([sql, ar_model, ar_model, ar_model]))
end

.sync_pending_records(ar_model, opts = {}, iteration = 1) ⇒ Object

Ex: from within ActiveRecord Model, call: SolrBatch.sync_pending_records(self, { :batch_size => 500 } );


43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/solr_batch.rb', line 43

def SolrBatch.sync_pending_records(ar_model, opts = {}, iteration=1)
  opts = { :batch_size => 500, :do_commit => true, :do_optimize => true }.merge(opts)
  batch_size = opts.delete(:batch_size)
  batch_size = 500 if batch_size < 1
  do_commit   = opts.delete(:do_commit)
  do_optimize = opts.delete(:do_optimize)
  raise "Invalid option keys: #{opts.keys}" unless opts.keys.size == 0

  RAILS_DEFAULT_LOGGER.debug("SolrBatch.sync_pending_records(#{ar_model})")
  items = get_pending_records(ar_model, batch_size)
  solr_log(ar_model, 'No records to process') and return if items.size == 0

  delete_batch = []
  add_batch = []
  processed = {}
  items.each do |o|
    solr_id = o.send(:solr_id)
    if !processed.has_key?(solr_id)  ## Duplicates are possible in the log table
      processed[solr_id] = true
      if o.solr_sync_record_deleted_p == 't'
        delete_batch << solr_id
      else
        add_batch << o.to_solr_doc
      end
    end
  end
  
  # We delete in bulk with a single query
  ActsAsSolr::Post.execute(Solr::Request::Delete.new(:query => id_query(delete_batch))) if delete_batch.size > 0
  ar_model.send(:solr_add, add_batch) if add_batch.size > 0
  SolrBatch.delete_from_queue(ar_model, items.last.solr_sync_records_log_id)

  solr_log(ar_model, "#{items.size} items have been batch added to index.")

  if (items.size >= batch_size && iteration < 10)
    ## More records of this class to process
    SolrBatch.sync_pending_records(ar_model, opts, iteration + 1)
  else 
    ar_model.send(:solr_commit)   if do_commit
    ar_model.send(:solr_optimize) if do_optimize
  end
end