Class: MoSQL::Streamer
- Inherits:
-
Object
- Object
- MoSQL::Streamer
- Includes:
- Logging
- Defined in:
- lib/mosql/streamer.rb
Constant Summary collapse
- BATCH =
1000
- NEW_KEYS =
[:options, :tailer, :mongo, :sql, :schema]
Instance Attribute Summary collapse
-
#options ⇒ Object
readonly
Returns the value of attribute options.
-
#tailer ⇒ Object
readonly
Returns the value of attribute tailer.
Instance Method Summary collapse
- #bulk_upsert(table, ns, items) ⇒ Object
- #collection_for_ns(ns) ⇒ Object
- #did_truncate ⇒ Object
- #handle_op(op) ⇒ Object
- #import ⇒ Object
- #import_collection(ns, collection, filter) ⇒ Object
- #initial_import ⇒ Object
-
#initialize(opts) ⇒ Streamer
constructor
A new instance of Streamer.
- #optail ⇒ Object
- #stop ⇒ Object
- #sync_object(ns, _id) ⇒ Object
- #track_time ⇒ Object
- #unsafe_handle_exceptions(ns, obj) ⇒ Object
- #with_retries(tries = 10) ⇒ Object
Methods included from Logging
Constructor Details
#initialize(opts) ⇒ Streamer
Returns a new instance of Streamer.
11 12 13 14 15 16 17 18 19 20 |
# File 'lib/mosql/streamer.rb', line 11 def initialize(opts) NEW_KEYS.each do |parm| unless opts.key?(parm) raise ArgumentError.new("Required argument `#{parm}' not provided to #{self.class.name}#new.") end instance_variable_set(:"@#{parm.to_s}", opts[parm]) end @done = false end |
Instance Attribute Details
#options ⇒ Object (readonly)
Returns the value of attribute options.
7 8 9 |
# File 'lib/mosql/streamer.rb', line 7 def @options end |
#tailer ⇒ Object (readonly)
Returns the value of attribute tailer.
7 8 9 |
# File 'lib/mosql/streamer.rb', line 7 def tailer @tailer end |
Instance Method Details
#bulk_upsert(table, ns, items) ⇒ Object
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
# File 'lib/mosql/streamer.rb', line 51 def bulk_upsert(table, ns, items) begin @schema.copy_data(table.db, ns, items) rescue Sequel::DatabaseError => e log.debug("Bulk insert error (#{e}), attempting invidual upserts...") cols = @schema.all_columns(@schema.find_ns(ns)) items.each do |it| h = {} cols.zip(it).each { |k,v| h[k] = v } unsafe_handle_exceptions(ns, h) do @sql.upsert!(table, @schema.primary_sql_key_for_ns(ns), h) end end end end |
#collection_for_ns(ns) ⇒ Object
32 33 34 35 |
# File 'lib/mosql/streamer.rb', line 32 def collection_for_ns(ns) dbname, collection = ns.split(".", 2) @mongo.db(dbname).collection(collection) end |
#did_truncate ⇒ Object
128 |
# File 'lib/mosql/streamer.rb', line 128 def did_truncate; @did_truncate ||= {}; end |
#handle_op(op) ⇒ Object
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 |
# File 'lib/mosql/streamer.rb', line 190 def handle_op(op) log.debug("processing op: #{op.inspect}") unless op['ns'] && op['op'] log.warn("Weird op: #{op.inspect}") return end unless @schema.find_ns(op['ns']) log.debug("Skipping op for unknown ns #{op['ns']}...") return end ns = op['ns'] dbname, collection_name = ns.split(".", 2) case op['op'] when 'n' log.debug("Skipping no-op #{op.inspect}") when 'i' if collection_name == 'system.indexes' log.info("Skipping index update: #{op.inspect}") else unsafe_handle_exceptions(ns, op['o']) do @sql.upsert_ns(ns, op['o']) end end when 'u' selector = op['o2'] update = op['o'] if update.keys.any? { |k| k.start_with? '$' } log.debug("resync #{ns}: #{selector['_id']} (update was: #{update.inspect})") sync_object(ns, selector['_id']) else log.debug("upsert #{ns}: _id=#{selector['_id']}") # The update operation replaces the existing object, but # preserves its _id field, so grab the _id off of the # 'query' field -- it's not guaranteed to be present on the # update. update = { '_id' => selector['_id'] }.merge(update) unsafe_handle_exceptions(ns, update) do @sql.upsert_ns(ns, update) end end when 'd' if [:ignore_delete] log.debug("Ignoring delete op on #{ns} as instructed.") else @sql.delete_ns(ns, op['o']) end else log.info("Skipping unknown op #{op.inspect}") end end |
#import ⇒ Object
26 27 28 29 30 |
# File 'lib/mosql/streamer.rb', line 26 def import if [:reimport] || tailer..seconds == 0 initial_import end end |
#import_collection(ns, collection, filter) ⇒ Object
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 |
# File 'lib/mosql/streamer.rb', line 130 def import_collection(ns, collection, filter) log.info("Importing for #{ns}...") count = 0 batch = [] table = @sql.table_for_ns(ns) unless [:no_drop_tables] || did_truncate[table.first_source] table.truncate did_truncate[table.first_source] = true end start = Time.now sql_time = 0 collection.find(filter, :batch_size => BATCH) do |cursor| with_retries do cursor.each do |obj| batch << @schema.transform(ns, obj) count += 1 if batch.length >= BATCH sql_time += track_time do bulk_upsert(table, ns, batch) end elapsed = Time.now - start log.info("Imported #{count} rows (#{elapsed}s, #{sql_time}s SQL)...") batch.clear exit(0) if @done end end end end unless batch.empty? bulk_upsert(table, ns, batch) end end |
#initial_import ⇒ Object
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
# File 'lib/mosql/streamer.rb', line 89 def initial_import @schema.create_schema(@sql.db, ![:no_drop_tables]) unless [:skip_tail] start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts'] end dbnames = [] if dbname = [:dbname] log.info "Skipping DB scan and using db: #{dbname}" dbnames = [ dbname ] else dbnames = @mongo.database_names end dbnames.each do |dbname| spec = @schema.find_db(dbname) if(spec.nil?) log.info("Mongd DB '#{dbname}' not found in config file. Skipping.") next end log.info("Importing for Mongo DB #{dbname}...") db = @mongo.db(dbname) collections = db.collections.select { |c| spec.key?(c.name) } collections.each do |collection| ns = "#{dbname}.#{collection.name}" import_collection(ns, collection, spec[collection.name][:meta][:filter]) exit(0) if @done end end tailer.(start_ts) unless [:skip_tail] end |
#optail ⇒ Object
166 167 168 169 170 171 172 173 174 175 |
# File 'lib/mosql/streamer.rb', line 166 def optail tailer.tail_from([:tail_from] ? BSON::Timestamp.new([:tail_from].to_i, 0) : nil) until @done tailer.stream(1000) do |op| handle_op(op) end end end |
#stop ⇒ Object
22 23 24 |
# File 'lib/mosql/streamer.rb', line 22 def stop @done = true end |
#sync_object(ns, _id) ⇒ Object
177 178 179 180 181 182 183 184 185 186 187 188 |
# File 'lib/mosql/streamer.rb', line 177 def sync_object(ns, _id) primary_sql_key = @schema.primary_sql_key_for_ns(ns) sqlid = @sql.transform_one_ns(ns, { '_id' => _id })[primary_sql_key] obj = collection_for_ns(ns).find_one({:_id => _id}) if obj unsafe_handle_exceptions(ns, obj) do @sql.upsert_ns(ns, obj) end else @sql.table_for_ns(ns).where(primary_sql_key.to_sym => sqlid).delete() end end |
#track_time ⇒ Object
83 84 85 86 87 |
# File 'lib/mosql/streamer.rb', line 83 def track_time start = Time.now yield Time.now - start end |
#unsafe_handle_exceptions(ns, obj) ⇒ Object
37 38 39 40 41 42 43 44 45 46 47 48 49 |
# File 'lib/mosql/streamer.rb', line 37 def unsafe_handle_exceptions(ns, obj) begin yield rescue Sequel::DatabaseError => e wrapped = e.wrapped_exception if wrapped.result && [:unsafe] log.warn("Ignoring row (#{obj.inspect}): #{e}") else log.error("Error processing #{obj.inspect} for #{ns}.") raise e end end end |
#with_retries(tries = 10) ⇒ Object
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
# File 'lib/mosql/streamer.rb', line 67 def with_retries(tries=10) tries.times do |try| begin yield rescue Mongo::ConnectionError, Mongo::ConnectionFailure, Mongo::OperationFailure => e # Duplicate key error raise if e.kind_of?(Mongo::OperationFailure) && [11000, 11001].include?(e.error_code) # Cursor timeout raise if e.kind_of?(Mongo::OperationFailure) && e. =~ /^Query response returned CURSOR_NOT_FOUND/ delay = 0.5 * (1.5 ** try) log.warn("Mongo exception: #{e}, sleeping #{delay}s...") sleep(delay) end end end |