Class: MongoOplogBackup::Backup

Inherits:
Object
  • Object
show all
Defined in:
lib/mongo_oplog_backup/backup.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config, backup_name = nil) ⇒ Backup

Returns a new instance of Backup.



21
22
23
24
25
26
27
28
29
30
 
# File 'lib/mongo_oplog_backup/backup.rb', line 21

def initialize(config, backup_name=nil)
  @config = config
  @backup_name = backup_name
  if backup_name.nil?
    state_file = config.global_state_file
    state = JSON.parse(File.read(state_file)) rescue nil
    state ||= {}
    @backup_name = state['backup']
  end
end

Instance Attribute Details

#backup_name ⇒ Object (readonly)

Returns the value of attribute backup_name.



10
11
12
 
# File 'lib/mongo_oplog_backup/backup.rb', line 10

def backup_name
  @backup_name
end

#config ⇒ Object (readonly)

Returns the value of attribute config.



10
11
12
 
# File 'lib/mongo_oplog_backup/backup.rb', line 10

def config
  @config
end

Instance Method Details

#backup_folder ⇒ Object 



12
13
14
15
 
# File 'lib/mongo_oplog_backup/backup.rb', line 12

def backup_folder
  return nil unless backup_name
  File.join(config.backup_dir, backup_name)
end

#backup_full ⇒ Object 



127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/mongo_oplog_backup/backup.rb', line 127

def backup_full
  position = latest_oplog_timestamp
  raise "Cannot backup with empty oplog" if position.nil?
  @backup_name = "backup-#{position}"
  if File.exists? backup_folder
    raise "Backup folder '#{backup_folder}' already exists; not performing backup."
  end
  dump_folder = File.join(backup_folder, 'dump')
  dump_args = ['--out', dump_folder]
  dump_args << '--gzip' if config.use_compression?
  result = config.mongodump(dump_args)
  unless File.directory? dump_folder
    MongoOplogBackup.log.error 'Backup folder does not exist'
    raise 'Full backup failed'
  end

  File.write(File.join(dump_folder, 'debug.log'), result.standard_output)

  unless result.standard_error.length == 0
    File.write(File.join(dump_folder, 'error.log'), result.standard_error)
  end

  write_state({
    'position' => position
  })

  return {
    position: position,
    backup: backup_name
  }
end

#backup_oplog(options = {}) ⇒ Object

Raises:

  • (ArgumentError) 


47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/mongo_oplog_backup/backup.rb', line 47

def backup_oplog(options={})
  raise ArgumentError, "No state in #{backup_name}" unless File.exists? state_file

  backup_state = JSON.parse(File.read(state_file))
  start_at = options[:start] || BSON::Timestamp.from_json(backup_state['position'])
  raise ArgumentError, ":start is required" unless start_at

  query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]

  dump_args = ['--out', config.oplog_dump_folder, '--db', 'local', '--collection', 'oplog.rs']
  dump_args += query
  dump_args << '--gzip' if config.use_compression?
  config.mongodump(dump_args)

  unless File.exists? config.oplog_dump
    raise "mongodump failed"
  end
  MongoOplogBackup.log.debug "Checking timestamps..."
  timestamps = Oplog.oplog_timestamps(config.oplog_dump)

  unless timestamps.increasing?
    raise "Something went wrong - oplog is not ordered."
  end

  first = timestamps[0]
  last = timestamps[-1]

  if first > start_at
    raise "Expected first oplog entry to be #{start_at.inspect} but was #{first.inspect}\n" +
      "The oplog is probably too small.\n" +
      "Increase the oplog size, the start with another full backup."
  elsif first < start_at
    raise "Expected first oplog entry to be #{start_at.inspect} but was #{first.inspect}\n" +
      "Something went wrong in our query."
  end

  result = {
    entries: timestamps.count,
    first: first,
    position: last
  }

  if timestamps.count == 1
    result[:empty] = true
  else
    outfile = "oplog-#{first}-#{last}.bson"
    outfile += '.gz' if config.use_compression?
    full_path = File.join(backup_folder, outfile)
    FileUtils.mkdir_p backup_folder
    FileUtils.mv config.oplog_dump, full_path

    write_state({
      'position' => result[:position]
    })
    result[:file] = full_path
    result[:empty] = false
  end

  FileUtils.rm_r config.oplog_dump_folder rescue nil
  result
end

#latest_oplog_timestamp ⇒ Object 



115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/mongo_oplog_backup/backup.rb', line 115

def latest_oplog_timestamp
  script = File.expand_path('../../oplog-last-timestamp.js', File.dirname(__FILE__))
  result_text = config.mongo('admin', script).standard_output
  begin
    response = JSON.parse(strip_warnings_which_should_be_in_stderr_anyway(result_text))
    return nil unless response['position']
    BSON::Timestamp.from_json(response['position'])
  rescue JSON::ParserError => e
    raise StandardError, "Failed to connect to MongoDB: #{result_text}"
  end
end

#latest_oplog_timestamp_moped ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/mongo_oplog_backup/backup.rb', line 196

def latest_oplog_timestamp_moped
  # Alternative implementation for `latest_oplog_timestamp`
  require 'moped'
  session = Moped::Session.new([ "127.0.0.1:27017" ])
  session.use 'local'
  oplog = session['oplog.rs']
  entry = oplog.find.limit(1).sort('$natural' => -1).one
  if entry
    entry['ts']
  else
    nil
  end
end

#lock(lockname, &block) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
 
# File 'lib/mongo_oplog_backup/backup.rb', line 36

def lock(lockname, &block)
  File.open(lockname, File::RDWR|File::CREAT, 0644) do |file|
    # Get a non-blocking lock
    got_lock = file.flock(File::LOCK_EX|File::LOCK_NB)
    if got_lock == false
      raise LockError, "Failed to acquire lock - another backup may be busy"
    end
    yield
  end
end

#perform(mode = :auto, options = {}) ⇒ Object 



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
 
# File 'lib/mongo_oplog_backup/backup.rb', line 159

def perform(mode=:auto, options={})
  FileUtils.mkdir_p config.backup_dir
  have_backup = backup_folder != nil

  if mode == :auto
    if have_backup
      mode = :oplog
    else
      mode = :full
    end
  end

  if mode == :oplog
    raise "Unknown backup position - cannot perform oplog backup. Have you completed a full backup?" unless have_backup
    MongoOplogBackup.log.info "Performing incremental oplog backup"
    lock(File.join(backup_folder, 'backup.lock')) do
      result = backup_oplog
      unless result[:empty]
        new_entries = result[:entries] - 1
        MongoOplogBackup.log.info "Backed up #{new_entries} new entries to #{result[:file]}"
      else
        MongoOplogBackup.log.info "Nothing new to backup"
      end
    end
  elsif mode == :full
    lock(config.global_lock_file) do
      MongoOplogBackup.log.info "Performing full backup"
      result = backup_full
      File.write(config.global_state_file, {
        'backup' => result[:backup]
      }.to_json)
      MongoOplogBackup.log.info "Performed full backup"
    end
    perform(:oplog, options)
  end
end

#state_file ⇒ Object 



17
18
19
 
# File 'lib/mongo_oplog_backup/backup.rb', line 17

def state_file
  File.join(backup_folder, 'state.json')
end

#strip_warnings_which_should_be_in_stderr_anyway(data) ⇒ Object

Because jira.mongodb.org/browse/SERVER-18643 Mongo shell warns (in stdout) about self-signed certs, regardless of ‘allowInvalidCertificates’ option.



111
112
113
 
# File 'lib/mongo_oplog_backup/backup.rb', line 111

def strip_warnings_which_should_be_in_stderr_anyway data
  data.gsub(/^.*[thread\d.*].* certificate.*$/,'')
end

#write_state(state) ⇒ Object 



32
33
34
 
# File 'lib/mongo_oplog_backup/backup.rb', line 32

def write_state(state)
  File.write(state_file, state.to_json)
end