Class: Flydata::FileUtil::SyncFileManager
- Inherits:
-
Object
- Object
- Flydata::FileUtil::SyncFileManager
- Defined in:
- lib/flydata/sync_file_manager.rb
Constant Summary collapse
- DUMP_DIR =
ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
- BACKUP_DIR =
ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
- TABLE_POSITIONS_DIR =
ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
Instance Method Summary collapse
- #backup_dump_dir ⇒ Object
- #binlog_path ⇒ Object
- #delete_table_binlog_pos(table_name) ⇒ Object
- #dump_file_path ⇒ Object
-
#dump_pos_path ⇒ Object
dump pos file for resume.
- #get_table_binlog_pos(table_name) ⇒ Object
-
#increment_and_save_table_position(table_name) ⇒ Object
Read a sequence number from the table’s position file, increment the number and pass the number to a block.
- #increment_table_rev(table_name, base_rev) ⇒ Object
-
#initialize(data_entry) ⇒ SyncFileManager
constructor
A new instance of SyncFileManager.
- #load_dump_pos ⇒ Object
- #load_sync_info ⇒ Object
- #move_table_binlog_files(tables) ⇒ Object
-
#mysql_table_marshal_dump_path ⇒ Object
MysqlTable marshal file.
- #reset_table_position_files(tables) ⇒ Object
-
#save_binlog(binlog_pos) ⇒ Object
binlog.pos file.
- #save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil) ⇒ Object
- #save_mysql_table_marshal_dump(mysql_table) ⇒ Object
- #save_sync_info(initial_sync, tables) ⇒ Object
- #save_table_binlog_pos(tables, binlog_pos) ⇒ Object
- #sync_info_file ⇒ Object
- #table_position_file_paths(*tables) ⇒ Object
- #table_positions_dir_path ⇒ Object
- #table_rev(table_name) ⇒ Object
- #table_rev_file_path(table_name) ⇒ Object
- #table_rev_file_paths(*tables) ⇒ Object
Constructor Details
#initialize(data_entry) ⇒ SyncFileManager
Returns a new instance of SyncFileManager.
7 8 9 |
# File 'lib/flydata/sync_file_manager.rb', line 7 def initialize(data_entry) @data_entry = data_entry end |
Instance Method Details
#backup_dump_dir ⇒ Object
184 185 186 187 188 189 190 |
# File 'lib/flydata/sync_file_manager.rb', line 184 def backup_dump_dir backup_dir = BACKUP_DIR.dup FileUtils.mkdir_p(backup_dir) unless Dir.exists?(backup_dir) dest_dir = File.join(backup_dir, Time.now.strftime("%Y%m%d%H%M%S")) FileUtils.mkdir(dest_dir) FileUtils.mv(dump_dir, dest_dir) end |
#binlog_path ⇒ Object
56 57 58 |
# File 'lib/flydata/sync_file_manager.rb', line 56 def binlog_path File.join(FLYDATA_HOME, @data_entry['name'] + ".binlog.pos") end |
#delete_table_binlog_pos(table_name) ⇒ Object
155 156 157 158 159 160 161 162 |
# File 'lib/flydata/sync_file_manager.rb', line 155 def delete_table_binlog_pos(table_name) file = File.join(table_positions_dir_path, table_name + ".binlog.pos") if File.exists?(file) FileUtils.rm(file, :force => true) else puts "#{file} does not exist. Something is wrong. Did you delete the file manually when flydata was running?" end end |
#dump_file_path ⇒ Object
11 12 13 |
# File 'lib/flydata/sync_file_manager.rb', line 11 def dump_file_path File.join(dump_dir, @data_entry['name']) + ".dump" end |
#dump_pos_path ⇒ Object
dump pos file for resume
16 17 18 |
# File 'lib/flydata/sync_file_manager.rb', line 16 def dump_pos_path dump_file_path + ".pos" end |
#get_table_binlog_pos(table_name) ⇒ Object
118 119 120 121 122 |
# File 'lib/flydata/sync_file_manager.rb', line 118 def get_table_binlog_pos(table_name) file = File.join(table_positions_dir_path, table_name + ".binlog.pos") return nil unless File.exists?(file) File.open(file, 'r').readline end |
#increment_and_save_table_position(table_name) ⇒ Object
Read a sequence number from the table’s position file, increment the number and pass the number to a block. After executing the block, saves the value to the position file.
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
# File 'lib/flydata/sync_file_manager.rb', line 80 def increment_and_save_table_position(table_name) file = File.join(table_positions_dir_path, table_name + ".pos") retry_count = 0 begin File.open(file, "r+") do |f| seq = f.read seq = seq.to_i + 1 yield(seq) f.rewind f.truncate(0) f.write(seq) end rescue Errno::ENOENT raise if retry_count > 0 # Already retried. Must be a differentfile causing the error # File not exist. Create one with initial value of '0' File.open(file, "w") {|f| f.write('0') } retry_count += 1 retry end end |
#increment_table_rev(table_name, base_rev) ⇒ Object
146 147 148 149 150 151 152 153 |
# File 'lib/flydata/sync_file_manager.rb', line 146 def increment_table_rev(table_name, base_rev) file = table_rev_file_path(table_name) new_rev = base_rev + 1 File.open(file, "w") do |f| f.write(new_rev) end new_rev end |
#load_dump_pos ⇒ Object
26 27 28 29 30 31 32 33 34 35 |
# File 'lib/flydata/sync_file_manager.rb', line 26 def load_dump_pos path = dump_pos_path return {} unless File.exists?(path) items = File.open(path, 'r').readline.split("\t") raise "Invalid dump.pos file: #{path}" unless items.length >= 5 && items.length <= 7 mysql_table = load_mysql_table_marshal_dump { status: items[0], table_name: items[1], last_pos: items[2].to_i, binlog_pos: {binfile: items[3], pos: items[4].to_i}, state: items[5], substate: items[6], mysql_table: mysql_table} end |
#load_sync_info ⇒ Object
111 112 113 114 115 116 |
# File 'lib/flydata/sync_file_manager.rb', line 111 def load_sync_info return nil unless File.exists?(sync_info_file) items = File.open(sync_info_file, 'r').readline.split("\t") { initial_sync: (items[0] == 'true'), tables: items[1] } end |
#move_table_binlog_files(tables) ⇒ Object
173 174 175 176 177 178 179 180 181 182 |
# File 'lib/flydata/sync_file_manager.rb', line 173 def move_table_binlog_files(tables) FileUtils.mkdir_p(table_positions_dir_path) unless Dir.exists?(table_positions_dir_path) tables.each do |table_name| file = File.join(dump_dir, table_name + ".binlog.pos") if ! File.exists?(file) raise "#{file} does not exist. Error!!" end FileUtils.mv(file, table_positions_dir_path) end end |
#mysql_table_marshal_dump_path ⇒ Object
MysqlTable marshal file
38 39 40 |
# File 'lib/flydata/sync_file_manager.rb', line 38 def mysql_table_marshal_dump_path dump_file_path + ".mysql_table" end |
#reset_table_position_files(tables) ⇒ Object
60 61 62 63 64 65 |
# File 'lib/flydata/sync_file_manager.rb', line 60 def reset_table_position_files(tables) tables.each do |table_name| file = File.join(table_positions_dir_path, table_name + ".pos") File.open(file, "w") {|f| f.write('0') } end end |
#save_binlog(binlog_pos) ⇒ Object
binlog.pos file
49 50 51 52 53 54 |
# File 'lib/flydata/sync_file_manager.rb', line 49 def save_binlog(binlog_pos) path = binlog_path File.open(path, 'w') do |f| f.write(binlog_content(binlog_pos)) end end |
#save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil) ⇒ Object
20 21 22 23 24 |
# File 'lib/flydata/sync_file_manager.rb', line 20 def save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil) File.open(dump_pos_path, 'w') do |f| f.write(dump_pos_content(status, table_name, last_pos, binlog_pos, state, substate)) end end |
#save_mysql_table_marshal_dump(mysql_table) ⇒ Object
42 43 44 45 46 |
# File 'lib/flydata/sync_file_manager.rb', line 42 def save_mysql_table_marshal_dump(mysql_table) File.open(mysql_table_marshal_dump_path, 'w') do |f| f.write Marshal.dump(mysql_table) end end |
#save_sync_info(initial_sync, tables) ⇒ Object
105 106 107 108 109 |
# File 'lib/flydata/sync_file_manager.rb', line 105 def save_sync_info(initial_sync, tables) File.open(sync_info_file, "w") do |f| f.write([initial_sync, tables].join("\t")) end end |
#save_table_binlog_pos(tables, binlog_pos) ⇒ Object
164 165 166 167 168 169 170 171 |
# File 'lib/flydata/sync_file_manager.rb', line 164 def save_table_binlog_pos(tables, binlog_pos) tables.split(" ").each do |table_name| file = File.join(dump_dir, table_name + ".binlog.pos") File.open(file, "w") do |f| f.write(binlog_content(binlog_pos)) end end end |
#sync_info_file ⇒ Object
101 102 103 |
# File 'lib/flydata/sync_file_manager.rb', line 101 def sync_info_file File.join(dump_dir, "sync.info") end |
#table_position_file_paths(*tables) ⇒ Object
71 72 73 74 |
# File 'lib/flydata/sync_file_manager.rb', line 71 def table_position_file_paths(*tables) tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.pos')) : tables.map{|table| File.join(table_positions_dir_path, table + '.pos')} end |
#table_positions_dir_path ⇒ Object
67 68 69 |
# File 'lib/flydata/sync_file_manager.rb', line 67 def table_positions_dir_path TABLE_POSITIONS_DIR end |
#table_rev(table_name) ⇒ Object
133 134 135 136 137 138 139 140 141 142 143 144 |
# File 'lib/flydata/sync_file_manager.rb', line 133 def table_rev(table_name) file = table_rev_file_path(table_name) return 1 unless File.exists?(file) #default revision is 1 File.open(file, "r+") do |f| seq = f.read if seq.empty? return 1 else return seq.to_i end end end |
#table_rev_file_path(table_name) ⇒ Object
124 125 126 |
# File 'lib/flydata/sync_file_manager.rb', line 124 def table_rev_file_path(table_name) File.join(table_positions_dir_path, table_name + ".rev") end |
#table_rev_file_paths(*tables) ⇒ Object
128 129 130 131 |
# File 'lib/flydata/sync_file_manager.rb', line 128 def table_rev_file_paths(*tables) tables.empty? ? Dir.glob(File.join(table_positions_dir_path, "*.rev")) : tables.map{|table| table_rev_file_path(table)} end |