Module: Taps::Utils
Instance Method Summary collapse
- #base64decode(data) ⇒ Object
- #base64encode(data) ⇒ Object
- #bin(cmd) ⇒ Object
- #blobs_to_string(row, columns) ⇒ Object
- #calculate_chunksize(old_chunksize) ⇒ Object
- #checksum(data) ⇒ Object
- #format_data(data, opts = {}) ⇒ Object
-
#incorrect_blobs(db, table) ⇒ Object
mysql text and blobs fields are handled the same way internally this is not true for other databases so we must check if the field is actually text and manually convert it back to a string.
- #load_indexes(database_url, index_data) ⇒ Object
- #load_schema(database_url, schema_data) ⇒ Object
- #order_by(db, table) ⇒ Object
- #primary_key(db, table) ⇒ Object
- #reraise_server_exception(e) ⇒ Object
- #schema_bin(*args) ⇒ Object
-
#server_error_handling ⇒ Object
try to detect server side errors to give the client a more useful error message.
- #single_integer_primary_key(db, table) ⇒ Object
- #valid_data?(data, crc32) ⇒ Boolean
- #windows? ⇒ Boolean
Instance Method Details
#base64decode(data) ⇒ Object
37 38 39 |
# File 'lib/taps/utils.rb', line 37 def base64decode(data) data.unpack('m').first end |
#base64encode(data) ⇒ Object
33 34 35 |
# File 'lib/taps/utils.rb', line 33 def base64encode(data) [data].pack('m') end |
#bin(cmd) ⇒ Object
20 21 22 23 |
# File 'lib/taps/utils.rb', line 20 def bin(cmd) cmd = "#{cmd}.cmd" if windows? cmd end |
#blobs_to_string(row, columns) ⇒ Object
85 86 87 88 89 90 91 |
# File 'lib/taps/utils.rb', line 85 def blobs_to_string(row, columns) return row if columns.empty? columns.each do |c| row[c] = row[c].to_s if row[c].is_a?(Sequel::SQL::Blob) end row end |
#calculate_chunksize(old_chunksize) ⇒ Object
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
# File 'lib/taps/utils.rb', line 93 def calculate_chunksize(old_chunksize) c = Taps::Chunksize.new(old_chunksize) begin c.start_time = Time.now c.time_in_db = yield c rescue Errno::EPIPE, RestClient::RequestFailed, RestClient::RequestTimeout c.retries += 1 raise if c.retries > 2 # we got disconnected, the chunksize could be too large # reset the chunksize based on the number of retries c.reset_chunksize retry end c.end_time = Time.now c.calc_new_chunksize end |
#checksum(data) ⇒ Object
25 26 27 |
# File 'lib/taps/utils.rb', line 25 def checksum(data) Zlib.crc32(data) end |
#format_data(data, opts = {}) ⇒ Object
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
# File 'lib/taps/utils.rb', line 41 def format_data(data, opts = {}) return {} if data.empty? string_columns = opts[:string_columns] || [] schema = opts[:schema] || [] table = opts[:table] max_lengths = schema.each_with_object({}) do |(column, ), hash| hash.update(column => Regexp.last_match(1).to_i) if [:db_type] =~ /^varchar\((\d+)\)/ end header = data[0].keys only_data = data.collect do |row| row = blobs_to_string(row, string_columns) row.each do |column, value| next unless value.to_s.length > (max_lengths[column] || value.to_s.length) raise Taps::InvalidData, <<-ERROR Detected value that exceeds the length limitation of its column. This is generally due to the fact that SQLite does not enforce length restrictions. Table : #{table} Column : #{column} Type : #{schema.detect { |s| s.first == column }.last[:db_type]} Value : #{value} ERROR end header.collect { |h| row[h] } end { header: header, data: only_data } end |
#incorrect_blobs(db, table) ⇒ Object
mysql text and blobs fields are handled the same way internally this is not true for other databases so we must check if the field is actually text and manually convert it back to a string
74 75 76 77 78 79 80 81 82 83 |
# File 'lib/taps/utils.rb', line 74 def incorrect_blobs(db, table) return [] if (db.url =~ /mysql:\/\//).nil? columns = [] db.schema(table).each do |data| column, cdata = data columns << column if cdata[:db_type] =~ /text/ end columns end |
#load_indexes(database_url, index_data) ⇒ Object
120 121 122 123 124 125 |
# File 'lib/taps/utils.rb', line 120 def load_indexes(database_url, index_data) Tempfile.open('taps') do |tmp| File.open(tmp.path, 'w') { |f| f.write(index_data) } schema_bin(:load_indexes, database_url, tmp.path) end end |
#load_schema(database_url, schema_data) ⇒ Object
113 114 115 116 117 118 |
# File 'lib/taps/utils.rb', line 113 def load_schema(database_url, schema_data) Tempfile.open('taps') do |tmp| File.open(tmp.path, 'w') { |f| f.write(schema_data) } schema_bin(:load, database_url, tmp.path) end end |
#order_by(db, table) ⇒ Object
142 143 144 145 146 147 148 149 150 |
# File 'lib/taps/utils.rb', line 142 def order_by(db, table) pkey = primary_key(db, table) if pkey pkey.is_a?(Array) ? pkey : [pkey.to_sym] else table = table.to_sym.identifier unless table.is_a?(Sequel::SQL::Identifier) db[table].columns end end |
#primary_key(db, table) ⇒ Object
132 133 134 |
# File 'lib/taps/utils.rb', line 132 def primary_key(db, table) db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c[0] } end |
#reraise_server_exception(e) ⇒ Object
164 165 166 167 168 169 170 171 172 173 174 175 176 177 |
# File 'lib/taps/utils.rb', line 164 def reraise_server_exception(e) if e.is_a?(RestClient::Exception) if e.respond_to?(:response) && e.response.headers[:content_type] == 'application/json' json = ::OkJson.decode(e.response.to_s) klass = begin eval(json['error_class']) rescue nil end raise klass.new(json['error_message'], backtrace: json['error_backtrace']) if klass end end raise e end |
#schema_bin(*args) ⇒ Object
127 128 129 130 |
# File 'lib/taps/utils.rb', line 127 def schema_bin(*args) bin_path = File.("#{File.dirname(__FILE__)}/../../bin/#{bin('schema2')}") `"#{bin_path}" #{args.map { |a| "'#{a}'" }.join(' ')}` end |
#server_error_handling ⇒ Object
try to detect server side errors to give the client a more useful error message
154 155 156 157 158 159 160 161 162 |
# File 'lib/taps/utils.rb', line 154 def server_error_handling yield rescue Sequel::DatabaseError => e if e. =~ /duplicate key value/i raise Taps::DuplicatePrimaryKeyError, e. else raise end end |
#single_integer_primary_key(db, table) ⇒ Object
136 137 138 139 140 |
# File 'lib/taps/utils.rb', line 136 def single_integer_primary_key(db, table) table = table.to_sym.identifier unless table.is_a?(Sequel::SQL::Identifier) keys = db.schema(table).select { |c| c[1][:primary_key] } !keys.nil? && (keys.size == 1) && (keys[0][1][:type] == :integer) end |
#valid_data?(data, crc32) ⇒ Boolean
29 30 31 |
# File 'lib/taps/utils.rb', line 29 def valid_data?(data, crc32) Zlib.crc32(data) == crc32.to_i end |
#windows? ⇒ Boolean
14 15 16 17 18 |
# File 'lib/taps/utils.rb', line 14 def windows? return @windows if defined?(@windows) require 'rbconfig' @windows = !!(RbConfig::CONFIG['host_os'] =~ /mswin|mingw/) end |