Module: Taps::Utils
Instance Method Summary collapse
- #bin(cmd) ⇒ Object
- #blobs_to_string(row, columns) ⇒ Object
- #calculate_chunksize(old_chunksize) ⇒ Object
- #checksum(data) ⇒ Object
- #format_data(data, string_columns) ⇒ Object
- #gunzip(gzip_data) ⇒ Object
- #gzip(data) ⇒ Object
-
#incorrect_blobs(db, table) ⇒ Object
mysql text and blobs fields are handled the same way internally this is not true for other databases so we must check if the field is actually text and manually convert it back to a string.
- #load_indexes(database_url, index_data) ⇒ Object
- #load_schema(database_url, schema_data) ⇒ Object
- #order_by(db, table) ⇒ Object
- #primary_key(db, table) ⇒ Object
- #schema_bin(*args) ⇒ Object
- #valid_data?(data, crc32) ⇒ Boolean
- #windows? ⇒ Boolean
Instance Method Details
#bin(cmd) ⇒ Object
14 15 16 17 |
# File 'lib/taps/utils.rb', line 14 def bin(cmd) cmd = "#{cmd}.cmd" if windows? cmd end |
#blobs_to_string(row, columns) ⇒ Object
67 68 69 70 71 72 73 |
# File 'lib/taps/utils.rb', line 67 def blobs_to_string(row, columns) return row if columns.size == 0 columns.each do |c| row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob) end row end |
#calculate_chunksize(old_chunksize) ⇒ Object
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
# File 'lib/taps/utils.rb', line 75 def calculate_chunksize(old_chunksize) chunksize = old_chunksize retries = 0 begin t1 = Time.now yield chunksize rescue Errno::EPIPE retries += 1 raise if retries > 1 # we got disconnected, the chunksize could be too large # so we're resetting to a very small value chunksize = 100 retry end t2 = Time.now diff = t2 - t1 new_chunksize = if diff > 3.0 (chunksize / 3).ceil elsif diff > 1.1 chunksize - 100 elsif diff < 0.8 chunksize * 2 else chunksize + 100 end new_chunksize = 100 if new_chunksize < 100 new_chunksize end |
#checksum(data) ⇒ Object
19 20 21 |
# File 'lib/taps/utils.rb', line 19 def checksum(data) Zlib.crc32(data) end |
#format_data(data, string_columns) ⇒ Object
43 44 45 46 47 48 49 50 51 |
# File 'lib/taps/utils.rb', line 43 def format_data(data, string_columns) return {} if data.size == 0 header = data[0].keys only_data = data.collect do |row| row = blobs_to_string(row, string_columns) header.collect { |h| row[h] } end { :header => header, :data => only_data } end |
#gunzip(gzip_data) ⇒ Object
35 36 37 38 39 40 41 |
# File 'lib/taps/utils.rb', line 35 def gunzip(gzip_data) io = StringIO.new(gzip_data) gz = Zlib::GzipReader.new(io) data = gz.read gz.close data end |
#gzip(data) ⇒ Object
27 28 29 30 31 32 33 |
# File 'lib/taps/utils.rb', line 27 def gzip(data) io = StringIO.new gz = Zlib::GzipWriter.new(io) gz.write data gz.close io.string end |
#incorrect_blobs(db, table) ⇒ Object
mysql text and blobs fields are handled the same way internally this is not true for other databases so we must check if the field is actually text and manually convert it back to a string
56 57 58 59 60 61 62 63 64 65 |
# File 'lib/taps/utils.rb', line 56 def incorrect_blobs(db, table) return [] unless db.class.to_s == "Sequel::MySQL::Database" columns = [] db.schema(table).each do |data| column, cdata = data columns << column if cdata[:db_type] =~ /text/ end columns end |
#load_indexes(database_url, index_data) ⇒ Object
114 115 116 117 118 119 |
# File 'lib/taps/utils.rb', line 114 def load_indexes(database_url, index_data) Tempfile.open('taps') do |tmp| File.open(tmp.path, 'w') { |f| f.write(index_data) } schema_bin(:load_indexes, database_url, tmp.path) end end |
#load_schema(database_url, schema_data) ⇒ Object
107 108 109 110 111 112 |
# File 'lib/taps/utils.rb', line 107 def load_schema(database_url, schema_data) Tempfile.open('taps') do |tmp| File.open(tmp.path, 'w') { |f| f.write(schema_data) } schema_bin(:load, database_url, tmp.path) end end |
#order_by(db, table) ⇒ Object
133 134 135 136 137 138 139 140 |
# File 'lib/taps/utils.rb', line 133 def order_by(db, table) pkey = primary_key(db, table) if pkey [pkey.to_sym] else db[table].columns end end |
#primary_key(db, table) ⇒ Object
125 126 127 128 129 130 131 |
# File 'lib/taps/utils.rb', line 125 def primary_key(db, table) if db.respond_to?(:primary_key) db.primary_key(table) else db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c.first }.shift end end |
#schema_bin(*args) ⇒ Object
121 122 123 |
# File 'lib/taps/utils.rb', line 121 def schema_bin(*args) `#{File.dirname(__FILE__)}/../../bin/#{bin('schema')} #{args.join(' ')}` end |
#valid_data?(data, crc32) ⇒ Boolean
23 24 25 |
# File 'lib/taps/utils.rb', line 23 def valid_data?(data, crc32) Zlib.crc32(data) == crc32.to_i end |
#windows? ⇒ Boolean
10 11 12 |
# File 'lib/taps/utils.rb', line 10 def windows? RUBY_PLATFORM =~ /mswin32/ end |