Module: Tapsoob::Utils

Extended by:
Utils
Included in:
Utils
Defined in:
lib/tapsoob/utils.rb

Instance Method Summary collapse

Instance Method Details

#base64decode(data) ⇒ Object 



35
36
37
 
# File 'lib/tapsoob/utils.rb', line 35

def base64decode(data)
  data.unpack("m").first
end

#base64encode(data) ⇒ Object 



31
32
33
 
# File 'lib/tapsoob/utils.rb', line 31

def base64encode(data)
  [data].pack("m")
end

#bin(cmd) ⇒ Object 



18
19
20
21
 
# File 'lib/tapsoob/utils.rb', line 18

def bin(cmd)
  cmd = "#{cmd}.cmd" if windows?
  cmd
end

#blobs_to_string(row, columns) ⇒ Object 



90
91
92
93
94
95
96
 
# File 'lib/tapsoob/utils.rb', line 90

def blobs_to_string(row, columns)
  return row if columns.size == 0
  columns.each do |c|
    row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
  end
  row
end

#calculate_chunksize(old_chunksize) ⇒ Object 



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/tapsoob/utils.rb', line 98

def calculate_chunksize(old_chunksize)
  c = Tapsoob::Chunksize.new(old_chunksize)

  begin
    c.start_time = Time.now
    c.time_in_db = yield c
  rescue Errno::EPIPE
    c.retries += 1
    raise if c.retries > 2

    # we got disconnected, the chunksize could be too large
    # reset the chunksize based on the number of retries
    c.reset_chunksize
    retry
  end

  c.end_time = Time.now
  c.calc_new_chunksize
end

#checksum(data) ⇒ Object 



23
24
25
 
# File 'lib/tapsoob/utils.rb', line 23

def checksum(data)
  Zlib.crc32(data)
end

#export_indexes(dump_path, table, index_data) ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/tapsoob/utils.rb', line 124

def export_indexes(dump_path, table, index_data)
  data = [index_data]
  if File.exists?(File.join(dump_path, "indexes", "#{table}.json"))
    previous_data = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table}.json")))
    data = data + previous_data
  end

  File.open(File.join(dump_path, "indexes", "#{table}.json"), 'w') do |file|
    file.write(JSON.generate(data))
  end
end

#export_rows(dump_path, table, row_data) ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/tapsoob/utils.rb', line 136

def export_rows(dump_path, table, row_data)
  data = row_data
  if File.exists?(File.join(dump_path, "data", "#{table}.json"))
    previous_data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
    data[:data] = previous_data["data"] + row_data[:data]
  end

  File.open(File.join(dump_path, "data", "#{table}.json"), 'w') do |file|
    file.write(JSON.generate(data))
  end
end

#export_schema(dump_path, table, schema_data) ⇒ Object 



118
119
120
121
122
 
# File 'lib/tapsoob/utils.rb', line 118

def export_schema(dump_path, table, schema_data)
  File.open(File.join(dump_path, "schemas", "#{table}.rb"), 'w') do |file|
    file.write(schema_data)
  end
end

#format_data(data, opts = {}) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/tapsoob/utils.rb', line 39

def format_data(data, opts = {})
  return {} if data.size == 0
  string_columns = opts[:string_columns] || []
  schema = opts[:schema] || []
  table = opts[:table]

  max_lengths = schema.inject({}) do |hash, (column, meta)|
    if meta[:db_type] =~ /^varchar\((\d+)\)/
      hash.update(column => $1.to_i)
    end
    hash
  end

  header = data[0].keys
  only_data = data.collect do |row|
    row = blobs_to_string(row, string_columns)
    row.each do |column, data|
      if data.to_s.length > (max_lengths[column] || data.to_s.length)
        raise Tapsoob::InvalidData.new(<<-ERROR)
Detected data that exceeds the length limitation of its column. This is
generally due to the fact that SQLite does not enforce length restrictions.

Table : #{table}
Column : #{column}
Type : #{schema.detect{|s| s.first == column}.last[:db_type]}
Data : #{data}
        ERROR
      end

      # Type conversion
      row[column] = data.strftime('%Y-%m-%d %H:%M:%S') if data.is_a?(Time)
    end
    header.collect { |h| row[h] }
  end
  { table_name: table, header: header, data: only_data }
end

#incorrect_blobs(db, table) ⇒ Object

mysql text and blobs fields are handled the same way internally this is not true for other databases so we must check if the field is actually text and manually convert it back to a string



79
80
81
82
83
84
85
86
87
88
 
# File 'lib/tapsoob/utils.rb', line 79

def incorrect_blobs(db, table)
  return [] if (db.url =~ /mysql:\/\//).nil?

  columns = []
  db.schema(table).each do |data|
    column, cdata = data
    columns << column if cdata[:db_type] =~ /text/
  end
  columns
end

#load_indexes(database_url, index) ⇒ Object 



153
154
155
 
# File 'lib/tapsoob/utils.rb', line 153

def load_indexes(database_url, index)
  Tapsoob::Schema.load_indexes(database_url, index)
end

#load_schema(dump_path, database_url, table) ⇒ Object 



148
149
150
151
 
# File 'lib/tapsoob/utils.rb', line 148

def load_schema(dump_path, database_url, table)
  schema = File.join(dump_path, "schemas", "#{table}.rb")
  schema_bin(:load, database_url, schema.to_s)
end

#order_by(db, table) ⇒ Object 



174
175
176
177
178
179
180
181
182
 
# File 'lib/tapsoob/utils.rb', line 174

def order_by(db, table)
  pkey = primary_key(db, table)
  if pkey
    pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
  else
    table = table.to_sym unless table.kind_of?(Sequel::SQL::Identifier)
    db[table].columns
  end
end

#primary_key(db, table) ⇒ Object 



164
165
166
 
# File 'lib/tapsoob/utils.rb', line 164

def primary_key(db, table)
  db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c[0] }
end

#schema_bin(command, *args) ⇒ Object 



157
158
159
160
161
162
 
# File 'lib/tapsoob/utils.rb', line 157

def schema_bin(command, *args)
  require 'tapsoob/cli'
  subcommand = "schema"
  script = Tapsoob::CLI::Schema.new
  script.invoke(command, args.map { |a| "#{a}" })
end

#single_integer_primary_key(db, table) ⇒ Object 



168
169
170
171
172
 
# File 'lib/tapsoob/utils.rb', line 168

def single_integer_primary_key(db, table)
  table = table.to_sym unless table.kind_of?(Sequel::SQL::Identifier)
  keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
  not keys.nil? and keys.size == 1
end

#valid_data?(data, crc32) ⇒ Boolean

Returns:

  • (Boolean) 


27
28
29
 
# File 'lib/tapsoob/utils.rb', line 27

def valid_data?(data, crc32)
  Zlib.crc32(data) == crc32.to_i
end

#windows? ⇒ Boolean

Returns:

  • (Boolean) 


12
13
14
15
16
 
# File 'lib/tapsoob/utils.rb', line 12

def windows?
  return @windows if defined?(@windows)
  require 'rbconfig'
  @windows = !!(::RbConfig::CONFIG['host_os'] =~ /mswin|mingw/)
end