Module: Zdm

Defined in:
lib/zdm.rb,
lib/version.rb

Defined Under Namespace

Classes: Migrator, Table

Constant Summary collapse

BATCH_SIZE = 
40_000
DECREASE_THROTTLER =

seconds

4
DECREASE_SIZE = 
5_000
MIN_BATCH_SIZE = 
10_000
PROGRESS_EVERY =

seconds

30
VERSION = 
'1.1.2'

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.io ⇒ Object

Returns the value of attribute io.



5
6
7
 
# File 'lib/zdm.rb', line 5

def io
  @io
end

Class Method Details

.change_table(name) {|table| ... } ⇒ Object

Yields:

  • (table) 


7
8
9
10
11
12
 
# File 'lib/zdm.rb', line 7

def change_table(name, &block)
  table = Table.new(name)
  yield table
  Migrator.new(table).migrate!
  cleanup if defined?(Rails) && Rails.env.development?
end

.cleanup(before: nil) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/zdm.rb', line 14

def cleanup(before: nil)
  zdm_tables = connection.send(tables_method).select { |name| name.starts_with?('zdm_') }
  zdm_tables.each { |name| Migrator.new(Table.new(name.sub(/^zdm_/, ''))).cleanup }

  zdm_archive_tables = connection.send(tables_method).select { |name| name.starts_with?('zdma_') }
  if before
    zdm_archive_tables.select! { |table|
      Time.strptime(table, 'zdma_%Y%m%d_%H%M%S%N') <= before
    }
  end
  zdm_archive_tables.each { |name| execute('DROP TABLE `%s`' % name) }
end

.execute_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block) ⇒ Object 



40
41
42
43
44
45
 
# File 'lib/zdm.rb', line 40

def execute_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
  find_in_batches(table_name, start: start, finish: finish, batch_size: batch_size, progress_every: progress_every) do |batch_start, batch_end|
    sql = yield batch_start, batch_end
    execute(sql) if sql
  end
end

.find_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/zdm.rb', line 47

def find_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
  min = start || connection.select_value('SELECT MIN(`id`) FROM %s' % table_name)
  return unless min
  max = finish || connection.select_value('SELECT MAX(`id`) FROM %s' % table_name)
  return unless max
  todo = max - min + 1
  return unless todo > 0

  batch_end = min - 1
  start_time = last_progress = Time.now
  while true
    batch_start = batch_end + 1
    batch_end = [batch_start + batch_size - 1, max].min
    start_batch_time = Time.now

    yield batch_start, batch_end

    if $exit
      write(table_name, 'Received SIGTERM, exiting...')
      cleanup
      exit 1
    end

    # The end!
    break if batch_end >= max

    # Throttle
    current_time = Time.now
    if (current_time - start_batch_time) > DECREASE_THROTTLER
      batch_size = [(batch_size - DECREASE_SIZE).to_i, MIN_BATCH_SIZE].max
    end

    # Periodically show progress
    if (current_time - last_progress) >= progress_every
      last_progress = current_time
      done = batch_end - min + 1
      write(table_name, "%.2f%% (#{done}/#{todo})" % (done.to_f / todo * 100.0))
    end
  end

  duration = Time.now - start_time
  duration = (duration < 2*60) ? "#{duration.to_i} secs" : "#{(duration / 60).to_i} mins"
  write(table_name, "Completed (#{duration})")
end

.rails5? ⇒ Boolean

Returns:

  • (Boolean) 


31
32
33
 
# File 'lib/zdm.rb', line 31

def rails5?
  ActiveRecord.version.to_s =~ /^5/
end

.tables_method ⇒ Object 



27
28
29
 
# File 'lib/zdm.rb', line 27

def tables_method
  rails5? ? :data_sources : :tables
end