Module: Brick::MigrationBuilder

Extended by:
FancyGets
Included in:
MigrationsGenerator, SalesforceSchema
Defined in:
lib/generators/brick/migration_builder.rb

Constant Summary collapse

SQL_TYPES =

Many SQL types are the same as their migration data type name:

text, integer, bigint, date, boolean, decimal, float

These however are not:

{ 'character varying' => 'string',
  'character' => 'string', # %%% Need to put in "limit: 1"
  'xml' => 'text',
  'bytea' => 'binary',
  'timestamp without time zone' => 'timestamp',
  'timestamp with time zone' => 'timestamp',
  'time without time zone' => 'time',
  'time with time zone' => 'time',
  'double precision' => 'float',
  'smallint' => 'integer', # %%% Need to put in "limit: 2"
  'ARRAY' => 'string', # Note that we'll also add ", array: true"
  # Oracle data types
  'VARCHAR2' => 'string',
  'CHAR' => 'string',
  ['NUMBER', 22] => 'integer',
  /^INTERVAL / => 'string', # Time interval stuff like INTERVAL YEAR(2) TO MONTH, INTERVAL '999' DAY(3), etc
  'XMLTYPE' => 'xml',
  'RAW' => 'binary',
  'SDO_GEOMETRY' => 'geometry',
  # MSSQL data types
  'int' => 'integer',
  'char' => 'string',
  'varchar' => 'string',
  'nvarchar' => 'string',
  'nchar' => 'string',
  'datetime2' => 'timestamp',
  'bit' => 'boolean',
  'varbinary' => 'binary',
  'tinyint' => 'integer', # %%% Need to put in "limit: 2"
  'year' => 'integer',
  'set' => 'string',
  # Sqlite data types
  'TEXT' => 'text',
  '' => 'string',
  'INTEGER' => 'integer',
  'REAL' => 'float',
  'BLOB' => 'binary',
  'TIMESTAMP' => 'timestamp',
  'DATETIME' => 'timestamp'
}

Class Method Summary collapse

Class Method Details

.check_folder(is_insert_versions = true, is_delete_versions = false) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/generators/brick/migration_builder.rb', line 50

def check_folder(is_insert_versions = true, is_delete_versions = false)
  versions_to_delete_or_append = nil
  if Dir.exist?(mig_path = ActiveRecord::Migrator.migrations_paths.first || "#{::Rails.root}/db/migrate")
    if Dir["#{mig_path}/**/*.rb"].present?
      puts "WARNING: migrations folder #{mig_path} appears to already have ruby files present."
      mig_path2 = "#{::Rails.root}/tmp/brick_migrations"
      is_insert_versions = false unless mig_path == mig_path2
      if Dir.exist?(mig_path2)
        if Dir["#{mig_path2}/**/*.rb"].present?
          puts "As well, temporary folder #{mig_path2} also has ruby files present."
          puts "Choose a destination -- all existing .rb files will be removed:"
          mig_path2 = gets_list(list: ['Cancel operation!', "Append migration files into #{mig_path} anyway", mig_path, mig_path2])
          return if mig_path2.start_with?('Cancel')

          existing_mig_files = Dir["#{mig_path2}/**/*.rb"]
          if (is_insert_versions = mig_path == mig_path2)
            versions_to_delete_or_append = existing_mig_files.map { |ver| ver.split('/').last.split('_').first }
          end
          if mig_path2.start_with?('Append migration files into ')
            mig_path2 = mig_path
          else
            is_delete_versions = true
            existing_mig_files.each { |rb| File.delete(rb) }
          end
        else
          puts "Using temporary folder #{mig_path2} for created migration files.\n\n"
        end
      else
        puts "Creating the temporary folder #{mig_path2} for created migration files.\n\n"
        Dir.mkdir(mig_path2)
      end
      mig_path = mig_path2
    else
      puts "Using standard migration folder #{mig_path} for created migration files.\n\n"
    end
  else
    puts "Creating standard ActiveRecord migration folder #{mig_path} to hold new migration files.\n\n"
    Dir.mkdir(mig_path)
  end
  [mig_path, is_insert_versions, is_delete_versions]
end

.generate_migrations(chosen, mig_path, is_insert_versions, is_delete_versions, relations = ::Brick.relations, do_fks_last: nil, do_schema_migrations: true) ⇒ Object 



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
 
# File 'lib/generators/brick/migration_builder.rb', line 92

def generate_migrations(chosen, mig_path, is_insert_versions, is_delete_versions,
                        relations = ::Brick.relations, do_fks_last: nil, do_schema_migrations: true)
  if do_fks_last.nil?
    puts 'Would you like for the foreign keys to be built inline inside of each migration file, or as a final migration?'
    options = ['Inline', 'Separate final migration for all FKs']
    options << 'Create "additional_references" entries in brick.rb that emulate foreign keys'
    do_fks = gets_list(list: options).split(' ').first
    do_fks_last = do_fks unless do_fks == 'Inline'
  end

  is_sqlite = ActiveRecord::Base.connection.adapter_name == 'SQLite'
  key_type = ((is_sqlite || ActiveRecord.version < ::Gem::Version.new('5.1')) ? 'integer' : 'bigint')
  is_4x_rails = ActiveRecord.version < ::Gem::Version.new('5.0')
  ar_version = "[#{ActiveRecord.version.segments[0..1].join('.')}]" unless is_4x_rails

  schemas = chosen.each_with_object({}) do |v, s|
    if (v_parts = v.split('.')).length > 1
      s[v_parts.first] = nil unless [::Brick.default_schema, 'public'].include?(v_parts.first)
    end
  end
  separator = ::Brick.config.salesforce_mode ? 'x' : nil
  # Start the timestamps back the same number of minutes from now as expected number of migrations to create
  current_mig_time = [Time.now - (schemas.length + chosen.length).minutes]
  done = []
  fks = {}
  stuck = {}
  indexes = {} # Track index names to make sure things are unique
  built_schemas = {} # Track all built schemas so we can place an appropriate drop_schema command only in the first
                    # migration in which that schema is referenced, thereby allowing rollbacks to function properly.
  versions_to_create = [] # Resulting versions to be used when updating the schema_migrations table
  # Start by making migrations for fringe tables (those with no foreign keys).
  # Continue layer by layer, creating migrations for tables that reference ones already done, until
  # no more migrations can be created.  (At that point hopefully all tables are accounted for.)
  after_fks = [] # Track foreign keys to add after table creation
  while (fringe = chosen.reject do |tbl|
                    snag_fks = []
                    snags = relations.fetch(tbl, nil)&.fetch(:fks, nil)&.select do |_k, v|
                      # Skip any foreign keys which should be deferred ...
                      !Brick.drfgs[tbl]&.any? do |drfg|
                        drfg[0] == v.fetch(:fk, nil) && drfg[1] == v.fetch(:inverse_table, nil)
                      end &&
                      v[:is_bt] && !v[:polymorphic] && # ... and polymorphics ...
                      tbl != v[:inverse_table] && # ... and self-referencing associations (stuff like "parent_id")
                      !done.include?(v[:inverse_table]) &&
                      ::Brick.config.ignore_migration_fks.exclude?(snag_fk = "#{tbl}.#{v[:fk]}") &&
                      snag_fks << snag_fk
                    end
                    if snags&.present?
                      # puts snag_fks.inspect
                      stuck[tbl] = snags
                    end
                  end
        ).present?
    fringe.each do |tbl|
      mig = gen_migration_columns(relations, tbl, (tbl_parts = tbl.split('.')), (add_fks = []), built_schemas, mig_path, current_mig_time,
                                  key_type, is_4x_rails, ar_version, do_fks_last, versions_to_create)
      after_fks.concat(add_fks) if do_fks_last
      current_mig_time[0] += 1.minute
      versions_to_create << migration_file_write(mig_path, "create_#{::Brick._brick_index(tbl, nil, separator)}", current_mig_time, ar_version, mig)
    end
    done.concat(fringe)
    chosen -= done
  end

  if do_fks_last
    # Write out any more tables that haven't been done yet
    chosen.each do |tbl|
      mig = gen_migration_columns(relations, tbl, (tbl_parts = tbl.split('.')), (add_fks = []), built_schemas, mig_path, current_mig_time,
                                  key_type, is_4x_rails, ar_version, do_fks_last, versions_to_create)
      after_fks.concat(add_fks)
      current_mig_time[0] += 1.minute
      versions_to_create << migration_file_write(mig_path, "create_#{::Brick._brick_index(tbl, :migration, separator)}", current_mig_time, ar_version, mig)
    end
    done.concat(chosen)
    chosen.clear

    case do_fks_last
    when 'Separate' # Add a final migration to create all the foreign keys
      mig = +"  def change\n"
      after_fks.each do |add_fk|
        next unless add_fk[2] # add_fk[2] holds the inverse relation

        unless (pk = add_fk[2][:pkey].values.flatten&.first)
          # No official PK, but if coincidentally there's a column of the same name, take a chance on it
          pk = (add_fk[2][:cols].key?(add_fk[1]) && add_fk[1]) || '???'
        end
        mig << "    add_foreign_key #{add_fk[3]}, " # The tbl_code
        #         to_table               column
        mig << "#{add_fk[0]}, column: :#{add_fk[1]}, primary_key: :#{pk}\n"
      end
      if after_fks.length > 500
        minutes = (after_fks.length + 1000) / 1500
        mig << "    if ActiveRecord::Base.connection.adapter_name == 'PostgreSQL'\n"
        mig << "      puts 'NOTE:  It could take around #{minutes} #{'minute'.pluralize(minutes)} on a FAST machine for Postgres to do all the final processing for these foreign keys.  Please be patient!'\n"

        mig << "      # Vacuum takes only about ten seconds when all the tables are empty,
# and about 2 minutes when the tables are fairly full.
execute('COMMIT')
execute('VACUUM FULL')
execute('BEGIN TRANSACTION')
    end\n"
      end
      mig << +"  end\n"
      current_mig_time[0] += 1.minute
      versions_to_create << migration_file_write(mig_path, 'create_brick_fks.rbx', current_mig_time, ar_version, mig)
      puts "Have written out a final migration called 'create_brick_fks.rbx' which creates #{after_fks.length} foreign keys.
  This file extension (.rbx) will cause it not to run yet when you do a 'rails db:migrate'.
  The idea here is to do all data loading first, and then rename that migration file back
  into having a .rb extension, and run a final db:migrate to put the foreign keys in place."

    when 'Create' # Show additional_references entries that can be added into brick.rb
      puts 'Place this block into your brick.rb file:'
      puts '  ::Brick.additional_references = ['
      after_fks.each do |add_fk|
        next unless add_fk[2] # add_fk[2] holds the inverse relation

        unless (pk = add_fk[2][:pkey].values.flatten&.first)
          # No official PK, but if coincidentally there's a column of the same name, take a chance on it
          pk = (add_fk[2][:cols].key?(add_fk[1]) && add_fk[1]) || '???'
        end
        from_table = add_fk[3]
        from_table = "'#{from_table[1..-1]}'" if from_table[0] == ':'
        to_table = add_fk[0]
        to_table = "'#{to_table[1..-1]}'" if to_table[0] == ':'
        puts "    [#{from_table}, #{add_fk[1].inspect}, #{to_table}],"
      end
      puts '  ]'
    end
  end

  stuck_counts = Hash.new { |h, k| h[k] = 0 }
  chosen.each do |leftover|
    puts "Can't do #{leftover} because:\n  #{stuck[leftover].map do |snag|
      stuck_counts[snag.last[:inverse_table]] += 1
      snag.last[:assoc_name]
    end.join(', ')}"
  end
  if mig_path.start_with?(cur_path = ::Rails.root.to_s)
    pretty_mig_path = mig_path[cur_path.length..-1]
  end
  puts "\n*** Created #{done.length} migration files under #{pretty_mig_path || mig_path} ***"
  if (stuck_sorted = stuck_counts.to_a.sort { |a, b| b.last <=> a.last }).length.positive?
    puts "-----------------------------------------"
    puts "Unable to create migrations for #{stuck_sorted.length} tables#{
      ".  Here's the top 5 blockers" if stuck_sorted.length > 5
    }:"
    pp stuck_sorted[0..4]
  elsif do_schema_migrations # Successful, and now we can update the schema_migrations table accordingly
    unless ActiveRecord::Migration.table_exists?(ActiveRecord::Base.schema_migrations_table_name)
      ActiveRecord::SchemaMigration.create_table
    end
    # Remove to_delete - to_create
    if ((versions_to_delete_or_append ||= []) - versions_to_create).present? && is_delete_versions
      ActiveRecord::Base.execute_sql("DELETE FROM #{
        ActiveRecord::Base.schema_migrations_table_name} WHERE version IN (#{
        (versions_to_delete_or_append - versions_to_create).map { |vtd| "'#{vtd}'" }.join(', ')}
      )")
    end
    # Add to_create - to_delete
    if is_insert_versions && ((versions_to_create ||= []) - versions_to_delete_or_append).present?
      ActiveRecord::Base.execute_sql("INSERT INTO #{
        ActiveRecord::Base.schema_migrations_table_name} (version) VALUES #{
        (versions_to_create - versions_to_delete_or_append).map { |vtc| "('#{vtc}')" }.join(', ')
      }")
    end
  end
end