Class: Gitlab::Database::BackgroundMigration::BatchedMigration

Inherits:
SharedModel
  • Object
show all
Defined in:
lib/gitlab/database/background_migration/batched_migration.rb

Constant Summary collapse

JOB_CLASS_MODULE = 
'Gitlab::BackgroundMigration'
BATCH_CLASS_MODULE = 
"#{JOB_CLASS_MODULE}::BatchingStrategies"
MAXIMUM_FAILED_RATIO = 
0.5
MINIMUM_JOBS = 
50
FINISHED_PROGRESS_VALUE = 
100

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from SharedModel

connection, #connection_db_config, connection_pool, using_connection

Class Method Details

.active_migrations_distinct_on_table(connection:, limit:) ⇒ Object 



104
105
106
107
108
109
110
111
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 104

def self.active_migrations_distinct_on_table(connection:, limit:)
  distinct_on_table = select('DISTINCT ON (table_name) id')
    .for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection))
    .executable
    .order(table_name: :asc, id: :asc)

  where(id: distinct_on_table).queue_order.limit(limit)
end

.find_executable(id, connection:) ⇒ Object 



99
100
101
102
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 99

def self.find_executable(id, connection:)
  for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection))
    .executable.find_by_id(id)
end

.find_for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments) ⇒ Object 



95
96
97
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 95

def self.find_for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments)
  for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments).first
end

.gitlab_schema_column_exists? ⇒ Boolean

Returns:

  • (Boolean) 


47
48
49
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 47

def self.gitlab_schema_column_exists?
  column_names.include?('gitlab_schema')
end

.successful_rows_counts(migrations) ⇒ Object 



113
114
115
116
117
118
119
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 113

def self.successful_rows_counts(migrations)
  BatchedJob
    .with_status(:succeeded)
    .where(batched_background_migration_id: migrations)
    .group(:batched_background_migration_id)
    .sum(:batch_size)
end

.valid_status ⇒ Object 



91
92
93
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 91

def self.valid_status
  state_machine.states.map(&:name)
end

Instance Method Details

#batch_class ⇒ Object 



175
176
177
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 175

def batch_class
  "#{BATCH_CLASS_MODULE}::#{batch_class_name}".constantize
end

#batch_class_name=(class_name) ⇒ Object 



183
184
185
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 183

def batch_class_name=(class_name)
  write_attribute(:batch_class_name, class_name.delete_prefix("::"))
end

#create_batched_job!(min, max) ⇒ Object 



134
135
136
137
138
139
140
141
142
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 134

def create_batched_job!(min, max)
  batched_jobs.create!(
    min_value: min,
    max_value: max,
    batch_size: batch_size,
    sub_batch_size: sub_batch_size,
    pause_ms: pause_ms
  )
end

#health_context ⇒ Object 



222
223
224
225
226
227
228
229
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 222

def health_context
  @health_context ||= Gitlab::Database::HealthStatus::Context.new(
    self,
    connection,
    [table_name],
    gitlab_schema.to_sym
  )
end

#hold!(until_time: 10.minutes.from_now) ⇒ Object 



231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 231

def hold!(until_time: 10.minutes.from_now)
  duration_s = (until_time - Time.current).round
  Gitlab::AppLogger.info(
    message: "#{self} put on hold until #{until_time}",
    migration_id: id,
    job_class_name: job_class_name,
    duration_s: duration_s
  )

  update!(on_hold_until: until_time)
end

#interval_elapsed?(variance: 0) ⇒ Boolean

Returns:

  • (Boolean) 


127
128
129
130
131
132
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 127

def interval_elapsed?(variance: 0)
  return true unless last_job

  interval_with_variance = interval - variance
  last_job.created_at <= Time.current - interval_with_variance
end

#job_class ⇒ Object 



171
172
173
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 171

def job_class
  "#{JOB_CLASS_MODULE}::#{job_class_name}".constantize
end

#job_class_name=(class_name) ⇒ Object 



179
180
181
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 179

def job_class_name=(class_name)
  write_attribute(:job_class_name, class_name.delete_prefix("::"))
end

#migrated_tuple_count ⇒ Object 



187
188
189
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 187

def migrated_tuple_count
  batched_jobs.with_status(:succeeded).sum(:batch_size)
end

#next_min_value ⇒ Object 



167
168
169
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 167

def next_min_value
  last_job&.max_value&.next || min_value
end

#on_hold? ⇒ Boolean

Returns:

  • (Boolean) 


243
244
245
246
247
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 243

def on_hold?
  return false unless on_hold_until

  on_hold_until > Time.zone.now
end

#optimize! ⇒ Object 



218
219
220
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 218

def optimize!
  BatchOptimizer.new(self).optimize!
end

#progress ⇒ Object

Computes an estimation of the progress of the migration in percents.

Because ‘total_tuple_count` is an estimation of the tuples based on DB statistics when the migration is complete there can actually be more or less tuples that initially estimated as `total_tuple_count` so the progress may not show 100%. For that reason when we know migration completed successfully, we just return the 100 value



259
260
261
262
263
264
265
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 259

def progress
  return FINISHED_PROGRESS_VALUE if finished?

  return unless total_tuple_count.to_i > 0

  100 * migrated_tuple_count / total_tuple_count
end

#prometheus_labels ⇒ Object 



191
192
193
194
195
196
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 191

def prometheus_labels
  @prometheus_labels ||= {
    migration_id: id,
    migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name]
  }
end

#reset_attempts_of_blocked_jobs! ⇒ Object 



121
122
123
124
125
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 121

def reset_attempts_of_blocked_jobs!
  batched_jobs.blocked_by_max_attempts.each_batch(of: 100) do |batch|
    batch.update_all(attempts: 0)
  end
end

#retry_failed_jobs! ⇒ Object 



144
145
146
147
148
149
150
151
152
153
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 144

def retry_failed_jobs!
  batched_jobs.with_status(:failed).each_batch(of: 100) do |batch|
    self.class.transaction do
      batch.lock.each(&:split_and_retry!)
      self.execute!
    end
  end

  self.execute!
end

#should_stop? ⇒ Boolean

Returns:

  • (Boolean) 


155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 155

def should_stop?
  return unless started_at

  total_jobs = batched_jobs.created_since(started_at).count

  return if total_jobs < MINIMUM_JOBS

  failed_jobs = batched_jobs.with_status(:failed).created_since(started_at).count

  failed_jobs.fdiv(total_jobs) > MAXIMUM_FAILED_RATIO
end

#smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2) ⇒ Object 



198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 198

def smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2)
  jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs).with_preloads

  return if jobs.size < number_of_jobs

  efficiencies = jobs.map(&:time_efficiency).reject(&:nil?).each_with_index

  dividend = efficiencies.reduce(0) do |total, (job_eff, i)|
    total + job_eff * (1 - alpha)**i
  end

  divisor = efficiencies.reduce(0) do |total, (job_eff, i)|
    total + (1 - alpha)**i
  end

  return if divisor == 0

  (dividend / divisor).round(2)
end

#to_s ⇒ Object 



249
250
251
 
# File 'lib/gitlab/database/background_migration/batched_migration.rb', line 249

def to_s
  "BatchedMigration[id: #{id}]"
end