Class: ContainerRepository

Inherits:
ApplicationRecord show all
Includes:
AfterCommitQueue, EachBatch, Gitlab::SQL::Pattern, Gitlab::Utils::StrongMemoize, Packages::Destructible, Sortable
Defined in:
app/models/container_repository.rb

Constant Summary collapse

WAITING_CLEANUP_STATUSES = 
%i[cleanup_scheduled cleanup_unfinished].freeze
REQUIRING_CLEANUP_STATUSES = 
%i[cleanup_unscheduled cleanup_scheduled].freeze
IDLE_MIGRATION_STATES = 
%w[default pre_import_done import_done import_aborted import_skipped].freeze
ACTIVE_MIGRATION_STATES = 
%w[pre_importing importing].freeze
MIGRATION_STATES = 
(IDLE_MIGRATION_STATES + ACTIVE_MIGRATION_STATES).freeze
ABORTABLE_MIGRATION_STATES = 
(ACTIVE_MIGRATION_STATES + %w[pre_import_done default]).freeze
SKIPPABLE_MIGRATION_STATES = 
(ABORTABLE_MIGRATION_STATES + %w[import_aborted]).freeze
MIGRATION_PHASE_1_STARTED_AT = 
Date.new(2021, 11, 4).freeze
MIGRATION_PHASE_1_ENDED_AT = 
Date.new(2022, 01, 23).freeze
MAX_TAGS_PAGES = 
2000
AUTH_TOKEN_USAGE_RESERVED_TIME_IN_SECS =

The Registry client uses JWT token to authenticate to Registry. We cache the client using expiration time of JWT token. However it’s possible that the token is valid but by the time the request is made to Regsitry, it’s already expired. To prevent this case, we are subtracting a few seconds, defined by this constant from the cache expiration time.

5
TooManyImportsError = 
Class.new(StandardError)

Constants included from Gitlab::SQL::Pattern

Gitlab::SQL::Pattern::MIN_CHARS_FOR_PARTIAL_MATCHING, Gitlab::SQL::Pattern::REGEX_QUOTED_TERM

Constants inherited from ApplicationRecord

ApplicationRecord::MAX_PLUCK

Constants included from ResetOnUnionError

ResetOnUnionError::MAX_RESET_PERIOD

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from AfterCommitQueue

#run_after_commit, #run_after_commit_or_now

Methods included from Gitlab::SQL::Pattern

split_query_to_search_terms

Methods inherited from ApplicationRecord

cached_column_list, #create_or_load_association, declarative_enum, default_select_columns, id_in, id_not_in, iid_in, pluck_primary_key, primary_key_in, #readable_by?, safe_ensure_unique, safe_find_or_create_by, safe_find_or_create_by!, #to_ability_name, underscore, where_exists, where_not_exists, with_fast_read_statement_timeout, without_order

Methods included from SensitiveSerializableHash

#serializable_hash

Instance Attribute Details

#path ⇒ Object

rubocop: enable CodeReuse/ServiceClass



432
433
434
435
 
# File 'app/models/container_repository.rb', line 432

def path
  @path ||= [project.full_path, name]
    .select(&:present?).join('/').downcase
end

Class Method Details

.all_migrated? ⇒ Boolean

Returns:

  • (Boolean) 


249
250
251
252
253
254
 
# File 'app/models/container_repository.rb', line 249

def self.all_migrated?
  # check that the set of non migrated repositories is empty
  where(created_at: ...MIGRATION_PHASE_1_ENDED_AT)
    .where.not(migration_state: 'import_done')
    .empty?
end

.build_from_path(path) ⇒ Object 



603
604
605
 
# File 'app/models/container_repository.rb', line 603

def self.build_from_path(path)
  self.new(project: path.repository_project, name: path.repository_name)
end

.build_root_repository(project) ⇒ Object 



617
618
619
 
# File 'app/models/container_repository.rb', line 617

def self.build_root_repository(project)
  self.new(project: project, name: '')
end

.exists_by_path?(path) ⇒ Boolean

Returns:

  • (Boolean) 


242
243
244
245
246
247
 
# File 'app/models/container_repository.rb', line 242

def self.exists_by_path?(path)
  where(
    project: path.repository_project,
    name: path.repository_name
  ).exists?
end

.find_by_path(path) ⇒ Object 



625
626
627
 
# File 'app/models/container_repository.rb', line 625

def self.find_by_path(path)
  self.find_by(project: path.repository_project, name: path.repository_name)
end

.find_by_path!(path) ⇒ Object 



621
622
623
 
# File 'app/models/container_repository.rb', line 621

def self.find_by_path!(path)
  self.find_by!(project: path.repository_project, name: path.repository_name)
end

.find_or_create_from_path(path) ⇒ Object 



607
608
609
610
611
612
613
614
615
 
# File 'app/models/container_repository.rb', line 607

def self.find_or_create_from_path(path)
  repository = safe_find_or_create_by(
    project: path.repository_project,
    name: path.repository_name
  )
  return repository if repository.persisted?

  find_by_path!(path)
end

.registry_client_expiration_time ⇒ Object 



296
297
298
 
# File 'app/models/container_repository.rb', line 296

def self.registry_client_expiration_time
  (Gitlab::CurrentSettings.container_registry_token_expire_delay * 60) - AUTH_TOKEN_USAGE_RESERVED_TIME_IN_SECS
end

.requiring_cleanup ⇒ Object 



261
262
263
264
265
266
 
# File 'app/models/container_repository.rb', line 261

def self.requiring_cleanup
  with_enabled_policy
    .where(container_repositories: { expiration_policy_cleanup_status: REQUIRING_CLEANUP_STATUSES })
    .where('container_repositories.expiration_policy_started_at IS NULL OR container_repositories.expiration_policy_started_at < container_expiration_policies.next_run_at')
    .where('container_expiration_policies.next_run_at < ?', Time.zone.now)
end

.with_enabled_policy ⇒ Object 



256
257
258
259
 
# File 'app/models/container_repository.rb', line 256

def self.with_enabled_policy
  joins('INNER JOIN container_expiration_policies ON container_repositories.project_id = container_expiration_policies.project_id')
    .where(container_expiration_policies: { enabled: true })
end

.with_stale_migration(before_timestamp) ⇒ Object 



272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
 
# File 'app/models/container_repository.rb', line 272

def self.with_stale_migration(before_timestamp)
  stale_pre_importing = with_migration_states(:pre_importing)
                          .with_migration_pre_import_started_at_nil_or_before(before_timestamp)
  stale_pre_import_done = with_migration_states(:pre_import_done)
                            .with_migration_pre_import_done_at_nil_or_before(before_timestamp)
  stale_importing = with_migration_states(:importing)
                      .with_migration_import_started_at_nil_or_before(before_timestamp)

  union = ::Gitlab::SQL::Union.new([
                                     stale_pre_importing,
                                     stale_pre_import_done,
                                     stale_importing
                                   ])
  from("(#{union.to_sql}) #{ContainerRepository.table_name}")
end

.with_target_import_tier ⇒ Object 



288
289
290
291
292
293
294
 
# File 'app/models/container_repository.rb', line 288

def self.with_target_import_tier
  # overridden in ee
  #
  # Repositories are being migrated by tier on Saas, so we need to
  # filter by plan/subscription which is not available in FOSS
  all
end

.with_unfinished_cleanup ⇒ Object 



268
269
270
 
# File 'app/models/container_repository.rb', line 268

def self.with_unfinished_cleanup
  with_enabled_policy.cleanup_unfinished
end

Instance Method Details

#blob(config) ⇒ Object 



491
492
493
 
# File 'app/models/container_repository.rb', line 491

def blob(config)
  ContainerRegistry::Blob.new(self, config)
end

#delete_tag_by_digest(digest) ⇒ Object 



511
512
513
 
# File 'app/models/container_repository.rb', line 511

def delete_tag_by_digest(digest)
  client.delete_repository_tag_by_digest(self.path, digest)
end

#delete_tag_by_name(name) ⇒ Object 



515
516
517
 
# File 'app/models/container_repository.rb', line 515

def delete_tag_by_name(name)
  client.delete_repository_tag_by_name(self.path, name)
end

#delete_tags! ⇒ Object 



503
504
505
506
507
508
509
 
# File 'app/models/container_repository.rb', line 503

def delete_tags!
  return unless has_tags?

  digests = tags.map { |tag| tag.digest }.compact.to_set

  digests.map { |digest| delete_tag_by_digest(digest) }.all?
end

#each_tags_page(page_size: 100, &block) ⇒ Object

Raises:

  • (ArgumentError) 


459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
 
# File 'app/models/container_repository.rb', line 459

def each_tags_page(page_size: 100, &block)
  raise ArgumentError, 'not a migrated repository' unless migrated?
  raise ArgumentError, 'block not given' unless block

  # dummy uri to initialize the loop
  next_page_uri = URI('')
  page_count = 0

  while next_page_uri && page_count < MAX_TAGS_PAGES
    last = Rack::Utils.parse_nested_query(next_page_uri.query)['last']
    current_page = gitlab_api_client.tags(self.path, page_size: page_size, last: last)

    if current_page&.key?(:response_body)
      yield transform_tags_page(current_page[:response_body])
      next_page_uri = current_page.dig(:pagination, :next, :uri)
    else
      # no current page. Break the loop
      next_page_uri = nil
    end

    page_count += 1
  end

  raise 'too many pages requested' if page_count >= MAX_TAGS_PAGES
end

#external_import_status ⇒ Object 



413
414
415
416
417
 
# File 'app/models/container_repository.rb', line 413

def external_import_status
  strong_memoize(:import_status) do
    gitlab_api_client.import_status(self.path)
  end
end

#finish_pre_import_and_start_import ⇒ Object 



328
329
330
331
 
# File 'app/models/container_repository.rb', line 328

def finish_pre_import_and_start_import
  # nothing to do between those two transitions for now.
  finish_pre_import && start_import
end

#force_migration_cancel ⇒ Object

This method is not meant for consumption by the code It is meant for manual use in the case that a migration needs to be cancelled by an admin or SRE



593
594
595
596
597
598
599
600
601
 
# File 'app/models/container_repository.rb', line 593

def force_migration_cancel
  return :error unless gitlab_api_client.supports_gitlab_api?

  response = gitlab_api_client.cancel_repository_import(self.path, force: true)

  skip_import(reason: :migration_forced_canceled) if response[:status] == :ok

  response
end

#has_tags? ⇒ Boolean

Returns:

  • (Boolean) 


495
496
497
 
# File 'app/models/container_repository.rb', line 495

def has_tags?
  tags.any?
end

#last_import_step_done_at ⇒ Object 



409
410
411
 
# File 'app/models/container_repository.rb', line 409

def last_import_step_done_at
  [migration_pre_import_done_at, migration_import_done_at, migration_aborted_at, migration_skipped_at].compact.max
end

#location ⇒ Object 



437
438
439
 
# File 'app/models/container_repository.rb', line 437

def location
  File.join(registry.path, path)
end

#manifest ⇒ Object 



445
446
447
 
# File 'app/models/container_repository.rb', line 445

def manifest
  @manifest ||= client.repository_tags(path)
end

#migrated? ⇒ Boolean

Returns:

  • (Boolean) 


405
406
407
 
# File 'app/models/container_repository.rb', line 405

def migrated?
  Gitlab.com_except_jh?
end

#migration_cancel ⇒ Object 



584
585
586
587
588
 
# File 'app/models/container_repository.rb', line 584

def migration_cancel
  return :error unless gitlab_api_client.supports_gitlab_api?

  gitlab_api_client.cancel_repository_import(self.path)
end

#migration_import ⇒ Object

Raises:



575
576
577
578
579
580
581
582
 
# File 'app/models/container_repository.rb', line 575

def migration_import
  return :error unless gitlab_api_client.supports_gitlab_api?

  response = gitlab_api_client.import_repository(self.path)
  raise TooManyImportsError if response == :too_many_imports

  response
end

#migration_importing? ⇒ Boolean

Returns:

  • (Boolean) 


558
559
560
 
# File 'app/models/container_repository.rb', line 558

def migration_importing?
  migration_state == 'importing'
end

#migration_in_active_state? ⇒ Boolean

Returns:

  • (Boolean) 


554
555
556
 
# File 'app/models/container_repository.rb', line 554

def migration_in_active_state?
  migration_state.in?(ACTIVE_MIGRATION_STATES)
end

#migration_pre_import ⇒ Object

Raises:



566
567
568
569
570
571
572
573
 
# File 'app/models/container_repository.rb', line 566

def migration_pre_import
  return :error unless gitlab_api_client.supports_gitlab_api?

  response = gitlab_api_client.pre_import_repository(self.path)
  raise TooManyImportsError if response == :too_many_imports

  response
end

#migration_pre_importing? ⇒ Boolean

Returns:

  • (Boolean) 


562
563
564
 
# File 'app/models/container_repository.rb', line 562

def migration_pre_importing?
  migration_state == 'pre_importing'
end

#nearing_or_exceeded_retry_limit? ⇒ Boolean

Returns:

  • (Boolean) 


401
402
403
 
# File 'app/models/container_repository.rb', line 401

def nearing_or_exceeded_retry_limit?
  migration_retries_count >= ContainerRegistry::Migration.max_retries - 1
end

#reconcile_import_status(status) ⇒ Object 



342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
 
# File 'app/models/container_repository.rb', line 342

def reconcile_import_status(status)
  case status
  when 'native'
    finish_import_as(:native_import)
  when 'pre_import_in_progress'
    return if pre_importing?

    start_pre_import(forced: true)
  when 'import_in_progress'
    return if importing?

    start_import(forced: true)
  when 'import_complete'
    finish_import
  when 'import_failed', 'import_canceled'
    retry_import
  when 'pre_import_complete'
    finish_pre_import_and_start_import
  when 'pre_import_failed', 'pre_import_canceled'
    retry_pre_import
  else
    yield
  end
end

#registry ⇒ Object

rubocop: disable CodeReuse/ServiceClass



420
421
422
423
424
425
426
427
428
429
 
# File 'app/models/container_repository.rb', line 420

def registry
  strong_memoize_with_expiration(:registry, self.class.registry_client_expiration_time) do
    token = Auth::ContainerRegistryAuthenticationService.full_access_token(path)

    url = Gitlab.config.registry.api_url
    host_port = Gitlab.config.registry.host_port

    ContainerRegistry::Registry.new(url, token: token, path: host_port)
  end
end

#retried_too_many_times? ⇒ Boolean

Returns:

  • (Boolean) 


397
398
399
 
# File 'app/models/container_repository.rb', line 397

def retried_too_many_times?
  migration_retries_count >= ContainerRegistry::Migration.max_retries
end

#retry_aborted_migration ⇒ Object 



333
334
335
336
337
338
339
340
 
# File 'app/models/container_repository.rb', line 333

def retry_aborted_migration
  return unless migration_state == 'import_aborted'

  reconcile_import_status(external_import_status) do
    # If the import_status request fails, use the timestamp to guess current state
    migration_pre_import_done_at ? retry_import : retry_pre_import
  end
end

#retry_import ⇒ Object 



322
323
324
325
326
 
# File 'app/models/container_repository.rb', line 322

def retry_import
  return false unless ContainerRegistry::Migration.enabled?

  super
end

#retry_pre_import ⇒ Object 



316
317
318
319
320
 
# File 'app/models/container_repository.rb', line 316

def retry_pre_import
  return false unless ContainerRegistry::Migration.enabled?

  super
end

#root_repository? ⇒ Boolean

Returns:

  • (Boolean) 


499
500
501
 
# File 'app/models/container_repository.rb', line 499

def root_repository?
  name.empty?
end

#set_delete_ongoing_status ⇒ Object 



537
538
539
540
541
542
543
544
 
# File 'app/models/container_repository.rb', line 537

def set_delete_ongoing_status
  now = Time.zone.now
  update_columns(
    status: :delete_ongoing,
    delete_started_at: now,
    status_updated_at: now
  )
end

#set_delete_scheduled_status ⇒ Object 



546
547
548
549
550
551
552
 
# File 'app/models/container_repository.rb', line 546

def set_delete_scheduled_status
  update_columns(
    status: :delete_scheduled,
    delete_started_at: nil,
    status_updated_at: Time.zone.now
  )
end

#size ⇒ Object 



527
528
529
530
531
532
533
534
535
 
# File 'app/models/container_repository.rb', line 527

def size
  strong_memoize(:size) do
    next unless Gitlab.com_except_jh?
    next if self.created_at.before?(MIGRATION_PHASE_1_STARTED_AT) && self.migration_state != 'import_done'
    next unless gitlab_api_client.supports_gitlab_api?

    gitlab_api_client.repository_details(self.path, sizing: :self)['size_bytes']
  end
end

#skip_import(reason:) ⇒ Object 



304
305
306
307
308
 
# File 'app/models/container_repository.rb', line 304

def skip_import(reason:)
  self.migration_skipped_reason = reason

  super
end

#start_expiration_policy! ⇒ Object 



519
520
521
522
523
524
525
 
# File 'app/models/container_repository.rb', line 519

def start_expiration_policy!
  update!(
    expiration_policy_started_at: Time.zone.now,
    last_cleanup_deleted_tags_count: nil,
    expiration_policy_cleanup_status: :cleanup_ongoing
  )
end

#start_pre_import(*args) ⇒ Object 



310
311
312
313
314
 
# File 'app/models/container_repository.rb', line 310

def start_pre_import(*args)
  return false unless ContainerRegistry::Migration.enabled?

  super(*args)
end

#tag(tag) ⇒ Object 



441
442
443
 
# File 'app/models/container_repository.rb', line 441

def tag(tag)
  ContainerRegistry::Tag.new(self, tag)
end

#tags ⇒ Object 



449
450
451
452
453
454
455
456
457
 
# File 'app/models/container_repository.rb', line 449

def tags
  return [] unless manifest && manifest['tags']

  strong_memoize(:tags) do
    manifest['tags'].sort.map do |tag|
      ContainerRegistry::Tag.new(self, tag)
    end
  end
end

#tags_count ⇒ Object 



485
486
487
488
489
 
# File 'app/models/container_repository.rb', line 485

def tags_count
  return 0 unless manifest && manifest['tags']

  manifest['tags'].size
end

#try_import ⇒ Object

Raises:

  • (ArgumentError) 


367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
 
# File 'app/models/container_repository.rb', line 367

def try_import
  raise ArgumentError, 'block not given' unless block_given?

  try_count = 0
  begin
    try_count += 1

    case yield
    when :ok
      return true
    when :not_found
      finish_import_as(:not_found)
    when :already_imported
      finish_import_as(:native_import)
    else
      abort_import
    end

    false
  rescue TooManyImportsError
    if try_count <= ::ContainerRegistry::Migration.start_max_retries
      sleep 0.1 * try_count
      retry
    else
      abort_import
      false
    end
  end
end