Module: Gitlab::GithubImport::BulkImporting

Included in:
Importer::LabelsImporter, Importer::MilestonesImporter, Importer::ReleasesImporter
Defined in:
lib/gitlab/github_import/bulk_importing.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#client ⇒ Object (readonly)

Returns the value of attribute client.



6
7
8
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 6

def client
  @client
end

#project ⇒ Object (readonly)

Returns the value of attribute project.



6
7
8
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 6

def project
  @project
end

Instance Method Details

#build_database_rows(enum) ⇒ Object

Builds and returns an Array of objects to bulk insert into the database and array of validation errors if object is invalid.

enum - An Enumerable that returns the objects to turn into database

rows.



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 21

def build_database_rows(enum)
  errors = []
  rows = enum.each_with_object([]) do |(object, _), result|
    next if already_imported?(object)

    attrs = build_attributes(object)
    build_record = model.new(attrs)

    if build_record.invalid?
      github_identifiers = github_identifiers(object)

      log_error(github_identifiers, build_record.errors.full_messages)
      errors << {
        validation_errors: build_record.errors,
        external_identifiers: github_identifiers
      }
      next
    end

    result << attrs
  end

  log_and_increment_counter(rows.size, :fetched)

  [rows, errors]
end

#bulk_insert(rows, batch_size: 100) ⇒ Object

Bulk inserts the given rows into the database.



49
50
51
52
53
54
55
56
57
58
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 49

def bulk_insert(rows, batch_size: 100)
  inserted_ids = []
  rows.each_slice(batch_size) do |slice|
    inserted_ids += ApplicationRecord.legacy_bulk_insert(model.table_name, slice, return_ids: true) # rubocop:disable Gitlab/BulkInsert -- required

    log_and_increment_counter(slice.size, :imported)
  end

  inserted_ids
end

#bulk_insert_failures(errors) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 64

def bulk_insert_failures(errors)
  rows = errors.map do |error|
    correlation_id_value = Labkit::Correlation::CorrelationId.current_or_new_id

    {
      source: self.class.name,
      exception_class: 'ActiveRecord::RecordInvalid',
      exception_message: error[:validation_errors].full_messages.first.truncate(255),
      correlation_id_value: correlation_id_value,
      retry_count: nil,
      created_at: Time.zone.now,
      external_identifiers: error[:external_identifiers]
    }
  end

  project.import_failures.insert_all(rows)
end

#initialize(project, client) ⇒ Object

project - An instance of Project. client - An instance of Gitlab::GithubImport::Client.



10
11
12
13
14
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 10

def initialize(project, client)
  @project = project
  @client = client
  @validation_errors = []
end

#object_type ⇒ Object

Raises:

  • (NotImplementedError) 


60
61
62
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 60

def object_type
  raise NotImplementedError
end