Module: Gitlab::GithubImport::BulkImporting

Included in:
Importer::LabelsImporter, Importer::MilestonesImporter, Importer::ReleasesImporter
Defined in:
lib/gitlab/github_import/bulk_importing.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#client ⇒ Object (readonly)

Returns the value of attribute client.



6
7
8
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 6

def client
  @client
end

#project ⇒ Object (readonly)

Returns the value of attribute project.



6
7
8
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 6

def project
  @project
end

Instance Method Details

#build_database_rows(enum) ⇒ Object

Builds and returns an Array of objects to bulk insert into the database and array of validation errors if object is invalid.

enum - An Enumerable that returns the objects to turn into database

rows.



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 21

def build_database_rows(enum)
  errors = []
  rows = enum.each_with_object([]) do |(object, _), result|
    next if already_imported?(object)

    attrs = build_attributes(object)
    build_record = model.new(attrs)

    if build_record.invalid?
      github_identifiers = github_identifiers(object)

      log_error(github_identifiers, build_record.errors.full_messages)
      errors << {
        validation_errors: build_record.errors,
        github_identifiers: github_identifiers
      }
      next
    end

    result << attrs
  end

  log_and_increment_counter(rows.size, :fetched)

  [rows, errors]
end

#bulk_insert(rows, batch_size: 100) ⇒ Object

Bulk inserts the given rows into the database.



49
50
51
52
53
54
55
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 49

def bulk_insert(rows, batch_size: 100)
  rows.each_slice(batch_size) do |slice|
    ApplicationRecord.legacy_bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert

    log_and_increment_counter(slice.size, :imported)
  end
end

#bulk_insert_failures(errors) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 61

def bulk_insert_failures(errors)
  rows = errors.map do |error|
    correlation_id_value = Labkit::Correlation::CorrelationId.current_or_new_id

    {
      source: self.class.name,
      exception_class: 'ActiveRecord::RecordInvalid',
      exception_message: error[:validation_errors].full_messages.first.truncate(255),
      correlation_id_value: correlation_id_value,
      retry_count: nil,
      created_at: Time.zone.now,
      external_identifiers: error[:github_identifiers]
    }
  end

  project.import_failures.insert_all(rows)
end

#initialize(project, client) ⇒ Object

project - An instance of ‘Project`. client - An instance of `Gitlab::GithubImport::Client`.



10
11
12
13
14
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 10

def initialize(project, client)
  @project = project
  @client = client
  @validation_errors = []
end

#object_type ⇒ Object

Raises:

  • (NotImplementedError) 


57
58
59
 
# File 'lib/gitlab/github_import/bulk_importing.rb', line 57

def object_type
  raise NotImplementedError
end