Class: Gitlab::BackgroundMigration::BackfillRootStorageStatisticsForkStorageSizes

Inherits:
BatchedMigrationJob
  • Object
show all
Defined in:
lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb

Overview

Backfill the following columns on the namespace_root_storage_statistics table:

- public_forks_storage_size
- internal_forks_storage_size
- private_forks_storage_size

Constant Summary collapse

VISIBILITY_LEVELS_TO_STORAGE_SIZE_COLUMNS = 
{
  0 => :private_forks_storage_size,
  10 => :internal_forks_storage_size,
  20 => :public_forks_storage_size
}.freeze

Constants inherited from BatchedMigrationJob

Gitlab::BackgroundMigration::BatchedMigrationJob::DEFAULT_FEATURE_CATEGORY

Constants included from Database::DynamicModelHelpers

Database::DynamicModelHelpers::BATCH_SIZE

Instance Method Summary collapse

Methods inherited from BatchedMigrationJob

#batch_metrics, feature_category, #filter_batch, generic_instance, #initialize, job_arguments, job_arguments_count, operation_name, scope_to

Methods included from Database::DynamicModelHelpers

#define_batchable_model, #each_batch, #each_batch_range

Constructor Details

This class inherits a constructor from Gitlab::BackgroundMigration::BatchedMigrationJob

Instance Method Details

#execute(sql) ⇒ Object 



56
57
58
 
# File 'lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb', line 56

def execute(sql)
  ::ApplicationRecord.connection.execute(sql)
end

#group_namespace_sql(namespace_id) ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb', line 77

def group_namespace_sql(namespace_id)
  <<~SQL
    SELECT
      SUM("project_statistics"."storage_size") AS sum_project_statistics_storage_size,
      "projects"."visibility_level" AS projects_visibility_level
    FROM
      "projects"
      INNER JOIN "project_statistics" ON "project_statistics"."project_id" = "projects"."id"
      INNER JOIN "fork_network_members" ON "fork_network_members"."project_id" = "projects"."id"
      INNER JOIN "fork_networks" ON "fork_networks"."id" = "fork_network_members"."fork_network_id"
    WHERE
      "projects"."namespace_id" IN (
        SELECT namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id
        FROM "namespaces"
        WHERE "namespaces"."type" = 'Group' AND (traversal_ids @> ('{#{namespace_id}}'))
      )
      AND (fork_networks.root_project_id != projects.id)
    GROUP BY "projects"."visibility_level"
  SQL
end

#has_fork_data?(root_storage_statistics) ⇒ Boolean

Returns:

  • (Boolean) 


46
47
48
49
50
 
# File 'lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb', line 46

def has_fork_data?(root_storage_statistics)
  root_storage_statistics.public_forks_storage_size != 0 ||
    root_storage_statistics.internal_forks_storage_size != 0 ||
    root_storage_statistics.private_forks_storage_size != 0
end

#perform ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb', line 19

def perform
  each_sub_batch do |sub_batch|
    sub_batch.each do |root_storage_statistics|
      next if has_fork_data?(root_storage_statistics)

      namespace_id = root_storage_statistics.namespace_id

      namespace_type = execute("SELECT type FROM namespaces WHERE id = #{namespace_id}").first&.fetch('type')

      next if namespace_type.nil?

      sql = if user_namespace?(namespace_type)
              user_namespace_sql(namespace_id)
            else
              group_namespace_sql(namespace_id)
            end

      stats = execute(sql)
        .map { |h| { h['projects_visibility_level'] => h['sum_project_statistics_storage_size'] } }
        .reduce({}) { |memo, h| memo.merge(h) }
        .transform_keys { |k| VISIBILITY_LEVELS_TO_STORAGE_SIZE_COLUMNS[k] }

      root_storage_statistics.update!(stats)
    end
  end
end

#user_namespace?(type) ⇒ Boolean

Returns:

  • (Boolean) 


52
53
54
 
# File 'lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb', line 52

def user_namespace?(type)
  type.nil? || type == 'User' || !(type == 'Group' || type == 'Project')
end

#user_namespace_sql(namespace_id) ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb', line 60

def user_namespace_sql(namespace_id)
  <<~SQL
    SELECT
      SUM("project_statistics"."storage_size") AS sum_project_statistics_storage_size,
      "projects"."visibility_level" AS projects_visibility_level
    FROM
      "projects"
      INNER JOIN "project_statistics" ON "project_statistics"."project_id" = "projects"."id"
      INNER JOIN "fork_network_members" ON "fork_network_members"."project_id" = "projects"."id"
      INNER JOIN "fork_networks" ON "fork_networks"."id" = "fork_network_members"."fork_network_id"
    WHERE
      "projects"."namespace_id" = #{namespace_id}
      AND (fork_networks.root_project_id != projects.id)
    GROUP BY "projects"."visibility_level"
  SQL
end