Class: Gitlab::Database::BatchCounter

Inherits:
Object
  • Object
show all
Defined in:
lib/gitlab/database/batch_count.rb

Constant Summary collapse

FALLBACK = 
-1
MIN_REQUIRED_BATCH_SIZE = 
1_250
DEFAULT_SUM_BATCH_SIZE = 
1_000
MAX_ALLOWED_LOOPS = 
10_000
SLEEP_TIME_IN_SECONDS =

10 msec sleep

0.01
ALLOWED_MODES = 
[:itself, :distinct].freeze
DEFAULT_DISTINCT_BATCH_SIZE =

Each query should take < 500ms gitlab.com/gitlab-org/gitlab/-/merge_requests/22705

10_000
DEFAULT_BATCH_SIZE = 
100_000

Instance Method Summary collapse

Constructor Details

#initialize(relation, column: nil, operation: :count, operation_args: nil) ⇒ BatchCounter

Returns a new instance of BatchCounter.


52
53
54
55
56
57
 
# File 'lib/gitlab/database/batch_count.rb', line 52

def initialize(relation, column: nil, operation: :count, operation_args: nil)
  @relation = relation
  @column = column || relation.primary_key
  @operation = operation
  @operation_args = operation_args
end

Instance Method Details

#batch_fetch(start, finish, mode) ⇒ Object 


101
102
103
104
 
# File 'lib/gitlab/database/batch_count.rb', line 101

def batch_fetch(start, finish, mode)
  # rubocop:disable GitlabSecurity/PublicSend
  @relation.select(@column).public_send(mode).where(between_condition(start, finish)).send(@operation, *@operation_args)
end

#count(batch_size: nil, mode: :itself, start: nil, finish: nil) ⇒ Object 


66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/gitlab/database/batch_count.rb', line 66

def count(batch_size: nil, mode: :itself, start: nil, finish: nil)
  raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open?

  check_mode!(mode)

  # non-distinct have better performance
  batch_size ||= batch_size_for_mode_and_operation(mode, @operation)

  start = actual_start(start)
  finish = actual_finish(finish)

  raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0
  return FALLBACK if unwanted_configuration?(finish, batch_size, start)

  counter = 0
  batch_start = start

  while batch_start <= finish
    begin
      counter += batch_fetch(batch_start, batch_start + batch_size, mode)
      batch_start += batch_size
    rescue ActiveRecord::QueryCanceled
      # retry with a safe batch size & warmer cache
      if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE
        batch_size /= 2
      else
        return FALLBACK
      end
    end
    sleep(SLEEP_TIME_IN_SECONDS)
  end

  counter
end

#unwanted_configuration?(finish, batch_size, start) ⇒ Boolean

Returns:

  • (Boolean) 

59
60
61
62
63
64
 
# File 'lib/gitlab/database/batch_count.rb', line 59

def unwanted_configuration?(finish, batch_size, start)
  (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) ||
    (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) ||
    (finish - start) / batch_size >= MAX_ALLOWED_LOOPS ||
    start > finish
end