Module: Gitlab::Utils::UsageData

Constant Summary collapse

FALLBACK =
-1
HISTOGRAM_FALLBACK =
{ '-1' => -1 }.freeze
DISTRIBUTED_HLL_FALLBACK =
-2
MAX_BUCKET_SIZE =
100

Instance Method Summary collapse

Instance Method Details

#add(*args) ⇒ Object

rubocop: enable CodeReuse/ActiveRecord



186
187
188
189
190
191
192
193
194
195
# File 'lib/gitlab/utils/usage_data.rb', line 186

def add(*args)
   do
    break -1 if args.any?(&:negative?)

    args.sum
  rescue StandardError => error
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
    FALLBACK
  end
end

#add_metric(metric, time_frame: 'none', options: {}) ⇒ Object



51
52
53
54
55
# File 'lib/gitlab/utils/usage_data.rb', line 51

def add_metric(metric, time_frame: 'none', options: {})
  metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize

  metric_class.new(time_frame: time_frame, options: options).value
end

#alt_usage_data(value = nil, fallback: FALLBACK, &block) ⇒ Object



197
198
199
200
201
202
203
204
205
206
207
208
# File 'lib/gitlab/utils/usage_data.rb', line 197

def alt_usage_data(value = nil, fallback: FALLBACK, &block)
   do
    if block
      yield
    else
      value
    end
  rescue StandardError => error
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
    fallback
  end
end

#average(relation, column, batch_size: nil, start: nil, finish: nil) ⇒ Object



107
108
109
110
111
112
113
114
# File 'lib/gitlab/utils/usage_data.rb', line 107

def average(relation, column, batch_size: nil, start: nil, finish: nil)
   do
    Gitlab::Database::BatchCount.batch_average(relation, column, batch_size: batch_size, start: start, finish: finish)
  rescue ActiveRecord::StatementInvalid => error
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
    FALLBACK
  end
end

#count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil, start_at: Time.current) ⇒ Object



57
58
59
60
61
62
63
64
65
66
67
68
# File 'lib/gitlab/utils/usage_data.rb', line 57

def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil, start_at: Time.current)
   do
    if batch
      Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish)
    else
      relation.count
    end
  rescue ActiveRecord::StatementInvalid => error
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
    FALLBACK
  end
end

#distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) ⇒ Object



70
71
72
73
74
75
76
77
78
79
80
81
# File 'lib/gitlab/utils/usage_data.rb', line 70

def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
   do
    if batch
      Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish)
    else
      relation.distinct_count_by(column)
    end
  rescue ActiveRecord::StatementInvalid => error
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
    FALLBACK
  end
end

#estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) ⇒ Object



83
84
85
86
87
88
89
90
91
92
93
94
95
96
# File 'lib/gitlab/utils/usage_data.rb', line 83

def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil)
   do
    buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter
                .new(relation, column)
                .execute(batch_size: batch_size, start: start, finish: finish)

    yield buckets if block_given?

    buckets.estimated_distinct_count
  rescue ActiveRecord::StatementInvalid => error
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
    FALLBACK
  end
end

#histogram(relation, column, buckets:, bucket_size: buckets.size) ⇒ Object

We don’t support batching with histograms. Please avoid using this method on large tables. See gitlab.com/gitlab-org/gitlab/-/issues/323949.

rubocop: disable CodeReuse/ActiveRecord



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
# File 'lib/gitlab/utils/usage_data.rb', line 121

def histogram(relation, column, buckets:, bucket_size: buckets.size)
   do
    # Using lambda to avoid exposing histogram specific methods
    parameters_valid = lambda do
      error_message =
        if buckets.first == buckets.last
          'Lower bucket bound cannot equal to upper bucket bound'
        elsif bucket_size == 0
          'Bucket size cannot be zero'
        elsif bucket_size > MAX_BUCKET_SIZE
          "Bucket size #{bucket_size} exceeds the limit of #{MAX_BUCKET_SIZE}"
        end

      break true unless error_message

      exception = ArgumentError.new(error_message)
      exception.set_backtrace(caller)
      Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception)

      false
    end

    break HISTOGRAM_FALLBACK unless parameters_valid.call

    count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped'))
    cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped)

    # For example, 9 segments gives 10 buckets
    bucket_segments = bucket_size - 1

    width_bucket = Arel::Nodes::NamedFunction
      .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments])
      .as('buckets')

    query = cte
      .table
      .project(width_bucket, cte.table[:count])
      .group('buckets')
      .order('buckets')
      .with(cte.to_arel)

    # Return the histogram as a Hash because buckets are unique.
    relation
      .connection
      .exec_query(query.to_sql)
      .rows
      .to_h
      # Keys are converted to strings in Usage Ping JSON
      .stringify_keys
  rescue ActiveRecord::StatementInvalid => e
    Gitlab::AppJsonLogger.error(
      event: 'histogram',
      relation: relation.table_name,
      operation: 'histogram',
      operation_args: [column, buckets.first, buckets.last, bucket_segments],
      query: query.to_sql,
      message: e.message
    )
    # Raises error for dev env
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
    HISTOGRAM_FALLBACK
  end
end

#maximum_id(model, column = nil) ⇒ Object



249
250
251
252
253
254
255
256
# File 'lib/gitlab/utils/usage_data.rb', line 249

def maximum_id(model, column = nil)
  key = :"#{model.name.downcase.gsub('::', '_')}_maximum_id"
  column_to_read = column || :id

  strong_memoize(key) do
    model.maximum(column_to_read)
  end
end

#measure_durationObject



231
232
233
234
235
236
237
# File 'lib/gitlab/utils/usage_data.rb', line 231

def measure_duration
  result = nil
  duration = Benchmark.realtime do
    result = yield
  end
  [result, duration]
end

#minimum_id(model, column = nil) ⇒ Object



258
259
260
261
262
263
264
265
# File 'lib/gitlab/utils/usage_data.rb', line 258

def minimum_id(model, column = nil)
  key = :"#{model.name.downcase.gsub('::', '_')}_minimum_id"
  column_to_read = column || :id

  strong_memoize(key) do
    model.minimum(column_to_read)
  end
end

#redis_usage_data(counter = nil, &block) ⇒ Object



210
211
212
213
214
215
216
217
218
# File 'lib/gitlab/utils/usage_data.rb', line 210

def redis_usage_data(counter = nil, &block)
   do
    if block
      redis_usage_counter(&block)
    elsif counter.present?
      redis_usage_data_totals(counter)
    end
  end
end

#sum(relation, column, batch_size: nil, start: nil, finish: nil) ⇒ Object



98
99
100
101
102
103
104
105
# File 'lib/gitlab/utils/usage_data.rb', line 98

def sum(relation, column, batch_size: nil, start: nil, finish: nil)
   do
    Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish)
  rescue ActiveRecord::StatementInvalid => error
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
    FALLBACK
  end
end

#track_usage_event(event_name, values) ⇒ Object

Parameters:

  • event_name (String, Symbol)

    the event name

  • values (Array|String)

    the values counted



245
246
247
# File 'lib/gitlab/utils/usage_data.rb', line 245

def track_usage_event(event_name, values)
  Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values)
end

#with_finished_at(key, &block) ⇒ Object



239
240
241
# File 'lib/gitlab/utils/usage_data.rb', line 239

def with_finished_at(key, &block)
  yield.merge(key => Time.current)
end

#with_metadataObject



47
48
49
# File 'lib/gitlab/utils/usage_data.rb', line 47

def 
  yield
end

#with_prometheus_client(fallback: {}, verify: true) ⇒ Object



220
221
222
223
224
225
226
227
228
229
# File 'lib/gitlab/utils/usage_data.rb', line 220

def with_prometheus_client(fallback: {}, verify: true)
   do
    client = prometheus_client(verify: verify)
    break fallback unless client

    yield client
  rescue StandardError
    fallback
  end
end