Class: Gitlab::Ci::Queue::Metrics

Inherits:
Object
  • Object
show all
Extended by:
Utils::StrongMemoize
Defined in:
lib/gitlab/ci/queue/metrics.rb

Constant Summary collapse

QUEUE_DURATION_SECONDS_BUCKETS = 
[1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
QUEUE_ACTIVE_RUNNERS_BUCKETS = 
[1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
QUEUE_DEPTH_TOTAL_BUCKETS = 
[1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
QUEUE_SIZE_TOTAL_BUCKETS = 
[1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze
QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = 
[0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 15, 20, 30, 60].freeze
METRICS_SHARD_TAG_PREFIX = 
'metrics_shard::'
DEFAULT_METRICS_SHARD = 
'default'
OPERATION_COUNTERS = 
[
  :build_can_pick,
  :build_not_pick,
  :build_not_pending,
  :build_queue_push,
  :build_queue_pop,
  :build_status_stale,
  :build_temporary_locked,
  :build_conflict_lock,
  :build_conflict_exception,
  :build_conflict_transition,
  :queue_attempt,
  :queue_conflict,
  :queue_iteration,
  :queue_depth_limit,
  :queue_replication_lag,
  :runner_pre_assign_checks_failed,
  :runner_pre_assign_checks_success,
  :runner_queue_tick,
  :shared_runner_build_new,
  :shared_runner_build_done
].to_set.freeze
QUEUE_DEPTH_HISTOGRAMS = 
[
  :found,
  :not_found,
  :conflict
].to_set.freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(runner) ⇒ Metrics

Returns a new instance of Metrics.



49
50
51
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 49

def initialize(runner)
  @runner = runner
end

Instance Attribute Details

#runner ⇒ Object (readonly)

Returns the value of attribute runner.



47
48
49
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 47

def runner
  @runner
end

Class Method Details

.attempt_counter ⇒ Object 



142
143
144
145
146
147
148
149
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 142

def self.attempt_counter
  strong_memoize(:attempt_counter) do
    name = :job_register_attempts_total
    comment = 'Counts the times a runner tries to register a job'

    Gitlab::Metrics.counter(name, comment)
  end
end

.current_queue_size ⇒ Object 



193
194
195
196
197
198
199
200
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 193

def self.current_queue_size
  strong_memoize(:current_queue_size) do
    name = :gitlab_ci_current_queue_size
    comment = 'Current size of initialized CI/CD builds queue'

    Gitlab::Metrics.gauge(name, comment)
  end
end

.failed_attempt_counter ⇒ Object 



133
134
135
136
137
138
139
140
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 133

def self.failed_attempt_counter
  strong_memoize(:failed_attempt_counter) do
    name = :job_register_attempts_failed_total
    comment = 'Counts the times a runner fails to register a job'

    Gitlab::Metrics.counter(name, comment)
  end
end

.increment_queue_operation(operation) ⇒ Object 



115
116
117
118
119
120
121
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 115

def self.increment_queue_operation(operation)
  if !Rails.env.production? && OPERATION_COUNTERS.exclude?(operation)
    raise ArgumentError, "unknown queue operation: #{operation}"
  end

  queue_operations_total.increment(operation: operation)
end

.increment_runner_tick(runner) ⇒ Object 



129
130
131
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 129

def self.increment_runner_tick(runner)
  self.new(runner).increment_queue_operation(:runner_queue_tick)
end

.job_queue_duration_seconds ⇒ Object 



151
152
153
154
155
156
157
158
159
160
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 151

def self.job_queue_duration_seconds
  strong_memoize(:job_queue_duration_seconds) do
    name = :job_queue_duration_seconds
    comment = 'Request handling execution time'
    buckets = QUEUE_DURATION_SECONDS_BUCKETS
    labels = {}

    Gitlab::Metrics.histogram(name, comment, labels, buckets)
  end
end

.observe_active_runners(runners_proc) ⇒ Object 



123
124
125
126
127
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 123

def self.observe_active_runners(runners_proc)
  return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops)

  queue_active_runners_total.observe({}, runners_proc.call.to_f)
end

.queue_active_runners_total ⇒ Object 



224
225
226
227
228
229
230
231
232
233
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 224

def self.queue_active_runners_total
  strong_memoize(:queue_active_runners_total) do
    name = :gitlab_ci_queue_active_runners_total
    comment = 'The amount of active runners that can process queue in a project'
    buckets = QUEUE_ACTIVE_RUNNERS_BUCKETS
    labels = {}

    Gitlab::Metrics.histogram(name, comment, labels, buckets)
  end
end

.queue_depth_total ⇒ Object 



171
172
173
174
175
176
177
178
179
180
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 171

def self.queue_depth_total
  strong_memoize(:queue_depth_total) do
    name = :gitlab_ci_queue_depth_total
    comment = 'Size of a CI/CD builds queue in relation to the operation result'
    buckets = QUEUE_DEPTH_TOTAL_BUCKETS
    labels = {}

    Gitlab::Metrics.histogram(name, comment, labels, buckets)
  end
end

.queue_iteration_duration_seconds ⇒ Object 



202
203
204
205
206
207
208
209
210
211
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 202

def self.queue_iteration_duration_seconds
  strong_memoize(:queue_iteration_duration_seconds) do
    name = :gitlab_ci_queue_iteration_duration_seconds
    comment = 'Time it takes to find a build in CI/CD queue'
    buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
    labels = {}

    Gitlab::Metrics.histogram(name, comment, labels, buckets)
  end
end

.queue_operations_total ⇒ Object 



162
163
164
165
166
167
168
169
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 162

def self.queue_operations_total
  strong_memoize(:queue_operations_total) do
    name = :gitlab_ci_queue_operations_total
    comment = 'Counts all the operations that are happening inside a queue'

    Gitlab::Metrics.counter(name, comment)
  end
end

.queue_retrieval_duration_seconds ⇒ Object 



213
214
215
216
217
218
219
220
221
222
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 213

def self.queue_retrieval_duration_seconds
  strong_memoize(:queue_retrieval_duration_seconds) do
    name = :gitlab_ci_queue_retrieval_duration_seconds
    comment = 'Time it takes to execute a SQL query to retrieve builds queue'
    buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
    labels = {}

    Gitlab::Metrics.histogram(name, comment, labels, buckets)
  end
end

.queue_size_total ⇒ Object 



182
183
184
185
186
187
188
189
190
191
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 182

def self.queue_size_total
  strong_memoize(:queue_size_total) do
    name = :gitlab_ci_queue_size_total
    comment = 'Size of initialized CI/CD builds queue'
    buckets = QUEUE_SIZE_TOTAL_BUCKETS
    labels = {}

    Gitlab::Metrics.histogram(name, comment, labels, buckets)
  end
end

Instance Method Details

#increment_queue_operation(operation) ⇒ Object 



72
73
74
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 72

def increment_queue_operation(operation)
  self.class.increment_queue_operation(operation)
end

#observe_queue_depth(queue, size) ⇒ Object 



76
77
78
79
80
81
82
83
84
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 76

def observe_queue_depth(queue, size)
  return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops)

  if !Rails.env.production? && QUEUE_DEPTH_HISTOGRAMS.exclude?(queue)
    raise ArgumentError, "unknown queue depth label: #{queue}"
  end

  self.class.queue_depth_total.observe({ queue: queue }, size.to_f)
end

#observe_queue_size(size_proc, runner_type) ⇒ Object 



86
87
88
89
90
91
92
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 86

def observe_queue_size(size_proc, runner_type)
  return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops)

  size = size_proc.call.to_f
  self.class.queue_size_total.observe({ runner_type: runner_type }, size)
  self.class.current_queue_size.set({ runner_type: runner_type }, size)
end

#observe_queue_time(metric, runner_type) ⇒ Object 



94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 94

def observe_queue_time(metric, runner_type)
  start_time = ::Gitlab::Metrics::System.monotonic_time

  result = yield

  return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops)

  seconds = ::Gitlab::Metrics::System.monotonic_time - start_time

  case metric
  when :process
    self.class.queue_iteration_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f)
  when :retrieve
    self.class.queue_retrieval_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f)
  else
    raise ArgumentError unless Rails.env.production?
  end

  result
end

#register_failure ⇒ Object 



53
54
55
56
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 53

def register_failure
  self.class.failed_attempt_counter.increment
  self.class.attempt_counter.increment
end

#register_success(job) ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/gitlab/ci/queue/metrics.rb', line 58

def register_success(job)
  labels = { shared_runner: runner.instance_type?,
             jobs_running_for_project: job.project_jobs_running_on_instance_runners_count,
             shard: DEFAULT_METRICS_SHARD }

  if runner.instance_type?
    shard = runner.tag_list.sort.find { |name| name.starts_with?(METRICS_SHARD_TAG_PREFIX) }
    labels[:shard] = shard.gsub(METRICS_SHARD_TAG_PREFIX, '') if shard
  end

  self.class.job_queue_duration_seconds.observe(labels, job.time_in_queue_seconds) unless job.queued_at.nil?
  self.class.attempt_counter.increment
end