Class: Gitlab::Ci::Queue::Metrics
- Inherits:
-
Object
- Object
- Gitlab::Ci::Queue::Metrics
- Extended by:
- Utils::StrongMemoize
- Defined in:
- lib/gitlab/ci/queue/metrics.rb
Constant Summary collapse
- QUEUE_DURATION_SECONDS_BUCKETS =
[1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
- QUEUE_ACTIVE_RUNNERS_BUCKETS =
[1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
- QUEUE_DEPTH_TOTAL_BUCKETS =
[1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
- QUEUE_SIZE_TOTAL_BUCKETS =
[1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze
- QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS =
[0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 15, 20, 30, 60].freeze
- METRICS_SHARD_TAG_PREFIX =
'metrics_shard::'
- DEFAULT_METRICS_SHARD =
'default'
- OPERATION_COUNTERS =
[ :build_can_pick, :build_not_pick, :build_not_pending, :build_queue_push, :build_queue_pop, :build_temporary_locked, :build_conflict_lock, :build_conflict_exception, :build_conflict_transition, :queue_attempt, :queue_conflict, :queue_iteration, :queue_depth_limit, :queue_replication_lag, :runner_pre_assign_checks_failed, :runner_pre_assign_checks_success, :runner_queue_tick, :shared_runner_build_new, :shared_runner_build_done ].to_set.freeze
- QUEUE_DEPTH_HISTOGRAMS =
[ :found, :not_found, :conflict ].to_set.freeze
Instance Attribute Summary collapse
-
#runner ⇒ Object
readonly
Returns the value of attribute runner.
Class Method Summary collapse
- .attempt_counter ⇒ Object
- .current_queue_size ⇒ Object
- .failed_attempt_counter ⇒ Object
- .increment_queue_operation(operation) ⇒ Object
- .increment_runner_tick(runner) ⇒ Object
- .job_queue_duration_seconds ⇒ Object
- .observe_active_runners(runners_proc) ⇒ Object
- .queue_active_runners_total ⇒ Object
- .queue_depth_total ⇒ Object
- .queue_iteration_duration_seconds ⇒ Object
- .queue_operations_total ⇒ Object
- .queue_retrieval_duration_seconds ⇒ Object
- .queue_size_total ⇒ Object
Instance Method Summary collapse
- #increment_queue_operation(operation) ⇒ Object
-
#initialize(runner) ⇒ Metrics
constructor
A new instance of Metrics.
- #observe_queue_depth(queue, size) ⇒ Object
- #observe_queue_size(size_proc, runner_type) ⇒ Object
- #observe_queue_time(metric, runner_type) ⇒ Object
- #register_failure ⇒ Object
- #register_success(job) ⇒ Object
Constructor Details
#initialize(runner) ⇒ Metrics
Returns a new instance of Metrics.
48 49 50 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 48 def initialize(runner) @runner = runner end |
Instance Attribute Details
#runner ⇒ Object (readonly)
Returns the value of attribute runner.
46 47 48 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 46 def runner @runner end |
Class Method Details
.attempt_counter ⇒ Object
141 142 143 144 145 146 147 148 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 141 def self.attempt_counter strong_memoize(:attempt_counter) do name = :job_register_attempts_total comment = 'Counts the times a runner tries to register a job' Gitlab::Metrics.counter(name, comment) end end |
.current_queue_size ⇒ Object
192 193 194 195 196 197 198 199 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 192 def self.current_queue_size strong_memoize(:current_queue_size) do name = :gitlab_ci_current_queue_size comment = 'Current size of initialized CI/CD builds queue' Gitlab::Metrics.gauge(name, comment) end end |
.failed_attempt_counter ⇒ Object
132 133 134 135 136 137 138 139 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 132 def self.failed_attempt_counter strong_memoize(:failed_attempt_counter) do name = :job_register_attempts_failed_total comment = 'Counts the times a runner fails to register a job' Gitlab::Metrics.counter(name, comment) end end |
.increment_queue_operation(operation) ⇒ Object
114 115 116 117 118 119 120 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 114 def self.increment_queue_operation(operation) if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) raise ArgumentError, "unknown queue operation: #{operation}" end queue_operations_total.increment(operation: operation) end |
.increment_runner_tick(runner) ⇒ Object
128 129 130 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 128 def self.increment_runner_tick(runner) self.new(runner).increment_queue_operation(:runner_queue_tick) end |
.job_queue_duration_seconds ⇒ Object
150 151 152 153 154 155 156 157 158 159 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 150 def self.job_queue_duration_seconds strong_memoize(:job_queue_duration_seconds) do name = :job_queue_duration_seconds comment = 'Request handling execution time' buckets = QUEUE_DURATION_SECONDS_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) end end |
.observe_active_runners(runners_proc) ⇒ Object
122 123 124 125 126 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 122 def self.observe_active_runners(runners_proc) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops) queue_active_runners_total.observe({}, runners_proc.call.to_f) end |
.queue_active_runners_total ⇒ Object
223 224 225 226 227 228 229 230 231 232 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 223 def self.queue_active_runners_total strong_memoize(:queue_active_runners_total) do name = :gitlab_ci_queue_active_runners_total comment = 'The amount of active runners that can process queue in a project' buckets = QUEUE_ACTIVE_RUNNERS_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) end end |
.queue_depth_total ⇒ Object
170 171 172 173 174 175 176 177 178 179 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 170 def self.queue_depth_total strong_memoize(:queue_depth_total) do name = :gitlab_ci_queue_depth_total comment = 'Size of a CI/CD builds queue in relation to the operation result' buckets = QUEUE_DEPTH_TOTAL_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) end end |
.queue_iteration_duration_seconds ⇒ Object
201 202 203 204 205 206 207 208 209 210 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 201 def self.queue_iteration_duration_seconds strong_memoize(:queue_iteration_duration_seconds) do name = :gitlab_ci_queue_iteration_duration_seconds comment = 'Time it takes to find a build in CI/CD queue' buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) end end |
.queue_operations_total ⇒ Object
161 162 163 164 165 166 167 168 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 161 def self.queue_operations_total strong_memoize(:queue_operations_total) do name = :gitlab_ci_queue_operations_total comment = 'Counts all the operations that are happening inside a queue' Gitlab::Metrics.counter(name, comment) end end |
.queue_retrieval_duration_seconds ⇒ Object
212 213 214 215 216 217 218 219 220 221 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 212 def self.queue_retrieval_duration_seconds strong_memoize(:queue_retrieval_duration_seconds) do name = :gitlab_ci_queue_retrieval_duration_seconds comment = 'Time it takes to execute a SQL query to retrieve builds queue' buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) end end |
.queue_size_total ⇒ Object
181 182 183 184 185 186 187 188 189 190 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 181 def self.queue_size_total strong_memoize(:queue_size_total) do name = :gitlab_ci_queue_size_total comment = 'Size of initialized CI/CD builds queue' buckets = QUEUE_SIZE_TOTAL_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) end end |
Instance Method Details
#increment_queue_operation(operation) ⇒ Object
71 72 73 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 71 def increment_queue_operation(operation) self.class.increment_queue_operation(operation) end |
#observe_queue_depth(queue, size) ⇒ Object
75 76 77 78 79 80 81 82 83 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 75 def observe_queue_depth(queue, size) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops) if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue) raise ArgumentError, "unknown queue depth label: #{queue}" end self.class.queue_depth_total.observe({ queue: queue }, size.to_f) end |
#observe_queue_size(size_proc, runner_type) ⇒ Object
85 86 87 88 89 90 91 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 85 def observe_queue_size(size_proc, runner_type) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops) size = size_proc.call.to_f self.class.queue_size_total.observe({ runner_type: runner_type }, size) self.class.current_queue_size.set({ runner_type: runner_type }, size) end |
#observe_queue_time(metric, runner_type) ⇒ Object
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 93 def observe_queue_time(metric, runner_type) start_time = ::Gitlab::Metrics::System.monotonic_time result = yield return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops) seconds = ::Gitlab::Metrics::System.monotonic_time - start_time case metric when :process self.class.queue_iteration_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f) when :retrieve self.class.queue_retrieval_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f) else raise ArgumentError unless Rails.env.production? end result end |
#register_failure ⇒ Object
52 53 54 55 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 52 def register_failure self.class.failed_attempt_counter.increment self.class.attempt_counter.increment end |
#register_success(job) ⇒ Object
57 58 59 60 61 62 63 64 65 66 67 68 69 |
# File 'lib/gitlab/ci/queue/metrics.rb', line 57 def register_success(job) labels = { shared_runner: runner.instance_type?, jobs_running_for_project: job.project_jobs_running_on_instance_runners_count, shard: DEFAULT_METRICS_SHARD } if runner.instance_type? shard = runner.tag_list.sort.find { |name| name.starts_with?(METRICS_SHARD_TAG_PREFIX) } labels[:shard] = shard.gsub(METRICS_SHARD_TAG_PREFIX, '') if shard end self.class.job_queue_duration_seconds.observe(labels, job.time_in_queue_seconds) unless job.queued_at.nil? self.class.attempt_counter.increment end |