Class: Gitlab::UsageData

Inherits:
Object
  • Object
show all
Extended by:
Gitlab::Usage::TimeFrame, Gitlab::Utils::StrongMemoize, Gitlab::Utils::UsageData
Defined in:
lib/gitlab/usage_data.rb,
lib/gitlab/usage_data/topology.rb

Direct Known Subclasses

UsageDataNonSqlMetrics, UsageDataQueries

Defined Under Namespace

Classes: Topology

Constant Summary collapse

MAX_GENERATION_TIME_FOR_SAAS = 
40.hours
CE_MEMOIZED_VALUES = 
%i[
  issue_minimum_id
  issue_maximum_id
  project_minimum_id
  project_maximum_id
  user_minimum_id
  user_maximum_id
  deployment_minimum_id
  deployment_maximum_id
  auth_providers
  recorded_at
].freeze

Constants included from Gitlab::Usage::TimeFrame

Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME, Gitlab::Usage::TimeFrame::DEFAULT_TIMESTAMP_COLUMN, Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME, Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME

Constants included from Gitlab::Utils::UsageData

Gitlab::Utils::UsageData::DISTRIBUTED_HLL_FALLBACK, Gitlab::Utils::UsageData::FALLBACK, Gitlab::Utils::UsageData::HISTOGRAM_FALLBACK, Gitlab::Utils::UsageData::MAX_BUCKET_SIZE

Class Method Summary collapse

Methods included from Gitlab::Usage::TimeFrame

monthly_time_range, monthly_time_range_db_params, weekly_time_range, weekly_time_range_db_params

Methods included from Gitlab::Utils::UsageData

add, add_metric, alt_usage_data, average, count, distinct_count, estimate_batch_distinct_count, histogram, maximum_id, measure_duration, metrics_collection_metadata, minimum_id, redis_usage_data, sum, track_usage_event, with_finished_at, with_metadata, with_prometheus_client

Class Method Details

.components_usage_data ⇒ Object 



146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/gitlab/usage_data.rb', line 146

def components_usage_data
  {
    git: { version: alt_usage_data(fallback: { major: -1 }) { Gitlab::Git.version } },
    gitaly: {
      version: alt_usage_data { Gitaly::Server.all.first.server_version },
      servers: alt_usage_data { Gitaly::Server.count },
      clusters: alt_usage_data { Gitaly::Server.gitaly_clusters },
      filesystems: alt_usage_data(fallback: ["-1"]) { Gitaly::Server.filesystems }
    },
    gitlab_pages: {
      enabled: alt_usage_data(fallback: nil) { Gitlab.config.pages.enabled },
      version: alt_usage_data { Gitlab::Pages::VERSION }
    },
    container_registry_server: {
      vendor: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_vendor },
      version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version }
    },
    database: {
      # rubocop: disable UsageData/LargeTable
      adapter: alt_usage_data { ApplicationRecord.database.adapter_name },
      version: alt_usage_data { ApplicationRecord.database.version },
      pg_system_id: alt_usage_data { ApplicationRecord.database.system_id },
      flavor: alt_usage_data { ApplicationRecord.database.flavor }
      # rubocop: enable UsageData/LargeTable
    },
    mail: {
      smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] }
    }
  }
end

.data ⇒ Object 



41
42
43
44
45
46
47
48
 
# File 'lib/gitlab/usage_data.rb', line 41

def data
  clear_memoized

  with_finished_at(:recording_ce_finished_at) do
    { recorded_at: recorded_at }
      .merge(usage_data_metrics)
  end
end

.features_usage_data ⇒ Object 



144
 
# File 'lib/gitlab/usage_data.rb', line 144

def features_usage_data = {}

.object_store_config(component) ⇒ Object 



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/gitlab/usage_data.rb', line 177

def object_store_config(component)
  config = alt_usage_data(fallback: nil) do
    Settings[component]['object_store']
  end

  if config.present?
    {
      enabled: alt_usage_data { Settings[component]['enabled'] },
      object_store: {
        enabled: alt_usage_data { config['enabled'] },
        direct_upload: alt_usage_data { config['direct_upload'] },
        background_upload: alt_usage_data { false }, # This setting no longer exists
        provider: alt_usage_data { config['connection']['provider'] }
      }
    }
  else
    {
      enabled: alt_usage_data { Settings[component]['enabled'] }
    }
  end
end

.object_store_usage_data ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
 
# File 'lib/gitlab/usage_data.rb', line 199

def object_store_usage_data
  {
    object_store: {
      artifacts: object_store_config('artifacts'),
      external_diffs: object_store_config('external_diffs'),
      lfs: object_store_config('lfs'),
      uploads: object_store_config('uploads'),
      packages: object_store_config('packages')
    }
  }
end

.recorded_at ⇒ Object 



50
51
52
 
# File 'lib/gitlab/usage_data.rb', line 50

def recorded_at
  @recorded_at ||= Time.current
end

.system_usage_data ⇒ Object

rubocop: disable Metrics/AbcSize rubocop: disable CodeReuse/ActiveRecord



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/gitlab/usage_data.rb', line 56

def system_usage_data
  {
    counts: {
      assignee_lists: count(List.assignee),
      ci_external_pipelines: count(::Ci::Pipeline.external),
      ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source),
      ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source),
      ci_triggers: count(::Ci::Trigger),
      ci_pipeline_schedules: count(::Ci::PipelineSchedule),
      auto_devops_enabled: count(::ProjectAutoDevops.enabled),
      auto_devops_disabled: count(::ProjectAutoDevops.disabled),
      deploy_keys: count(DeployKey),
      feature_flags: count(Operations::FeatureFlag),
      environments: count(::Environment),
      clusters: count(::Clusters::Cluster),
      clusters_enabled: count(::Clusters::Cluster.enabled),
      project_clusters_enabled: count(::Clusters::Cluster.enabled.project_type),
      group_clusters_enabled: count(::Clusters::Cluster.enabled.group_type),
      instance_clusters_enabled: count(::Clusters::Cluster.enabled.instance_type),
      clusters_disabled: count(::Clusters::Cluster.disabled),
      project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type),
      group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type),
      instance_clusters_disabled: count(::Clusters::Cluster.disabled.instance_type),
      clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled),
      clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
      clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
      clusters_management_project: count(::Clusters::Cluster.with_management_project),
      kubernetes_agents: count(::Clusters::Agent),
      kubernetes_agents_with_token: distinct_count(::Clusters::AgentToken, :agent_id),
      in_review_folder: count(::Environment.in_review_folder),
      groups: count(Group),
      issues: add_metric('CountIssuesMetric', time_frame: 'all'),
      issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
      issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
      issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
      incident_issues: count(::Issue.with_issue_type(:incident), start: minimum_id(Issue), finish: maximum_id(Issue)),
      alert_bot_incident_issues: count(::Issue.authored(::Users::Internal.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)),
      keys: count(Key),
      label_lists: count(List.label),
      lfs_objects: count(LfsObject),
      milestone_lists: count(List.milestone),
      milestones: count(Milestone),
      projects_with_packages: distinct_count(::Packages::Package, :project_id),
      packages: count(::Packages::Package),
      pages_domains: count(PagesDomain),
      pool_repositories: count(PoolRepository),
      projects: count(Project),
      projects_creating_incidents: distinct_count(Issue.with_issue_type(:incident), :project_id),
      projects_imported_from_github: count(Project.where(import_type: 'github')),
      projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)),
      projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
      projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id),
      projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.of_report_type(:terraform), :project_id),
      projects_with_terraform_states: distinct_count(::Terraform::State, :project_id),
      protected_branches: count(ProtectedBranch),
      protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])),
      releases: count(Release),
      remote_mirrors: count(RemoteMirror),
      suggestions: count(Suggestion),
      terraform_reports: count(::Ci::JobArtifact.of_report_type(:terraform)),
      terraform_states: count(::Terraform::State),
      todos: count(Todo),
      uploads: count(Upload),
      web_hooks: count(WebHook),
      labels: count(Label),
      merge_requests: count(MergeRequest),
      notes: count(Note)
    }.merge(
      user_preferences_usage,
      service_desk_counts
    )
  }
end

.system_usage_data_license ⇒ Object

rubocop: enable Metrics/AbcSize rubocop: enable CodeReuse/ActiveRecord



132
133
134
135
136
 
# File 'lib/gitlab/usage_data.rb', line 132

def system_usage_data_license
  {
    license: {}
  }
end

.system_usage_data_weekly ⇒ Object 



138
139
140
141
142
 
# File 'lib/gitlab/usage_data.rb', line 138

def system_usage_data_weekly
  {
    counts_weekly: {}
  }
end

.topology_usage_data ⇒ Object 



211
212
213
 
# File 'lib/gitlab/usage_data.rb', line 211

def topology_usage_data
  Gitlab::UsageData::Topology.new.topology_usage_data
end

.usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) ⇒ Object

Source: gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv



223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
 
# File 'lib/gitlab/usage_data.rb', line 223

def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {})
  {
    key => {
      configure: usage_activity_by_stage_configure(time_period),
      create: usage_activity_by_stage_create(time_period),
      enablement: usage_activity_by_stage_enablement(time_period),
      manage: usage_activity_by_stage_manage(time_period),
      monitor: usage_activity_by_stage_monitor(time_period),
      package: usage_activity_by_stage_package(time_period),
      plan: usage_activity_by_stage_plan(time_period),
      release: usage_activity_by_stage_release(time_period),
      verify: usage_activity_by_stage_verify(time_period)
    }
  }
end

.usage_activity_by_stage_configure(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord rubocop: disable UsageData/LargeTable



241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
 
# File 'lib/gitlab/usage_data.rb', line 241

def usage_activity_by_stage_configure(time_period)
  {
    clusters_management_project: clusters_user_distinct_count(::Clusters::Cluster.with_management_project, time_period),
    clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled, time_period),
    clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled, time_period),
    clusters_platforms_gke: clusters_user_distinct_count(::Clusters::Cluster.gcp_installed.enabled, time_period),
    clusters_platforms_eks: clusters_user_distinct_count(::Clusters::Cluster.aws_installed.enabled, time_period),
    clusters_platforms_user: clusters_user_distinct_count(::Clusters::Cluster.user_provided.enabled, time_period),
    instance_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.instance_type, time_period),
    instance_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.instance_type, time_period),
    group_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.group_type, time_period),
    group_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.group_type, time_period),
    project_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.project_type, time_period),
    project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period),
    # These two `projects_slack_x` metrics are owned by the Manage stage, but are in this method as their key paths can't change.
    # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123442#note_1427961339.
    projects_slack_notifications_active: distinct_count(::Project.with_slack_integration.where(time_period), :creator_id),
    projects_slack_slash_active: distinct_count(::Project.with_slack_slash_commands_integration.where(time_period), :creator_id)
  }
end

.usage_activity_by_stage_create(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord



265
266
267
268
269
270
271
272
273
274
 
# File 'lib/gitlab/usage_data.rb', line 265

def usage_activity_by_stage_create(time_period)
  {
    deploy_keys: distinct_count(::DeployKey.where(time_period), :user_id),
    keys: distinct_count(::Key.regular_keys.where(time_period), :user_id),
    projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))),
    projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))),
    remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id),
    snippets: distinct_count(::Snippet.where(time_period), :author_id)
  }
end

.usage_activity_by_stage_enablement(time_period) ⇒ Object

Empty placeholder allows this to match the pattern used by other sections



278
279
280
 
# File 'lib/gitlab/usage_data.rb', line 278

def usage_activity_by_stage_enablement(time_period)
  {}
end

.usage_activity_by_stage_manage(time_period) ⇒ Object

Omitted because no user, creator or author associated: ‘campaigns_imported_from_github`, `ldap_group_links` rubocop: disable CodeReuse/ActiveRecord



284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/gitlab/usage_data.rb', line 284

def usage_activity_by_stage_manage(time_period)
  {
    # rubocop: disable UsageData/LargeTable
    events: stage_manage_events(time_period),
    groups: distinct_count(::GroupMember.where(time_period), :user_id),
    users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)),
    omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
    user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
    bulk_imports: {
      gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab))
    },
    group_imports: group_imports(time_period)
  }
end

.usage_activity_by_stage_monitor(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord



301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
 
# File 'lib/gitlab/usage_data.rb', line 301

def usage_activity_by_stage_monitor(time_period)
  # Calculate histogram only for overall as other time periods aren't available/useful here.
  integrations_histogram = time_period.empty? ? histogram(::AlertManagement::HttpIntegration.active, :project_id, buckets: 1..100) : nil

  {
    clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
    operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
      start: minimum_id(User),
      finish: maximum_id(User)),
    projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id),
    projects_with_incidents: distinct_count(::Issue.with_issue_type(:incident).where(time_period), :project_id),
    # We are making an assumption here that all alert_management_alerts are associated with an issue of type
    # incident. In reality this is very close to the truth and allows more efficient queries.
    # More info in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121297#note_1416999956
    projects_with_alert_incidents: distinct_count(::AlertManagement::Alert.where(time_period).where.not(issue_id: nil), :project_id),
    projects_with_enabled_alert_integrations_histogram: integrations_histogram
  }.compact
end

.usage_activity_by_stage_package(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord



322
323
324
325
326
 
# File 'lib/gitlab/usage_data.rb', line 322

def usage_activity_by_stage_package(time_period)
  {
    projects_with_packages: distinct_count(::Project.with_packages.where(time_period), :creator_id)
  }
end

.usage_activity_by_stage_plan(time_period) ⇒ Object

Omitted because no user, creator or author associated: ‘boards`, `labels`, `milestones`, `uploads` Omitted because too expensive: `epics_deepest_relationship_level` rubocop: disable CodeReuse/ActiveRecord



332
333
334
335
336
337
338
339
340
341
342
343
344
 
# File 'lib/gitlab/usage_data.rb', line 332

def usage_activity_by_stage_plan(time_period)
  time_frame = metric_time_period(time_period)
  {
    issues: add_metric('CountUsersCreatingIssuesMetric', time_frame: time_frame),
    notes: distinct_count(::Note.where(time_period), :author_id),
    projects: distinct_count(::Project.where(time_period), :creator_id),
    todos: distinct_count(::Todo.where(time_period), :author_id),
    service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period),
    service_desk_issues: count(::Issue.service_desk.where(time_period)),
    projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).where(time_period), :creator_id),
    projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_server.where(time_period), :creator_id)
  }
end

.usage_activity_by_stage_release(time_period) ⇒ Object

Omitted because no user, creator or author associated: ‘environments`, `feature_flags`, `in_review_folder`, `pages_domains` rubocop: disable CodeReuse/ActiveRecord



349
350
351
352
353
354
355
356
357
358
 
# File 'lib/gitlab/usage_data.rb', line 349

def usage_activity_by_stage_release(time_period)
  time_frame = metric_time_period(time_period)
  {
    deployments: distinct_count(::Deployment.where(time_period), :user_id),
    failed_deployments: distinct_count(::Deployment.failed.where(time_period), :user_id),
    releases: distinct_count(::Release.where(time_period), :author_id),
    successful_deployments: distinct_count(::Deployment.success.where(time_period), :user_id),
    releases_with_milestones: add_metric('CountUsersAssociatingMilestonesToReleasesMetric', time_frame: time_frame)
  }
end

.usage_activity_by_stage_verify(time_period) ⇒ Object

Omitted because no user, creator or author associated: ‘ci_runners` rubocop: disable CodeReuse/ActiveRecord



363
364
365
366
367
368
369
370
371
372
373
374
 
# File 'lib/gitlab/usage_data.rb', line 363

def usage_activity_by_stage_verify(time_period)
  {
    ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
    ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
    ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id)
  }
end

.user_preferences_usage ⇒ Object

augmented in EE



216
217
218
219
220
 
# File 'lib/gitlab/usage_data.rb', line 216

def user_preferences_usage
  {
    user_preferences_user_gitpod_enabled: count(UserPreference.with_user.gitpod_enabled.merge(User.active))
  }
end

.with_metadata ⇒ Object

rubocop: enable CodeReuse/ActiveRecord



377
378
379
380
381
382
383
384
385
386
387
388
389
 
# File 'lib/gitlab/usage_data.rb', line 377

def with_metadata
  result = nil
  error = nil

  duration = Benchmark.realtime do
    result = yield
  rescue StandardError => e
    error = e
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
  end

  ::Gitlab::Usage::ServicePing::LegacyMetricMetadataDecorator.new(result, duration, error: error)
end