Module: Gitlab::SidekiqConfig
- Extended by:
- CliMethods
- Defined in:
- lib/gitlab/sidekiq_config.rb,
lib/gitlab/sidekiq_config/worker.rb,
lib/gitlab/sidekiq_config/cli_methods.rb,
lib/gitlab/sidekiq_config/dummy_worker.rb,
lib/gitlab/sidekiq_config/worker_router.rb,
lib/gitlab/sidekiq_config/worker_matcher.rb
Defined Under Namespace
Modules: CliMethods Classes: DummyWorker, Worker, WorkerMatcher, WorkerRouter
Constant Summary collapse
- FOSS_QUEUE_CONFIG_PATH =
'app/workers/all_queues.yml'
- EE_QUEUE_CONFIG_PATH =
'ee/app/workers/all_queues.yml'
- JH_QUEUE_CONFIG_PATH =
'jh/app/workers/all_queues.yml'
- SIDEKIQ_QUEUES_PATH =
'config/sidekiq_queues.yml'
- JH_SIDEKIQ_QUEUES_PATH =
'jh/config/sidekiq_queues.yml'
- QUEUE_CONFIG_PATHS =
[ FOSS_QUEUE_CONFIG_PATH, (EE_QUEUE_CONFIG_PATH if Gitlab.ee?), (JH_QUEUE_CONFIG_PATH if Gitlab.jh?) ].compact.freeze
- DEFAULT_WORKERS =
This maps workers not in our application code to queues. We need these queues in our YAML files to ensure we don’t accidentally miss jobs from these queues.
The default queue should be unused, which is why it maps to an invalid class name. We keep it in the YAML file for safety, just in case anything does get scheduled to run there.
{ '_' => DummyWorker.new( queue: 'default', weight: 1, tags: [] ), 'ActionMailer::MailDeliveryJob' => DummyWorker.new( name: 'ActionMailer::MailDeliveryJob', queue: 'mailers', urgency: 'low', weight: 2, tags: [] ) }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false, jh: false) }.freeze
Class Method Summary collapse
-
.all_queues_yml_outdated? ⇒ Boolean
YAML.load_file is OK here as we control the file contents.
- .config_queues ⇒ Object
- .cron_jobs ⇒ Object
- .cron_workers ⇒ Object
-
.current_worker_queue_mappings ⇒ Object
Like worker_queue_mappings, but only for the queues running in the current Sidekiq process.
-
.jh_queues_for_sidekiq_queues_yml ⇒ Object
Override in JH repo.
- .queues_for_sidekiq_queues_yml ⇒ Object
- .redis_queues ⇒ Object
-
.routing_queues ⇒ Object
Get the list of queues from all available workers following queue routing rules.
-
.sidekiq_queues_yml_outdated? ⇒ Boolean
YAML.load_file is OK here as we control the file contents.
-
.worker_queue_mappings ⇒ Object
Returns a hash of worker class name => mapped queue name.
- .workers ⇒ Object
- .workers_for_all_queues_yml ⇒ Object
Methods included from CliMethods
clear_memoization!, expand_queues, query_queues, worker_metadatas, worker_queues
Class Method Details
.all_queues_yml_outdated? ⇒ Boolean
YAML.load_file is OK here as we control the file contents
115 116 117 118 119 120 121 122 123 |
# File 'lib/gitlab/sidekiq_config.rb', line 115 def all_queues_yml_outdated? foss_workers, ee_workers, jh_workers = workers_for_all_queues_yml return true if foss_workers != YAML.load_file(FOSS_QUEUE_CONFIG_PATH) return true if Gitlab.ee? && ee_workers != YAML.load_file(EE_QUEUE_CONFIG_PATH) Gitlab.jh? && File.exist?(JH_QUEUE_CONFIG_PATH) && jh_workers != YAML.load_file(JH_QUEUE_CONFIG_PATH) end |
.config_queues ⇒ Object
49 50 51 52 53 54 |
# File 'lib/gitlab/sidekiq_config.rb', line 49 def config_queues @config_queues ||= begin config = YAML.load_file(Rails.root.join(SIDEKIQ_QUEUES_PATH)) config[:queues].map(&:first) end end |
.cron_jobs ⇒ Object
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
# File 'lib/gitlab/sidekiq_config.rb', line 56 def cron_jobs @cron_jobs ||= begin Gitlab.config.load_dynamic_cron_schedules! jobs = Gitlab.config.cron_jobs.to_hash jobs.delete('poll_interval') # Would be interpreted as a job otherwise # Settingslogic (former gem used for yaml configuration) didn't allow 'class' key # Therefore, we configure cron jobs with `job_class` as a workaround. required_keys = %w[job_class cron] jobs.each do |k, v| if jobs[k] && required_keys.all? { |s| jobs[k].key?(s) } jobs[k]['class'] = jobs[k].delete('job_class') else jobs.delete(k) Gitlab::AppLogger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.") end end jobs end end |
.cron_workers ⇒ Object
80 81 82 |
# File 'lib/gitlab/sidekiq_config.rb', line 80 def cron_workers @cron_workers ||= cron_jobs.map { |job_name, | ['class'].constantize } end |
.current_worker_queue_mappings ⇒ Object
Like worker_queue_mappings, but only for the queues running in the current Sidekiq process
162 163 164 165 166 |
# File 'lib/gitlab/sidekiq_config.rb', line 162 def current_worker_queue_mappings worker_queue_mappings .select { |worker, queue| Sidekiq[:queues].include?(queue) } .to_h end |
.jh_queues_for_sidekiq_queues_yml ⇒ Object
Override in JH repo
142 143 144 |
# File 'lib/gitlab/sidekiq_config.rb', line 142 def jh_queues_for_sidekiq_queues_yml [] end |
.queues_for_sidekiq_queues_yml ⇒ Object
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 |
# File 'lib/gitlab/sidekiq_config.rb', line 125 def queues_for_sidekiq_queues_yml namespaces_with_equal_weights = workers .reject { |worker| worker.jh? } .group_by(&:queue_namespace) .map(&:last) .select { |workers| workers.map(&:get_weight).uniq.count == 1 } .map(&:first) namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set remaining_queues = workers.reject { |worker| worker.jh? }.reject { |worker| namespaces.include?(worker.queue_namespace) } (namespaces_with_equal_weights.map(&:namespace_and_weight) + remaining_queues.map(&:queue_and_weight)).sort end |
.redis_queues ⇒ Object
44 45 46 47 |
# File 'lib/gitlab/sidekiq_config.rb', line 44 def redis_queues # Not memoized, because this can change during the life of the application Sidekiq::Queue.all.map(&:name) end |
.routing_queues ⇒ Object
Get the list of queues from all available workers following queue routing rules. Sidekiq::Queue.all fetches the list of queues from Redis. It may contain some redundant, obsolete queues from previous iterations of GitLab.
172 173 174 175 176 177 178 179 180 |
# File 'lib/gitlab/sidekiq_config.rb', line 172 def routing_queues @routing_queues ||= workers.map do |worker| if worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) worker.queue else ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass) end end.uniq.sort end |
.sidekiq_queues_yml_outdated? ⇒ Boolean
YAML.load_file is OK here as we control the file contents
147 148 149 150 151 |
# File 'lib/gitlab/sidekiq_config.rb', line 147 def sidekiq_queues_yml_outdated? config_queues = YAML.load_file(SIDEKIQ_QUEUES_PATH)[:queues] queues_for_sidekiq_queues_yml != config_queues end |
.worker_queue_mappings ⇒ Object
Returns a hash of worker class name => mapped queue name
154 155 156 157 158 |
# File 'lib/gitlab/sidekiq_config.rb', line 154 def worker_queue_mappings workers .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) } .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] } end |
.workers ⇒ Object
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
# File 'lib/gitlab/sidekiq_config.rb', line 84 def workers @workers ||= begin result = [] result.concat(DEFAULT_WORKERS.values) result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false, jh: false)) if Gitlab.ee? result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true, jh: false)) end if Gitlab.jh? result.concat(find_workers(Rails.root.join('jh', 'app', 'workers'), ee: false, jh: true)) end result end end |
.workers_for_all_queues_yml ⇒ Object
102 103 104 105 106 107 108 109 110 111 112 |
# File 'lib/gitlab/sidekiq_config.rb', line 102 def workers_for_all_queues_yml workers.each_with_object([[], [], []]) do |worker, array| if worker.jh? array[2].push(worker) elsif worker.ee? array[1].push(worker) else array[0].push(worker) end end.map(&:sort) end |