Module: Gitlab::GitalyClient

Defined in:
lib/gitlab/gitaly_client.rb,
lib/gitlab/gitaly_client/call.rb,
lib/gitlab/gitaly_client/diff.rb,
lib/gitlab/gitaly_client/util.rb,
lib/gitlab/gitaly_client/diff_blob.rb,
lib/gitlab/gitaly_client/wiki_page.rb,
lib/gitlab/gitaly_client/ref_service.rb,
lib/gitlab/gitaly_client/blob_service.rb,
lib/gitlab/gitaly_client/diff_service.rb,
lib/gitlab/gitaly_client/diff_stitcher.rb,
lib/gitlab/gitaly_client/attributes_bag.rb,
lib/gitlab/gitaly_client/blobs_stitcher.rb,
lib/gitlab/gitaly_client/commit_service.rb,
lib/gitlab/gitaly_client/gitaly_context.rb,
lib/gitlab/gitaly_client/list_refs_sort.rb,
lib/gitlab/gitaly_client/remote_service.rb,
lib/gitlab/gitaly_client/server_service.rb,
lib/gitlab/gitaly_client/cleanup_service.rb,
lib/gitlab/gitaly_client/analysis_service.rb,
lib/gitlab/gitaly_client/queue_enumerator.rb,
lib/gitlab/gitaly_client/storage_settings.rb,
lib/gitlab/gitaly_client/conflicts_service.rb,
lib/gitlab/gitaly_client/operation_service.rb,
lib/gitlab/gitaly_client/list_blobs_adapter.rb,
lib/gitlab/gitaly_client/repository_service.rb,
lib/gitlab/gitaly_client/diff_blobs_stitcher.rb,
lib/gitlab/gitaly_client/object_pool_service.rb,
lib/gitlab/gitaly_client/health_check_service.rb,
lib/gitlab/gitaly_client/praefect_info_service.rb,
lib/gitlab/gitaly_client/conflict_files_stitcher.rb,
lib/gitlab/gitaly_client/with_feature_flag_actors.rb,
lib/gitlab/gitaly_client/commit_collection_with_next_cursor.rb

Defined Under Namespace

Modules: AttributesBag, Util, WithFeatureFlagActors Classes: AnalysisService, BlobService, BlobsStitcher, Call, CleanupService, CommitCollectionWithNextCursor, CommitService, ConflictFilesStitcher, ConflictsService, Diff, DiffBlob, DiffBlobsStitcher, DiffService, DiffStitcher, GitalyContext, HealthCheckService, ListBlobsAdapter, ListRefsSort, ObjectPoolService, OperationService, PraefectInfoService, QueueEnumerator, RefService, RemoteService, RepositoryService, ServerService, StorageSettings, TooManyInvocationsError, WikiPage

Constant Summary collapse

SERVER_VERSION_FILE = 
'GITALY_SERVER_VERSION'
MAXIMUM_GITALY_CALLS = 
30
CLIENT_NAME = 
(Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
GITALY_METADATA_FILENAME = 
'.gitaly-metadata'
MUTEX = 
Mutex.new

Class Method Summary collapse

Class Method Details

.add_call_details(details) ⇒ Object 



537
538
539
540
 
# File 'lib/gitlab/gitaly_client.rb', line 537

def self.add_call_details(details)
  Gitlab::SafeRequestStore['gitaly_call_details'] ||= []
  Gitlab::SafeRequestStore['gitaly_call_details'] << details
end

.add_query_time(duration) ⇒ Object 



315
316
317
318
319
320
 
# File 'lib/gitlab/gitaly_client.rb', line 315

def self.add_query_time(duration)
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0
  Gitlab::SafeRequestStore[:gitaly_query_time] += duration
end

.address(storage) ⇒ Object 



243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
 
# File 'lib/gitlab/gitaly_client.rb', line 243

def self.address(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  address = params['gitaly_address']
  unless address.present?
    raise "storage #{storage.inspect} is missing a gitaly_address"
  end

  unless %w[tcp unix tls dns].include?(URI(address).scheme)
    raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls' or 'dns'"
  end

  address
end

.address_metadata(storage) ⇒ Object 



259
260
261
 
# File 'lib/gitlab/gitaly_client.rb', line 259

def self.address_metadata(storage)
  Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
end

.allow_n_plus_1_calls ⇒ Object 



476
477
478
479
480
481
482
483
484
485
 
# File 'lib/gitlab/gitaly_client.rb', line 476

def self.allow_n_plus_1_calls
  return yield unless Gitlab::SafeRequestStore.active?

  begin
    increment_call_count(:gitaly_call_count_exception_block_depth)
    yield
  ensure
    decrement_call_count(:gitaly_call_count_exception_block_depth)
  end
end

.allow_ref_name_caching ⇒ Object

Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it’s possible the branch is mutated afterwards. However, for read-only requests that never mutate the branch, this method allows caching of the ref name directly.



491
492
493
494
495
496
497
498
499
500
501
 
# File 'lib/gitlab/gitaly_client.rb', line 491

def self.allow_ref_name_caching
  return yield unless Gitlab::SafeRequestStore.active?
  return yield if ref_name_caching_allowed?

  begin
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
    yield
  ensure
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
  end
end

.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, gitaly_context: {}, &block) ⇒ Object

All Gitaly RPC call sites should use GitalyClient.call. This method makes sure that per-request authentication headers are set.

This method optionally takes a block which receives the keyword arguments hash ‘kwargs’ that will be passed to gRPC. This allows the caller to modify or augment the keyword arguments. The block must return a hash.

For example:

GitalyClient.call(storage, service, rpc, request) do |kwargs|

kwargs.merge(deadline: Time.now + 10)

end

The optional remote_storage keyword argument is used to enable inter-gitaly calls. Say you have an RPC that needs to pull data from one repository to another. For example, to fetch a branch from a (non-deduplicated) fork into the fork parent. In that case you would send an RPC call to the Gitaly server hosting the fork parent, and in the request, you would tell that Gitaly server to pull Git data from the fork. How does that Gitaly server connect to the Gitaly server the forked repo lives on? This is the problem remote_storage: solves: it adds address and authentication information to the call, as gRPC metadata (under the gitaly-servers header). The request would say “pull from repo X on gitaly-2”. In the Ruby code you pass ‘remote_storage: ’gitaly-2’‘. And then the metadata would say “gitaly-2 is at network address tcp://10.0.1.2:8075”.



295
296
297
 
# File 'lib/gitlab/gitaly_client.rb', line 295

def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, gitaly_context: {}, &block)
  Gitlab::GitalyClient::Call.new(storage, service, rpc, request, remote_storage, timeout, gitaly_context: gitaly_context).call(&block)
end

.clear_stubs! ⇒ Object 



231
232
233
234
235
236
237
 
# File 'lib/gitlab/gitaly_client.rb', line 231

def self.clear_stubs!
  MUTEX.synchronize do
    @channels&.each_value(&:close)
    @stubs = nil
    @channels = nil
  end
end

.connection_data(storage) ⇒ Object 



263
264
265
 
# File 'lib/gitlab/gitaly_client.rb', line 263

def self.connection_data(storage)
  { 'address' => address(storage), 'token' => token(storage) }
end

.create_channel(storage) ⇒ Object

Cache gRPC servers by storage. All the client stubs in the same process can share the underlying connection to the same host thanks to HTTP2 framing protocol that gRPC is built on top. This method is not thread-safe. It is intended to be a part of stub, method behind a mutex protection.



224
225
226
227
228
229
 
# File 'lib/gitlab/gitaly_client.rb', line 224

def self.create_channel(storage)
  @channels ||= {}
  @channels[storage] ||= GRPC::ClientStub.setup_channel(
    nil, stub_address(storage), stub_creds(storage), channel_args
  )
end

.decode_detailed_error(err) ⇒ Object 



633
634
635
636
637
638
639
640
641
642
643
644
645
646
 
# File 'lib/gitlab/gitaly_client.rb', line 633

def self.decode_detailed_error(err)
  # details could have more than one in theory, but we only have one to worry about for now.
  detailed_error = err.to_rpc_status&.details&.first

  return unless detailed_error.present?

  prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)}
  error_type = prefix.match(detailed_error.type_url)[:error_type]

  Gitaly.const_get(error_type, false).decode(detailed_error.value)
rescue NameError, NoMethodError
  # Error Class might not be known to ruby yet
  nil
end

.default_timeout ⇒ Object

The default timeout on all Gitaly calls



558
559
560
 
# File 'lib/gitlab/gitaly_client.rb', line 558

def self.default_timeout
  timeout(:gitaly_timeout_default)
end

.enforce_gitaly_request_limits(call_site) ⇒ Object

Ensures that Gitaly is not being abuse through n+1 misuse etc

Raises:



437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
 
# File 'lib/gitlab/gitaly_client.rb', line 437

def self.enforce_gitaly_request_limits(call_site)
  # Only count limits in request-response environments
  return unless Gitlab::SafeRequestStore.active?

  # This is this actual number of times this call was made. Used for information purposes only
  actual_call_count = increment_call_count("gitaly_#{call_site}_actual")

  return unless enforce_gitaly_request_limits?

  # Check if this call is nested within a allow_n_plus_1_calls
  # block and skip check if it is
  return if get_call_count(:gitaly_call_count_exception_block_depth) > 0

  # This is the count of calls outside of a `allow_n_plus_1_calls` block
  # It is used for enforcement but not statistics
  permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")

  count_stack

  return if permitted_call_count <= MAXIMUM_GITALY_CALLS

  raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end

.execute(storage, service, rpc, request, remote_storage:, timeout:, gitaly_context: {}) ⇒ Object 



299
300
301
302
303
304
305
306
307
308
 
# File 'lib/gitlab/gitaly_client.rb', line 299

def self.execute(storage, service, rpc, request, remote_storage:, timeout:, gitaly_context: {})
  enforce_gitaly_request_limits(:call)
  Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!
  raise_if_concurrent_ruby!

  kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage, gitaly_context: gitaly_context)
  kwargs = yield(kwargs) if block_given?

  stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
end

.expected_server_version ⇒ Object 



548
549
550
551
 
# File 'lib/gitlab/gitaly_client.rb', line 548

def self.expected_server_version
  path = Rails.root.join(SERVER_VERSION_FILE)
  path.read.chomp
end

.fast_timeout ⇒ Object 



562
563
564
 
# File 'lib/gitlab/gitaly_client.rb', line 562

def self.fast_timeout
  timeout(:gitaly_timeout_fast)
end

.feature_flag_actors ⇒ Object 



673
674
675
676
677
678
679
 
# File 'lib/gitlab/gitaly_client.rb', line 673

def self.feature_flag_actors
  if Gitlab::SafeRequestStore.active?
    Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {}
  else
    Thread.current[:gitaly_feature_flag_actors] ||= {}
  end
end

.fetch_relative_path ⇒ Object

The GitLab internal/allowed/ API sets the :gitlab_git_relative_path variable. This provides the repository relative path which can be used to locate snapshot repositories in Gitaly which act as a quarantine repository until a transaction is committed.



384
385
386
387
388
389
 
# File 'lib/gitlab/gitaly_client.rb', line 384

def self.fetch_relative_path
  return unless Gitlab::SafeRequestStore.active?
  return if Gitlab::SafeRequestStore[:gitlab_git_relative_path].blank?

  Gitlab::SafeRequestStore.fetch(:gitlab_git_relative_path)
end

.filesystem_disk_available(storage) ⇒ Object 



582
583
584
 
# File 'lib/gitlab/gitaly_client.rb', line 582

def self.filesystem_disk_available(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available
end

.filesystem_disk_used(storage) ⇒ Object 



586
587
588
 
# File 'lib/gitlab/gitaly_client.rb', line 586

def self.filesystem_disk_used(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used
end

.filesystem_id(storage) ⇒ Object 



578
579
580
 
# File 'lib/gitlab/gitaly_client.rb', line 578

def self.filesystem_id(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id
end

.get_request_count ⇒ Object

Returns the of the number of Gitaly calls made for this request



526
527
528
 
# File 'lib/gitlab/gitaly_client.rb', line 526

def self.get_request_count
  get_call_count("gitaly_call_actual")
end

.list_call_details ⇒ Object 



542
543
544
545
546
 
# File 'lib/gitlab/gitaly_client.rb', line 542

def self.list_call_details
  return [] unless Gitlab::PerformanceBar.enabled_for_request?

  Gitlab::SafeRequestStore['gitaly_call_details'] || []
end

.long_timeout ⇒ Object 



570
571
572
573
574
575
576
 
# File 'lib/gitlab/gitaly_client.rb', line 570

def self.long_timeout
  if Gitlab::Runtime.puma?
    default_timeout
  else
    6.hours
  end
end

.medium_timeout ⇒ Object 



566
567
568
 
# File 'lib/gitlab/gitaly_client.rb', line 566

def self.medium_timeout
  timeout(:gitaly_timeout_medium)
end

.query_time ⇒ Object 



310
311
312
313
 
# File 'lib/gitlab/gitaly_client.rb', line 310

def self.query_time
  query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0
  query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end

.random_storage ⇒ Object 



239
240
241
 
# File 'lib/gitlab/gitaly_client.rb', line 239

def self.random_storage
  Gitlab.config.repositories.storages.keys.sample
end

.ref_name_caching_allowed? ⇒ Boolean

Returns:

  • (Boolean) 


503
504
505
 
# File 'lib/gitlab/gitaly_client.rb', line 503

def self.ref_name_caching_allowed?
  Gitlab::SafeRequestStore[:allow_ref_name_caching]
end

.request_kwargs(storage, timeout:, remote_storage: nil, gitaly_context: {}) ⇒ Object 



342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
 
# File 'lib/gitlab/gitaly_client.rb', line 342

def self.request_kwargs(storage, timeout:, remote_storage: nil, gitaly_context: {})
  metadata = {
    'authorization' => "Bearer #{authorization_token(storage)}",
    'client_name' => CLIENT_NAME
  }

  relative_path = fetch_relative_path

  gitaly_context = GitalyContext.current_context.merge(gitaly_context)

  ::Gitlab::Auth::Identity.currently_linked do |identity|
    gitaly_context['scoped-user-id'] = identity.scoped_user.id.to_s
  end

  context_data = Gitlab::ApplicationContext.current

  feature_stack = Thread.current[:gitaly_feature_stack]
  feature = feature_stack && feature_stack[0]
  metadata['call_site'] = feature.to_s if feature
  metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
  metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
  metadata['gitaly-session-id'] = session_id
  metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil)
  metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil)
  metadata[Labkit::Fields::GL_USER_ID] = context_data['meta.gl_user_id'].to_s if context_data&.fetch('meta.gl_user_id', nil)
  metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil)
  metadata['relative-path-bin'] = relative_path if relative_path
  metadata['gitaly-client-context-bin'] = gitaly_context.to_json if gitaly_context.present?

  metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors))
  metadata.merge!(route_to_primary)

  deadline_info = request_deadline(timeout)
  metadata.merge!(deadline_info.slice(:deadline_type))

  { metadata: metadata, deadline: deadline_info[:deadline] }
end

.reset_counts ⇒ Object 



530
531
532
533
534
535
 
# File 'lib/gitlab/gitaly_client.rb', line 530

def self.reset_counts
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore["gitaly_call_actual"] = 0
  Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0
end

.retry_policy ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/gitlab/gitaly_client.rb', line 57

def self.retry_policy
  ## Default max retry time is 6 seconds:
  ##  0.4 +
  ##  (0.4 * 2) = 0.8 +
  ##  (0.8 * 2) = 1.6 +
  ##  (1.6 * 2) = 3.2
  ##  ==
  ##  0.4 + 0.8 + 1.6 + 3.2 = 6
  {
    maxAttempts: Gitlab.config.gitaly.try(:client_max_attempts) || 4,
    initialBackoff: '0.4s',
    maxBackoff: Gitlab.config.gitaly.try(:client_max_backoff) || '1.4s',
    backoffMultiplier: 2,
    retryableStatusCodes: %w[UNAVAILABLE ABORTED]
  }
end

.session_id ⇒ Object 



425
426
427
 
# File 'lib/gitlab/gitaly_client.rb', line 425

def self.session_id
  Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
end

.stub(name, storage) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/gitlab/gitaly_client.rb', line 34

def self.stub(name, storage)
  MUTEX.synchronize do
    @stubs ||= {}
    @stubs[storage] ||= {}
    @stubs[storage][name] ||= begin
      klass = stub_class(name)
      channel = create_channel(storage)
      klass.new(channel.target, nil, interceptors: interceptors, channel_override: channel)
    end
  end
end

.stub_address(storage) ⇒ Object 



217
218
219
 
# File 'lib/gitlab/gitaly_client.rb', line 217

def self.stub_address(storage)
  address(storage).sub(%r{^tcp://|^tls://}, '')
end

.stub_class(name) ⇒ Object 



209
210
211
212
213
214
215
 
# File 'lib/gitlab/gitaly_client.rb', line 209

def self.stub_class(name)
  if name == :health_check
    Grpc::Health::V1::Health::Stub
  else
    Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
  end
end

.stub_creds(storage) ⇒ Object 



201
202
203
204
205
206
207
 
# File 'lib/gitlab/gitaly_client.rb', line 201

def self.stub_creds(storage)
  if URI(address(storage)).scheme == 'tls'
    GRPC::Core::ChannelCredentials.new ::Gitlab::X509::Certificate.ca_certs_bundle
  else
    :this_channel_is_insecure
  end
end

.timestamp(time) ⇒ Object 



553
554
555
 
# File 'lib/gitlab/gitaly_client.rb', line 553

def self.timestamp(time)
  Google::Protobuf::Timestamp.new(seconds: time.to_i)
end

.token(storage) ⇒ Object 



429
430
431
432
433
434
 
# File 'lib/gitlab/gitaly_client.rb', line 429

def self.token(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end

.unwrap_detailed_error(err) ⇒ Object

This method attempts to unwrap a detailed error from a Gitaly RPC error. It first decodes the detailed error using decode_detailed_error. If successful, it tries to extract the unwrapped error by calling the method named by the error attribute on the decoded error object.



652
653
654
655
656
657
658
659
660
 
# File 'lib/gitlab/gitaly_client.rb', line 652

def self.unwrap_detailed_error(err)
  e = decode_detailed_error(err)

  return e if e.nil? || !e.respond_to?(:error) || e.error.nil? || !e.error.respond_to?(:to_s)

  unwrapped_error = e[e.error.to_s]

  unwrapped_error || e
end

.with_context ⇒ Object 



46
47
48
 
# File 'lib/gitlab/gitaly_client.rb', line 46

def self.with_context(...)
  GitalyContext.with_context(...)
end

.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block) ⇒ Object 



662
663
664
665
666
667
668
669
670
671
 
# File 'lib/gitlab/gitaly_client.rb', line 662

def self.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block)
  feature_flag_actors[:repository] = repository
  feature_flag_actors[:user] = user
  feature_flag_actors[:project] = project
  feature_flag_actors[:group] = group

  yield
ensure
  feature_flag_actors.clear
end