Module: Gitlab::Database

Defined in:
lib/gitlab/database.rb,
lib/gitlab/database/count.rb,
lib/gitlab/database/grant.rb,
lib/gitlab/database/median.rb,
lib/gitlab/database/date_time.rb,
lib/gitlab/database/reindexing.rb,
lib/gitlab/database/batch_count.rb,
lib/gitlab/database/bulk_update.rb,
lib/gitlab/database/sha_attribute.rb,
lib/gitlab/database/postgres_index.rb,
lib/gitlab/database/schema_cleaner.rb,
lib/gitlab/database/schema_helpers.rb,
lib/gitlab/database/connection_timer.rb,
lib/gitlab/database/custom_structure.rb,
lib/gitlab/database/sha256_attribute.rb,
lib/gitlab/database/similarity_score.rb,
lib/gitlab/database/migration_helpers.rb,
lib/gitlab/database/with_lock_retries.rb,
lib/gitlab/database/postgres_partition.rb,
lib/gitlab/database/read_only_relation.rb,
lib/gitlab/database/schema_version_files.rb,
lib/gitlab/database/dynamic_model_helpers.rb,
lib/gitlab/database/reindexing/coordinator.rb,
lib/gitlab/database/background_migration_job.rb,
lib/gitlab/database/multi_threaded_migration.rb,
lib/gitlab/database/obsolete_ignored_columns.rb,
lib/gitlab/database/reindexing/reindex_action.rb,
lib/gitlab/database/count/exact_count_strategy.rb,
lib/gitlab/database/partitioning/replace_table.rb,
lib/gitlab/database/postgres_partitioned_table.rb,
lib/gitlab/database/partitioning/time_partition.rb,
lib/gitlab/database/x509_serial_number_attribute.rb,
lib/gitlab/database/partitioning/monthly_strategy.rb,
lib/gitlab/database/reindexing/concurrent_reindex.rb,
lib/gitlab/database/count/reltuples_count_strategy.rb,
lib/gitlab/database/partitioning/partition_creator.rb,
lib/gitlab/database/partitioning_migration_helpers.rb,
lib/gitlab/database/count/tablesample_count_strategy.rb,
lib/gitlab/database/partitioning/partition_monitoring.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1.rb,
lib/gitlab/database/postgres_hll_batch_distinct_counter.rb,
lib/gitlab/database/migrations/background_migration_helpers.rb,
lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb,
lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb,
lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb,
lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb,
lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb,
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb,
lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb,
lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb,
lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb,
lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb

Defined Under Namespace

Modules: ActiveRecordBaseTransactionMetrics, BatchCount, BulkUpdate, Count, DateTime, DynamicModelHelpers, Median, MigrationHelpers, Migrations, MultiThreadedMigration, Partitioning, PartitioningMigrationHelpers, PostgresqlAdapter, PostgresqlDatabaseTasks, ReadOnlyRelation, Reindexing, RenameReservedPathsMigration, SchemaHelpers Classes: BackgroundMigrationJob, BatchCounter, ConnectionTimer, CustomStructure, Grant, ObsoleteIgnoredColumns, PostgresHllBatchDistinctCounter, PostgresIndex, PostgresPartition, PostgresPartitionedTable, SchemaCleaner, SchemaVersionFiles, Sha256Attribute, ShaAttribute, SimilarityScore, WithLockRetries, X509SerialNumberAttribute

Constant Summary collapse

MINIMUM_POSTGRES_VERSION =

Minimum PostgreSQL version requirement per documentation: docs.gitlab.com/ee/install/requirements.html#postgresql-requirements

11
MAX_INT_VALUE =

www.postgresql.org/docs/9.2/static/datatype-numeric.html

2147483647
MIN_INT_VALUE = 
-2147483648
MAX_TIMESTAMP_VALUE =

The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: www.postgresql.org/docs/9.1/static/datatype-datetime.html dev.mysql.com/doc/refman/5.7/en/datetime.html FIXME: this should just be the max value of timestampz

Time.at((1 << 31) - 1).freeze
MAX_TEXT_SIZE_LIMIT =

The maximum number of characters for text fields, to avoid DoS attacks via parsing huge text fields gitlab.com/gitlab-org/gitlab-foss/issues/61974

1_000_000
MIN_SCHEMA_VERSION =

Minimum schema version from which migrations are supported Migrations before this version may have been removed

20190506135400
MIN_SCHEMA_GITLAB_VERSION = 
'11.11.0'
DYNAMIC_PARTITIONS_SCHEMA =

Schema we store dynamically managed partitions in (e.g. for time partitioning)

:gitlab_partitions_dynamic
STATIC_PARTITIONS_SCHEMA =

Schema we store static partitions in (e.g. for hash partitioning)

:gitlab_partitions_static
EXTRA_SCHEMAS =

This is an extensive list of postgres schemas owned by GitLab It does not include the default public schema

[DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze
BINARY_TYPE =

PostgreSQL defines its own class with slightly different behaviour from the default Binary type.

ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea

Class Method Summary collapse

Class Method Details

.adapter_name ⇒ Object 


50
51
52
 
# File 'lib/gitlab/database.rb', line 50

def self.adapter_name
  config['adapter']
end

.add_post_migrate_path_to_rails(force: false) ⇒ Object 


269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/gitlab/database.rb', line 269

def self.add_post_migrate_path_to_rails(force: false)
  return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force

  Rails.application.config.paths['db'].each do |db_path|
    path = Rails.root.join(db_path, 'post_migrate').to_s

    unless Rails.application.config.paths['db/migrate'].include? path
      Rails.application.config.paths['db/migrate'] << path

      # Rails memoizes migrations at certain points where it won't read the above
      # path just yet. As such we must also update the following list of paths.
      ActiveRecord::Migrator.migrations_paths << path
    end
  end
end

.bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) ⇒ Object

Bulk inserts a number of rows into a table, optionally returning their IDs.

table - The name of the table to insert the rows into. rows - An Array of Hash instances, each mapping the columns to their

values.

return_ids - When set to true the return value will be an Array of IDs of

the inserted rows

disable_quote - A key or an Array of keys to exclude from quoting (You

become responsible for protection from SQL injection for
these keys!)

on_conflict - Defines an upsert. Values can be: :disabled (default) or

:do_nothing


169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/gitlab/database.rb', line 169

def self.bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil)
  return if rows.empty?

  keys = rows.first.keys
  columns = keys.map { |key| connection.quote_column_name(key) }

  disable_quote = Array(disable_quote).to_set
  tuples = rows.map do |row|
    keys.map do |k|
      disable_quote.include?(k) ? row[k] : connection.quote(row[k])
    end
  end

  sql = <<-EOF
    INSERT INTO #{table} (#{columns.join(', ')})
    VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
  EOF

  sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing

  sql = "#{sql} RETURNING id" if return_ids

  result = connection.execute(sql)

  if return_ids
    result.values.map { |tuple| tuple[0].to_i }
  else
    []
  end
end

.cached_column_exists?(table_name, column_name) ⇒ Boolean

Returns:

  • (Boolean) 

231
232
233
 
# File 'lib/gitlab/database.rb', line 231

def self.cached_column_exists?(table_name, column_name)
  connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s)
end

.cached_table_exists?(table_name) ⇒ Boolean

Returns:

  • (Boolean) 

235
236
237
 
# File 'lib/gitlab/database.rb', line 235

def self.cached_table_exists?(table_name)
  exists? && connection.schema_cache.data_source_exists?(table_name)
end

.check_postgres_version_and_print_warning ⇒ Object 


99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/gitlab/database.rb', line 99

def self.check_postgres_version_and_print_warning
  return if Gitlab::Database.postgresql_minimum_supported_version?
  return if Gitlab::Runtime.rails_runner?

  Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result

              ██     ██  █████  ██████  ███    ██ ██ ███    ██  ██████ 
              ██     ██ ██   ██ ██   ██ ████   ██ ██ ████   ██ ██      
              ██  █  ██ ███████ ██████  ██ ██  ██ ██ ██ ██  ██ ██   ███ 
              ██ ███ ██ ██   ██ ██   ██ ██  ██ ██ ██ ██  ██ ██ ██    ██ 
               ███ ███  ██   ██ ██   ██ ██   ████ ██ ██   ████  ██████  

    ******************************************************************************
      You are using PostgreSQL <%= Gitlab::Database.version %>, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>
      is required for this version of GitLab.
      <% if Rails.env.development? || Rails.env.test? %>
      If using gitlab-development-kit, please find the relevant steps here:
        https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/postgresql.md#upgrade-postgresql
      <% end %>
      Please upgrade your environment to a supported PostgreSQL version, see
      https://docs.gitlab.com/ee/install/requirements.html#database for details.
    ******************************************************************************
  EOS
rescue ActiveRecord::ActiveRecordError, PG::Error
  # ignore - happens when Rake tasks yet have to create a database, e.g. for testing
end

.config ⇒ Object 


38
39
40
 
# File 'lib/gitlab/database.rb', line 38

def self.config
  ActiveRecord::Base.configurations[Rails.env]
end

.create_connection_pool(pool_size, host = nil, port = nil) ⇒ Object

pool_size - The size of the DB pool. host - An optional host name to use instead of the default one.


206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/gitlab/database.rb', line 206

def self.create_connection_pool(pool_size, host = nil, port = nil)
  env = Rails.env
  original_config = ActiveRecord::Base.configurations.to_h

  env_config = original_config[env].merge('pool' => pool_size)
  env_config['host'] = host if host
  env_config['port'] = port if port

  config = ActiveRecord::DatabaseConfigurations.new(
    original_config.merge(env => env_config)
  )

  spec =
    ActiveRecord::
      ConnectionAdapters::
      ConnectionSpecification::Resolver.new(config).spec(env.to_sym)

  ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec)
end

.database_name ⇒ Object 


46
47
48
 
# File 'lib/gitlab/database.rb', line 46

def self.database_name
  config['database']
end

.db_read_only? ⇒ Boolean

Check whether the underlying database is in read-only mode

Returns:

  • (Boolean) 

76
77
78
79
80
81
82
83
84
85
 
# File 'lib/gitlab/database.rb', line 76

def self.db_read_only?
  pg_is_in_recovery =
    ActiveRecord::Base
      .connection
      .execute('SELECT pg_is_in_recovery()')
      .first
      .fetch('pg_is_in_recovery')

  Gitlab::Utils.to_boolean(pg_is_in_recovery)
end

.db_read_write? ⇒ Boolean

Returns:

  • (Boolean) 

87
88
89
 
# File 'lib/gitlab/database.rb', line 87

def self.db_read_write?
  !self.db_read_only?
end

.exists? ⇒ Boolean

Returns:

  • (Boolean) 

245
246
247
248
249
250
251
 
# File 'lib/gitlab/database.rb', line 245

def self.exists?
  connection

  true
rescue
  false
end

.false_value ⇒ Object 


142
143
144
 
# File 'lib/gitlab/database.rb', line 142

def self.false_value
  "'f'"
end

.get_write_location(ar_connection) ⇒ Object 


259
260
261
262
263
264
265
 
# File 'lib/gitlab/database.rb', line 259

def self.get_write_location(ar_connection)
  row = ar_connection
    .select_all("SELECT pg_current_wal_insert_lsn()::text AS location")
    .first

  row['location'] if row
end

.human_adapter_name ⇒ Object 


54
55
56
57
58
59
60
 
# File 'lib/gitlab/database.rb', line 54

def self.human_adapter_name
  if postgresql?
    'PostgreSQL'
  else
    'Unknown'
  end
end

.inside_transaction? ⇒ Boolean

inside_transaction? will return true if the caller is running within a transaction. Handles special cases when running inside a test environment, where tests may be wrapped in transactions

Returns:

  • (Boolean) 

287
288
289
290
291
292
293
 
# File 'lib/gitlab/database.rb', line 287

def self.inside_transaction?
  if Rails.env.test?
    ActiveRecord::Base.connection.open_transactions > open_transactions_baseline
  else
    ActiveRecord::Base.connection.open_transactions > 0
  end
end

.install_monkey_patches ⇒ Object

Monkeypatch rails with upgraded database observability


312
313
314
 
# File 'lib/gitlab/database.rb', line 312

def self.install_monkey_patches
  ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics)
end

.nulls_first_order(field, direction = 'ASC') ⇒ Object 


130
131
132
 
# File 'lib/gitlab/database.rb', line 130

def self.nulls_first_order(field, direction = 'ASC')
  Arel.sql("#{field} #{direction} NULLS FIRST")
end

.nulls_last_order(field, direction = 'ASC') ⇒ Object 


126
127
128
 
# File 'lib/gitlab/database.rb', line 126

def self.nulls_last_order(field, direction = 'ASC')
  Arel.sql("#{field} #{direction} NULLS LAST")
end

.observe_transaction_duration(duration_seconds) ⇒ Object

observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to record transaction durations.


318
319
320
321
322
323
324
325
326
327
 
# File 'lib/gitlab/database.rb', line 318

def self.observe_transaction_duration(duration_seconds)
  if current_transaction = ::Gitlab::Metrics::Transaction.current
    current_transaction.observe(:gitlab_database_transaction_seconds, duration_seconds) do
      docstring "Time spent in database transactions, in seconds"
    end
  end
rescue Prometheus::Client::LabelSetValidator::LabelSetError => err
  # Ensure that errors in recording these metrics don't affect the operation of the application
  Gitlab::AppLogger.error("Unable to observe database transaction duration: #{err}")
end

.postgresql? ⇒ Boolean

Deprecated.

Returns:

  • (Boolean) 

63
64
65
 
# File 'lib/gitlab/database.rb', line 63

def self.postgresql?
  adapter_name.casecmp('postgresql') == 0
end

.postgresql_minimum_supported_version? ⇒ Boolean

Returns:

  • (Boolean) 

95
96
97
 
# File 'lib/gitlab/database.rb', line 95

def self.postgresql_minimum_supported_version?
  version.to_f >= MINIMUM_POSTGRES_VERSION
end

.random ⇒ Object 


134
135
136
 
# File 'lib/gitlab/database.rb', line 134

def self.random
  "RANDOM()"
end

.read_only? ⇒ Boolean

Returns:

  • (Boolean) 

67
68
69
 
# File 'lib/gitlab/database.rb', line 67

def self.read_only?
  false
end

.read_write? ⇒ Boolean

Returns:

  • (Boolean) 

71
72
73
 
# File 'lib/gitlab/database.rb', line 71

def self.read_write?
  !self.read_only?
end

.reset_open_transactions_baseline ⇒ Object 


302
303
304
 
# File 'lib/gitlab/database.rb', line 302

def self.reset_open_transactions_baseline
  @open_transactions_baseline = 0
end

.sanitize_timestamp(timestamp) ⇒ Object 


200
201
202
 
# File 'lib/gitlab/database.rb', line 200

def self.sanitize_timestamp(timestamp)
  MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup
end

.set_open_transactions_baseline ⇒ Object

These methods that access @open_transactions_baseline are not thread-safe. These are fine though because we only call these in RSpec's main thread. If we decide to run specs multi-threaded, we would need to use something like ThreadGroup to keep track of this value


298
299
300
 
# File 'lib/gitlab/database.rb', line 298

def self.set_open_transactions_baseline
  @open_transactions_baseline = ActiveRecord::Base.connection.open_transactions
end

.system_id ⇒ Object 


253
254
255
256
257
 
# File 'lib/gitlab/database.rb', line 253

def self.system_id
  row = connection.execute('SELECT system_identifier FROM pg_control_system()').first

  row['system_identifier']
end

.true_value ⇒ Object 


138
139
140
 
# File 'lib/gitlab/database.rb', line 138

def self.true_value
  "'t'"
end

.username ⇒ Object 


42
43
44
 
# File 'lib/gitlab/database.rb', line 42

def self.username
  config['username'] || ENV['USER']
end

.version ⇒ Object 


91
92
93
 
# File 'lib/gitlab/database.rb', line 91

def self.version
  @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
end

.with_connection_pool(pool_size) ⇒ Object 


146
147
148
149
150
151
152
153
154
 
# File 'lib/gitlab/database.rb', line 146

def self.with_connection_pool(pool_size)
  pool = create_connection_pool(pool_size)

  begin
    yield(pool)
  ensure
    pool.disconnect!
  end
end