Class: Ci::BuildTraceChunk
Constant Summary
collapse
- CHUNK_SIZE =
128.kilobytes
- WRITE_LOCK_RETRY =
10
- WRITE_LOCK_SLEEP =
0.01.seconds
- WRITE_LOCK_TTL =
1.minute
- FailedToPersistDataError =
Class.new(StandardError)
- DATA_STORES =
{
redis: 1,
database: 2,
fog: 3,
redis_trace_chunks: 4
}.freeze
- STORE_TYPES =
DATA_STORES.keys.index_with do |store|
"Ci::BuildTraceChunks::#{store.to_s.camelize}".constantize
end.freeze
- LIVE_STORES =
%i[redis redis_trace_chunks].freeze
Gitlab::OptimisticLocking::MAX_RETRIES
Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError
FastDestroyAll::ForbiddenActionError
Partitionable::MUTEX
ApplicationRecord::MAX_PLUCK
ResetOnUnionError::MAX_RESET_PERIOD
Class Method Summary
collapse
Instance Method Summary
collapse
log_optimistic_lock_retries, retry_lock, retry_lock_histogram, retry_lock_logger
#in_lock
model_name, table_name_prefix
cached_column_list, #create_or_load_association, declarative_enum, default_select_columns, id_in, id_not_in, iid_in, pluck_primary_key, primary_key_in, #readable_by?, safe_ensure_unique, safe_find_or_create_by, safe_find_or_create_by!, #to_ability_name, underscore, where_exists, where_not_exists, with_fast_read_statement_timeout, without_order
#serializable_hash
Class Method Details
.all_stores ⇒ Object
48
49
50
|
# File 'app/models/ci/build_trace_chunk.rb', line 48
def all_stores
STORE_TYPES.keys
end
|
.begin_fast_destroy ⇒ Object
66
67
68
69
70
71
72
73
|
# File 'app/models/ci/build_trace_chunk.rb', line 66
def begin_fast_destroy
all_stores.each_with_object({}) do |store, result|
relation = public_send(store) keys = get_store_class(store).keys(relation)
result[store] = keys if keys.present?
end
end
|
.finalize_fast_destroy(keys) ⇒ Object
77
78
79
80
81
|
# File 'app/models/ci/build_trace_chunk.rb', line 77
def finalize_fast_destroy(keys)
keys.each do |store, value|
get_store_class(store).delete_keys(value)
end
end
|
.get_store_class(store) ⇒ Object
56
57
58
59
60
61
62
|
# File 'app/models/ci/build_trace_chunk.rb', line 56
def get_store_class(store)
store = store.to_sym
raise "Unknown store type: #{store}" unless STORE_TYPES.key?(store)
STORE_TYPES[store].new
end
|
Sometimes we do not want to read raw data. This method makes it easier to find attributes that are just metadata excluding raw data.
97
98
99
|
# File 'app/models/ci/build_trace_chunk.rb', line 97
def metadata_attributes
attribute_names - %w[raw_data]
end
|
.persistable_store ⇒ Object
52
53
54
|
# File 'app/models/ci/build_trace_chunk.rb', line 52
def persistable_store
STORE_TYPES[:fog].available? ? :fog : :database
end
|
.with_read_consistency(build, &block) ⇒ Object
Sometime we need to ensure that the first read goes to a primary database, what is especially important in EE. This method does not change the behavior in CE.
88
89
90
91
|
# File 'app/models/ci/build_trace_chunk.rb', line 88
def with_read_consistency(build, &block)
::Gitlab::Database::Consistency
.with_read_consistency(&block)
end
|
Instance Method Details
#<=>(other) ⇒ Object
210
211
212
213
214
|
# File 'app/models/ci/build_trace_chunk.rb', line 210
def <=>(other)
return unless build_id == other.build_id
chunk_index <=> other.chunk_index
end
|
#append(new_data, offset) ⇒ Object
117
118
119
120
121
122
123
124
125
|
# File 'app/models/ci/build_trace_chunk.rb', line 117
def append(new_data, offset)
raise ArgumentError, 'New data is missing' unless new_data
raise ArgumentError, 'Offset is out of range' if offset < 0 || offset > size
raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize)
in_lock(lock_key, **lock_params) { unsafe_append_data!(new_data, offset) }
schedule_to_persist! if full?
end
|
#crc32 ⇒ Object
106
107
108
|
# File 'app/models/ci/build_trace_chunk.rb', line 106
def crc32
checksum.to_i
end
|
#data ⇒ Object
102
103
104
|
# File 'app/models/ci/build_trace_chunk.rb', line 102
def data
@data ||= get_data.to_s
end
|
#end_offset ⇒ Object
135
136
137
|
# File 'app/models/ci/build_trace_chunk.rb', line 135
def end_offset
start_offset + size
end
|
#final? ⇒ Boolean
Build trace chunk is final (the last one that we do not expect to ever become full) when a runner submitted a build pending state and there is no chunk with higher index in the database.
194
195
196
|
# File 'app/models/ci/build_trace_chunk.rb', line 194
def final?
build.pending_state.present? && chunks_max_index == chunk_index
end
|
#flushed? ⇒ Boolean
198
199
200
|
# File 'app/models/ci/build_trace_chunk.rb', line 198
def flushed?
!live?
end
|
#live? ⇒ Boolean
206
207
208
|
# File 'app/models/ci/build_trace_chunk.rb', line 206
def live?
LIVE_STORES.include?(data_store.to_sym)
end
|
#migrated? ⇒ Boolean
202
203
204
|
# File 'app/models/ci/build_trace_chunk.rb', line 202
def migrated?
flushed?
end
|
#persist_data! ⇒ Object
It is possible that we run into two concurrent migrations. It might happen that a chunk gets migrated after being loaded by another worker but before the worker acquires a lock to perform the migration.
We are using Redis locking to ensure that we perform this operation inside an exclusive lock, but this does not prevent us from running into race conditions related to updating a model representation in the database. Optimistic locking is another mechanism that help here.
We are using optimistic locking combined with Redis locking to ensure that a chunk gets migrated properly.
We are using until_executed deduplication strategy for workers, which should prevent duplicated workers running in parallel for the same build trace, and causing an exception related to an exclusive lock not being acquired
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
# File 'app/models/ci/build_trace_chunk.rb', line 167
def persist_data!
in_lock(lock_key, **lock_params) do raise FailedToPersistDataError, 'Modifed build trace chunk detected' if has_changes_to_save?
self.class.with_read_consistency(build) do
reset.unsafe_persist_data!
end
end
rescue FailedToObtainLockError
metrics.increment_trace_operation(operation: :stalled)
raise FailedToPersistDataError, 'Data migration failed due to a worker duplication'
rescue ActiveRecord::StaleObjectError
raise FailedToPersistDataError, <<~MSG
Data migration race condition detected
store: #{data_store}
build: #{build.id}
index: #{chunk_index}
MSG
end
|
#range ⇒ Object
139
140
141
|
# File 'app/models/ci/build_trace_chunk.rb', line 139
def range
(start_offset...end_offset)
end
|
#schedule_to_persist! ⇒ Object
143
144
145
146
147
|
# File 'app/models/ci/build_trace_chunk.rb', line 143
def schedule_to_persist!
return if flushed?
Ci::BuildTraceChunkFlushWorker.perform_async(id)
end
|
#size ⇒ Object
127
128
129
|
# File 'app/models/ci/build_trace_chunk.rb', line 127
def size
@size ||= @data&.bytesize || current_store.size(self) || data&.bytesize
end
|
#start_offset ⇒ Object
131
132
133
|
# File 'app/models/ci/build_trace_chunk.rb', line 131
def start_offset
chunk_index * CHUNK_SIZE
end
|
#truncate(offset = 0) ⇒ Object
110
111
112
113
114
115
|
# File 'app/models/ci/build_trace_chunk.rb', line 110
def truncate(offset = 0)
raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
return if offset == size
append(+"", offset)
end
|