Class: PEROBS::EquiBlobsFile

Inherits:
Object
  • Object
show all
Defined in:
lib/perobs/EquiBlobsFile.rb

Overview

This class implements persistent storage space for same size data blobs. The blobs can be stored and retrieved and can be deleted again. The EquiBlobsFile manages the storage of the blobs and free storage spaces. The files grows and shrinks as needed. A blob is referenced by its address. The address is an Integer that must be larger than 0. The value 0 is used to represent an undefined address or nil. The file has a 4 * 8 bytes long header that stores the total entry count, the total space count, the offset of the first entry and the offset of the first space. The header is followed by a custom entry section. Each entry is also 8 bytes long. After the custom entry section the data blobs start.

Constant Summary collapse

TOTAL_ENTRIES_OFFSET = 
0
TOTAL_SPACES_OFFSET = 
8
FIRST_ENTRY_OFFSET = 
2 * 8
FIRST_SPACE_OFFSET = 
3 * 8
HEADER_SIZE = 
4 * 8

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(dir, name, progressmeter, entry_bytes, first_entry_default = 0) ⇒ EquiBlobsFile

Create a new stack file in the given directory with the given file name.

Parameters:

  • dir (String) —

    Directory

  • name (String) —

    File name

  • progressmeter (ProgressMeter) —

    Reference to a progress meter object

  • entry_bytes (Integer) —

    Number of bytes each entry must have

  • first_entry_default (Integer) (defaults to: 0) —

    Default address of the first blob



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/perobs/EquiBlobsFile.rb', line 61

def initialize(dir, name, progressmeter, entry_bytes,
               first_entry_default = 0)
  @name = name
  @file_name = File.join(dir, name + '.blobs')
  @progressmeter = progressmeter
  if entry_bytes < 8
    PEROBS.log.fatal "EquiBlobsFile entry size must be at least 8"
  end
  @entry_bytes = entry_bytes
  @first_entry_default = first_entry_default
  clear_custom_data
  reset_counters

  # The File handle.
  @f = nil
end

Instance Attribute Details

#file_name ⇒ Object (readonly)

Returns the value of attribute file_name.



52
53
54
 
# File 'lib/perobs/EquiBlobsFile.rb', line 52

def file_name
  @file_name
end

#first_entry ⇒ Object

Returns the value of attribute first_entry.



52
53
54
 
# File 'lib/perobs/EquiBlobsFile.rb', line 52

def first_entry
  @first_entry
end

#total_entries ⇒ Object (readonly)

Returns the value of attribute total_entries.



52
53
54
 
# File 'lib/perobs/EquiBlobsFile.rb', line 52

def total_entries
  @total_entries
end

#total_spaces ⇒ Object (readonly)

Returns the value of attribute total_spaces.



52
53
54
 
# File 'lib/perobs/EquiBlobsFile.rb', line 52

def total_spaces
  @total_spaces
end

Instance Method Details

#check ⇒ Boolean

Check the file for logical errors.

Returns:

  • (Boolean) —

    true of file has no errors, false otherwise.



361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
 
# File 'lib/perobs/EquiBlobsFile.rb', line 361

def check
  sync

  return false unless check_spaces
  return false unless check_entries

  expected_size = address_to_offset(@total_entries + @total_spaces + 1)
  actual_size = @f.size
  if actual_size != expected_size
    PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}. " +
      "Expected #{expected_size} bytes but found #{actual_size} bytes."
    return false
  end

  true
end

#clear ⇒ Object

Delete all data.



187
188
189
190
191
192
 
# File 'lib/perobs/EquiBlobsFile.rb', line 187

def clear
  @f.truncate(0)
  @f.flush
  reset_counters
  write_header
end

#clear_custom_data ⇒ Object

Reset (delete) all custom data labels that have been registered.



132
133
134
135
136
137
138
139
140
141
 
# File 'lib/perobs/EquiBlobsFile.rb', line 132

def clear_custom_data
  unless @f.nil?
    PEROBS.log.fatal "clear_custom_data should only be called when " +
      "the file is not opened"
  end

  @custom_data_labels = []
  @custom_data_values = []
  @custom_data_defaults = []
end

#close ⇒ Object

Close the blob file. This method must be called before the program is terminated to avoid data loss.



101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/perobs/EquiBlobsFile.rb', line 101

def close
  begin
    if @f
      @f.flush
      @f.flock(File::LOCK_UN)
      @f.fsync
      @f.close
      @f = nil
    end
  rescue IOError => e
    PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"
  end
end

#delete_blob(address) ⇒ Object

Delete the blob at the given address.

Parameters:

  • address (Integer) —

    Address of blob to delete



325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
 
# File 'lib/perobs/EquiBlobsFile.rb', line 325

def delete_blob(address)
  unless address >= 0
    PEROBS.log.fatal "Blob address must be larger than 0, " +
      "not #{address}"
  end

  offset = address_to_offset(address)
  begin
    @f.seek(offset)
    if (marker = read_char) != 1 && marker != 2
      PEROBS.log.fatal "Cannot delete blob stored at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Blob is " +
        (marker == 0 ? 'empty' : 'corrupted') + '.'
    end
    @f.seek(address_to_offset(address))
    write_char(0)
    write_unsigned_int(@first_space)
  rescue IOError => e
    PEROBS.log.fatal "Cannot delete blob at address #{address}: " +
      e.message
  end

  @first_space = offset
  @total_spaces += 1
  @total_entries -= 1 unless marker == 1
  write_header

  if offset == @f.size - 1 - @entry_bytes
    # We have deleted the last entry in the file. Make sure that all empty
    # entries are removed up to the now new last used entry.
    trim_file
  end
end

#erase ⇒ Object

Erase the backing store. This method should only be called when the file is not currently open.



168
169
170
171
172
 
# File 'lib/perobs/EquiBlobsFile.rb', line 168

def erase
  @f = nil
  File.delete(@file_name) if File.exist?(@file_name)
  reset_counters
end

#file_exist? ⇒ Boolean

Check if the file exists and is larger than 0.

Returns:

  • (Boolean) 


379
380
381
 
# File 'lib/perobs/EquiBlobsFile.rb', line 379

def file_exist?
  File.exist?(@file_name) && File.size(@file_name) > 0
end

#free_address ⇒ Integer

Return the address of a free blob storage space. Addresses start at 0 and increase linearly.

Returns:

  • (Integer) —

    address of a free blob space



204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/perobs/EquiBlobsFile.rb', line 204

def free_address
  if @first_space == 0
    # There is currently no free entry. Create a new reserved entry at the
    # end of the file.
    begin
      offset = @f.size
      @f.seek(offset)
      write_n_bytes([1] + ::Array.new(@entry_bytes, 0))
      write_header
      return offset_to_address(offset)
    rescue IOError => e
      PEROBS.log.fatal "Cannot create reserved space at #{@first_space} " +
        "in EquiBlobsFile #{@file_name}: #{e.message}"
    end
  else
    begin
      free_space_address = offset_to_address(@first_space)
      @f.seek(@first_space)
      marker = read_char
      @first_space = read_unsigned_int
      unless marker == 0
        PEROBS.log.fatal "Free space list of EquiBlobsFile #{@file_name} " +
          "points to non-empty entry at address #{@first_space}"
      end
      # Mark entry as reserved by setting the mark byte to 1.
      @f.seek(-(1 + 8), IO::SEEK_CUR)
      write_char(1)

      # Update the file header
      @total_spaces -= 1
      write_header
      return free_space_address
    rescue IOError => e
      PEROBS.log.fatal "Cannot mark reserved space at " +
        "#{free_space_address} in EquiBlobsFile #{@file_name}: " +
        "#{e.message}"
    end
  end
end

#get_custom_data(name) ⇒ Integer

Get the registered custom data field value.

Parameters:

  • name (String) —

    Label of the offset

Returns:

  • (Integer) —

    Value of the custom data field



158
159
160
161
162
163
164
 
# File 'lib/perobs/EquiBlobsFile.rb', line 158

def get_custom_data(name)
  unless @custom_data_labels.include?(name)
    PEROBS.log.fatal "Unknown custom data field #{name}"
  end

  @custom_data_values[@custom_data_labels.index(name)]
end

#open ⇒ Object

Open the blob file.



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/perobs/EquiBlobsFile.rb', line 79

def open
  begin
    if File.exist?(@file_name)
      # Open an existing file.
      @f = File.open(@file_name, 'rb+')
      read_header
    else
      # Create a new file by writing a new header.
      @f = File.open(@file_name, 'wb+')
      write_header
    end
  rescue IOError => e
    PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
  end
  unless @f.flock(File::LOCK_NB | File::LOCK_EX)
    PEROBS.log.fatal 'Database blob file is locked by another process'
  end
  @f.sync = true
end

#register_custom_data(name, default_value = 0) ⇒ Object

In addition to the standard offsets for the first entry and the first space any number of additional data fields can be registered. This must be done right after the object is instanciated and before the open() method is called. Each field represents a 64 bit unsigned integer.

Parameters:

  • name (String) —

    The label for this offset

  • default_value (Integer) (defaults to: 0) —

    The default value for the offset



121
122
123
124
125
126
127
128
129
 
# File 'lib/perobs/EquiBlobsFile.rb', line 121

def register_custom_data(name, default_value = 0)
  if @custom_data_labels.include?(name)
    PEROBS.log.fatal "Custom data field #{name} has already been registered"
  end

  @custom_data_labels << name
  @custom_data_values << default_value
  @custom_data_defaults << default_value
end

#retrieve_blob(address) ⇒ String

Retrieve a blob from the given address.

Parameters:

  • address (Integer) —

    Address to store the blob

Returns:

  • (String) —

    blob bytes



295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
 
# File 'lib/perobs/EquiBlobsFile.rb', line 295

def retrieve_blob(address)
  unless address > 0
    PEROBS.log.fatal "Blob retrieval address must be larger than 0, " +
      "not #{address}"
  end

  begin
    if (offset = address_to_offset(address)) >= @f.size
      PEROBS.log.fatal "Cannot retrieve blob at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Address is beyond end of file."
    end

    @f.seek(address_to_offset(address))
    if (marker = read_char) != 2
      PEROBS.log.fatal "Cannot retrieve blob at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Blob is " +
        (marker == 0 ? 'empty' : marker == 1 ? 'reserved' : 'corrupted') +
        '.'
    end
    bytes = @f.read(@entry_bytes)
  rescue IOError => e
    PEROBS.log.fatal "Cannot retrieve blob at adress #{address} " +
      "of EquiBlobsFile #{@file_name}: " + e.message
  end

  bytes
end

#set_custom_data(name, value) ⇒ Object

Set the registered custom data field to the given value.

Parameters:

  • name (String) —

    Label of the offset

  • value (Integer) —

    Value



146
147
148
149
150
151
152
153
 
# File 'lib/perobs/EquiBlobsFile.rb', line 146

def set_custom_data(name, value)
  unless @custom_data_labels.include?(name)
    PEROBS.log.fatal "Unknown custom data field #{name}"
  end

  @custom_data_values[@custom_data_labels.index(name)] = value
  write_header if @f
end

#store_blob(address, bytes) ⇒ Object

Store the given byte blob at the specified address. If the blob space is already in use the content will be overwritten.

Parameters:

  • address (Integer) —

    Address to store the blob

  • bytes (String) —

    bytes to store



248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
 
# File 'lib/perobs/EquiBlobsFile.rb', line 248

def store_blob(address, bytes)
  unless address >= 0
    PEROBS.log.fatal "Blob storage address must be larger than 0, " +
      "not #{address}"
  end
  if bytes.length != @entry_bytes
    PEROBS.log.fatal "All stack entries must be #{@entry_bytes} " +
      "long. This entry is #{bytes.length} bytes long."
  end

  marker = 1
  begin
    offset = address_to_offset(address)
    if offset > (file_size = @f.size)
      PEROBS.log.fatal "Cannot store blob at address #{address} in " +
        "EquiBlobsFile #{@file_name}. Address is larger than file size. " +
        "Offset: #{offset}  File size: #{file_size}"
    end

    @f.seek(offset)
    # The first byte is the marker byte. It's set to 2 for cells that hold
    # a blob. 1 for reserved cells and 0 for empty cells. The cell must be
    # either already be in use or be reserved. It must not be 0.
    if file_size > offset &&
       (marker = read_char) != 1 && marker != 2
      PEROBS.log.fatal "Marker for entry at address #{address} of " +
        "EquiBlobsFile #{@file_name} must be 1 or 2 but is #{marker}"
    end
    @f.seek(offset)
    write_char(2)
    @f.write(bytes)
    @f.flush
  rescue IOError => e
    PEROBS.log.fatal "Cannot store blob at address #{address} in " +
      "EquiBlobsFile #{@file_name}: #{e.message}"
  end

  # Update the entries counter if we inserted a new blob.
  if marker == 1
    @total_entries += 1
    write_header
  end
end

#sync ⇒ Object

Flush out all unwritten data.



175
176
177
178
179
180
181
182
183
184
 
# File 'lib/perobs/EquiBlobsFile.rb', line 175

def sync
  begin
    if @f
      @f.flush
      @f.fsync
    end
  rescue IOError => e
    PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
  end
end