Class: PEROBS::EquiBlobsFile

Inherits:
Object
  • Object
show all
Defined in:
lib/perobs/EquiBlobsFile.rb

Overview

This class implements persistent storage space for same size data blobs. The blobs can be stored and retrieved and can be deleted again. The EquiBlobsFile manages the storage of the blobs and free storage spaces. The files grows and shrinks as needed. A blob is referenced by its address. The address is an Integer that must be larger than 0. The value 0 is used to represent an undefined address or nil. The file has a 4 * 8 bytes long header that stores the total entry count, the total space count, the offset of the first entry and the offset of the first space.

Constant Summary collapse

TOTAL_ENTRIES_OFFSET = 
0
TOTAL_SPACES_OFFSET = 
8
FIRST_ENTRY_OFFSET = 
2 * 8
FIRST_SPACE_OFFSET = 
3 * 8
HEADER_SIZE = 
4 * 8

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(dir, name, entry_bytes, first_entry_default = 0) ⇒ EquiBlobsFile

Create a new stack file in the given directory with the given file name.

Parameters:

  • dir (String) —

    Directory

  • name (String) —

    File name

  • entry_bytes (Integer) —

    Number of bytes each entry must have

  • first_entry_default (Integer) (defaults to: 0) —

    Default address of the first blob



55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/perobs/EquiBlobsFile.rb', line 55

def initialize(dir, name, entry_bytes, first_entry_default = 0)
  @file_name = File.join(dir, name + '.blobs')
  if entry_bytes < 8
    PEROBS.log.fatal "EquiBlobsFile entry size must be at least 8"
  end
  @entry_bytes = entry_bytes
  @first_entry_default = first_entry_default
  reset_counters

  # The File handle.
  @f = nil
end

Instance Attribute Details

#file_name ⇒ Object (readonly)

Returns the value of attribute file_name.



48
49
50
 
# File 'lib/perobs/EquiBlobsFile.rb', line 48

def file_name
  @file_name
end

#first_entry ⇒ Object

Returns the value of attribute first_entry.



48
49
50
 
# File 'lib/perobs/EquiBlobsFile.rb', line 48

def first_entry
  @first_entry
end

#total_entries ⇒ Object (readonly)

Returns the value of attribute total_entries.



48
49
50
 
# File 'lib/perobs/EquiBlobsFile.rb', line 48

def total_entries
  @total_entries
end

#total_spaces ⇒ Object (readonly)

Returns the value of attribute total_spaces.



48
49
50
 
# File 'lib/perobs/EquiBlobsFile.rb', line 48

def total_spaces
  @total_spaces
end

Instance Method Details

#check ⇒ Boolean

Check the file for logical errors.

Returns:

  • (Boolean) —

    true of file has no errors, false otherwise.



295
296
297
298
299
300
301
302
303
304
305
306
 
# File 'lib/perobs/EquiBlobsFile.rb', line 295

def check
  return false unless check_spaces
  return false unless check_entries

  if @f.size != HEADER_SIZE + (@total_entries + @total_spaces) *
                              (1 + @entry_bytes)
    PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}"
    return false
  end

  true
end

#clear ⇒ Object

Delete all data.



121
122
123
124
125
126
 
# File 'lib/perobs/EquiBlobsFile.rb', line 121

def clear
  @f.truncate(0)
  @f.flush
  reset_counters
  write_header
end

#close ⇒ Object

Close the blob file. This method must be called before the program is terminated to avoid data loss.



90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/perobs/EquiBlobsFile.rb', line 90

def close
  begin
    if @f
      @f.flush
      @f.flock(File::LOCK_UN)
      @f.close
      @f = nil
    end
  rescue IOError => e
    PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"
  end
end

#delete_blob(address) ⇒ Object

Delete the blob at the given address.

Parameters:

  • address (Integer) —

    Address of blob to delete



259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
 
# File 'lib/perobs/EquiBlobsFile.rb', line 259

def delete_blob(address)
  unless address >= 0
    PEROBS.log.fatal "Blob address must be larger than 0, " +
      "not #{address}"
  end

  offset = address_to_offset(address)
  begin
    @f.seek(offset)
    if (marker = read_char) != 1 && marker != 2
      PEROBS.log.fatal "Cannot delete blob stored at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Blob is " +
        (marker == 0 ? 'empty' : 'corrupted') + '.'
    end
    @f.seek(address_to_offset(address))
    write_char(0)
    write_unsigned_int(@first_space)
  rescue IOError => e
    PEROBS.log.fatal "Cannot delete blob at address #{address}: " +
      e.message
  end

  @first_space = offset
  @total_spaces += 1
  @total_entries -= 1
  write_header

  if offset == @f.size - 1 - @entry_bytes
    # We have deleted the last entry in the file. Make sure that all empty
    # entries are removed up to the now new last used entry.
    trim_file
  end
end

#erase ⇒ Object

Erase the backing store. This method should only be called when the file is not currently open.



105
106
107
108
109
 
# File 'lib/perobs/EquiBlobsFile.rb', line 105

def erase
  PEROBS.log.fatal 'Cannot call EquiBlobsFile::erase while it is open' if @f
  File.delete(@file_name) if File.exist?(@file_name)
  reset_counters
end

#file_exist? ⇒ Boolean

Check if the file exists and is larger than 0.

Returns:

  • (Boolean) 


309
310
311
 
# File 'lib/perobs/EquiBlobsFile.rb', line 309

def file_exist?
  File.exist?(@file_name) && File.size(@file_name) > 0
end

#free_address ⇒ Integer

Return the address of a free blob storage space. Addresses start at 0 and increase linearly.

Returns:

  • (Integer) —

    address of a free blob space



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/perobs/EquiBlobsFile.rb', line 138

def free_address
  if @first_space == 0
    # There is currently no free entry. Create a new reserved entry at the
    # end of the file.
    begin
      offset = @f.size
      @f.seek(offset)
      write_n_bytes([1] + ::Array.new(@entry_bytes, 0))
      write_header
      return offset_to_address(offset)
    rescue IOError => e
      PEROBS.log.fatal "Cannot create reserved space at #{@first_space} " +
        "in EquiBlobsFile #{@file_name}: #{e.message}"
    end
  else
    begin
      free_space_address = offset_to_address(@first_space)
      @f.seek(@first_space)
      marker = read_char
      @first_space = read_unsigned_int
      unless marker == 0
        PEROBS.log.fatal "Free space list of EquiBlobsFile #{@file_name} " +
          "points to non-empty entry at address #{@first_space}"
      end
      # Mark entry as reserved by setting the mark byte to 1.
      @f.seek(-(1 + 8), IO::SEEK_CUR)
      write_char(1)

      # Update the file header
      @total_spaces -= 1
      write_header
      return free_space_address
    rescue IOError => e
      PEROBS.log.fatal "Cannot mark reserved space at " +
        "#{free_space_address} in EquiBlobsFile #{@file_name}: " +
        "#{e.message}"
    end
  end
end

#open ⇒ Object

Open the blob file.



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/perobs/EquiBlobsFile.rb', line 69

def open
  begin
    if File.exist?(@file_name)
      # Open an existing file.
      @f = File.open(@file_name, 'rb+')
      read_header
    else
      # Create a new file by writing a new header.
      @f = File.open(@file_name, 'wb+')
      write_header
    end
  rescue IOError => e
    PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
  end
  unless @f.flock(File::LOCK_NB | File::LOCK_EX)
    PEROBS.log.fatal 'Database blob file is locked by another process'
  end
end

#retrieve_blob(address) ⇒ String

Retrieve a blob from the given address.

Parameters:

  • address (Integer) —

    Address to store the blob

Returns:

  • (String) —

    blob bytes



229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/perobs/EquiBlobsFile.rb', line 229

def retrieve_blob(address)
  unless address > 0
    PEROBS.log.fatal "Blob retrieval address must be larger than 0, " +
      "not #{address}"
  end

  begin
    if (offset = address_to_offset(address)) >= @f.size
      PEROBS.log.fatal "Cannot retrieve blob at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Address is beyond end of file."
    end

    @f.seek(address_to_offset(address))
    if (marker = read_char) != 2
      PEROBS.log.fatal "Cannot retrieve blob at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Blob is " +
        (marker == 0 ? 'empty' : marker == 1 ? 'reserved' : 'corrupted') +
        '.'
    end
    bytes = @f.read(@entry_bytes)
  rescue IOError => e
    PEROBS.log.fatal "Cannot retrieve blob at adress #{address} " +
      "of EquiBlobsFile #{@file_name}: " + e.message
  end

  bytes
end

#store_blob(address, bytes) ⇒ Object

Store the given byte blob at the specified address. If the blob space is already in use the content will be overwritten.

Parameters:

  • address (Integer) —

    Address to store the blob

  • bytes (String) —

    bytes to store



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/perobs/EquiBlobsFile.rb', line 182

def store_blob(address, bytes)
  unless address >= 0
    PEROBS.log.fatal "Blob storage address must be larger than 0, " +
      "not #{address}"
  end
  if bytes.length != @entry_bytes
    PEROBS.log.fatal "All stack entries must be #{@entry_bytes} " +
      "long. This entry is #{bytes.length} bytes long."
  end

  marker = 1
  begin
    offset = address_to_offset(address)
    if offset > (file_size = @f.size)
      PEROBS.log.fatal "Cannot store blob at address #{address} in " +
        "EquiBlobsFile #{@file_name}. Address is larger than file size. " +
        "Offset: #{offset}  File size: #{file_size}"
    end

    @f.seek(offset)
    # The first byte is the marker byte. It's set to 2 for cells that hold
    # a blob. 1 for reserved cells and 0 for empty cells. The cell must be
    # either already be in use or be reserved. It must not be 0.
    if file_size > offset &&
       (marker = read_char) != 1 && marker != 2
      PEROBS.log.fatal "Marker for entry at address #{address} of " +
        "EquiBlobsFile #{@file_name} must be 1 or 2 but is #{marker}"
    end
    @f.seek(offset)
    write_char(2)
    @f.write(bytes)
    @f.flush
  rescue IOError => e
    PEROBS.log.fatal "Cannot store blob at address #{address} in " +
      "EquiBlobsFile #{@file_name}: #{e.message}"
  end

  # Update the entries counter if we inserted a new blob.
  if marker == 1
    @total_entries += 1
    write_header
  end
end

#sync ⇒ Object

Flush out all unwritten data.



112
113
114
115
116
117
118
 
# File 'lib/perobs/EquiBlobsFile.rb', line 112

def sync
  begin
    @f.flush if @f
  rescue IOError => e
    PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
  end
end