Class: BlockFile

Inherits:
Object
  • Object
show all
Defined in:
lib/geotree/blockfile.rb

Overview

Block file.

For storing data in a collection of ‘blocks’, fixed-length arrays of bytes, with facility for recycling blocks that are no longer used.

Base class is in-memory only; subclass to persist blocks, e.g., to a file system. The DiskBlockFile class does exactly this.

Each block has a unique ‘name’. This is a positive integer. The word ‘name’ is chosen instead of ‘id’ to avoid conflicting with the use of ‘id’ in some languages.

Each block file header includes space for four integers, for use by the application.

Direct Known Subclasses

DiskBlockFile

Constant Summary collapse

FIRST_BLOCK_ID = 
2
USER_HEADER_INTS = 
4
VERSION_ =

—- All constants ending with ‘_’ will be made private

1965
INT_BYTES_ =

Header block fields; each is an integer

4
HDR_VERSION_ = 
0
HDR_BLOCKSIZE_ = 
1
HDR_MAXINDEX_ = 
2
HDR_RECYCLEINDEX_ = 
3
HDR_USERSTART_ = 
4
HDR_SIZE_BYTES_ = 
(HDR_USERSTART_ + USER_HEADER_INTS) * INT_BYTES_
RC_PREV_DIR_NAME_ =

Fields of a recycle directory block

0
RC_ENTRIES_USED_ = 
1
RC_ENTRIES_START_ = 
2

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(block_size) ⇒ BlockFile

Constructor. Constructs a file, initially closed.

Parameters:

  • block_size —

    size of blocks, in bytes



47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/geotree/blockfile.rb', line 47

def initialize(block_size)
  block_size = [block_size, HDR_SIZE_BYTES_].max

  @header_data = nil
  @recycle_data = nil
  @header_modified = false

  # for in-memory version only:
  @mem_file = nil

  @block_size = block_size
end

Instance Attribute Details

#block_size ⇒ Object (readonly)

Size of blocks, in bytes



44
45
46
 
# File 'lib/geotree/blockfile.rb', line 44

def block_size
  @block_size
end

Class Method Details

.clear_block(block) ⇒ Object

Clear block to zeros



276
277
278
 
# File 'lib/geotree/blockfile.rb', line 276

def self.clear_block(block)
  block[0..-1] = zero_bytes(block.size)
end

.copy_block(dest, src) ⇒ Object 



280
281
282
 
# File 'lib/geotree/blockfile.rb', line 280

def BlockFile.copy_block(dest, src)
  dest[0..-1] = src
end

.read_int(block, int_offset) ⇒ Object

Read an integer from a block of bytes



254
255
256
257
258
259
260
261
262
263
264
 
# File 'lib/geotree/blockfile.rb', line 254

def BlockFile.read_int(block, int_offset)
  #  assert!(block)
  j = int_offset*INT_BYTES_

  # We must treat the most significant byte as a signed byte
  high_byte = block[j].ord
  if high_byte > 127
    high_byte = high_byte - 256
  end
  (high_byte << 24) | (block[j+1].ord << 16) | (block[j+2].ord << 8) | block[j+3].ord
end

.write_int(block, int_offset, value) ⇒ Object

Write an integer into a block of bytes



267
268
269
270
271
272
273
 
# File 'lib/geotree/blockfile.rb', line 267

def BlockFile.write_int(block, int_offset, value)
  j = int_offset * INT_BYTES_
  block[j] = ((value >> 24) & 0xff).chr
  block[j+1] = ((value >> 16) & 0xff).chr
  block[j+2] = ((value >> 8) & 0xff).chr
  block[j+3] = (value & 0xff).chr
end

Instance Method Details

#alloc(src = nil) ⇒ Object

Allocate a new block. First block allocated will have name = FIRST_BLOCK_ID.

Parameters:

  • src (defaults to: nil) —

    block data to write; if null, allocates and writes zeros

Returns:

  • name of block



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/geotree/blockfile.rb', line 100

def alloc(src = nil)

  ensure_open

  src ||= alloc_buffer

  # get index of last recycle block directory
  r_index = rdir_head_name

  # any entries remain in this directory?
  n_ent = get_rdir_slots_used

  if n_ent == 0
    prev_rb_block = get_rdir_next_name

    if prev_rb_block > 0
      # use directory as new block
      ret = r_index
      r_index = prev_rb_block
      write_hdr(HDR_RECYCLEINDEX_, r_index)
      read(prev_rb_block, @recycle_data)
      append_or_replace(ret, src)
    else
      ret = name_max
      append_or_replace(ret, src)
    end
  else
    slot = n_ent - 1;
    ret = get_rdir_slot(slot)
    set_rdir_slot(slot,0)
    set_rdir_slots_used(slot)
    append_or_replace(r_index, @recycle_data)
    append_or_replace(ret,src)
  end
  ret
end

#alloc_buffer ⇒ Object

Create an array of bytes, all zeros, of length equal to this block file’s block length



249
250
251
 
# File 'lib/geotree/blockfile.rb', line 249

def alloc_buffer
  zero_bytes(@block_size)
end

#close ⇒ Object 



165
166
167
168
169
170
171
172
173
 
# File 'lib/geotree/blockfile.rb', line 165

def close
  ensure_open
  aux_flush
  close_storage

  @header_data = nil
  @recycle_data = nil
  @mem_file = nil
end

#close_storage ⇒ Object

Close underlying storage



327
328
329
 
# File 'lib/geotree/blockfile.rb', line 327

def close_storage
  @mem_file = nil
end

#dump(block_name) ⇒ Object 



243
244
245
246
 
# File 'lib/geotree/blockfile.rb', line 243

def dump(block_name)
  b = read(block_name)
  hex_dump(b,"Block #{block_name}")
end

#flush ⇒ Object

Flush underlying storage



333
334
 
# File 'lib/geotree/blockfile.rb', line 333

def flush
end

#free(block_name) ⇒ Object

Free up a block

Raises:

  • (ArgumentError) 


138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/geotree/blockfile.rb', line 138

def free(block_name)
  ensure_open

  raise(ArgumentError,"no such block: #{block_name}") if block_name >=  name_max

  slot = get_rdir_slots_used()

  # if there is a free slot in the current recycle block, use it

  if slot < get_rdir_capacity
    set_rdir_slot(slot,block_name)
    set_rdir_slots_used(slot+1)
    append_or_replace(rdir_head_name, @recycle_data)
  else
    # use freed block as next recycle page
    old_dir = rdir_head_name

    write_hdr(HDR_RECYCLEINDEX_, block_name)

    read(block_name, @recycle_data)
    BlockFile.clear_block(@recycle_data)

    set_rdir_next_name(old_dir)
    append_or_replace(block_name, @recycle_data)
  end
end

#inspect ⇒ Object 



193
194
195
 
# File 'lib/geotree/blockfile.rb', line 193

def inspect
  to_s
end

#name_max ⇒ Object

Get 1 + name of highest block ever created



339
340
341
 
# File 'lib/geotree/blockfile.rb', line 339

def name_max
  BlockFile.read_int(@header_data, HDR_MAXINDEX_)
end

#open ⇒ Object

Open file. Creates underlying storage if necessary. File must not already be open.

Returns:

  • true if underlying storage already existed



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/geotree/blockfile.rb', line 69

def open
  !open? || raise(IllegalStateException)
  existed = open_storage
  if !existed
    @header_data = alloc_buffer
    BlockFile.write_int(@header_data, HDR_VERSION_, VERSION_)
    BlockFile.write_int(@header_data, HDR_BLOCKSIZE_, block_size)
    BlockFile.write_int(@header_data, HDR_RECYCLEINDEX_, 1)
    append_or_replace(0, @header_data)

    @recycle_data = alloc_buffer
    append_or_replace(rdir_head_name, @recycle_data)

    aux_flush
  else
    @header_data = read(0)
    if BlockFile.read_int(@header_data,HDR_VERSION_) != VERSION_
      raise ArgumentError,"bad version"
    end
    if BlockFile.read_int(@header_data,HDR_BLOCKSIZE_) != block_size
      raise ArgumentError,"unexpected block size"
    end
    @recycle_data = read(rdir_head_name)
  end
  existed
end

#open? ⇒ Boolean

Determine if file is open

Returns:

  • (Boolean) 


61
62
63
 
# File 'lib/geotree/blockfile.rb', line 61

def open?
  return @header_data != nil
end

#open_storage ⇒ Object

Open underlying storage; create it if necessary

Returns:

  • true if underlying storage already existed



320
321
322
323
 
# File 'lib/geotree/blockfile.rb', line 320

def open_storage
  @mem_file = []
  false
end

#read(block_name, dest_buffer = nil) ⇒ Object

Read block from storage.

Parameters:

  • block_name —

    index of block

  • dest_buffer (defaults to: nil) —

    where to store data; if nil, you should call alloc_buffer to create it

Returns:

  • buffer



295
296
297
298
299
300
301
302
303
304
 
# File 'lib/geotree/blockfile.rb', line 295

def read(block_name, dest_buffer = nil)
  dest_buffer ||= alloc_buffer
  if block_name >= @mem_file.size
    raise ArgumentError,"No such block name #{block_name} exists (size=#{@mem_file.size})"
  end

  src = @mem_file[block_name]
  BlockFile.copy_block(dest_buffer, src)
  dest_buffer
end

#read_user(int_index) ⇒ Object

Read one of the user values from the header

Parameters:

  • int_index —

    index of user value (0..3)

Raises:

  • (ArgumentError) 


177
178
179
180
181
 
# File 'lib/geotree/blockfile.rb', line 177

def read_user(int_index)
  ensure_open
  raise ArgumentError if !(int_index >= 0 && int_index < USER_HEADER_INTS)
  BlockFile.read_int(@header_data, HDR_USERSTART_ + int_index)
end

#to_s ⇒ Object 



197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/geotree/blockfile.rb', line 197

def to_s
  s = ''
  s << "BlockFile blockSize:#{block_size} "
  if open?
    s << "\n name_max=#{name_max}"
    s << "\n rdir_head_name=#{rdir_head_name}"

    s << "\n"

    # Dump a map of currently allocated blocks
    usage = {}
    usage[0] = 'H'
    ri = rdir_head_name
    while ri != 0
      usage[ri] = 'R'
      rd = read(ri)

      next_ri = BlockFile.read_int(rd, RC_PREV_DIR_NAME_)
      used = BlockFile.read_int(rd, RC_ENTRIES_USED_)
      used.times do |i|
        rblock = BlockFile.read_int(rd,RC_ENTRIES_START_+i)
        usage[rblock] = 'r'
      end
      ri = next_ri
    end

    row_size = 64

    puts("------------- Block Map --------------")
    name_max.times do |i|
      if (i % row_size) == 0
        printf("%04x: ",i)
      elsif (i % 4 == 0)
        print('  ')
      end
      label = usage[i]
      label ||= '.'
      print label
      print "\n" if ((i+1) % row_size) == 0
    end
    print "\n" if (name_max % row_size != 0)
    puts("--------------------------------------")
  end
  s
end

#write(block_name, src_buffer) ⇒ Object

Write block to storage. Name is either index of existing block, or

number of existing blocks (to append to end of existing ones)

Parameters:

  • block_name —

    name of block

  • src_buffer —

    data to write



311
312
313
314
315
316
 
# File 'lib/geotree/blockfile.rb', line 311

def write(block_name, src_buffer)
  if  block_name == @mem_file.size
    @mem_file << alloc_buffer
  end
  BlockFile.copy_block(@mem_file[block_name], src_buffer)
end

#write_user(int_index, value) ⇒ Object

Write a user value

Parameters:

  • int_index —

    index of user value (0..3)

  • value —

    value to write

Raises:

  • (ArgumentError) 


186
187
188
189
190
191
 
# File 'lib/geotree/blockfile.rb', line 186

def write_user(int_index, value)
  ensure_open
  raise ArgumentError if !(int_index >= 0 && int_index < USER_HEADER_INTS)
  BlockFile.write_int(@header_data, HDR_USERSTART_ + int_index, value)
  @header_modified = true
end