Class: S3DataPacker::JSONBatch

Inherits:
Object
  • Object
show all
Defined in:
lib/s3_data_packer/json_batch.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ JSONBatch

Returns a new instance of JSONBatch.



5
6
7
8
9
10
11
12
13
 
# File 'lib/s3_data_packer/json_batch.rb', line 5

def initialize opts = {}
  @delimitter = "\r\n"
  @workdir = opts[:workdir]
  @filename_generator = opts[:filename_generator]
  @pre_processor = opts[:pre_processor] # Should be a proc
  @size = opts[:size]
  @item_count = 0
  init_workdir!
end

Instance Attribute Details

#batch ⇒ Object (readonly)

Returns the value of attribute batch.



3
4
5
 
# File 'lib/s3_data_packer/json_batch.rb', line 3

def batch
  @batch
end

#delimitter ⇒ Object (readonly)

Returns the value of attribute delimitter.



3
4
5
 
# File 'lib/s3_data_packer/json_batch.rb', line 3

def delimitter
  @delimitter
end

#item_count ⇒ Object (readonly)

Returns the value of attribute item_count.



3
4
5
 
# File 'lib/s3_data_packer/json_batch.rb', line 3

def item_count
  @item_count
end

Instance Method Details

#append_data!(data) ⇒ Object 



41
42
43
44
45
 
# File 'lib/s3_data_packer/json_batch.rb', line 41

def append_data! data
  digested = pre_proccess_data(data)
  batch << "#{digested}#{delimitter}"
  @item_count += 1
end

#close! ⇒ Object 



51
52
53
 
# File 'lib/s3_data_packer/json_batch.rb', line 51

def close!
  batch.close
end

#delete! ⇒ Object 



55
56
57
58
59
 
# File 'lib/s3_data_packer/json_batch.rb', line 55

def delete!
  close! if !@batch.closed?
  File.delete(path) if File.exist?(path)
  reset!
end

#filename_generator ⇒ Object 



23
24
25
 
# File 'lib/s3_data_packer/json_batch.rb', line 23

def filename_generator
  @filename_generator ||= S3DataPacker::FilenameGenerator.new
end

#finalize! ⇒ Object 



61
62
63
64
65
66
67
 
# File 'lib/s3_data_packer/json_batch.rb', line 61

def finalize!
  close! if !batch.closed?
  final_path = batch.path
  final_path = compress! if S3DataPacker.config.compress_batch?
  reset!
  final_path
end

#full? ⇒ Boolean

Returns:

  • (Boolean) 


27
28
29
 
# File 'lib/s3_data_packer/json_batch.rb', line 27

def full?
  item_count >= size
end

#generate_filename ⇒ Object 



31
32
33
34
 
# File 'lib/s3_data_packer/json_batch.rb', line 31

def generate_filename
  name = filename_generator.generate!
  "#{workdir}/#{name}.json"
end

#new_file! ⇒ Object 



36
37
38
39
 
# File 'lib/s3_data_packer/json_batch.rb', line 36

def new_file!
  close! if @batch
  @batch = File.open(generate_filename, 'w')
end

#path ⇒ Object 



47
48
49
 
# File 'lib/s3_data_packer/json_batch.rb', line 47

def path
  batch.path
end

#size ⇒ Object 



15
16
17
 
# File 'lib/s3_data_packer/json_batch.rb', line 15

def size
  @size ||= S3DataPacker.config.batch_size
end

#workdir ⇒ Object 



19
20
21
 
# File 'lib/s3_data_packer/json_batch.rb', line 19

def workdir
  @workdir ||= S3DataPacker.config.workdir
end