Class: S3DataPacker::Configuration

Inherits:
Object
  • Object
show all
Defined in:
lib/s3_data_packer/configuration.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Configuration

Returns a new instance of Configuration.



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/s3_data_packer/configuration.rb', line 57

def initialize
  @thread_count = 2
  @thread_sleep_time = 1
  @thread_lock_wait_time = 1
  @max_queue_size = 10000
  @max_queue_wait = 5
  @batch_size = 100000
  @workdir = 'tmp/s3_data_packer'
  @cleanup_batch = true
  @compress_batch = true
  @output_filename_prefix = nil
  @output_filename_suffix = 'batch'
  @output_filename_pattern = %i[timecode_int suffix]
  @output_filename_splitter = '_'
end

Instance Attribute Details

#batch_size ⇒ Object

Number of items of the final batch size



39
40
41
 
# File 'lib/s3_data_packer/configuration.rb', line 39

def batch_size
  @batch_size
end

#cleanup_batch ⇒ Object

Whether to keep or delete the finalized batch file. Set to false if you want to keep the output files in the workdir.



32
33
34
 
# File 'lib/s3_data_packer/configuration.rb', line 32

def cleanup_batch
  @cleanup_batch
end

#compress_batch ⇒ Object

Whether to compress the final batch file or not. If set to true, the output file will be compressed with GZip, and the uncompressed file will be removed.



36
37
38
 
# File 'lib/s3_data_packer/configuration.rb', line 36

def compress_batch
  @compress_batch
end

#logger ⇒ Object

Standard logger to output information



4
5
6
 
# File 'lib/s3_data_packer/configuration.rb', line 4

def logger
  @logger
end

#max_queue_size ⇒ Object

Maximum number of items to maintain in queue to not overflow while workers process items.



19
20
21
 
# File 'lib/s3_data_packer/configuration.rb', line 19

def max_queue_size
  @max_queue_size
end

#max_queue_wait ⇒ Object

Time in seconds to wait when the queue reached max_queue_size to keep adding new items.



23
24
25
 
# File 'lib/s3_data_packer/configuration.rb', line 23

def max_queue_wait
  @max_queue_wait
end

#output_filename_pattern ⇒ Object

Desired pattern to construct output filenames



52
53
54
 
# File 'lib/s3_data_packer/configuration.rb', line 52

def output_filename_pattern
  @output_filename_pattern
end

#output_filename_prefix ⇒ Object

String prefix to include in output filenames for the batches



46
47
48
 
# File 'lib/s3_data_packer/configuration.rb', line 46

def output_filename_prefix
  @output_filename_prefix
end

#output_filename_splitter ⇒ Object

Splitter character to concat different values to generate a filename



55
56
57
 
# File 'lib/s3_data_packer/configuration.rb', line 55

def output_filename_splitter
  @output_filename_splitter
end

#output_filename_suffix ⇒ Object

String suffix to include in output filenames for the batches



49
50
51
 
# File 'lib/s3_data_packer/configuration.rb', line 49

def output_filename_suffix
  @output_filename_suffix
end

#s3_api_key ⇒ Object

Returns the value of attribute s3_api_key.



41
42
43
 
# File 'lib/s3_data_packer/configuration.rb', line 41

def s3_api_key
  @s3_api_key
end

#s3_api_secret ⇒ Object

Returns the value of attribute s3_api_secret.



42
43
44
 
# File 'lib/s3_data_packer/configuration.rb', line 42

def s3_api_secret
  @s3_api_secret
end

#s3_region ⇒ Object

Returns the value of attribute s3_region.



43
44
45
 
# File 'lib/s3_data_packer/configuration.rb', line 43

def s3_region
  @s3_region
end

#thread_count ⇒ Object

How many threads to run for reading and processing items. This needs to be balanced out with the speed at which item keys are gathered to prevent emptying the queue too early.



9
10
11
 
# File 'lib/s3_data_packer/configuration.rb', line 9

def thread_count
  @thread_count
end

#thread_lock_wait_time ⇒ Object

Time in seconds for thread to wait when locked.



15
16
17
 
# File 'lib/s3_data_packer/configuration.rb', line 15

def thread_lock_wait_time
  @thread_lock_wait_time
end

#thread_sleep_time ⇒ Object

Time in seconds to let a thread sleep when there’s no pending items in queue.



12
13
14
 
# File 'lib/s3_data_packer/configuration.rb', line 12

def thread_sleep_time
  @thread_sleep_time
end

#workdir ⇒ Object

Directory to keep working files. Make sure you have permissions on the path set. If the path does not exist, the packer will try to create it before using it.



28
29
30
 
# File 'lib/s3_data_packer/configuration.rb', line 28

def workdir
  @workdir
end

Instance Method Details

#cleanup_batch? ⇒ Boolean

Returns:

  • (Boolean) 


77
78
79
 
# File 'lib/s3_data_packer/configuration.rb', line 77

def cleanup_batch?
  cleanup_batch == true
end

#compress_batch? ⇒ Boolean

Returns:

  • (Boolean) 


73
74
75
 
# File 'lib/s3_data_packer/configuration.rb', line 73

def compress_batch?
  compress_batch == true
end

#default_s3_credentials ⇒ Object 



85
86
87
88
89
 
# File 'lib/s3_data_packer/configuration.rb', line 85

def default_s3_credentials
  return nil unless s3_credentials?

  Aws::Credentials.new(s3_api_key, s3_api_secret)
end

#filename_generator_defaults ⇒ Object 



91
92
93
94
95
96
 
# File 'lib/s3_data_packer/configuration.rb', line 91

def filename_generator_defaults
  { prefix: output_filename_prefix,
    suffix: output_filename_suffix,
    pattern: output_filename_pattern,
    splitter: output_filename_splitter }
end

#s3_credentials? ⇒ Boolean

Returns:

  • (Boolean) 


81
82
83
 
# File 'lib/s3_data_packer/configuration.rb', line 81

def s3_credentials?
  s3_api_key && s3_api_secret
end