Module: IOStreams

Defined in:
lib/io_streams/version.rb,
lib/iostreams.rb,
lib/io_streams/pgp.rb,
lib/io_streams/streams.rb,
lib/io_streams/csv/reader.rb,
lib/io_streams/csv/writer.rb,
lib/io_streams/io_streams.rb,
lib/io_streams/pgp/reader.rb,
lib/io_streams/pgp/writer.rb,
lib/io_streams/zip/reader.rb,
lib/io_streams/zip/writer.rb,
lib/io_streams/file/reader.rb,
lib/io_streams/file/writer.rb,
lib/io_streams/gzip/reader.rb,
lib/io_streams/gzip/writer.rb,
lib/io_streams/sftp/reader.rb,
lib/io_streams/sftp/writer.rb,
lib/io_streams/xlsx/reader.rb,
lib/io_streams/delimited/reader.rb,
lib/io_streams/delimited/writer.rb

Overview

:nodoc

Defined Under Namespace

Modules: CSV, Delimited, File, Gzip, Pgp, SFTP, Xlsx, Zip Classes: Extension, StreamStruct, Streams

Constant Summary collapse

VERSION = 
'0.10.0'
UTF8_ENCODING = 
Encoding.find('UTF-8').freeze
BINARY_ENCODING = 
Encoding.find('BINARY').freeze
@@extensions =

A registry to hold formats for processing files during upload or download

Concurrent::Hash.new

Class Method Summary collapse

Class Method Details

.compressed?(file_name) ⇒ Boolean

Returns [true|false] whether the file is compressed Note: Currently only looks at the file name extension

Returns:

  • (Boolean) 


200
201
202
 
# File 'lib/io_streams/io_streams.rb', line 200

def self.compressed?(file_name)
  !(file_name =~ /\.(zip|gz|gzip|xls.|)\z/i).nil?
end

.copy(source_stream, target_stream, buffer_size = 65536) ⇒ Object

Copies the source stream to the target stream Returns [Integer] the number of bytes copied

Example:

IOStreams.reader('a.csv') do |source_stream|
  IOStreams.writer('b.csv.enc') do |target_stream|
    IOStreams.copy(source_stream, target_stream)
  end
end



178
179
180
181
182
183
184
185
186
 
# File 'lib/io_streams/io_streams.rb', line 178

def self.copy(source_stream, target_stream, buffer_size=65536)
  bytes = 0
  while data = source_stream.read(buffer_size)
    break if data.size == 0
    bytes += data.size
    target_stream.write(data)
  end
  bytes
end

.delete_stream(stream, streams) ⇒ Object

Deletes the specified stream from the supplied streams if present Returns deleted stream, or nil if not found

Raises:

  • (ArgumentError) 


206
207
208
209
210
211
212
213
 
# File 'lib/io_streams/io_streams.rb', line 206

def self.delete_stream(stream, streams)
  raise(ArgumentError, "Argument :stream must be a symbol: #{stream.inspect}") unless stream.is_a?(Symbol)

  Array(streams).delete_if do |_stream|
    stream_key = _stream.is_a?(Symbol) ? _stream : _stream.keys.first
    stream == stream_key
  end
end

.delimited_stream?(streams) ⇒ Boolean

Returns [true|false] whether the stream starts with a delimited reader or writer

Returns:

  • (Boolean) 


216
217
218
219
220
221
222
 
# File 'lib/io_streams/io_streams.rb', line 216

def self.delimited_stream?(streams)
  stream = Array(streams).first
  return false unless stream

  # TODO Need to figure out a way so that this is not hard-coded
  [:xlsx, :xlsm, :delimited].include?(stream.is_a?(Symbol) ? stream : stream.keys.first)
end

.deregister_extension(extension) ⇒ Object

De-Register a file extension

Returns [Symbol] the extension removed, or nil if the extension was not registered

Example:

register_extension(:xls)

Raises:

  • (ArgumentError) 


64
65
66
67
 
# File 'lib/io_streams/io_streams.rb', line 64

def self.deregister_extension(extension)
  raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
  @@extensions.delete(extension.to_sym)
end

.reader(file_name_or_io, streams = nil, &block) ⇒ Object

Returns a Reader for reading a file / stream

Parameters

file_name_or_io [String|IO]
  The file_name of the file to write to, or an IO Stream that implements
  #read.

streams [Symbol|Array]
  The formats/streams that be used to convert the data whilst it is
  being read.
  When nil, the file_name will be inspected to try and determine what
  streams should be applied.
  Default: nil

Stream types / extensions supported:

.zip       Zip File                                   [ :zip ]
.gz, .gzip GZip File                                  [ :gzip ]
.enc       File Encrypted using symmetric encryption  [ :enc ]
other      All other extensions will be returned as:  [ :file ]

When a file is encrypted, it may also be compressed:

.zip.enc  [ :zip, :enc ]
.gz.enc   [ :gz,  :enc ]

Example: Zip

IOStreams.reader('myfile.zip') do |stream|
  puts stream.read
end

Example: Encrypted Zip

IOStreams.reader('myfile.zip.enc') do |stream|
  puts stream.read
end

Example: Explicitly set the streams

IOStreams.reader('myfile.zip.enc', [:zip, :enc]) do |stream|
  puts stream.read
end

Example: Supply custom options

# Encrypt the file and get Symmetric Encryption to also compress it
IOStreams.reader('myfile.csv.enc', [:enc]) do |stream|
  puts stream.read
end



113
114
115
 
# File 'lib/io_streams/io_streams.rb', line 113

def self.reader(file_name_or_io, streams = nil, &block)
  stream(:reader, file_name_or_io, streams, &block)
end

.reader_stream?(file_name_or_io) ⇒ Boolean

Returns [true|false] whether the supplied file_name_or_io is a reader stream

Returns:

  • (Boolean) 


189
190
191
 
# File 'lib/io_streams/io_streams.rb', line 189

def self.reader_stream?(file_name_or_io)
  file_name_or_io.respond_to?(:read)
end

.register_extension(extension, reader_class, writer_class) ⇒ Object

Register a file extension and the reader and writer classes to use to format it

Example:

# MyXls::Reader and MyXls::Writer must implement .open
register_extension(:xls, MyXls::Reader, MyXls::Writer)

Raises:

  • (ArgumentError) 


53
54
55
56
 
# File 'lib/io_streams/io_streams.rb', line 53

def self.register_extension(extension, reader_class, writer_class)
  raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
  @@extensions[extension.to_sym] = Extension.new(reader_class, writer_class)
end

.streams_for_file_name(file_name) ⇒ Object

Returns [Array] the formats required to process the file by looking at its extension(s)

Extensions supported:

.zip       Zip File                                   [ :zip ]
.gz, .gzip GZip File                                  [ :gzip ]
.enc       File Encrypted using symmetric encryption  [ :enc ]
other      All other extensions will be returned as:  [ :file ]

When a file is encrypted, it may also be compressed:

.zip.enc  [ :zip, :enc ]
.gz.enc   [ :gz,  :enc ]

Example Zip file:

RocketJob::Formatter::Formats.streams_for_file_name('myfile.zip')
=> [ :zip ]

Example Encrypted Gzip file:

RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv.gz.enc')
=> [ :gz, :enc ]

Example plain text / binary file:

RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv')
=> [ :file ]

Raises:

  • (ArgumentError) 


33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/io_streams/io_streams.rb', line 33

def self.streams_for_file_name(file_name)
  raise ArgumentError.new('File name cannot be nil') if file_name.nil?
  raise ArgumentError.new("File name must be a string: #{file_name.inspect}, class: #{file_name.class}") unless file_name.is_a?(String)
  parts      = file_name.split('.')
  extensions = []
  while extension = parts.pop
    break unless @@extensions[extension.to_sym]
    extensions.unshift(extension.to_sym)
  end
  extensions << :file if extensions.size == 0
  extensions
end

.writer(file_name_or_io, streams = nil, &block) ⇒ Object

Returns a Writer for writing to a file / stream

Parameters

file_name_or_io [String|IO]
  The file_name of the file to write to, or an IO Stream that implements
  #write.

streams [Symbol|Array]
  The formats/streams that be used to convert the data whilst it is
  being written.
  When nil, the file_name will be inspected to try and determine what
  streams should be applied.
  Default: nil

Stream types / extensions supported:

.zip       Zip File                                   [ :zip ]
.gz, .gzip GZip File                                  [ :gzip ]
.enc       File Encrypted using symmetric encryption  [ :enc ]
other      All other extensions will be returned as:  [ :file ]

When a file is encrypted, it may also be compressed:

.zip.enc  [ :zip, :enc ]
.gz.enc   [ :gz,  :enc ]

Example: Zip

IOStreams.writer('myfile.zip') do |stream|
  stream.write(data)
end

Example: Encrypted Zip

IOStreams.writer('myfile.zip.enc') do |stream|
  stream.write(data)
end

Example: Explicitly set the streams

IOStreams.writer('myfile.zip.enc', [:zip, :enc]) do |stream|
  stream.write(data)
end

Example: Supply custom options

IOStreams.writer('myfile.csv.enc', [enc: { compress: true }]) do |stream|
  stream.write(data)
end

Example: Set internal filename when creating a zip file

IOStreams.writer('myfile.csv.zip', zip: { zip_file_name: 'myfile.csv' }) do |stream|
  stream.write(data)
end



165
166
167
 
# File 'lib/io_streams/io_streams.rb', line 165

def self.writer(file_name_or_io, streams = nil, &block)
  stream(:writer, file_name_or_io, streams, &block)
end

.writer_stream?(file_name_or_io) ⇒ Boolean

Returns [true|false] whether the supplied file_name_or_io is a reader stream

Returns:

  • (Boolean) 


194
195
196
 
# File 'lib/io_streams/io_streams.rb', line 194

def self.writer_stream?(file_name_or_io)
  file_name_or_io.respond_to?(:write)
end