Class: Stockboy::Providers::FTP

Inherits:
Stockboy::Provider show all
Defined in:
lib/stockboy/providers/ftp.rb

Overview

Get data from a remote FTP server

Allows for selecting the appropriate file to be read from the given directory by glob pattern or regex pattern (glob string is more efficient for listing files from FTP). By default the :last file in the list is used, but can be controlled by sorting and reducing with the #pick option.

Job template DSL

provider :ftp do
  host      'ftp.example.com'
  username  'example'
  password  '424242'
  file_dir  'data/daily'
  file_name /report-[0-9]+\.csv/
  pick      ->(list) { list[-2] }
end

Defined Under Namespace

Classes: FTPAdapter, SFTPAdapter

Options collapse

Attributes inherited from Stockboy::Provider

#data, #data_size, #data_time, #errors, #logger

Instance Method Summary collapse

Methods inherited from Stockboy::Provider

#data?, #inspect, #reload, #valid?

Constructor Details

#initialize(opts = {}, &block) ⇒ FTP

Initialize a new FTP provider



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/stockboy/providers/ftp.rb', line 98

def initialize(opts={}, &block)
  super(opts, &block)
  @host         = opts[:host]
  @passive      = opts[:passive]
  @username     = opts[:username]
  @password     = opts[:password]
  @secure       = opts[:secure]
  @binary       = opts[:binary]
  @file_dir     = opts[:file_dir]
  @file_name    = opts[:file_name]
  @file_newer   = opts[:file_newer]
  @file_smaller = opts[:file_smaller]
  @file_larger  = opts[:file_larger]
  @pick         = opts[:pick] || :last
  DSL.new(self).instance_eval(&block) if block_given?
  @open_adapter  = nil
end

Instance Attribute Details

#binary ⇒ Boolean

Use binary mode for file transfers

Examples:

binary true

Returns:

  • (Boolean) 


73
 
# File 'lib/stockboy/providers/ftp.rb', line 73

dsl_attr :binary

#file_dir ⇒ String

Path where data files can be found. This should be an absolute path.

Examples:

file_dir "/data"

Returns:

  • (String) 


85
 
# File 'lib/stockboy/providers/ftp.rb', line 85

dsl_attr :file_name

#file_name ⇒ String, Regexp

A string (glob) or regular expression matching files. E.g. one of:

Examples:

file_name "export-latest.csv"
file_name "export-*.csv"
file_name /export-\d{4}-\d{2}-\d{2}.csv/

Returns:

  • (String, Regexp) 


85
 
# File 'lib/stockboy/providers/ftp.rb', line 85

dsl_attr :file_name

#file_newer ⇒ Time, Date

Validates that the file to be processed is recent enough. To guard against processing an old file (even if it’s the latest one), this should be set to the frequency you expect to receive new files for periodic processing. This validation option is applied after a matching file is picked.

Examples:

since Date.today

Returns:

  • (Time, Date) 


85
 
# File 'lib/stockboy/providers/ftp.rb', line 85

dsl_attr :file_name

#host ⇒ String

Host name or IP address for FTP server connection

Examples:

host "ftp.example.com"

Returns:

  • (String) 


37
 
# File 'lib/stockboy/providers/ftp.rb', line 37

dsl_attr :host

#passive ⇒ Boolean

Use a passive or active connection

Examples:

passive true

Returns:

  • (Boolean) 


46
 
# File 'lib/stockboy/providers/ftp.rb', line 46

dsl_attr :passive

#password ⇒ String

Password for connection credentials

Examples:

password "424242"

Returns:

  • (String) 


64
 
# File 'lib/stockboy/providers/ftp.rb', line 64

dsl_attr :password

#pick ⇒ Object

Method for choosing which file to process from potential matches.

@example
  pick :last
  pick :first
  pick ->(list) {
    list.max_by { |name| Time.strptime(name[/\d+/], "%m%d%Y").to_i }
  }



92
 
# File 'lib/stockboy/providers/ftp.rb', line 92

dsl_attr :pick

#secure ⇒ Boolean

Use SFTP protocol for file transfers

Examples:

secure true

Returns:

  • (Boolean) 


82
 
# File 'lib/stockboy/providers/ftp.rb', line 82

dsl_attr :secure

#username ⇒ String

User name for connection credentials

Examples:

username "arthur"

Returns:

  • (String) 


55
 
# File 'lib/stockboy/providers/ftp.rb', line 55

dsl_attr :username

Instance Method Details

#adapter ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/stockboy/providers/ftp.rb', line 120

def adapter
  return yield @open_adapter if @open_adapter

  adapter_class.new(self).open do |ftp|
    @open_adapter = ftp
    ftp.chdir file_dir if file_dir
    response = yield ftp
    @open_adapter = nil
    response
  end

rescue adapter_class.exception_class => e
  errors << e.message
  logger.warn e.message
  nil
end

#adapter_class ⇒ Object 



116
117
118
 
# File 'lib/stockboy/providers/ftp.rb', line 116

def adapter_class
  secure ? SFTPAdapter : FTPAdapter
end

#clear ⇒ Object 



158
159
160
161
162
163
 
# File 'lib/stockboy/providers/ftp.rb', line 158

def clear
  super
  @matching_file = nil
  @data_time = nil
  @data_size = nil
end

#client ⇒ Object 



137
138
139
 
# File 'lib/stockboy/providers/ftp.rb', line 137

def client
  adapter { |ftp| yield ftp.client }
end

#delete_data ⇒ Object

Raises:



149
150
151
152
153
154
155
156
 
# File 'lib/stockboy/providers/ftp.rb', line 149

def delete_data
  raise Stockboy::OutOfSequence, "must confirm #matching_file or calling #data" unless picked_matching_file?
  adapter do |ftp|
    logger.info "FTP deleting file #{host} #{file_dir}/#{matching_file}"
    ftp.delete matching_file
    matching_file
  end
end

#matching_file ⇒ Object 



141
142
143
144
145
146
147
 
# File 'lib/stockboy/providers/ftp.rb', line 141

def matching_file
  return @matching_file if @matching_file
  adapter do |ftp|
    file_listing = ftp.list_files
    @matching_file = pick_from file_listing.select(&file_name_matcher)
  end
end