Class: Stockboy::Providers::FTP

Inherits:
Stockboy::Provider show all
Defined in:
lib/stockboy/providers/ftp.rb

Overview

Get data from a remote FTP server

Allows for selecting the appropriate file to be read from the given directory by glob pattern or regex pattern (glob string is more efficient for listing files from FTP). By default the :last file in the list is used, but can be controlled by sorting and reducing with the #pick option.

Job template DSL

provider :ftp do
  host      'ftp.example.com'
  username  'example'
  password  '424242'
  file_dir  'data/daily'
  file_name /report-[0-9]+\.csv/
  pick      ->(list) { list[-2] }
end

Options collapse

Attributes inherited from Stockboy::Provider

#data, #data_size, #data_time, #errors, #logger

Instance Method Summary collapse

Methods inherited from Stockboy::Provider

#data?, #inspect, #reload, #valid?

Constructor Details

#initialize(opts = {}, &block) ⇒ FTP

Initialize a new FTP provider



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/stockboy/providers/ftp.rb', line 88

def initialize(opts={}, &block)
  super(opts, &block)
  @host         = opts[:host]
  @passive      = opts[:passive]
  @username     = opts[:username]
  @password     = opts[:password]
  @binary       = opts[:binary]
  @file_dir     = opts[:file_dir]
  @file_name    = opts[:file_name]
  @file_newer   = opts[:file_newer]
  @file_smaller = opts[:file_smaller]
  @file_larger  = opts[:file_larger]
  @pick         = opts[:pick] || :last
  DSL.new(self).instance_eval(&block) if block_given?
end

Instance Attribute Details

#binary ⇒ Boolean

Use binary mode for file transfers

Examples:

binary true

Returns:

  • (Boolean) 


72
 
# File 'lib/stockboy/providers/ftp.rb', line 72

dsl_attr :binary

#file_dir ⇒ String

Path where data files can be found. This should be an absolute path.

Examples:

file_dir "/data"

Returns:

  • (String) 


75
 
# File 'lib/stockboy/providers/ftp.rb', line 75

dsl_attr :file_name

#file_name ⇒ String, Regexp

A string (glob) or regular expression matching files. E.g. one of:

Examples:

file_name "export-latest.csv"
file_name "export-*.csv"
file_name /export-\d{4}-\d{2}-\d{2}.csv/

Returns:

  • (String, Regexp) 


75
 
# File 'lib/stockboy/providers/ftp.rb', line 75

dsl_attr :file_name

#file_newer ⇒ Time, Date

Validates that the file to be processed is recent enough. To guard against processing an old file (even if it’s the latest one), this should be set to the frequency you expect to receive new files for periodic processing. This validation option is applied after a matching file is picked.

Examples:

since Date.today

Returns:

  • (Time, Date) 


75
 
# File 'lib/stockboy/providers/ftp.rb', line 75

dsl_attr :file_name

#host ⇒ String

Host name or IP address for FTP server connection

Examples:

host "ftp.example.com"

Returns:

  • (String) 


36
 
# File 'lib/stockboy/providers/ftp.rb', line 36

dsl_attr :host

#passive ⇒ Boolean

Use a passive or active connection

Examples:

passive true

Returns:

  • (Boolean) 


45
 
# File 'lib/stockboy/providers/ftp.rb', line 45

dsl_attr :passive

#password ⇒ String

Password for connection credentials

Examples:

password "424242"

Returns:

  • (String) 


63
 
# File 'lib/stockboy/providers/ftp.rb', line 63

dsl_attr :password

#pick ⇒ Object

Method for choosing which file to process from potential matches.

@example
  pick :last
  pick :first
  pick ->(list) {
    list.max_by { |name| Time.strptime(name[/\d+/], "%m%d%Y").to_i }
  }



82
 
# File 'lib/stockboy/providers/ftp.rb', line 82

dsl_attr :pick

#username ⇒ String

User name for connection credentials

Examples:

username "arthur"

Returns:

  • (String) 


54
 
# File 'lib/stockboy/providers/ftp.rb', line 54

dsl_attr :username

Instance Method Details

#clear ⇒ Object 



139
140
141
142
143
144
 
# File 'lib/stockboy/providers/ftp.rb', line 139

def clear
  super
  @matching_file = nil
  @data_time = nil
  @data_size = nil
end

#client ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/stockboy/providers/ftp.rb', line 104

def client
  return yield @open_client if @open_client

  Net::FTP.open(host, username, password) do |ftp|
    ftp.binary = binary
    ftp.passive = passive
    ftp.chdir file_dir if file_dir
    @open_client = ftp
    response = yield ftp
    @open_client = nil
    response
  end
rescue Net::FTPError => e
  errors << e.message
  logger.warn e.message
  nil
end

#delete_data ⇒ Object

Raises:



130
131
132
133
134
135
136
137
 
# File 'lib/stockboy/providers/ftp.rb', line 130

def delete_data
  raise Stockboy::OutOfSequence, "must confirm #matching_file or calling #data" unless picked_matching_file?
  client do |ftp|
    logger.info "FTP deleting file #{host} #{file_dir}/#{matching_file}"
    ftp.delete matching_file
    matching_file
  end
end

#matching_file ⇒ Object 



122
123
124
125
126
127
128
 
# File 'lib/stockboy/providers/ftp.rb', line 122

def matching_file
  return @matching_file if @matching_file
  client do |ftp|
    file_listing = ftp.nlst.sort
    @matching_file = pick_from file_listing.select(&file_name_matcher)
  end
end