Class: TerrImporter::Application::Downloader

Inherits:
Object
  • Object
show all
Includes:
DownloadHelper
Defined in:
lib/terrimporter/downloader.rb

Instance Method Summary collapse

Methods included from DownloadHelper

#create_directory

Constructor Details

#initialize(base_uri) ⇒ Downloader

Returns a new instance of Downloader.



9
10
11
12
 
# File 'lib/terrimporter/downloader.rb', line 9

def initialize(base_uri)
  @base_uri = base_uri
  LOG.debug "Downloader initialized to uri: #{@base_uri}"
end

Instance Method Details

#batch_download(remote_path, local_path, type_filter = "", statistics_key = nil) ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/terrimporter/downloader.rb', line 41

def batch_download(remote_path, local_path, type_filter = "", statistics_key = nil)
  source_path = url(remote_path)
  create_directory local_path
  LOG.debug "Download multiple files from #{source_path} to #{local_path} #{"allowed extensions: " + type_filter unless type_filter.empty?}"

  files = html_directory_list(source_path)

  unless type_filter.empty?
    LOG.debug "Apply type filter: #{type_filter}"
    files = files.find_all { |file| file =~ Regexp.new(".*" + type_filter.robust_split.join("|") + "$") }
  end

  LOG.info "Download #{files.size} files..."
  files.each do |file|
    local_file_path = File.join(local_path.to_s, file)
    self.download(File.join(source_path.to_s, file), local_file_path)
    STAT.add(statistics_key) unless statistics_key.nil?
  end
end

#download(remote_path, local_path = nil) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/terrimporter/downloader.rb', line 14

def download(remote_path, local_path = nil)
  remote_url = url(remote_path)
  begin
    if local_path.nil? #download to buffer
      download_to_buffer(remote_url)
    else
      download_to_file(remote_url, local_path)
      STAT.add(:download)
    end
  rescue Exception => e
    raise DefaultError, "Error opening url: #{remote_url}, message: #{e.message}"
  end
end

#download_to_buffer(remote_url) ⇒ Object 



28
29
30
31
32
33
 
# File 'lib/terrimporter/downloader.rb', line 28

def download_to_buffer(remote_url)
  LOG.debug "Download #{remote_url} to buffer"
  data = StringIO.new
    remote_url.open { |io| data = io.read }
    data.to_s
end

#download_to_file(remote_url, local_path) ⇒ Object 



35
36
37
38
39
 
# File 'lib/terrimporter/downloader.rb', line 35

def download_to_file(remote_url, local_path)
  LOG.info "Download #{remote_url} to local path #{local_path}"
  create_directory File.dirname(local_path)
  open(local_path, "wb") { |file| file.write(remote_url.open.read) }
end