Class: BrandEins::Utils::Fetcher

Inherits:
Object
  • Object
show all
Includes:
Singleton
Defined in:
lib/brandeins/utils/fetcher.rb

Overview

Used as a centralized resource fetcher with a caching mechanism

Constant Summary collapse

ContentNotFetchedError = 
Class.new StandardError

Instance Method Summary collapse

Instance Method Details

#add_to_cache(file_name, file_content) ⇒ Object 



104
105
106
107
 
# File 'lib/brandeins/utils/fetcher.rb', line 104

def add_to_cache(file_name, file_content)
  cache_file_path = cache_path_for_file_name(file_name)
  cache_files[cache_file_path.to_s] = File.mtime(cache_file_path)
end

#cache_files ⇒ Object 



109
110
111
112
113
114
115
116
117
 
# File 'lib/brandeins/utils/fetcher.rb', line 109

def cache_files
  @cache_files ||= begin
                     files = Dir[cache_path + './*']
                     files_with_mtime = files.map do |file_path|
                       [file_path, File.mtime(file_path)]
                     end
                     Hash[files_with_mtime]
                   end
end

#cache_has_file?(file_name) ⇒ Boolean

Returns:

  • (Boolean) 


63
64
65
66
 
# File 'lib/brandeins/utils/fetcher.rb', line 63

def cache_has_file?(file_name)
  cache_path = cache_path_for_file_name(file_name)
  cache_files.key? cache_path
end

#cache_limit_in_bytes ⇒ Object 



77
78
79
 
# File 'lib/brandeins/utils/fetcher.rb', line 77

def cache_limit_in_bytes
  BrandEins::Config['cache_limit_bytes']
end

#cache_limit_reached? ⇒ Boolean

Returns:

  • (Boolean) 


85
86
87
 
# File 'lib/brandeins/utils/fetcher.rb', line 85

def cache_limit_reached?
  cache_size_in_bytes > cache_limit_in_bytes
end

#cache_path ⇒ Object 



81
82
83
 
# File 'lib/brandeins/utils/fetcher.rb', line 81

def cache_path
  BrandEins::Config['cache_path']
end

#cache_path_for_file_name(file_name) ⇒ Object 



68
69
70
 
# File 'lib/brandeins/utils/fetcher.rb', line 68

def cache_path_for_file_name(file_name)
  cache_path + escaped_file_name(file_name)
end

#cache_size_in_bytes ⇒ Object 



89
90
91
92
93
94
 
# File 'lib/brandeins/utils/fetcher.rb', line 89

def cache_size_in_bytes
  cache_files.reduce(0) do |sum, (file, _)|
    next unless file
    sum += File.size?(file) || 0
  end
end

#cli ⇒ Object 



119
120
121
 
# File 'lib/brandeins/utils/fetcher.rb', line 119

def cli
  @cli ||= BrandEins::Utils::CliOutput.instance
end

#escaped_file_name(file_name) ⇒ Object 



72
73
74
75
 
# File 'lib/brandeins/utils/fetcher.rb', line 72

def escaped_file_name(file_name)
  uri = URI.parse(file_name)
  [uri.host, File.basename(uri.path)].compact.join('-')
end

#fetch(url) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/brandeins/utils/fetcher.rb', line 21

def fetch(url)
  remove_oldest_cache_file if cache_limit_reached?
  if cache_has_file?(url)
    cli.debug "Fetching file from cache: #{url}" do
      file_from_cache(url)
    end
  else
    cli.debug "Fetching file from url: #{url}" do
      http_get_resource(url)
    end
  end
end

#file_from_cache(file_name) ⇒ Object 



55
56
57
58
59
60
61
 
# File 'lib/brandeins/utils/fetcher.rb', line 55

def file_from_cache(file_name)
  cache_file_path = cache_path_for_file_name(file_name)
  cli.statusline "Reading file from cache: #{file_name}" do
    cache_file_path = cache_path_for_file_name(file_name)
    File.binread(cache_file_path)
  end
end

#http_get_resource(url) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/brandeins/utils/fetcher.rb', line 34

def http_get_resource(url)
  cli.statusline "Fetching: #{url}" do
    uri  = URI.parse(url)
    resp = Net::HTTP.get_response(uri)
    unless resp.class == Net::HTTPOK
      raise ContentNotFetchedError, "Received #{resp.code}: #{resp.msg}"
    end
    write_to_cache(url, resp.body)
    resp.body
  end
end

#remove_oldest_cache_file ⇒ Object 



96
97
98
99
100
101
102
 
# File 'lib/brandeins/utils/fetcher.rb', line 96

def remove_oldest_cache_file
  oldest_file = cache_files.sort_by { |file, time| time }.last.first
  cli.debug "Removing cached file: #{oldest_file}" do
    FileUtils.rm oldest_file
    cache_files.delete(oldest_file)
  end
end

#write_to_cache(file_name, file_content) ⇒ Object 



46
47
48
49
50
51
52
53
 
# File 'lib/brandeins/utils/fetcher.rb', line 46

def write_to_cache(file_name, file_content)
  cache_file_path = cache_path_for_file_name(file_name)
  cli.debug "Writing file to cache: #{cache_file_path}" do
    result = !!File.binwrite(cache_file_path, file_content)
    add_to_cache(file_name, file_content)
    return result
  end
end