Module: SilentFetcher

Defined in:
lib/silent_fetcher.rb,
lib/silent_fetcher/version.rb,
lib/silent_fetcher/configuration.rb

Defined Under Namespace

Classes: Configuration, ExpectedError

Constant Summary collapse

DEFAULT_CHARSET = 
'UTF-8'
DEFAULT_RETRY_COUNT = 
3
DEFAULT_TIMEOUT = 
60
EXPECTED_ERRORS = 
{
  'URI::InvalidURIError'         => [/the scheme http does not accept registry part/, /bad URI/],
  'ArgumentError'                => [/invalid byte sequence/],
  'SocketError'                  => [/Hostname not known/],
  'RuntimeError'                 => [/HTTP redirection loop/],
  'EOFError'                     => [/end of file reached/],
  'Errno::EHOSTUNREACH'          => [/No route to host/],
  'Errno::ECONNRESET'            => [/Connection reset by peer/],
  'Errno::ECONNREFUSED'          => [/Connection refused/],
  'Errno::ENETUNREACH'           => [/Network is unreachable/],
  'Errno::ETIMEDOUT'             => [],
  'HTTParty::RedirectionTooDeep' => [],
  'OpenURI::HTTPError'           => [],
  'OpenSSL::SSL::SSLError'       => [/SSL_connect returned=1 errno=0 state=SSLv3/]
}
RETRYABLE_ERRORS = 
[Net::OpenTimeout, Net::ReadTimeout]
VERSION = 
"0.0.3"

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.configuration ⇒ Object

Returns the value of attribute configuration.



38
39
40
 
# File 'lib/silent_fetcher.rb', line 38

def configuration
  @configuration
end

Class Method Details

.configure {|configuration| ... } ⇒ Object

Yields:



78
79
80
81
 
# File 'lib/silent_fetcher.rb', line 78

def configure
  self.configuration ||= Configuration.new
  yield(configuration)
end

.expected_error_classes ⇒ Object 



83
84
85
 
# File 'lib/silent_fetcher.rb', line 83

def expected_error_classes
  EXPECTED_ERRORS.keys.map(&:constantize)
end

.fetch(url, retry_count: DEFAULT_RETRY_COUNT, allow_no_response: false) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/silent_fetcher.rb', line 50

def fetch(url, retry_count: DEFAULT_RETRY_COUNT, allow_no_response: false)
  response = HTTParty.get(url, fetch_options)

  if response.body.size == 0 && !allow_no_response
    raise SilentFetcher::ExpectedError, "response.body.size == 0: #{url}"
  end

  response.body

rescue *RETRYABLE_ERRORS => e
  if retry_count > 0
    retry_count -= 1
    retry
  else
    raise SilentFetcher::ExpectedError, "#{e.message}: #{url}"
  end

rescue => e
  class_string = e.class.to_s

  if EXPECTED_ERRORS[class_string] &&
    (EXPECTED_ERRORS[class_string].none? || EXPECTED_ERRORS[class_string].any? {|m| e.message =~ m })
    raise SilentFetcher::ExpectedError, "#{e.class}: #{e.message} by #{url}"
  else
    raise e
  end
end

.parse_feed(url) ⇒ Object 



44
45
46
47
48
 
# File 'lib/silent_fetcher.rb', line 44

def parse_feed(url)
  Feedjira::Feed.parse(
    fetch(url)
  )
end

.parse_html(url, charset: DEFAULT_CHARSET) ⇒ Object 



40
41
42
 
# File 'lib/silent_fetcher.rb', line 40

def parse_html(url, charset: DEFAULT_CHARSET)
  Nokogiri::HTML(fetch(url), nil, charset)
end