Method: Mechanize::HTTP::Agent#fetch

Defined in:

lib/mechanize/http/agent.rb

#fetch(uri, method = :get, headers = {}, params = [], referer = current_page, redirects = 0) ⇒ Object

Retrieves uri and parses it into a page or other object according to PluggableParser. If the URI is an HTTP or HTTPS scheme URI the given HTTP method is used to retrieve it, along with the HTTP headers, request params and HTTP referer.

The final URI to access is built with uri and params, the latter of which is formatted into a string using Mechanize::Util.build_query_string, which see.

redirects tracks the number of redirects experienced when retrieving the page. If it is over the redirection_limit an error will be raised.

# File 'lib/mechanize/http/agent.rb', line 244

def fetch uri, method = :get, headers = {}, params = [],
          referer = current_page, redirects = 0

  referer_uri = referer ? referer.uri : nil
  uri         = resolve uri, referer
  uri, params = resolve_parameters uri, method, params
  request     = http_request uri, method, params
  connection  = connection_for uri

  request_auth             request, uri
  disable_keep_alive       request
  enable_gzip              request
  request_language_charset request
  request_cookies          request, uri
  request_host             request, uri
  request_referer          request, uri, referer_uri
  request_user_agent       request
  request_add_headers      request, headers
  pre_connect              request

  # Consult robots.txt
  if robots && uri.is_a?(URI::HTTP)
    robots_allowed?(uri) or raise Mechanize::RobotsDisallowedError.new(uri)
  end

  # Add If-Modified-Since if page is in history
  if page = visited_page(uri) and last_modified = page.response['Last-Modified']
    request['If-Modified-Since'] = last_modified
  end if @conditional_requests

  # Specify timeouts if supplied and our connection supports them
  if @open_timeout && connection.respond_to?(:open_timeout=)
    connection.open_timeout = @open_timeout
  end
  if @read_timeout && connection.respond_to?(:read_timeout=)
    connection.read_timeout = @read_timeout
  end
  if @write_timeout && connection.respond_to?(:write_timeout=)
    connection.write_timeout = @write_timeout
  end

  request_log request

  response_body_io = nil

  # Send the request
  begin
    response = connection.request(uri, request) { |res|
      response_log res

      response_body_io = response_read res, request, uri

      res
    }
  rescue Mechanize::ChunkedTerminationError => e
    raise unless @ignore_bad_chunking

    response = e.response
    response_body_io = e.body_io
  end

  hook_content_encoding response, uri, response_body_io

  response_body_io = response_content_encoding response, response_body_io if
    request.response_body_permitted?

  post_connect uri, response, response_body_io

  page = response_parse response, response_body_io, uri

  response_cookies response, uri, page

  meta = response_follow_meta_refresh response, uri, page, redirects
  return meta if meta

  if robots && page.is_a?(Mechanize::Page)
    page.parser.noindex? and raise Mechanize::RobotsDisallowedError.new(uri)
  end

  case response
  when Net::HTTPSuccess
    page
  when Mechanize::FileResponse
    page
  when Net::HTTPNotModified
    log.debug("Got cached page") if log
    visited_page(uri) || page
  when Net::HTTPRedirection
    response_redirect response, method, page, redirects, headers, referer
  when Net::HTTPUnauthorized
    response_authenticate(response, page, uri, request, headers, params,
                          referer)
  else
    if @allowed_error_codes.any? {|code| code.to_s == page.code} then
      page
    else
      raise Mechanize::ResponseCodeError.new(page, 'unhandled response')
    end
  end
end