Class: Down::NetHttp

Inherits:
Backend show all
Defined in:
lib/down/net_http.rb

Overview

Provides streaming downloads implemented with Net::HTTP and open-uri.

Defined Under Namespace

Modules: DownloadedFile

Constant Summary collapse

URI_NORMALIZER = 
-> (url) do
  addressable_uri = Addressable::URI.parse(url)
  addressable_uri.normalize.to_s
end

Instance Method Summary collapse

Methods inherited from Backend

download, open

Constructor Details

#initialize(*args, **options) ⇒ NetHttp

Initializes the backend with common defaults.



21
22
23
24
25
26
27
28
29
 
# File 'lib/down/net_http.rb', line 21

def initialize(*args, **options)
  @options = merge_options({
    headers:        { "User-Agent" => "Down/#{Down::VERSION}" },
    max_redirects:  2,
    open_timeout:   30,
    read_timeout:   30,
    uri_normalizer: URI_NORMALIZER,
  }, *args, **options)
end

Instance Method Details

#download(url, *args, **options) ⇒ Object

Downloads a remote file to disk using open-uri. Accepts any open-uri options, and a few more.



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/down/net_http.rb', line 33

def download(url, *args, **options)
  options = merge_options(@options, *args, **options)

  max_size            = options.delete(:max_size)
  max_redirects       = options.delete(:max_redirects)
  progress_proc       = options.delete(:progress_proc)
  content_length_proc = options.delete(:content_length_proc)
  destination         = options.delete(:destination)
  headers             = options.delete(:headers)
  uri_normalizer      = options.delete(:uri_normalizer)
  extension           = options.delete(:extension)

  # Use open-uri's :content_lenth_proc or :progress_proc to raise an
  # exception early if the file is too large.
  #
  # Also disable following redirects, as we'll provide our own
  # implementation that has the ability to limit the number of redirects.
  open_uri_options = {
    content_length_proc: proc { |size|
      if size && max_size && size > max_size
        raise Down::TooLarge, "file is too large (#{size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
      end
      content_length_proc.call(size) if content_length_proc
    },
    progress_proc: proc { |current_size|
      if max_size && current_size > max_size
        raise Down::TooLarge, "file is too large (#{current_size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
      end
      progress_proc.call(current_size) if progress_proc
    },
    redirect: false,
  }

  # Handle basic authentication in the :proxy option.
  if options[:proxy]
    proxy    = URI(options.delete(:proxy))
    user     = proxy.user
    password = proxy.password

    if user || password
      proxy.user     = nil
      proxy.password = nil

      open_uri_options[:proxy_http_basic_authentication] = [proxy.to_s, user, password]
    else
      open_uri_options[:proxy] = proxy.to_s
    end
  end

  open_uri_options.merge!(options)
  open_uri_options.merge!(headers)

  uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))

  # Handle basic authentication in the remote URL.
  if uri.user || uri.password
    open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
    uri.user = nil
    uri.password = nil
  end

  open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)

  # Handle the fact that open-uri returns StringIOs for small files.
  extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
  tempfile = ensure_tempfile(open_uri_file, extname)
  OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
  tempfile.extend Down::NetHttp::DownloadedFile

  download_result(tempfile, destination)
end

#open(url, *args, **options) ⇒ Object

Starts retrieving the remote file using Net::HTTP and returns an IO-like object which downloads the response body on-demand.



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/down/net_http.rb', line 107

def open(url, *args, **options)
  options = merge_options(@options, *args, **options)

  max_redirects  = options.delete(:max_redirects)
  uri_normalizer = options.delete(:uri_normalizer)

  uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))

  # Create a Fiber that halts when response headers are received.
  request = Fiber.new do
    net_http_request(uri, options, follows_remaining: max_redirects) do |response|
      Fiber.yield response
    end
  end

  response = request.resume

  response_error!(response) unless response.is_a?(Net::HTTPSuccess)

  # Build an IO-like object that will retrieve response body on-demand.
  Down::ChunkedIO.new(
    chunks:     enum_for(:stream_body, response),
    size:       response["Content-Length"] && response["Content-Length"].to_i,
    encoding:   response.type_params["charset"],
    rewindable: options.fetch(:rewindable, true),
    on_close:   -> { request.resume }, # close HTTP connnection
    data: {
      status:   response.code.to_i,
      headers:  normalize_headers(response.each_header),
      response: response,
    },
  )
end