Class: Down::Httpx

Inherits:
Backend show all
Defined in:
lib/down/httpx.rb

Overview

Provides streaming downloads implemented with HTTPX.

Defined Under Namespace

Modules: DownloadedFile

Constant Summary collapse

USER_AGENT =

Initializes the backend

"Down/#{Down::VERSION}"
BASIC_AUTH = 
HTTPX::VERSION >= "1.0.0" ? :basic_auth : :basic_authentication

Instance Method Summary collapse

Methods inherited from Backend

download, open

Constructor Details

#initialize(**options, &block) ⇒ Httpx

Returns a new instance of Httpx.



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/down/httpx.rb', line 19

def initialize(**options, &block)
  @method = options.delete(:method) || :get
  headers = options.delete(:headers) || {}
  @client = HTTPX
      .plugin(:follow_redirects, max_redirects: 2)
      .plugin(BASIC_AUTH)
      .plugin(:stream)
      .with(
        headers: { "user-agent": USER_AGENT }.merge(headers),
        timeout: { connect_timeout: 30, write_timeout: 30, read_timeout: 30 },
        **options
      )

  @client = block.call(@client) if block
end

Instance Method Details

#download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block) ⇒ Object

Downlods the remote file to disk. Accepts HTTPX options via a hash or a block, and some additional options as well.



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/down/httpx.rb', line 38

def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
  client = @client

  response = request(client, url, **options, &block)

  content_length = nil

  if response.headers.key?("content-length")
    content_length = response.headers["content-length"].to_i

    content_length_proc.call(content_length) if content_length_proc

    if max_size && content_length > max_size
      response.close
      raise Down::TooLarge, "file is too large (#{content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
    end
  end

  extname  = extension ? ".#{extension}" : File.extname(response.uri.path)
  tempfile = Tempfile.new(["down-http", extname], binmode: true)

  stream_body(response) do |chunk|
    tempfile.write(chunk)
    chunk.clear # deallocate string

    progress_proc.call(tempfile.size) if progress_proc

    if max_size && tempfile.size > max_size
      raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
    end
  end

  tempfile.open # flush written content

  tempfile.extend DownloadedFile
  tempfile.url     = response.uri.to_s
  tempfile.headers = normalize_headers(response.headers.to_h)
  tempfile.content_type = response.content_type.mime_type
  tempfile.charset = response.body.encoding

  download_result(tempfile, destination)
rescue
  tempfile.close! if tempfile
  raise
end

#open(url, rewindable: true, **options, &block) ⇒ Object

Starts retrieving the remote file and returns an IO-like object which downloads the response body on-demand. Accepts HTTP.rb options via a hash or a block.



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/down/httpx.rb', line 87

def open(url, rewindable: true, **options, &block)
  response = request(@client, url, stream: true, **options, &block)
  size = response.headers["content-length"]
  size = size.to_i if size
  Down::ChunkedIO.new(
    chunks:     enum_for(:stream_body, response),
    size:       size,
    encoding:   response.body.encoding,
    rewindable: rewindable,
    data:       {
      status:   response.status,
      headers:  normalize_headers(response.headers.to_h),
      response: response
    },
  )
end