Class: EventMachine::HttpClient

Inherits:
Connection
  • Object
show all
Includes:
Deferrable, HttpEncoding
Defined in:
lib/em-http/client.rb

Direct Known Subclasses

MockHttpRequest::FakeHttpClient

Constant Summary collapse

TRANSFER_ENCODING = 
"TRANSFER_ENCODING"
CONTENT_ENCODING = 
"CONTENT_ENCODING"
CONTENT_LENGTH = 
"CONTENT_LENGTH"
CONTENT_TYPE = 
"CONTENT_TYPE"
LAST_MODIFIED = 
"LAST_MODIFIED"
KEEP_ALIVE = 
"CONNECTION"
"SET_COOKIE"
LOCATION = 
"LOCATION"
HOST = 
"HOST"
ETAG = 
"ETAG"
CRLF = 
"\r\n"

Constants included from HttpEncoding

EventMachine::HttpEncoding::FIELD_ENCODING, EventMachine::HttpEncoding::HTTP_REQUEST_HEADER

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from HttpEncoding

#bytesize, #encode_auth, #encode_cookie, #encode_field, #encode_headers, #encode_host, #encode_param, #encode_query, #encode_request, #escape, #form_encode_body, #munge_header_keys, #unescape

Instance Attribute Details

#content_charset ⇒ Object (readonly)

Returns the value of attribute content_charset.



30
31
32
 
# File 'lib/em-http/client.rb', line 30

def content_charset
  @content_charset
end

#error ⇒ Object (readonly)

Returns the value of attribute error.



30
31
32
 
# File 'lib/em-http/client.rb', line 30

def error
  @error
end

#last_effective_url ⇒ Object (readonly)

Returns the value of attribute last_effective_url.



30
31
32
 
# File 'lib/em-http/client.rb', line 30

def last_effective_url
  @last_effective_url
end

#method ⇒ Object

Returns the value of attribute method.



29
30
31
 
# File 'lib/em-http/client.rb', line 29

def method
  @method
end

#options ⇒ Object

Returns the value of attribute options.



29
30
31
 
# File 'lib/em-http/client.rb', line 29

def options
  @options
end

#redirects ⇒ Object (readonly)

Returns the value of attribute redirects.



30
31
32
 
# File 'lib/em-http/client.rb', line 30

def redirects
  @redirects
end

#response ⇒ Object (readonly)

Returns the value of attribute response.



30
31
32
 
# File 'lib/em-http/client.rb', line 30

def response
  @response
end

#response_header ⇒ Object (readonly)

Returns the value of attribute response_header.



30
31
32
 
# File 'lib/em-http/client.rb', line 30

def response_header
  @response_header
end

#uri ⇒ Object

Returns the value of attribute uri.



29
30
31
 
# File 'lib/em-http/client.rb', line 29

def uri
  @uri
end

Instance Method Details

#connect_proxy? ⇒ Boolean

determines if a http-proxy should be used with the CONNECT verb

Returns:

  • (Boolean) 


155
 
# File 'lib/em-http/client.rb', line 155

def connect_proxy?; http_proxy? && (@options[:proxy][:use_connect] == true); end

#connection_completed ⇒ Object

start HTTP request once we establish connection to host



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/em-http/client.rb', line 52

def connection_completed
  # if a socks proxy is specified, then a connection request
  # has to be made to the socks server and we need to wait
  # for a response code
  if socks_proxy? and @state == :response_header
    @state = :connect_socks_proxy
    send_socks_handshake

    # if we need to negotiate the proxy connection first, then
    # issue a CONNECT query and wait for 200 response
  elsif connect_proxy? and @state == :response_header
    @state = :connect_http_proxy
    send_request_header

    # if connecting via proxy, then state will be :proxy_connected,
    # indicating successful tunnel. from here, initiate normal http
    # exchange

  else
    @state = :response_header
    ssl = @options[:tls] || @options[:ssl] || {}
    start_tls(ssl) if @uri.scheme == "https" or @uri.port == 443
    send_request_header
    send_request_body
  end
end

#disconnect(&blk) ⇒ Object

assign disconnect callback for websocket



106
107
108
 
# File 'lib/em-http/client.rb', line 106

def disconnect(&blk)
  @disconnect = blk
end

#dispatch ⇒ Object

Response processing



319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
 
# File 'lib/em-http/client.rb', line 319

def dispatch
  while case @state
      when :connect_socks_proxy
        parse_socks_response
      when :connect_http_proxy
        parse_response_header
      when :response_header
        parse_response_header
      when :chunk_header
        parse_chunk_header
      when :chunk_body
        process_chunk_body
      when :chunk_footer
        process_chunk_footer
      when :response_footer
        process_response_footer
      when :body
        process_body
      when :websocket
        process_websocket
      when :finished, :invalid
        break
      else raise RuntimeError, "invalid state: #{@state}"
    end
  end
end

#finished? ⇒ Boolean

Returns:

  • (Boolean) 


281
282
283
 
# File 'lib/em-http/client.rb', line 281

def finished?
  @state == :finished || (@state == :body && @bytes_remaining.nil?)
end

#has_bytes?(num) ⇒ Boolean

determines if there is enough data in the buffer

Returns:

  • (Boolean) 


140
141
142
 
# File 'lib/em-http/client.rb', line 140

def has_bytes?(num)
  @data.size >= num
end

#headers(&blk) ⇒ Object

assign a headers parse callback



111
112
113
 
# File 'lib/em-http/client.rb', line 111

def headers(&blk)
  @headers = blk
end

#http_proxy? ⇒ Boolean

determines if a proxy should be used that uses http-headers as proxy-mechanism

this is the default proxy type if none is specified

Returns:

  • (Boolean) 


151
 
# File 'lib/em-http/client.rb', line 151

def http_proxy?; proxy? && [nil, :http].include?(@options[:proxy][:type]); end

#normalize_body ⇒ Object 



129
130
131
132
133
134
135
136
137
 
# File 'lib/em-http/client.rb', line 129

def normalize_body
  @normalized_body ||= begin
    if @options[:body].is_a? Hash
      form_encode_body(@options[:body])
    else
      @options[:body]
    end
  end
end

#on_body_data(data) ⇒ Object

Called when part of the body has been read



260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/em-http/client.rb', line 260

def on_body_data(data)
  if @content_decoder
    begin
      @content_decoder << data
    rescue HttpDecoders::DecoderError
      on_error "Content-decoder error"
    end
  else
    on_decoded_body_data(data)
  end
end

#on_decoded_body_data(data) ⇒ Object 



272
273
274
275
276
277
278
279
 
# File 'lib/em-http/client.rb', line 272

def on_decoded_body_data(data)
  data.force_encoding @content_charset if @content_charset
  if @stream
    @stream.call(data)
  else
    @response << data
  end
end

#on_error(msg, dns_error = false) ⇒ Object Also known as: close

request failed, invoke errback



91
92
93
94
95
96
97
 
# File 'lib/em-http/client.rb', line 91

def on_error(msg, dns_error = false)
  @error = msg

  # no connection signature on DNS failures
  # fail the connection directly
  dns_error == true ? fail(self) : unbind
end

#on_request_complete ⇒ Object

request is done, invoke the callback



80
81
82
83
84
85
86
87
88
 
# File 'lib/em-http/client.rb', line 80

def on_request_complete
  begin
    @content_decoder.finalize! if @content_decoder
  rescue HttpDecoders::DecoderError
    on_error "Content-decoder error"
  end

  close_connection
end

#parse_chunk_header ⇒ Object 



565
566
567
568
569
570
571
572
573
 
# File 'lib/em-http/client.rb', line 565

def parse_chunk_header
  return false unless parse_header(@chunk_header)

  @bytes_remaining = @chunk_header.chunk_size
  @chunk_header = HttpChunkHeader.new

  @state = @bytes_remaining > 0 ? :chunk_body : :response_footer
  true
end

#parse_header(header) ⇒ Object 



346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
 
# File 'lib/em-http/client.rb', line 346

def parse_header(header)
  return false if @data.empty?

  begin
    @parser_nbytes = @parser.execute(header, @data.to_str, @parser_nbytes)
  rescue EventMachine::HttpClientParserError
    @state = :invalid
    on_error "invalid HTTP format, parsing fails"
  end

  return false unless @parser.finished?

  # Clear parsed data from the buffer
  @data.read(@parser_nbytes)
  @parser.reset
  @parser_nbytes = 0

  true
end

#parse_response_header ⇒ Object 



366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
 
# File 'lib/em-http/client.rb', line 366

def parse_response_header
  return false unless parse_header(@response_header)

  # invoke headers callback after full parse if one
  # is specified by the user
  @headers.call(@response_header) if @headers

  unless @response_header.http_status and @response_header.http_reason
    @state = :invalid
    on_error "no HTTP response"
    return false
  end

  if @state == :connect_http_proxy
    # when a successfull tunnel is established, the proxy responds with a
    # 200 response code. from here, the tunnel is transparent.
    if @response_header.http_status.to_i == 200
      @response_header = HttpResponseHeader.new
      connection_completed
      return true
    else
      @state = :invalid
      on_error "proxy not accessible"
      return false
    end
  end

  # correct location header - some servers will incorrectly give a relative URI
  if @response_header.location
    begin
      location = Addressable::URI.parse(@response_header.location)

      if location.relative?
        location = @uri.join(location)
        @response_header[LOCATION] = location.to_s
      else
        # if redirect is to an absolute url, check for correct URI structure
        raise if location.host.nil?
      end

      # store last url on any sign of redirect
      @last_effective_url = location

    rescue
      on_error "Location header format error"
      return false
    end
  end

  # Fire callbacks immediately after recieving header requests
  # if the request method is HEAD. In case of a redirect, terminate
  # current connection and reinitialize the process.
  if @method == "HEAD"
    @state = :finished
    close_connection
    return false
  end

  if websocket?
    if @response_header.status == 101
      @state = :websocket
      succeed
    else
      fail "websocket handshake failed"
    end

  elsif @response_header.chunked_encoding?
    @state = :chunk_header
  elsif @response_header.content_length
    @state = :body
    @bytes_remaining = @response_header.content_length
  else
    @state = :body
    @bytes_remaining = nil
  end

  if decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
    begin
      @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
    rescue HttpDecoders::DecoderError
      on_error "Content-decoder error"
    end
  end

  if ''.respond_to?(:force_encoding) && /;\s*charset=\s*(.+?)\s*(;|$)/.match(response_header[CONTENT_TYPE])
    @content_charset = Encoding.find($1.gsub(/^\"|\"$/, '')) rescue Encoding.default_external
  end

  true
end

#parse_socks_response ⇒ Object

parses socks 5 server responses as specified on www.faqs.org/rfcs/rfc1928.html



474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
 
# File 'lib/em-http/client.rb', line 474

def parse_socks_response
  if @socks_state == :method_negotiation
    return false unless has_bytes? 2

    _, method = @data.read(2).unpack('CC')

    if socks_methods.include?(method)
      if method == 0
        @socks_state = :connecting

        return send_socks_connect_request

      elsif method == 2
        @socks_state = :authenticating

        credentials = @options[:proxy][:authorization]
        if credentials.size < 2
          @state = :invalid
          on_error "username and password are not supplied"
          return false
        end

        username, password = credentials

        send_data [5, username.length, username, password.length, password].pack('CCA*CA*')
      end

    else
      @state = :invalid
      on_error "proxy did not accept method"
      return false
    end

  elsif @socks_state == :authenticating
    return false unless has_bytes? 2

    _, status_code = @data.read(2).unpack('CC')

    if status_code == 0
      # success
      @socks_state = :connecting

      return send_socks_connect_request

    else
      # error
      @state = :invalid
      on_error "access denied by proxy"
      return false
    end

  elsif @socks_state == :connecting
    return false unless has_bytes? 10

    _, response_code, _, address_type, _, _ = @data.read(10).unpack('CCCCNn')

    if response_code == 0
      # success
      @socks_state = :connected
      @state = :proxy_connected

      @response_header = HttpResponseHeader.new

      # connection_completed will invoke actions to
      # start sending all http data transparently
      # over the socks connection
      connection_completed

    else
      # error
      @state = :invalid

      error_messages = {
        1 => "general socks server failure",
        2 => "connection not allowed by ruleset",
        3 => "network unreachable",
        4 => "host unreachable",
        5 => "connection refused",
        6 => "TTL expired",
        7 => "command not supported",
        8 => "address type not supported"
      }
      error_message = error_messages[response_code] || "unknown error (code: #{response_code})"
      on_error "socks5 connect error: #{error_message}"
      return false
    end
  end

  true
end

#post_init ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/em-http/client.rb', line 32

def post_init
  @parser = HttpClientParser.new
  @data = EventMachine::Buffer.new
  @chunk_header = HttpChunkHeader.new
  @response_header = HttpResponseHeader.new
  @parser_nbytes = 0
  @redirects = 0
  @response = ''
  @error = ''
  @headers = nil
  @last_effective_url = nil
  @content_decoder = nil
  @content_charset = nil
  @stream = nil
  @disconnect = nil
  @state = :response_header
  @socks_state = nil
end

#process_body ⇒ Object 



621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
 
# File 'lib/em-http/client.rb', line 621

def process_body
  if @bytes_remaining.nil?
    on_body_data @data.read
    return false
  end

  if @bytes_remaining.zero?
    @state = :finished
    on_request_complete
    return false
  end

  if @data.size < @bytes_remaining
    @bytes_remaining -= @data.size
    on_body_data @data.read
    return false
  end

  on_body_data @data.read(@bytes_remaining)
  @bytes_remaining = 0

  # If Keep-Alive is enabled, the server may be pushing more data to us
  # after the first request is complete. Hence, finish first request, and
  # reset state.
  if @response_header.keep_alive?
    @data.clear # hard reset, TODO: add support for keep-alive connections!
    @state = :finished
    on_request_complete

  else

    @data.clear
    @state = :finished
    on_request_complete
  end

  false
end

#process_chunk_body ⇒ Object 



575
576
577
578
579
580
581
582
583
584
585
586
587
 
# File 'lib/em-http/client.rb', line 575

def process_chunk_body
  if @data.size < @bytes_remaining
    @bytes_remaining -= @data.size
    on_body_data @data.read
    return false
  end

  on_body_data @data.read(@bytes_remaining)
  @bytes_remaining = 0

  @state = :chunk_footer
  true
end

#process_chunk_footer ⇒ Object 



589
590
591
592
593
594
595
596
597
598
599
600
 
# File 'lib/em-http/client.rb', line 589

def process_chunk_footer
  return false if @data.size < 2

  if @data.read(2) == CRLF
    @state = :chunk_header
  else
    @state = :invalid
    on_error "non-CRLF chunk footer"
  end

  true
end

#process_response_footer ⇒ Object 



602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
 
# File 'lib/em-http/client.rb', line 602

def process_response_footer
  return false if @data.size < 2

  if @data.read(2) == CRLF
    if @data.empty?
      @state = :finished
      on_request_complete
    else
      @state = :invalid
      on_error "garbage at end of chunked response"
    end
  else
    @state = :invalid
    on_error "non-CRLF response footer"
  end

  false
end

#process_websocket ⇒ Object 



660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
 
# File 'lib/em-http/client.rb', line 660

def process_websocket
  return false if @data.empty?

  # slice the message out of the buffer and pass in
  # for processing, and buffer data otherwise
  buffer = @data.read
  while msg = buffer.slice!(/\000([^\377]*)\377/n)
    msg.gsub!(/\A\x00|\xff\z/n, '')
    @stream.call(msg)
  end

  # store remainder if message boundary has not yet
  # been received
  @data << buffer if not buffer.empty?

  false
end

#proxy? ⇒ Boolean

Returns:

  • (Boolean) 


145
 
# File 'lib/em-http/client.rb', line 145

def proxy?; !@options[:proxy].nil?; end

#receive_data(data) ⇒ Object 



254
255
256
257
 
# File 'lib/em-http/client.rb', line 254

def receive_data(data)
  @data << data
  dispatch
end

#send(data) ⇒ Object

raw data push from the client (WebSocket) should only be invoked after handshake, otherwise it will inject data into the header exchange

frames need to start with 0x00-0x7f byte and end with an 0xFF byte. Per spec, we can also set the first byte to a value betweent 0x80 and 0xFF, followed by a leading length indicator



123
124
125
126
127
 
# File 'lib/em-http/client.rb', line 123

def send(data)
  if @state == :websocket
    send_data("\x00#{data}\xff")
  end
end

#send_request_body ⇒ Object 



244
245
246
247
248
249
250
251
252
 
# File 'lib/em-http/client.rb', line 244

def send_request_body
  if @options[:body]
    body = normalize_body
    send_data body
    return
  elsif @options[:file]
    stream_file_data @options[:file], :http_chunks => false
  end
end

#send_request_header ⇒ Object 



178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/em-http/client.rb', line 178

def send_request_header
  query   = @options[:query]
  head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
  file    = @options[:file]
  proxy   = @options[:proxy]
  body    = normalize_body

  request_header = nil

  if http_proxy?
    # initialize headers for the http proxy
    head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
    head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]

    # if we need to negotiate the tunnel connection first, then
    # issue a CONNECT query to the proxy first. This is an optional
    # flag, by default we will provide full URIs to the proxy
    if @state == :connect_http_proxy
      request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
    end
  end

  if websocket?
    head['upgrade'] = 'WebSocket'
    head['connection'] = 'Upgrade'
    head['origin'] = @options[:origin] || @uri.host

  else
    # Set the Content-Length if file is given
    head['content-length'] = File.size(file) if file

    # Set the Content-Length if body is given
    head['content-length'] =  body.bytesize if body

    # Set the cookie header if provided
    if cookie = head.delete('cookie')
      head['cookie'] = encode_cookie(cookie)
    end

    # Set content-type header if missing and body is a Ruby hash
    if not head['content-type'] and options[:body].is_a? Hash
      head['content-type'] = 'application/x-www-form-urlencoded'
    end

    # Set connection close unless keepalive
    unless options[:keepalive]
      head['connection'] = 'close'
    end
  end

  # Set the Host header if it hasn't been specified already
  head['host'] ||= encode_host

  # Set the User-Agent if it hasn't been specified
  head['user-agent'] ||= "EventMachine HttpClient"

  # Record last seen URL
  @last_effective_url = @uri

  # Build the request headers
  request_header ||= encode_request(@method, @uri, query, proxy)
  request_header << encode_headers(head)
  request_header << CRLF
  send_data request_header
end

#send_socks_connect_request ⇒ Object 



457
458
459
460
461
462
463
464
465
466
467
468
469
470
 
# File 'lib/em-http/client.rb', line 457

def send_socks_connect_request
  # TO-DO: Implement address types for IPv6 and Domain
  begin
    ip_address = Socket.gethostbyname(@uri.host).last
    send_data [5, 1, 0, 1, ip_address, @uri.port].flatten.pack('CCCCA4n')

  rescue
    @state = :invalid
    on_error "could not resolve host", true
    return false
  end

  true
end

#send_socks_handshake ⇒ Object 



168
169
170
171
172
173
174
175
176
 
# File 'lib/em-http/client.rb', line 168

def send_socks_handshake
  # Method Negotiation as described on
  # http://www.faqs.org/rfcs/rfc1928.html Section 3

  @socks_state = :method_negotiation

  methods = socks_methods
  send_data [5, methods.size].pack('CC') + methods.pack('C*')
end

#socks_methods ⇒ Object 



160
161
162
163
164
165
166
 
# File 'lib/em-http/client.rb', line 160

def socks_methods
  methods = []
  methods << 2 if !options[:proxy][:authorization].nil? # 2 => Username/Password Authentication
  methods << 0 # 0 => No Authentication Required

  methods
end

#socks_proxy? ⇒ Boolean

determines if a SOCKS5 proxy should be used

Returns:

  • (Boolean) 


158
 
# File 'lib/em-http/client.rb', line 158

def socks_proxy?; proxy? && (@options[:proxy][:type] == :socks); end

#stream(&blk) ⇒ Object

assign a stream processing block



101
102
103
 
# File 'lib/em-http/client.rb', line 101

def stream(&blk)
  @stream = blk
end

#unbind ⇒ Object 



285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
 
# File 'lib/em-http/client.rb', line 285

def unbind
  if finished? && (@last_effective_url != @uri) && (@redirects < @options[:redirects])
    begin
      # update uri to redirect location if we're allowed to traverse deeper
      @uri = @last_effective_url

      # keep track of the depth of requests we made in this session
      @redirects += 1

      # swap current connection and reassign current handler
      req = HttpOptions.new(@method, @uri, @options)
      reconnect(req.host, req.port)

      @response_header = HttpResponseHeader.new
      @state = :response_header
      @response = ''
      @data.clear
    rescue EventMachine::ConnectionError => e
      on_error(e.message, true)
    end
  else
    if finished?
      succeed(self)
    else
      @disconnect.call(self) if @state == :websocket and @disconnect
      fail(self)
    end
  end
end

#websocket? ⇒ Boolean

Returns:

  • (Boolean) 


144
 
# File 'lib/em-http/client.rb', line 144

def websocket?; @uri.scheme == 'ws'; end