Class: Arachni::HTTP::Client

Inherits:
Object
  • Object
show all
Includes:
Support::Mixins::Observable, UI::Output, Utilities, Singleton
Defined in:
lib/arachni/http/client.rb

Overview

Provides a system-wide, simple and high-performance HTTP client.

Author:

Defined Under Namespace

Classes: Error

Constant Summary collapse

MAX_CONCURRENCY =

Default maximum concurrency for HTTP requests.

20
HTTP_TIMEOUT =

Default 1 minute timeout for HTTP requests.

60_000
CUSTOM_404_CACHE_SIZE =

Maximum size of the cache that holds 404 signatures.

50
CUSTOM_404_SIGNATURE_THRESHOLD =

Maximum allowed difference ratio when comparing custom 404 signatures. The fact that we refine the signatures allows us to set this threshold really low and still maintain good accuracy.

0.1

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Support::Mixins::Observable

included

Methods included from Utilities

#available_port, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_document, #cookies_from_file, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_document, #forms_from_response, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_document, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods included from UI::Output

#debug?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Constructor Details

#initialize ⇒ Client

Returns a new instance of Client.



123
124
125
126
 
# File 'lib/arachni/http/client.rb', line 123

def initialize
    super
    reset
end

Instance Attribute Details

#burst_response_count ⇒ Integer (readonly)

Returns Amount of responses received for the running requests (of the current burst).

Returns:

  • (Integer) —

    Amount of responses received for the running requests (of the current burst).



121
122
123
 
# File 'lib/arachni/http/client.rb', line 121

def burst_response_count
  @burst_response_count
end

#burst_response_time_sum ⇒ Integer (readonly)

Returns Sum of the response times for the running requests (of the current burst).

Returns:

  • (Integer) —

    Sum of the response times for the running requests (of the current burst).



117
118
119
 
# File 'lib/arachni/http/client.rb', line 117

def burst_response_time_sum
  @burst_response_time_sum
end

#headers ⇒ Hash (readonly)

Returns Default headers for requests.

Returns:



101
102
103
 
# File 'lib/arachni/http/client.rb', line 101

def headers
  @headers
end

#request_count ⇒ Integer (readonly)

Returns Amount of performed requests.

Returns:

  • (Integer) —

    Amount of performed requests.



105
106
107
 
# File 'lib/arachni/http/client.rb', line 105

def request_count
  @request_count
end

#response_count ⇒ Integer (readonly)

Returns Amount of received responses.

Returns:

  • (Integer) —

    Amount of received responses.



109
110
111
 
# File 'lib/arachni/http/client.rb', line 109

def response_count
  @response_count
end

#time_out_count ⇒ Integer (readonly)

Returns Amount of timed-out requests.

Returns:

  • (Integer) —

    Amount of timed-out requests.



113
114
115
 
# File 'lib/arachni/http/client.rb', line 113

def time_out_count
  @time_out_count
end

#url ⇒ String (readonly)

Returns Framework target URL, used as reference.

Returns:

  • (String) —

    Framework target URL, used as reference.



97
98
99
 
# File 'lib/arachni/http/client.rb', line 97

def url
  @url
end

Class Method Details

.method_missing(sym, *args, &block) ⇒ Object 



577
578
579
 
# File 'lib/arachni/http/client.rb', line 577

def self.method_missing( sym, *args, &block )
    instance.send( sym, *args, &block )
end

Instance Method Details

#_404_cache ⇒ Object 



582
583
584
 
# File 'lib/arachni/http/client.rb', line 582

def _404_cache
    @_404
end

#abort ⇒ Object

Aborts the running requests on a best effort basis.



260
261
262
 
# File 'lib/arachni/http/client.rb', line 260

def abort
    exception_jail { @hydra.abort }
end

#after_each_run(&block) ⇒ Arachni::HTTP

Returns self.

Parameters:

  • block (Block) —

    Called after each #run.

Returns:



57
 
# File 'lib/arachni/http/client.rb', line 57

advertise :after_each_run

#after_run(&block) ⇒ Arachni::HTTP::Client

Returns ‘self`.

Parameters:

  • block (Block) —

    Called after the next #run.

Returns:



49
 
# File 'lib/arachni/http/client.rb', line 49

advertise :after_run

#burst_average_response_time ⇒ Float

Returns Average response time for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Average response time for the running requests (i.e. the current burst).



295
296
297
298
 
# File 'lib/arachni/http/client.rb', line 295

def burst_average_response_time
    return 0 if @burst_response_count == 0
    @burst_response_time_sum / Float( @burst_response_count )
end

#burst_responses_per_second ⇒ Float

Returns Responses/second for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Responses/second for the running requests (i.e. the current burst).



302
303
304
305
306
307
 
# File 'lib/arachni/http/client.rb', line 302

def burst_responses_per_second
    if @burst_response_count > 0 && burst_runtime > 0
        return @burst_response_count / burst_runtime
    end
    0
end

#burst_runtime ⇒ Float

Returns Amount of time (in seconds) that the current burst has been running.

Returns:

  • (Float) —

    Amount of time (in seconds) that the current burst has been running.



288
289
290
291
 
# File 'lib/arachni/http/client.rb', line 288

def burst_runtime
    @burst_runtime.to_i > 0 ?
        @burst_runtime : Time.now - (@burst_runtime_start || Time.now)
end

#checked_but_not_custom_404?(url) ⇒ Bool

Returns ‘true` if the `url` has been checked for the existence of a custom-404 handler but none was identified, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    ‘true` if the `url` has been checked for the existence of a custom-404 handler but none was identified, `false` otherwise.



563
564
565
 
# File 'lib/arachni/http/client.rb', line 563

def checked_but_not_custom_404?( url )
    @with_regular_404_handler.include?( url_for_custom_404( url ) )
end

#checked_for_custom_404?(url) ⇒ Bool

Returns ‘true` if the `url` has been checked for the existence of a custom-404 handler, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    ‘true` if the `url` has been checked for the existence of a custom-404 handler, `false` otherwise.



553
554
555
 
# File 'lib/arachni/http/client.rb', line 553

def checked_for_custom_404?( url )
    _404_data_for_url( url )[:analyzed]
end

#cookie(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the cookies in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



416
417
418
419
 
# File 'lib/arachni/http/client.rb', line 416

def cookie( url = @url, options = {}, &block )
    options[:cookies] = (options.delete( :parameters ) || {}).dup
    request( url, options, &block )
end

#cookie_jar ⇒ CookieJar

Returns:



192
193
194
 
# File 'lib/arachni/http/client.rb', line 192

def cookie_jar
    State.http.cookie_jar
end

#cookies ⇒ Array<Arachni::Element::Cookie>

Returns All cookies in the jar.

Returns:



323
324
325
 
# File 'lib/arachni/http/client.rb', line 323

def cookies
    cookie_jar.cookies
end

#custom_404?(response, &block) ⇒ Boolean

Parameters:

  • response (Response) —

    Checks whether or not the provided response is a custom 404 page.

  • block (Block) —

    To be passed true or false depending on the result.

Returns:

  • (Boolean) 


463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
 
# File 'lib/arachni/http/client.rb', line 463

def custom_404?( response, &block )
    url = response.url

    if checked_for_custom_404?( url )
        result = is_404?( url, response.body )
        print_debug "#{__method__} [cached]: #{block} #{url} #{result}"
        return block.call( result )
    end

    # If someone else is already checking that resource don't bother
    # duplicating the effort, just let them know that we're waiting on the
    # results too.
    if _404_data_for_url( url )[:in_progress]
        print_debug "#{__method__} [waiting]: #{url} #{block}"
        _404_data_for_url( url )[:waiting] << [url, response.body, block]
        return
    end

    # Call dibs on fingerprinting this url.
    _404_data_for_url( url )[:in_progress] = true

    print_debug "#{__method__} [checking]: #{url} #{block}"

    precision  = 2
    generators = custom_404_probe_generators( url, precision )

    real_404s          = 0
    gathered_responses = 0
    expected_responses = generators.size * precision

    generators.each.with_index do |generator, i|
        _404_signatures_for_url( url )[i] ||= {}

        precision.times do
            get( generator.call,
                 follow_location: true,
                 # This is important, helps us reduce waiting callers.
                 high_priority: true
            ) do |c_res|

                gathered_responses += 1
                real_404s += 1 if c_res.code == 404

                if _404_signatures_for_url( url )[i][:body]
                    _404_signatures_for_url( url )[i][:rdiff] =
                        _404_signatures_for_url( url )[i][:body].
                            refine( c_res.body )

                    next if gathered_responses != expected_responses

                    # If we get real 404s flag that there's no handler.
                    if real_404s == expected_responses
                        @with_regular_404_handler << url_for_custom_404( url )
                    end

                    checked_for_custom_404( url )

                    result = is_404?( url, response.body )
                    print_debug "#{__method__} [checked]: #{block} #{url} #{result}"
                    block.call result

                    # Process other's request too.
                    while (waiting = _404_data_for_url( url )[:waiting].pop)
                        url, body, callback = waiting
                        result = is_404?( url, body )

                        print_debug "#{__method__} [notify]: #{callback} #{url} #{result}"
                        callback.call result
                    end

                    _404_data_for_url( url )[:in_progress] = false
                else
                    _404_signatures_for_url( url )[i][:body] =
                        Support::Signature.new(
                            c_res.body, threshold: CUSTOM_404_SIGNATURE_THRESHOLD
                        )
                end
            end
        end
    end

    nil
end

#get(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



384
385
386
 
# File 'lib/arachni/http/client.rb', line 384

def get( url = @url, options = {}, &block )
    request( url, options, &block )
end

#header(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the headers in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



427
428
429
430
431
 
# File 'lib/arachni/http/client.rb', line 427

def header( url = @url, options = {}, &block )
    options[:headers] ||= {}
    options[:headers].merge!( (options.delete( :parameters ) || {}).dup )
    request( url, options, &block )
end

#max_concurrency ⇒ Integer

Returns Current maximum concurrency of HTTP requests.

Returns:

  • (Integer) —

    Current maximum concurrency of HTTP requests.



317
318
319
 
# File 'lib/arachni/http/client.rb', line 317

def max_concurrency
    @hydra.max_concurrency
end

#max_concurrency=(concurrency) ⇒ Object

Parameters:

  • concurrency (Integer) —

    Sets the maximum concurrency of HTTP requests.



311
312
313
 
# File 'lib/arachni/http/client.rb', line 311

def max_concurrency=( concurrency )
    @hydra.max_concurrency = concurrency
end

#needs_custom_404_check?(url) ⇒ Bool

Returns ‘true` if the `url` needs to be checked for a #custom_404?, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    ‘true` if the `url` needs to be checked for a #custom_404?, `false` otherwise.



573
574
575
 
# File 'lib/arachni/http/client.rb', line 573

def needs_custom_404_check?( url )
    !checked_for_custom_404?( url ) || !checked_but_not_custom_404?( url )
end

#on_complete(&block) ⇒ Object 



69
 
# File 'lib/arachni/http/client.rb', line 69

advertise :on_complete

#on_new_cookies(&block) ⇒ Object

Parameters:

  • block (Block) —

    To be passed the new cookies and the response that set them



66
 
# File 'lib/arachni/http/client.rb', line 66

advertise :on_new_cookies

#on_queue(&block) ⇒ Object 



60
 
# File 'lib/arachni/http/client.rb', line 60

advertise :on_queue

#parse_and_set_cookies(response) ⇒ Object

Note:

Runs #on_new_cookies callbacks.

Parameters:

  • response (Response) —

    Extracts cookies from ‘response` and updates the cookie-jar.



452
453
454
455
456
457
 
# File 'lib/arachni/http/client.rb', line 452

def parse_and_set_cookies( response )
    cookies = Cookie.from_response( response )
    update_cookies( cookies )

    notify_on_new_cookies( cookies, response )
end

#post(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘POST` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



394
395
396
397
 
# File 'lib/arachni/http/client.rb', line 394

def post( url = @url, options = {}, &block )
    options[:body] = (options.delete( :parameters ) || {}).dup
    request( url, options.merge( method: :post ), &block )
end

#queue(request) ⇒ Object

Parameters:

  • request (Request) —

    Request to queue.



435
436
437
438
 
# File 'lib/arachni/http/client.rb', line 435

def queue( request )
    notify_on_queue( request )
    forward_request( request )
end

#request(url = @url, options = {}, &block) ⇒ Request, Response

Queues/performs a generic request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Options Hash (options):

  • :cookies (Hash) — default: {} —

    Extra cookies to use for this request.

  • :no_cookie_jar (Hash) — default: false —

    Do not include cookies from the #cookie_jar.

Returns:



342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
 
# File 'lib/arachni/http/client.rb', line 342

def request( url = @url, options = {}, &block )
    fail ArgumentError, 'URL cannot be empty.' if !url

    options = options.dup
    cookies = options.delete( :cookies ) || {}

    exception_jail false do
        if !options.delete( :no_cookie_jar )
            cookies = begin
                cookie_jar.for_url( url ).inject({}) do |h, c|
                    h[c.name] = c.value
                    h
                end.merge( cookies )
            rescue => e
                print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})."
                print_error_backtrace e
                cookies
            end
        end

        request = Request.new( options.merge(
            url:     url,
            headers: headers.merge( options.delete( :headers ) || {} ),
            cookies: cookies
        ))

        if block_given?
            request.on_complete( &block )
        end

        queue( request )
        return request.run if request.blocking?
        request
    end
end

#reset(hooks_too = true) ⇒ Arachni::HTTP

Returns Reset ‘self`.

Returns:



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/arachni/http/client.rb', line 130

def reset( hooks_too = true )
    clear_observers if hooks_too
    State.http.clear

    opts = Options

    @url = opts.url.to_s
    @url = nil if @url.empty?

    @hydra = Typhoeus::Hydra.new( max_concurrency: opts.http.request_concurrency || MAX_CONCURRENCY )

    headers.merge!(
        'Accept'     => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'User-Agent' => opts.http.user_agent
    )
    headers['From'] = opts.authorized_by if opts.authorized_by
    headers.merge!( opts.http.request_headers )

    cookie_jar.load( opts.http.cookie_jar_filepath ) if opts.http.cookie_jar_filepath
    update_cookies( opts.http.cookies )
    update_cookies( opts.http.cookie_string ) if opts.http.cookie_string

    reset_burst_info

    @request_count  = 0
    @response_count = 0
    @time_out_count = 0

    @total_response_time_sum = 0
    @total_runtime           = 0

    @queue_size = 0

    @with_regular_404_handler = Support::LookUp::HashSet.new
    @_404  = Hash.new

    self
end

#run ⇒ Object

Runs all queued requests



201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
 
# File 'lib/arachni/http/client.rb', line 201

def run
    exception_jail false do
        @burst_runtime = nil

        begin
            hydra_run

            duped_after_run = observers_for( :after_run ).dup
            observers_for( :after_run ).clear
            duped_after_run.each { |block| block.call }
        end while @queue_size > 0

        notify_after_each_run

        # Prune the custom 404 cache after callbacks have been called.
        prune_custom_404_cache

        @curr_res_time = 0
        @curr_res_cnt  = 0

        true
    end
end

#sandbox(&block) ⇒ Object

Note:

Cookies or new callbacks set as a result of the block won’t affect the HTTP singleton.

Return value of the block.

Parameters:

  • block (Block) —

    Block to executes inside a sandbox.

Returns:

  • (Object) —

    Return value of the block.



233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
 
# File 'lib/arachni/http/client.rb', line 233

def sandbox( &block )
    h = {}
    instance_variables.each do |iv|
        val = instance_variable_get( iv )
        h[iv] = val.deep_clone rescue val.dup rescue val
    end

    saved_observers = dup_observers

    pre_cookies = cookies.deep_clone
    pre_headers = headers.deep_clone

    ret = block.call( self )

    cookie_jar.clear
    update_cookies pre_cookies

    headers.clear
    headers.merge! pre_headers

    h.each { |iv, val| instance_variable_set( iv, val ) }
    set_observers( saved_observers )

    ret
end

#statistics ⇒ Hash

Returns Hash including HTTP client statistics including:

Returns:



183
184
185
186
187
188
189
 
# File 'lib/arachni/http/client.rb', line 183

def statistics
   [:request_count, :response_count, :time_out_count,
    :total_responses_per_second, :burst_response_time_sum,
    :burst_response_count, :burst_responses_per_second,
    :burst_average_response_time, :total_average_response_time,
    :max_concurrency].inject({}) { |h, k| h[k] = send(k); h }
end

#total_average_response_time ⇒ Float

Returns Average response time for all requests.

Returns:

  • (Float) —

    Average response time for all requests.



273
274
275
276
 
# File 'lib/arachni/http/client.rb', line 273

def total_average_response_time
    return 0 if @response_count == 0
    @total_response_time_sum / Float( @response_count )
end

#total_responses_per_second ⇒ Float

Returns Responses/second.

Returns:

  • (Float) —

    Responses/second.



279
280
281
282
283
284
 
# File 'lib/arachni/http/client.rb', line 279

def total_responses_per_second
    if @response_count > 0 && total_runtime > 0
        return @response_count / Float( total_runtime )
    end
    0
end

#total_runtime ⇒ Integer

Returns Amount of time (in seconds) that has been devoted to performing requests and getting responses.

Returns:

  • (Integer) —

    Amount of time (in seconds) that has been devoted to performing requests and getting responses.



267
268
269
 
# File 'lib/arachni/http/client.rb', line 267

def total_runtime
    @total_runtime > 0 ? @total_runtime : burst_runtime
end

#trace(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘TRACE` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



405
406
407
 
# File 'lib/arachni/http/client.rb', line 405

def trace( url = @url, options = {}, &block )
    request( url, options.merge( method: :trace ), &block )
end

#update_cookies(cookies) ⇒ Object Also known as: set_cookies

Parameters:



442
443
444
445
 
# File 'lib/arachni/http/client.rb', line 442

def update_cookies( cookies )
    cookie_jar.update( cookies )
    cookie_jar.cookies
end

#url_for_custom_404(url) ⇒ Object 



586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
 
# File 'lib/arachni/http/client.rb', line 586

def url_for_custom_404( url )
    parsed = Arachni::URI( url )

    # If we're dealing with a file resource, then its parent directory will
    # be the applicable custom-404 handler...
    if parsed.resource_extension
        trv_back = Arachni::URI( parsed.up_to_path ).path

    # ...however, if we're dealing with a directory, the applicable handler
    # will be its parent directory.
    else
        trv_back = File.dirname( Arachni::URI( parsed.up_to_path ).path )
    end

    trv_back += '/' if trv_back[-1] != '/'

    parsed = parsed.dup
    parsed.path = trv_back
    parsed.to_s
end