Class: Arachni::HTTP

Inherits:
Object show all
Includes:
Mixins::Observable, Module::Output, Utilities, Singleton
Defined in:
lib/arachni/http.rb,
lib/arachni/http/cookie_jar.rb

Overview

Provides a system-wide, simple and high-performance HTTP interface.

Author:

Defined Under Namespace

Classes: CookieJar, Error

Constant Summary collapse

MAX_CONCURRENCY =

Default maximum concurrency for HTTP requests.

20
REDIRECT_LIMIT =

Default maximum redirect limit.

20
HTTP_TIMEOUT =

Default 1 minute timeout for HTTP requests.

60_000
CUSTOM_404_CACHE_SIZE =

Maximum size of the cache that holds 404 signatures.

250
CUSTOM_404_SIGNATURE_THRESHOLD =

Maximum allowed difference (in tokens) when comparing custom 404 signatures.

25

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Mixins::Observable

#clear_observers, #method_missing

Methods included from Utilities

#available_port, #cookie_encode, #cookies_from_document, #cookies_from_file, #cookies_from_response, #exception_jail, #exclude_path?, #extract_domain, #follow_protocol?, #form_decode, #form_encode, #form_parse_request_body, #forms_from_document, #forms_from_response, #generate_token, #get_path, #html_decode, #html_encode, #include_path?, #links_from_document, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_query, #parse_set_cookie, #parse_url_vars, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #redundant_path?, #remove_constants, #seed, #skip_page?, #skip_path?, #skip_resource?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parser, #url_sanitize

Methods included from Module::Output

#fancy_name, #print_bad, #print_debug, #print_error, #print_info, #print_line, #print_ok, #print_status, #print_verbose

Methods included from UI::Output

#debug?, #debug_off, #debug_on, #disable_only_positives, #error_logfile, #flush_buffer, #log_error, #mute, #muted?, old_reset_output_options, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_pp, #print_error, #print_error_backtrace, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #set_buffer_cap, #set_error_logfile, #uncap_buffer, #unmute, #verbose, #verbose?

Constructor Details

#initialize ⇒ HTTP

Returns a new instance of HTTP.



91
92
93
 
# File 'lib/arachni/http.rb', line 91

def initialize
    reset
end

Dynamic Method Handling

This class handles dynamic methods through the method_missing method in the class Arachni::Mixins::Observable

Instance Attribute Details

Returns:



74
75
76
 
# File 'lib/arachni/http.rb', line 74

def cookie_jar
  @cookie_jar
end

#curr_res_cnt ⇒ Integer (readonly)

Returns amount of responses received for the running requests (of the current burst).

Returns:

  • (Integer) —

    amount of responses received for the running requests (of the current burst)



89
90
91
 
# File 'lib/arachni/http.rb', line 89

def curr_res_cnt
  @curr_res_cnt
end

#curr_res_time ⇒ Integer (readonly)

Returns sum of the response times of the running requests (of the current burst).

Returns:

  • (Integer) —

    sum of the response times of the running requests (of the current burst)



86
87
88
 
# File 'lib/arachni/http.rb', line 86

def curr_res_time
  @curr_res_time
end

#headers ⇒ Hash (readonly)

Returns default headers for each request.

Returns:

  • (Hash) —

    default headers for each request



71
72
73
 
# File 'lib/arachni/http.rb', line 71

def headers
  @headers
end

#request_count ⇒ Integer (readonly)

Returns amount of performed requests.

Returns:

  • (Integer) —

    amount of performed requests



77
78
79
 
# File 'lib/arachni/http.rb', line 77

def request_count
  @request_count
end

#response_count ⇒ Integer (readonly)

Returns amount of received responses.

Returns:

  • (Integer) —

    amount of received responses



80
81
82
 
# File 'lib/arachni/http.rb', line 80

def response_count
  @response_count
end

#time_out_count ⇒ Integer (readonly)

Returns amount of timed-out requests.

Returns:

  • (Integer) —

    amount of timed-out requests



83
84
85
 
# File 'lib/arachni/http.rb', line 83

def time_out_count
  @time_out_count
end

#url ⇒ String (readonly)

Returns framework seed/target URL.

Returns:

  • (String) —

    framework seed/target URL



68
69
70
 
# File 'lib/arachni/http.rb', line 68

def url
  @url
end

Class Method Details

.method_missing(sym, *args, &block) ⇒ Object 



537
538
539
 
# File 'lib/arachni/http.rb', line 537

def self.method_missing( sym, *args, &block )
    instance.send( sym, *args, &block )
end

Instance Method Details

#abort ⇒ Object

Aborts the running requests on a best effort basis



198
199
200
 
# File 'lib/arachni/http.rb', line 198

def abort
    exception_jail { @hydra.abort }
end

#after_run(&block) ⇒ Arachni::HTTP

Gets called each time a hydra run finishes.

Returns:



249
250
251
252
 
# File 'lib/arachni/http.rb', line 249

def after_run( &block )
    @after_run << block
    self
end

#after_run_persistent(&block) ⇒ Arachni::HTTP

Like #after_run but will not be removed after it’s run.

Returns:



259
260
261
262
 
# File 'lib/arachni/http.rb', line 259

def after_run_persistent( &block )
    add_after_run_persistent( &block )
    self
end

#average_res_time ⇒ Integer

Returns Average response time for the running requests (i.e. the current burst).

Returns:

  • (Integer) —

    Average response time for the running requests (i.e. the current burst).



211
212
213
214
 
# File 'lib/arachni/http.rb', line 211

def average_res_time
    return 0 if @curr_res_cnt == 0
    @curr_res_time / @curr_res_cnt
end

#burst_runtime ⇒ Integer

Returns Amount of time (in seconds) that the current burst has been running.

Returns:

  • (Integer) —

    Amount of time (in seconds) that the current burst has been running.



204
205
206
207
 
# File 'lib/arachni/http.rb', line 204

def burst_runtime
    @burst_runtime.to_i > 0 ?
        @burst_runtime : Time.now - (@burst_runtime_start || Time.now)
end

#cookie(url = @url, opts = {}, &block) ⇒ Typhoeus::Request

Gets a url with cookies and url variables

Parameters:

  • url (URI) (defaults to: @url)
  • opts (Hash) (defaults to: {}) —

    Request options.

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



407
408
409
410
411
 
# File 'lib/arachni/http.rb', line 407

def cookie( url = @url, opts = {}, &block )
    opts[:cookies] = (opts[:params] || {}).dup
    opts[:params]  = nil
    request( url, opts, &block )
end

#cookies ⇒ Array<Arachni::Element::Cookie>

Returns All cookies in the jar.

Returns:



240
241
242
 
# File 'lib/arachni/http.rb', line 240

def cookies
    @cookie_jar.cookies
end

#curr_res_per_second ⇒ Integer

Returns Responses/second for the running requests (i.e. the current burst).

Returns:

  • (Integer) —

    Responses/second for the running requests (i.e. the current burst).



218
219
220
221
222
223
 
# File 'lib/arachni/http.rb', line 218

def curr_res_per_second
    if @curr_res_cnt > 0 && burst_runtime > 0
        return (@curr_res_cnt / burst_runtime).to_i
    end
    0
end

#custom_404?(res, &block) ⇒ Boolean

Checks whether or not the provided response is a custom 404 page

Parameters:

  • res (Typhoeus::Response) —

    The response to check.

  • block (Block) —

    To be passed true or false depending on the result.

Returns:

  • (Boolean) 


497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
 
# File 'lib/arachni/http.rb', line 497

def custom_404?( res, &block )
    path = get_path( res.effective_url )

    return block.call( is_404?( path, res.body ) ) if has_custom_404_signature?( path )

    precision  = 2
    generators = custom_404_probe_generators( res.effective_url, precision )

    gathered_responses = 0
    expected_responses = generators.size * precision

    generators.each.with_index do |generator, i|
        _404_signatures_for_path( path )[i] ||= {}

        precision.times do
            get( generator.call, follow_location: true ) do |c_res|
                gathered_responses += 1

                if _404_signatures_for_path( path )[i][:body]
                    _404_signatures_for_path( path )[i][:rdiff] =
                        _404_signatures_for_path( path )[i][:body].
                            refine( c_res.body )

                    next if gathered_responses != expected_responses

                    has_custom_404_signature( path )
                    block.call is_404?( path, res.body )
                else
                    _404_signatures_for_path( path )[i][:body] =
                        Support::Signature.new(
                            c_res.body, threshold: CUSTOM_404_SIGNATURE_THRESHOLD
                        )
                end
            end
        end
    end

    nil
end

#get(url = @url, opts = {}, &block) ⇒ Typhoeus::Request

Gets a URL passing the provided query parameters.

Parameters:

  • url (URI) (defaults to: @url)
  • opts (Hash) (defaults to: {}) —

    Request options.

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



370
371
372
 
# File 'lib/arachni/http.rb', line 370

def get( url = @url, opts = {}, &block )
    request( url, opts, &block )
end

#header(url = @url, opts = {}, &block) ⇒ Typhoeus::Request

Gets a url with optional url variables and modified headers

Parameters:

  • url (URI) (defaults to: @url)
  • opts (Hash) (defaults to: {}) —

    Request options.

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



421
422
423
424
425
426
427
 
# File 'lib/arachni/http.rb', line 421

def header( url = @url, opts = {}, &block )
    opts[:headers] ||= {}
    opts[:headers].merge! ((opts[:params] || {}).dup )

    opts[:params]  = nil
    request( url, opts, &block )
end

#max_concurrency ⇒ Integer

Returns Current maximum concurrency of HTTP requests.

Returns:

  • (Integer) —

    Current maximum concurrency of HTTP requests.



235
236
237
 
# File 'lib/arachni/http.rb', line 235

def max_concurrency
    @hydra.max_concurrency
end

#max_concurrency=(concurrency) ⇒ Object

Sets the maximum concurrency of HTTP requests

Parameters:

  • concurrency (Integer) 


230
231
232
 
# File 'lib/arachni/http.rb', line 230

def max_concurrency=( concurrency )
    @hydra.max_concurrency = concurrency
end

#on_new_cookies(&block) ⇒ Object

Parameters:

  • block (Block) —

    To be passed the new cookies and the response that set them



486
487
488
 
# File 'lib/arachni/http.rb', line 486

def on_new_cookies( &block )
    add_on_new_cookies( &block )
end

#parse_and_set_cookies(res) ⇒ Object

Extracts cookies from an HTTP response and updates the cookie-jar.

It also executes callbacks added with ‘add_on_new_cookies( &block )`.

Parameters:



477
478
479
480
481
482
 
# File 'lib/arachni/http.rb', line 477

def parse_and_set_cookies( res )
    cookies = Cookie.from_response( res )
    update_cookies( cookies )

    call_on_new_cookies( cookies, res )
end

#post(url = @url, opts = {}, &block) ⇒ Typhoeus::Request

Posts a form to a URL with the provided query parameters.

Parameters:

  • url (URI) (defaults to: @url)
  • opts (Hash) (defaults to: {}) —

    Request options.

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



382
383
384
 
# File 'lib/arachni/http.rb', line 382

def post( url = @url, opts = {}, &block )
    request( url, opts.merge( method: :post ), &block )
end

#request(url = @url, opts = {}, &block) ⇒ Typhoeus::Request

Makes a generic request.

Parameters:

  • url (URI) (defaults to: @url)
  • opts (Hash) (defaults to: {}) —

    Request options.

  • block (Block) —

    Callback to be passed the response.

Options Hash (opts):

  • :params (Hash) — default: {} —

    Request parameters.

  • :train (Hash) — default: false —

    Force Arachni to analyze the HTML code looking for new elements.

  • :async (Hash) — default: true —

    Make the request async?

  • :headers (Hash) — default: {} —

    Extra HTTP request headers.

Returns:



278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
 
# File 'lib/arachni/http.rb', line 278

def request( url = @url, opts = {}, &block )
    fail ArgumentError, 'URL cannot be empty.' if !url

    params    = opts[:params] || {}
    train     = opts[:train]
    timeout   = opts[:timeout]
    cookies   = opts[:cookies] || {}
    async     = opts[:async]
    async     = true if async.nil?
    headers   = opts[:headers] || {}

    update_cookies  = opts[:update_cookies]
    follow_location = opts[:follow_location] || false

    username = opts.delete( :username )
    password = opts.delete( :password )

    exception_jail( false ) do

        if !opts[:no_cookiejar]
            cookies = begin
                @cookie_jar.for_url( url ).inject({}) do |h, c|
                    h[c.name] = c.value
                    h
                end.merge( cookies )
            rescue => e
                print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})."
                print_error_backtrace e
                cookies
            end
        end

        headers           = @headers.merge( headers )
        headers['Cookie'] ||= cookies.map do |k, v|
            "#{cookie_encode( k, :name )}=#{cookie_encode( v )}"
        end.join( ';' )

        headers.delete( 'Cookie' ) if headers['Cookie'].empty?
        headers.each { |k, v| headers[k] = Header.encode( v ) if v }

        # There are cases where the url already has a query and we also have
        # some params to work with. Some webapp frameworks will break
        # or get confused...plus the url will not be RFC compliant.
        #
        # Thus we need to merge the provided params with the
        # params of the url query and remove the latter from the url.
        cparams = params.dup
        curl    = normalize_url( url ).dup

        if opts[:method] != :post
            begin
                parsed = uri_parse( curl )
                cparams = parse_url_vars( curl ).merge( cparams )
                curl.gsub!( "?#{parsed.query}", '' ) if parsed.query
            rescue
                return
            end
        else
            cparams = cparams.inject( {} ) do |h, (k, v)|
                h[form_encode( k )] = form_encode( v ) if v && k
                h
            end
        end

        opts = {
            headers: headers,
            params:  cparams.empty? ? nil : cparams,
            method:  opts[:method].nil? ? :get : opts[:method],
            body:    opts[:body]
        }.merge( @opts )

        opts[:follow_location] = follow_location if follow_location
        opts[:timeout]         = timeout  if timeout
        opts[:username]        = username if username
        opts[:password]        = password if password

        req = Typhoeus::Request.new( curl, opts )
        req.train if train
        req.update_cookies if update_cookies
        queue( req, async, &block )
        req
    end
end

#reset(hooks_too = true) ⇒ Arachni::HTTP

Re-initializes the singleton

Returns:



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/arachni/http.rb', line 100

def reset( hooks_too = true )
    clear_observers if hooks_too

    opts = Options

    req_limit = opts.http_req_limit || MAX_CONCURRENCY

    hydra_opts = {
        max_concurrency: req_limit,
        method:          :auto
    }

    @url = opts.url.to_s
    @url = nil if @url.empty?

    @hydra      = Typhoeus::Hydra.new( hydra_opts )
    @hydra_sync = Typhoeus::Hydra.new( hydra_opts.merge( max_concurrency: 1 ) )

    @hydra.disable_memoization
    @hydra_sync.disable_memoization

    @headers = {
        'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Encoding' => 'gzip, deflate',
        'User-Agent'      => opts.user_agent
    }
    @headers['From'] = opts.authed_by if opts.authed_by

    @headers.merge!( opts.custom_headers )

    @cookie_jar = CookieJar.new( opts.cookie_jar )
    update_cookies( opts.cookies ) if opts.cookies
    update_cookies( opts.cookie_string ) if opts.cookie_string

    proxy_opts = {}
    proxy_opts = {
        proxy:          "#{opts.proxy_host}:#{opts.proxy_port}",
        proxy_username: opts.proxy_username,
        proxy_password: opts.proxy_password,
        proxy_type:     opts.proxy_type
    } if opts.proxy_host

    opts.redirect_limit ||= REDIRECT_LIMIT
    @opts = {
        follow_location:               false,
        max_redirects:                 opts.redirect_limit,
        disable_ssl_peer_verification: true,
        disable_ssl_host_verification: true,
        timeout:                       opts.http_timeout || HTTP_TIMEOUT,
        username:                      opts.http_username,
        password:                      opts.http_password
    }.merge( proxy_opts )

    @request_count  = 0
    @response_count = 0
    @time_out_count = 0

    @curr_res_time = 0
    @curr_res_cnt  = 0
    @burst_runtime = 0

    @queue_size = 0

    @after_run = []

    @_404 = Hash.new
    self
end

#run ⇒ Object

Runs all queued requests



170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/arachni/http.rb', line 170

def run
    exception_jail {
        @burst_runtime = nil

        begin
            hydra_run

            duped_after_run = @after_run.dup
            @after_run.clear
            duped_after_run.each { |block| block.call }
        end while @queue_size > 0

        call_after_run_persistent

        # Prune the custom 404 cache after callbacks have been called.
        prune_custom_404_cache

        @curr_res_time = 0
        @curr_res_cnt  = 0
        true
    }
rescue SystemExit
    raise
rescue
    nil
end

#sandbox(&block) ⇒ Object

Executes a ‘block` under a sandbox.

Cookies or new callbacks set as a result of the block won’t affect the HTTP singleton.

Parameters:

  • block (Block)

Returns:

  • (Object) —

    Return value of the block.



439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
 
# File 'lib/arachni/http.rb', line 439

def sandbox( &block )
    h = {}
    instance_variables.each do |iv|
        val = instance_variable_get( iv )
        h[iv] = val.deep_clone rescue val.dup rescue val
    end

    hooks = {}
    @__hooks.each { |k, v| hooks[k] = v.dup }

    ret = block.call( self )

    h.each { |iv, val| instance_variable_set( iv, val ) }
    @__hooks = hooks

    ret
end

#trace(url = @url, opts = {}, &block) ⇒ Typhoeus::Request

Sends an HTTP TRACE request to “url”.

Parameters:

  • url (URI) (defaults to: @url)
  • opts (Hash) (defaults to: {}) —

    Request options.

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



394
395
396
 
# File 'lib/arachni/http.rb', line 394

def trace( url = @url, opts = {}, &block )
    request( url, opts.merge( method: :trace ), &block )
end

#update_cookies(cookies) ⇒ Object Also known as: set_cookies

Updates the cookie-jar with the passed cookies.

Parameters:



462
463
464
465
466
467
 
# File 'lib/arachni/http.rb', line 462

def update_cookies( cookies )
    @cookie_jar.update( cookies )

    # Update framework cookies.
    Arachni::Options.cookies = @cookie_jar.cookies
end