Class: Arachni::HTTP::Client

Inherits:
Object
  • Object
show all
Includes:
Support::Mixins::Observable, UI::Output, Utilities, Singleton
Defined in:
lib/arachni/http/client.rb,
lib/arachni/http/client/dynamic_404_handler.rb

Overview

Provides a system-wide, simple and high-performance HTTP client.

Author:

Defined Under Namespace

Classes: Dynamic404Handler, Error

Constant Summary collapse

MAX_CONCURRENCY =

Default maximum concurrency for HTTP requests.

20
HTTP_TIMEOUT =

Default 1 minute timeout for HTTP requests.

60_000

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Support::Mixins::Observable

included

Methods included from Utilities

#available_port, #bytes_to_kilobytes, #bytes_to_megabytes, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_document, #cookies_from_file, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_document, #forms_from_response, #full_and_absolute_url?, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_document, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #regexp_array_match, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods included from UI::Output

#debug?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Constructor Details

#initialize ⇒ Client

Returns a new instance of Client.



119
120
121
122
 
# File 'lib/arachni/http/client.rb', line 119

def initialize
    super
    reset
end

Instance Attribute Details

#burst_response_count ⇒ Integer (readonly)

Returns Amount of responses received for the running requests (of the current burst).

Returns:

  • (Integer) —

    Amount of responses received for the running requests (of the current burst).



114
115
116
 
# File 'lib/arachni/http/client.rb', line 114

def burst_response_count
  @burst_response_count
end

#burst_response_time_sum ⇒ Integer (readonly)

Returns Sum of the response times for the running requests (of the current burst).

Returns:

  • (Integer) —

    Sum of the response times for the running requests (of the current burst).



110
111
112
 
# File 'lib/arachni/http/client.rb', line 110

def burst_response_time_sum
  @burst_response_time_sum
end

#dynamic_404_handler ⇒ Dynamic404Handler (readonly)

Returns:



117
118
119
 
# File 'lib/arachni/http/client.rb', line 117

def dynamic_404_handler
  @dynamic_404_handler
end

#headers ⇒ Hash (readonly)

Returns Default headers for requests.

Returns:



94
95
96
 
# File 'lib/arachni/http/client.rb', line 94

def headers
  @headers
end

#request_count ⇒ Integer (readonly)

Returns Amount of performed requests.

Returns:

  • (Integer) —

    Amount of performed requests.



98
99
100
 
# File 'lib/arachni/http/client.rb', line 98

def request_count
  @request_count
end

#response_count ⇒ Integer (readonly)

Returns Amount of received responses.

Returns:

  • (Integer) —

    Amount of received responses.



102
103
104
 
# File 'lib/arachni/http/client.rb', line 102

def response_count
  @response_count
end

#time_out_count ⇒ Integer (readonly)

Returns Amount of timed-out requests.

Returns:

  • (Integer) —

    Amount of timed-out requests.



106
107
108
 
# File 'lib/arachni/http/client.rb', line 106

def time_out_count
  @time_out_count
end

#url ⇒ String (readonly)

Returns Framework target URL, used as reference.

Returns:

  • (String) —

    Framework target URL, used as reference.



90
91
92
 
# File 'lib/arachni/http/client.rb', line 90

def url
  @url
end

Class Method Details

.method_missing(sym, *args, &block) ⇒ Object 



456
457
458
 
# File 'lib/arachni/http/client.rb', line 456

def self.method_missing( sym, *args, &block )
    instance.send( sym, *args, &block )
end

Instance Method Details

#abort ⇒ Object

Aborts the running requests on a best effort basis.



257
258
259
 
# File 'lib/arachni/http/client.rb', line 257

def abort
    exception_jail { client_abort }
end

#after_each_run(&block) ⇒ Arachni::HTTP

Returns self.

Parameters:

  • block (Block) —

    Called after each #run.

Returns:



58
 
# File 'lib/arachni/http/client.rb', line 58

advertise :after_each_run

#after_run(&block) ⇒ Arachni::HTTP::Client

Returns ‘self`.

Parameters:

  • block (Block) —

    Called after the next #run.

Returns:



50
 
# File 'lib/arachni/http/client.rb', line 50

advertise :after_run

#burst_average_response_time ⇒ Float

Returns Average response time for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Average response time for the running requests (i.e. the current burst).



292
293
294
295
 
# File 'lib/arachni/http/client.rb', line 292

def burst_average_response_time
    return 0 if @burst_response_count == 0
    @burst_response_time_sum / Float( @burst_response_count )
end

#burst_responses_per_second ⇒ Float

Returns Responses/second for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Responses/second for the running requests (i.e. the current burst).



299
300
301
302
303
304
 
# File 'lib/arachni/http/client.rb', line 299

def burst_responses_per_second
    if @burst_response_count > 0 && burst_runtime > 0
        return @burst_response_count / burst_runtime
    end
    0
end

#burst_runtime ⇒ Float

Returns Amount of time (in seconds) that the current burst has been running.

Returns:

  • (Float) —

    Amount of time (in seconds) that the current burst has been running.



285
286
287
288
 
# File 'lib/arachni/http/client.rb', line 285

def burst_runtime
    @burst_runtime.to_i > 0 ?
        @burst_runtime : Time.now - (@burst_runtime_start || Time.now)
end

#cookie(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the cookies in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



413
414
415
416
 
# File 'lib/arachni/http/client.rb', line 413

def cookie( url = @url, options = {}, &block )
    options[:cookies] = (options.delete( :parameters ) || {}).dup
    request( url, options, &block )
end

#cookie_jar ⇒ CookieJar

Returns:



189
190
191
 
# File 'lib/arachni/http/client.rb', line 189

def cookie_jar
    State.http.cookie_jar
end

#cookies ⇒ Array<Arachni::Element::Cookie>

Returns All cookies in the jar.

Returns:



320
321
322
 
# File 'lib/arachni/http/client.rb', line 320

def cookies
    cookie_jar.cookies
end

#get(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



381
382
383
 
# File 'lib/arachni/http/client.rb', line 381

def get( url = @url, options = {}, &block )
    request( url, options, &block )
end

#header(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the headers in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



424
425
426
427
428
 
# File 'lib/arachni/http/client.rb', line 424

def header( url = @url, options = {}, &block )
    options[:headers] ||= {}
    options[:headers].merge!( (options.delete( :parameters ) || {}).dup )
    request( url, options, &block )
end

#inspect ⇒ Object 



460
461
462
463
464
 
# File 'lib/arachni/http/client.rb', line 460

def inspect
    s = "#<#{self.class} "
    statistics.each { |k, v| s << "@#{k}=#{v.inspect} " }
    s << '>'
end

#max_concurrency ⇒ Integer

Returns Current maximum concurrency of HTTP requests.

Returns:

  • (Integer) —

    Current maximum concurrency of HTTP requests.



314
315
316
 
# File 'lib/arachni/http/client.rb', line 314

def max_concurrency
    @hydra.max_concurrency
end

#max_concurrency=(concurrency) ⇒ Object

Parameters:

  • concurrency (Integer) —

    Sets the maximum concurrency of HTTP requests.



308
309
310
 
# File 'lib/arachni/http/client.rb', line 308

def max_concurrency=( concurrency )
    @hydra.max_concurrency = concurrency
end

#on_complete(&block) ⇒ Object 



70
 
# File 'lib/arachni/http/client.rb', line 70

advertise :on_complete

#on_new_cookies(&block) ⇒ Object

Parameters:

  • block (Block) —

    To be passed the new cookies and the response that set them



67
 
# File 'lib/arachni/http/client.rb', line 67

advertise :on_new_cookies

#on_queue(&block) ⇒ Object 



61
 
# File 'lib/arachni/http/client.rb', line 61

advertise :on_queue

#parse_and_set_cookies(response) ⇒ Object

Note:

Runs #on_new_cookies callbacks.

Parameters:

  • response (Response) —

    Extracts cookies from ‘response` and updates the cookie-jar.



449
450
451
452
453
454
 
# File 'lib/arachni/http/client.rb', line 449

def parse_and_set_cookies( response )
    cookies = Cookie.from_response( response )
    update_cookies( cookies )

    notify_on_new_cookies( cookies, response )
end

#post(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘POST` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



391
392
393
394
 
# File 'lib/arachni/http/client.rb', line 391

def post( url = @url, options = {}, &block )
    options[:body] = (options.delete( :parameters ) || {}).dup
    request( url, options.merge( method: :post ), &block )
end

#queue(request) ⇒ Object

Parameters:

  • request (Request) —

    Request to queue.



432
433
434
435
 
# File 'lib/arachni/http/client.rb', line 432

def queue( request )
    notify_on_queue( request )
    forward_request( request )
end

#request(url = @url, options = {}, &block) ⇒ Request, Response

Queues/performs a generic request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Options Hash (options):

  • :cookies (Hash) — default: {} —

    Extra cookies to use for this request.

  • :no_cookie_jar (Hash) — default: false —

    Do not include cookies from the #cookie_jar.

Returns:



339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
 
# File 'lib/arachni/http/client.rb', line 339

def request( url = @url, options = {}, &block )
    fail ArgumentError, 'URL cannot be empty.' if !url

    options = options.dup
    cookies = options.delete( :cookies ) || {}

    exception_jail false do
        if !options.delete( :no_cookie_jar )
            cookies = begin
                cookie_jar.for_url( url ).inject({}) do |h, c|
                    h[c.name] = c.value
                    h
                end.merge( cookies )
            rescue => e
                print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})."
                print_error_backtrace e
                cookies
            end
        end

        request = Request.new( options.merge(
            url:     url,
            headers: headers.merge( options.delete( :headers ) || {} ),
            cookies: cookies
        ))

        if block_given?
            request.on_complete( &block )
        end

        queue( request )
        return request.run if request.blocking?
        request
    end
end

#reset(hooks_too = true) ⇒ Arachni::HTTP

Returns Reset ‘self`.

Returns:



126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/arachni/http/client.rb', line 126

def reset( hooks_too = true )
    clear_observers if hooks_too
    State.http.clear

    @url = Options.url.to_s
    @url = nil if @url.empty?

    client_initialize

    headers.merge!(
        'Accept'     => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'User-Agent' => Options.http.user_agent
    )
    headers['From'] = Options.authorized_by if Options.authorized_by
    headers.merge!( Options.http.request_headers )

    cookie_jar.load( Options.http.cookie_jar_filepath ) if Options.http.cookie_jar_filepath

    Options.http.cookies.each do |name, value|
        update_cookies( name => value )
    end

    update_cookies( Options.http.cookie_string ) if Options.http.cookie_string

    reset_burst_info

    @request_count  = 0
    @response_count = 0
    @time_out_count = 0

    @total_response_time_sum = 0
    @total_runtime           = 0

    @queue_size = 0

    @dynamic_404_handler = Dynamic404Handler.new

    self
end

#run ⇒ Object

Runs all queued requests



198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/arachni/http/client.rb', line 198

def run
    exception_jail false do
        @burst_runtime = nil

        begin
            run_and_update_statistics

            duped_after_run = observers_for( :after_run ).dup
            observers_for( :after_run ).clear
            duped_after_run.each { |block| block.call }
        end while @queue_size > 0 || observers_for( :after_run ).any?

        notify_after_each_run

        # Prune the custom 404 cache after callbacks have been called.
        @dynamic_404_handler.prune

        @curr_res_time = 0
        @curr_res_cnt  = 0

        true
    end
end

#sandbox(&block) ⇒ Object

Note:

Cookies or new callbacks set as a result of the block won’t affect the HTTP singleton.

Return value of the block.

Parameters:

  • block (Block) —

    Block to executes inside a sandbox.

Returns:

  • (Object) —

    Return value of the block.



230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
 
# File 'lib/arachni/http/client.rb', line 230

def sandbox( &block )
    h = {}
    instance_variables.each do |iv|
        val = instance_variable_get( iv )
        h[iv] = val.deep_clone rescue val.dup rescue val
    end

    saved_observers = dup_observers

    pre_cookies = cookies.deep_clone
    pre_headers = headers.deep_clone

    ret = block.call( self )

    cookie_jar.clear
    update_cookies pre_cookies

    headers.clear
    headers.merge! pre_headers

    h.each { |iv, val| instance_variable_set( iv, val ) }
    set_observers( saved_observers )

    ret
end

#statistics ⇒ Hash

Returns Hash including HTTP client statistics including:

Returns:



180
181
182
183
184
185
186
 
# File 'lib/arachni/http/client.rb', line 180

def statistics
   [:request_count, :response_count, :time_out_count,
    :total_responses_per_second, :burst_response_time_sum,
    :burst_response_count, :burst_responses_per_second,
    :burst_average_response_time, :total_average_response_time,
    :max_concurrency].inject({}) { |h, k| h[k] = send(k); h }
end

#total_average_response_time ⇒ Float

Returns Average response time for all requests.

Returns:

  • (Float) —

    Average response time for all requests.



270
271
272
273
 
# File 'lib/arachni/http/client.rb', line 270

def total_average_response_time
    return 0 if @response_count == 0
    @total_response_time_sum / Float( @response_count )
end

#total_responses_per_second ⇒ Float

Returns Responses/second.

Returns:

  • (Float) —

    Responses/second.



276
277
278
279
280
281
 
# File 'lib/arachni/http/client.rb', line 276

def total_responses_per_second
    if @response_count > 0 && total_runtime > 0
        return @response_count / Float( total_runtime )
    end
    0
end

#total_runtime ⇒ Integer

Returns Amount of time (in seconds) that has been devoted to performing requests and getting responses.

Returns:

  • (Integer) —

    Amount of time (in seconds) that has been devoted to performing requests and getting responses.



264
265
266
 
# File 'lib/arachni/http/client.rb', line 264

def total_runtime
    @total_runtime > 0 ? @total_runtime : burst_runtime
end

#trace(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘TRACE` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



402
403
404
 
# File 'lib/arachni/http/client.rb', line 402

def trace( url = @url, options = {}, &block )
    request( url, options.merge( method: :trace ), &block )
end

#update_cookies(cookies) ⇒ Object Also known as: set_cookies

Parameters:



439
440
441
442
 
# File 'lib/arachni/http/client.rb', line 439

def update_cookies( cookies )
    cookie_jar.update( cookies )
    cookie_jar.cookies
end