Module: Arachni::Utilities

Extended by:
Utilities
Included in:
Element::Base, Element::Capabilities::Auditable, ElementFilter, Framework, HTTP, HTTP::CookieJar, Mixins::Observable, Module::Base, Module::Base, Module::Manager, Module::Utilities, Parser, Platform::Fingerprinter, Platform::Manager, Platform::Manager, Plugin::Manager, Plugin::Manager, Processes::Dispatchers, Processes::Instances, RPC::Server::Dispatcher, RPC::Server::Framework, RPC::Server::Instance, Report::Manager, Report::Manager, Session, Spider, Trainer, Arachni::UI::CLI, Arachni::UI::CLI::Utilities, URI, URI, Utilities
Defined in:
lib/arachni/utilities.rb

Overview

Includes some useful methods for the system, the modules etc.

Author:

Instance Method Summary collapse

Instance Method Details

#available_port ⇒ Fixnum

Returns Random available port number.

Returns:

  • (Fixnum) —

    Random available port number.



371
372
373
374
 
# File 'lib/arachni/utilities.rb', line 371

def available_port
    nil while !port_available?( port = rand_port )
    port
end

#cookie_encode(*args) ⇒ Object

See Also:



100
101
102
 
# File 'lib/arachni/utilities.rb', line 100

def cookie_encode( *args )
    Cookie.encode( *args )
end

#cookies_from_document(*args) ⇒ Object

See Also:



85
86
87
 
# File 'lib/arachni/utilities.rb', line 85

def cookies_from_document( *args )
    Cookie.from_document( *args )
end

#cookies_from_file(*args) ⇒ Object

See Also:



95
96
97
 
# File 'lib/arachni/utilities.rb', line 95

def cookies_from_file( *args )
    Cookie.from_file( *args )
end

#cookies_from_response(*args) ⇒ Object

See Also:



80
81
82
 
# File 'lib/arachni/utilities.rb', line 80

def cookies_from_response( *args )
    Cookie.from_response( *args )
end

#exception_jail(raise_exception = true, &block) ⇒ Object

Wraps the “block” in exception handling code and runs it.

Parameters:

  • raise_exception (Bool) (defaults to: true) —

    re-raise exception

  • block (Block) —

    to call



415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
 
# File 'lib/arachni/utilities.rb', line 415

def exception_jail( raise_exception = true, &block )
    block.call
rescue Exception => e
    begin
        print_error e.inspect
        print_error_backtrace e
        print_error
        print_error 'Parent:'
        print_error  self.class.to_s
        print_error
        print_error 'Block:'
        print_error block.to_s
        print_error
        print_error 'Caller:'
        ::Kernel.caller.each { |l| print_error l }
        print_error '-' * 80
    rescue
    end
    raise e if raise_exception
end

#exclude_path?(url) ⇒ Bool

Decides whether the given ‘url` matches any framework exclusion rules.

Parameters:

Returns:

  • (Bool)

See Also:



221
222
223
 
# File 'lib/arachni/utilities.rb', line 221

def exclude_path?( url )
    uri_parse( url ).exclude?( Options.exclude )
end

#extract_domain(url) ⇒ String

Returns Domain name.

Parameters:

Returns:

See Also:



178
179
180
 
# File 'lib/arachni/utilities.rb', line 178

def extract_domain( url )
    uri_parse( url ).domain
end

#follow_protocol?(url, reference = Options.url) ⇒ Bool

Decides whether the given ‘url` has an acceptable protocol.

Parameters:

  • url (String)
  • reference (String) (defaults to: Options.url) —

    Reference URL.

Returns:

  • (Bool)

See Also:



266
267
268
269
270
271
272
273
274
275
276
277
 
# File 'lib/arachni/utilities.rb', line 266

def follow_protocol?( url, reference = Options.url )
    return true if !reference
    check_scheme = uri_parse( url ).scheme

    return false if !%(http https).include?( check_scheme.to_s.downcase )

    ref_scheme   = uri_parse( reference ).scheme
    return true if ref_scheme && ref_scheme != 'https'
    return true if ref_scheme == check_scheme

    !Options.https_only?
end

#form_decode(*args) ⇒ Object

See Also:



51
52
53
 
# File 'lib/arachni/utilities.rb', line 51

def form_decode( *args )
    Form.decode( *args )
end

#form_encode(*args) ⇒ Object

See Also:



46
47
48
 
# File 'lib/arachni/utilities.rb', line 46

def form_encode( *args )
    Form.encode( *args )
end

#form_parse_request_body(*args) ⇒ Object Also known as: parse_request_body

See Also:



56
57
58
 
# File 'lib/arachni/utilities.rb', line 56

def form_parse_request_body( *args )
    Form.parse_request_body( *args )
end

#forms_from_document(*args) ⇒ Object

See Also:



41
42
43
 
# File 'lib/arachni/utilities.rb', line 41

def forms_from_document( *args )
    Form.from_document( *args )
end

#forms_from_response(*args) ⇒ Object

See Also:



36
37
38
 
# File 'lib/arachni/utilities.rb', line 36

def forms_from_response( *args )
    Form.from_response( *args )
end

#generate_token ⇒ Object 



385
386
387
388
389
 
# File 'lib/arachni/utilities.rb', line 385

def generate_token
    secret = ''
    1000.times { secret << rand( 9999 ).to_s }
    Digest::SHA2.hexdigest( secret )
end

#get_path(url) ⇒ String

Returns path Full URL up to the path component (no resource, query etc.).

Parameters:

Returns:

  • (String) —

    path Full URL up to the path component (no resource, query etc.).

See Also:



167
168
169
 
# File 'lib/arachni/utilities.rb', line 167

def get_path( url )
    uri_parse( url ).up_to_path
end

#html_decode(str) ⇒ Object Also known as: html_unescape 



114
115
116
 
# File 'lib/arachni/utilities.rb', line 114

def html_decode( str )
    ::CGI.unescapeHTML( str.to_s )
end

#html_encode(str) ⇒ Object Also known as: html_escape 



119
120
121
 
# File 'lib/arachni/utilities.rb', line 119

def html_encode( str )
    ::CGI.escapeHTML( str.to_s )
end

#include_path?(url) ⇒ Bool

Decides whether the given ‘url` matches any framework inclusion rules.

Parameters:

Returns:

  • (Bool)

See Also:



235
236
237
 
# File 'lib/arachni/utilities.rb', line 235

def include_path?( url )
    uri_parse( url ).include?( Options.include )
end

#links_from_document(*args) ⇒ Object

See Also:



67
68
69
 
# File 'lib/arachni/utilities.rb', line 67

def links_from_document( *args )
    Link.from_document( *args )
end

#links_from_response(*args) ⇒ Object

See Also:



62
63
64
 
# File 'lib/arachni/utilities.rb', line 62

def links_from_response( *args )
    Link.from_response( *args )
end

#normalize_url(url) ⇒ Object

See Also:



150
151
152
 
# File 'lib/arachni/utilities.rb', line 150

def normalize_url( url )
    URI.normalize( url )
end

#page_from_response(*args) ⇒ Object

See Also:



105
106
107
 
# File 'lib/arachni/utilities.rb', line 105

def page_from_response( *args )
    Page.from_response( *args )
end

#page_from_url(*args, &block) ⇒ Object

See Also:



110
111
112
 
# File 'lib/arachni/utilities.rb', line 110

def page_from_url( *args, &block )
    Page.from_url( *args, &block )
end

#parse_query(*args) ⇒ Object 



75
76
77
 
# File 'lib/arachni/utilities.rb', line 75

def parse_query( *args )
    Link.parse_query_vars( *args )
end

#parse_set_cookie(*args) ⇒ Object

See Also:



90
91
92
 
# File 'lib/arachni/utilities.rb', line 90

def parse_set_cookie( *args )
    Cookie.parse_set_cookie( *args )
end

#parse_url_vars(*args) ⇒ Object

See Also:



72
73
74
 
# File 'lib/arachni/utilities.rb', line 72

def parse_url_vars( *args )
    Link.parse_query_vars( *args )
end

#path_in_domain?(url, reference = Options.url) ⇒ Bool

Compares 2 urls in order to decide whether or not they belong to the same domain.

Parameters:

  • url (String)
  • reference (String) (defaults to: Options.url)

Returns:

  • (Bool) —

    ‘true` if self is in the same domain as the `reference` URL, false otherwise.

See Also:



207
208
209
 
# File 'lib/arachni/utilities.rb', line 207

def path_in_domain?( url, reference = Options.url )
    uri_parse( url ).in_domain?( !Options.follow_subdomains, reference )
end

#path_too_deep?(url) ⇒ Bool

Returns ‘true` is the path exceeds the framework limit, `false` otherwise.

Parameters:

Returns:

  • (Bool) —

    ‘true` is the path exceeds the framework limit, `false` otherwise.

See Also:



191
192
193
 
# File 'lib/arachni/utilities.rb', line 191

def path_too_deep?( url )
    uri_parse( url ).too_deep?( Options.depth_limit )
end

#port_available?(port) ⇒ Bool

Checks whether the port number is available.

Parameters:

  • port (Fixnum)

Returns:

  • (Bool) 


398
399
400
401
402
403
404
405
406
407
 
# File 'lib/arachni/utilities.rb', line 398

def port_available?( port )
    begin
        socket = Socket.new( :INET, :STREAM, 0 )
        socket.bind( Addrinfo.tcp( '127.0.0.1', port ) )
        socket.close
        true
    rescue
        false
    end
end

#rand_port ⇒ Integer

Returns Random port within the user specified range.

Returns:

  • (Integer) —

    Random port within the user specified range.

See Also:



378
379
380
381
382
383
 
# File 'lib/arachni/utilities.rb', line 378

def rand_port
    first, last = Options.rpc_instance_port_range
    range = (first..last).to_a

    range[ rand( range.last - range.first ) ]
end

#redundant_path?(url, &block) ⇒ Bool

Checks if the provided URL matches a redundant filter and decreases its counter if so.

If a filter’s counter has reached 0 the method returns true.

Parameters:

Returns:

  • (Bool) —

    ‘true` if the `url` is redundant, `false` otherwise.

See Also:



251
252
253
 
# File 'lib/arachni/utilities.rb', line 251

def redundant_path?( url, &block )
    Options.redundant?( url, &block )
end

#remove_constants(mod, skip = [], children_only = true) ⇒ Object 



436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
 
# File 'lib/arachni/utilities.rb', line 436

def remove_constants( mod, skip = [], children_only = true )
    return if skip.include?( mod )
    return if !(mod.is_a?( Class ) || !mod.is_a?( Module )) ||
        !mod.to_s.start_with?( 'Arachni' )

    parent = Object
    mod.to_s.split( '::' )[0..-2].each do |ancestor|
        parent = parent.const_get( ancestor.to_sym )
    end

    mod.constants.each { |m| mod.send( :remove_const, m ) }

    return if children_only
    parent.send( :remove_const, mod.to_s.split( ':' ).last.to_sym )
end

#seed ⇒ String

Returns random HEX (SHA2) string.

Returns:

  • (String) —

    random HEX (SHA2) string



31
32
33
 
# File 'lib/arachni/utilities.rb', line 31

def seed
    @@seed ||= Digest::SHA2.hexdigest( srand( 1000 ).to_s )
end

#skip_page?(page_or_response) ⇒ Bool Also known as: skip_response?

Determines whether or not a given Page or Typhoeus::Response should be ignored.

Parameters:

Returns:

  • (Bool) —

    ‘true` if the `#body` of the given object matches any of the exclusion patterns, `false` otherwise.

See Also:



326
327
328
329
330
 
# File 'lib/arachni/utilities.rb', line 326

def skip_page?( page_or_response )
    (Options.exclude_binaries? && !page_or_response.text?) ||
        skip_path?( page_or_response.url ) ||
        Options.exclude_page?( page_or_response.body )
end

#skip_path?(path) ⇒ Bool

Note:

Does not call #redundant_path?.

Decides whether or not the provided ‘path` should be skipped based on:

Parameters:

Returns:

  • (Bool) 


293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
 
# File 'lib/arachni/utilities.rb', line 293

def skip_path?( path )
    return true if !path

    parsed = uri_parse( path.to_s )

    begin
        return true if !follow_protocol?( parsed )
        return true if !path_in_domain?( parsed )
        return true if path_too_deep?( parsed )
        return true if !include_path?( parsed )
        return true if exclude_path?( parsed )
        false
    rescue => e
        ap e
        ap e.backtrace
        true
    end
end

#skip_resource?(resource) ⇒ Bool

Determines whether or not the given ‘resource` should be ignored depending on its type and content.

Parameters:

  • resource (Page, Typhoeus::Response, String) —

    If given a:

    * {Page}: both its URL and body will be examined.
* {Typhoeus::Response}: both its effective URL and body will be examined.
* {String}: if multi-line it will be treated as a response body,
    otherwise as a path.

Returns:

  • (Bool) —

    ‘true` if the resource should be ignore,`false` otherwise.

See Also:

  • #skip_path?
  • ignore_page?
  • ignore_response?
  • Options#ignore?


353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
 
# File 'lib/arachni/utilities.rb', line 353

def skip_resource?( resource )
    case resource
        when Page
            skip_page?( resource )

        when Typhoeus::Response
            skip_response?( resource )

        else
            if (s = resource.to_s) =~ /[\r\n]/
                Options.exclude_page? s
            else
                skip_path? s
            end
    end
end

#to_absolute(relative_url, reference_url = Options.instance.url.to_s) ⇒ Object

See Also:



145
146
147
 
# File 'lib/arachni/utilities.rb', line 145

def to_absolute( relative_url, reference_url = Options.instance.url.to_s )
    URI.to_absolute( relative_url, reference_url )
end

#uri_decode(url) ⇒ Object

See Also:



140
141
142
 
# File 'lib/arachni/utilities.rb', line 140

def uri_decode( url )
    URI.decode( url )
end

#uri_encode(string, bad_characters = nil) ⇒ Object

See Also:



135
136
137
 
# File 'lib/arachni/utilities.rb', line 135

def uri_encode( string, bad_characters = nil )
    URI.encode( string, bad_characters )
end

#uri_parse(url) ⇒ Object

See Also:



130
131
132
 
# File 'lib/arachni/utilities.rb', line 130

def uri_parse( url )
    URI.parse( url )
end

#uri_parser ⇒ URI::Parser

Returns cached URI parser.

Returns:

  • (URI::Parser) —

    cached URI parser



125
126
127
 
# File 'lib/arachni/utilities.rb', line 125

def uri_parser
    URI.parser
end

#url_sanitize(url) ⇒ Object

See Also:



155
156
157
 
# File 'lib/arachni/utilities.rb', line 155

def url_sanitize( url )
    normalize_url( url )
end