Class: CMSScanner::Browser

Inherits:
Object
  • Object
show all
Extended by:
Actions
Defined in:
lib/cms_scanner/browser.rb,
lib/cms_scanner/browser/actions.rb,
lib/cms_scanner/browser/options.rb

Overview

Options available in the Browser

Defined Under Namespace

Modules: Actions

Constant Summary collapse

OPTIONS = 
[
  :cache_ttl,
  :cookie_jar,
  :cookie_string,
  :connect_timeout,
  :http_auth,
  :max_threads,
  :proxy,
  :proxy_auth,
  :random_user_agent,
  :request_timeout,
  :throttle,
  :user_agent,
  :user_agents_list,
  :vhost
]

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Actions

get, get_and_follow_location, head, post

Constructor Details

#initialize(parsed_options = {}) ⇒ Void

Parameters:

  • parsed_options (Hash) (defaults to: {}) 


12
13
14
15
16
 
# File 'lib/cms_scanner/browser.rb', line 12

def initialize(parsed_options = {})
  self.throttle = 0

  load_options(parsed_options)
end

Class Method Details

.instance(parsed_options = {}) ⇒ Browser

Returns The instance.

Parameters:

  • parsed_options (Hash) (defaults to: {})

Returns:



23
24
25
 
# File 'lib/cms_scanner/browser.rb', line 23

def self.instance(parsed_options = {})
  @@instance ||= new(parsed_options)
end

.reset ⇒ Object 



27
28
29
 
# File 'lib/cms_scanner/browser.rb', line 27

def self.reset
  @@instance = nil
end

Instance Method Details

#default_request_params ⇒ Hash

Returns:

  • (Hash) 


48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/cms_scanner/browser.rb', line 48

def default_request_params
  params = {
    # Disable SSL-Certificate checks, see http://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html
    ssl_verifypeer: false, ssl_verifyhost: 0,
    headers: { 'User-Agent' => user_agent },
    accept_encoding: 'gzip, deflate',
    method: :get
  }

  typhoeus_to_browser_opts.each do |typhoeus_opt, browser_opt|
    attr_value = public_send(browser_opt)
    params[typhoeus_opt] = attr_value unless attr_value.nil?
  end

  params[:proxyauth] = "#{proxy_auth[:username]}:#{proxy_auth[:password]}" if proxy_auth
  params[:userpwd]   = "#{http_auth[:username]}:#{http_auth[:password]}" if http_auth

  params[:headers].merge!('Host' => vhost) if vhost

  params
end

#default_user_agent ⇒ String

Returns:

  • (String) 


24
25
26
 
# File 'lib/cms_scanner/browser/options.rb', line 24

def default_user_agent
  "#{NS} v#{NS::VERSION}"
end

#forge_request(url, params = {}) ⇒ Typhoeus::Request

Parameters:

  • url (String)
  • params (Hash) (defaults to: {})

Returns:

  • (Typhoeus::Request) 


35
36
37
 
# File 'lib/cms_scanner/browser.rb', line 35

def forge_request(url, params = {})
  Typhoeus::Request.new(url, request_params(params))
end

#hydra ⇒ Object 



28
29
30
 
# File 'lib/cms_scanner/browser/options.rb', line 28

def hydra
  @hydra ||= Typhoeus::Hydra.new(max_concurrency: max_threads || 1)
end

#load_options(options = {}) ⇒ Object

Parameters:

  • options (Hash) (defaults to: {}) 


33
34
35
36
37
 
# File 'lib/cms_scanner/browser/options.rb', line 33

def load_options(options = {})
  OPTIONS.each do |sym|
    send("#{sym}=", options[sym]) if options.key?(sym)
  end
end

#max_threads=(number) ⇒ Object

Set the threads attribute and update the max_concurrency of Typhoeus::Hydra

If the throttle attribute is > 0, max_threads will be forced to 1

Parameters:

  • number (Integer) 


45
46
47
48
49
 
# File 'lib/cms_scanner/browser/options.rb', line 45

def max_threads=(number)
  @max_threads = number.to_i > 0 && throttle == 0 ? number.to_i : 1

  hydra.max_concurrency = @max_threads
end

#request_params(params = {}) ⇒ Hash

Parameters:

  • params (Hash) (defaults to: {})

Returns:

  • (Hash) 


73
74
75
76
77
 
# File 'lib/cms_scanner/browser.rb', line 73

def request_params(params = {})
  default_request_params.merge(params) do |key, oldval, newval|
    key == :headers ? oldval.merge(newval) : newval
  end
end

#throttle=(value) ⇒ Object

if value > 0, the max_threads will be set to 1

Parameters:

  • The (value) —

    throttle time in milliseconds



78
79
80
81
82
 
# File 'lib/cms_scanner/browser/options.rb', line 78

def throttle=(value)
  @throttle = value.to_i.abs / 1000.0

  self.max_threads = 1 if @throttle > 0
end

#trottle! ⇒ Object 



84
85
86
 
# File 'lib/cms_scanner/browser/options.rb', line 84

def trottle!
  sleep(throttle) if throttle > 0
end

#typhoeus_to_browser_opts ⇒ Hash

Returns:

  • (Hash) 


40
41
42
43
44
45
 
# File 'lib/cms_scanner/browser.rb', line 40

def typhoeus_to_browser_opts
  { connecttimeout: :connect_timeout, cache_ttl: :cache_ttl,
    proxy: :proxy, timeout: :request_timeout, cookiejar: :cookie_jar,
    cookiefile: :cookie_jar, cookie: :cookie_string
  }
end

#user_agent ⇒ String

Returns The user agent.

Returns:

  • (String) —

    The user agent



52
53
54
 
# File 'lib/cms_scanner/browser/options.rb', line 52

def user_agent
  @user_agent ||= random_user_agent ? user_agents.sample : default_user_agent
end

#user_agents ⇒ Array<String>

Returns:

  • (Array<String>) 


57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/cms_scanner/browser/options.rb', line 57

def user_agents
  return @user_agents if @user_agents

  @user_agents = []

  File.open(user_agents_list).each do |line|
    next if line == "\n" || line[0, 1] == '#'
    @user_agents << line.chomp
  end

  @user_agents
end

#user_agents_list ⇒ String

Returns The path to the user agents list.

Returns:

  • (String) —

    The path to the user agents list



71
72
73
 
# File 'lib/cms_scanner/browser/options.rb', line 71

def user_agents_list
  @user_agents_list ||= File.join(APP_DIR, 'user_agents.txt')
end