Class: CMSScanner::Browser

Inherits:
Object
  • Object
show all
Extended by:
Actions
Defined in:
lib/cms_scanner/browser.rb,
lib/cms_scanner/browser/actions.rb,
lib/cms_scanner/browser/options.rb

Overview

Options available in the Browser

Defined Under Namespace

Modules: Actions

Constant Summary collapse

OPTIONS = 
%i[
  cache_ttl
  cookie_jar
  cookie_string
  connect_timeout
  disable_tls_checks
  headers
  http_auth
  max_threads
  proxy
  proxy_auth
  random_user_agent
  request_timeout
  throttle
  url
  user_agent
  user_agents_list
  vhost
].freeze

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Actions

get, get_and_follow_location, head, post

Constructor Details

#initialize(parsed_options = {}) ⇒ Void

Parameters:

  • parsed_options (Hash) (defaults to: {}) 


14
15
16
17
18
 
# File 'lib/cms_scanner/browser.rb', line 14

def initialize(parsed_options = {})
  self.throttle = 0

  load_options(parsed_options.dup)
end

Class Method Details

.instance(parsed_options = {}) ⇒ Browser

Returns The instance.

Parameters:

  • parsed_options (Hash) (defaults to: {})

Returns:



25
26
27
 
# File 'lib/cms_scanner/browser.rb', line 25

def self.instance(parsed_options = {})
  @@instance ||= new(parsed_options)
end

.reset ⇒ Object 



29
30
31
 
# File 'lib/cms_scanner/browser.rb', line 29

def self.reset
  @@instance = nil
end

Instance Method Details

#default_connect_request_params ⇒ Hash

Returns The request params used to connect tothe target as well as potential other systems such as API.

Returns:

  • (Hash) —

    The request params used to connect tothe target as well as potential other systems such as API



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/cms_scanner/browser.rb', line 42

def default_connect_request_params
  params = {}

  if disable_tls_checks
    # See http://curl.haxx.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html
    params[:ssl_verifypeer] = false
    params[:ssl_verifyhost] = 0
    # TLSv1.0 and plus, allows to use a protocol potentially lower than the OS default
    params[:sslversion] = :tlsv1
  end

  {
    connecttimeout: :connect_timeout, cache_ttl: :cache_ttl,
    proxy: :proxy, timeout: :request_timeout
  }.each do |typhoeus_opt, browser_opt|
    attr_value = public_send(browser_opt)
    params[typhoeus_opt] = attr_value unless attr_value.nil?
  end

  params
end

#default_request_params ⇒ Hash

The params are not cached (using @params ||= for example), so that they are set accordingly if updated by a controller/other piece of code

Returns:

  • (Hash) 


67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/cms_scanner/browser.rb', line 67

def default_request_params
  params = default_connect_request_params.merge(
    headers: { 'User-Agent' => user_agent, 'Referer' => url }.merge(headers || {}),
    accept_encoding: 'gzip, deflate',
    method: :get
  )

  { cookiejar: :cookie_jar, cookiefile: :cookie_jar, cookie: :cookie_string }.each do |typhoeus_opt, browser_opt|
    attr_value = public_send(browser_opt)
    params[typhoeus_opt] = attr_value unless attr_value.nil?
  end

  params[:proxyuserpwd] = "#{proxy_auth[:username]}:#{proxy_auth[:password]}" if proxy_auth
  params[:userpwd] = "#{http_auth[:username]}:#{http_auth[:password]}" if http_auth

  params[:headers]['Host'] = vhost if vhost

  params
end

#default_user_agent ⇒ String

Returns:

  • (String) 


29
30
31
 
# File 'lib/cms_scanner/browser/options.rb', line 29

def default_user_agent
  "#{NS} v#{NS::VERSION}"
end

#forge_request(url, params = {}) ⇒ Typhoeus::Request

Parameters:

  • url (String)
  • params (Hash) (defaults to: {})

Returns:

  • (Typhoeus::Request) 


37
38
39
 
# File 'lib/cms_scanner/browser.rb', line 37

def forge_request(url, params = {})
  Typhoeus::Request.new(url, request_params(params))
end

#hydra ⇒ Typhoeus::Hydra

Returns:



34
35
36
 
# File 'lib/cms_scanner/browser/options.rb', line 34

def hydra
  @hydra ||= Typhoeus::Hydra.new(max_concurrency: max_threads || 1)
end

#load_options(options = {}) ⇒ Object

Parameters:

  • options (Hash) (defaults to: {}) 


39
40
41
42
43
 
# File 'lib/cms_scanner/browser/options.rb', line 39

def load_options(options = {})
  OPTIONS.each do |sym|
    send("#{sym}=", options[sym]) if options.key?(sym)
  end
end

#max_threads=(number) ⇒ Object

Set the threads attribute and update hydra accordinly If the throttle attribute is > 0, max_threads will be forced to 1

Parameters:

  • number (Integer) 


49
50
51
52
53
 
# File 'lib/cms_scanner/browser/options.rb', line 49

def max_threads=(number)
  @max_threads = number.to_i.positive? && throttle.zero? ? number.to_i : 1

  hydra.max_concurrency = @max_threads
end

#request_params(params = {}) ⇒ Hash

Parameters:

  • params (Hash) (defaults to: {})

Returns:

  • (Hash) 


90
91
92
93
94
 
# File 'lib/cms_scanner/browser.rb', line 90

def request_params(params = {})
  default_request_params.merge(params) do |key, oldval, newval|
    key == :headers ? oldval.merge(newval) : newval
  end
end

#throttle=(value) ⇒ Object

if value > 0, the max_threads will be set to 1

Parameters:

  • The (value) —

    throttle time in milliseconds



80
81
82
83
84
 
# File 'lib/cms_scanner/browser/options.rb', line 80

def throttle=(value)
  @throttle = value.to_i.abs / 1000.0

  self.max_threads = 1 if @throttle.positive?
end

#trottle! ⇒ Object 



86
87
88
 
# File 'lib/cms_scanner/browser/options.rb', line 86

def trottle!
  sleep(throttle) if throttle.positive?
end

#user_agent ⇒ String

Returns The user agent.

Returns:

  • (String) —

    The user agent



56
57
58
 
# File 'lib/cms_scanner/browser/options.rb', line 56

def user_agent
  @user_agent ||= random_user_agent ? user_agents.sample : default_user_agent
end

#user_agents ⇒ Array<String>

Returns:

  • (Array<String>) 


61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/cms_scanner/browser/options.rb', line 61

def user_agents
  return @user_agents if @user_agents

  @user_agents = []

  # The user_agents_list is managed by the CLI options, with the default being
  # APP_DIR/user_agents.txt
  File.open(user_agents_list).each do |line|
    next if line == "\n" || line[0, 1] == '#'

    @user_agents << line.chomp
  end

  @user_agents
end