Class: Watobo::Crawler::Agent

Inherits:
Mechanize
  • Object
show all
Defined in:
plugins/crawler/lib/engine.rb

Instance Method Summary collapse

Constructor Details

#initialize(opts) ⇒ Agent

Returns a new instance of Agent.



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'plugins/crawler/lib/engine.rb', line 28

def initialize(opts)
  super()

   
   self.verify_mode = OpenSSL::SSL::VERIFY_NONE
   self.ignore_bad_chunking = true
   self.keep_alive = false
   
   self.user_agent = opts[:user_agent] if opts.has_key?(:user_agent)
   
   if opts.has_key? :username and opts.has_key? :password
     unless opts[:username].empty? and opts[:password].empty?
       
        user = opts[:username]
        pw = opts[:password]
        uri = opts[:auth_uri]
      # puts "Got Credentials for #{uri}: #{user} / #{pw}" 
       self.add_auth(uri, user , pw )
       # TODO: remove this workaround for a Mechanize Bug (#243)
       p = self.get uri
     end
   end
   
   if ( opts.has_key? :proxy_host ) && ( opts.has_key? :proxy_port )
       self.set_proxy( opts[:proxy_host], opts[:proxy_port] )
   end

   if opts.has_key? :pre_connect_hook
      self.pre_connect_hooks << opts[:pre_connect_hook] if opts[:pre_connect_hook].respond_to? :call
   end       
   
   unless opts[:cookie_jar].nil?
     clean_jar = Mechanize::CookieJar.new
     opts[:cookie_jar].each{ |cookie|
       clean_jar.add! cookie
       }
       self.cookie_jar = clean_jar
   end
   
end