Class: Watobo::Crawler::Engine

Inherits:

Object

• Object
• Watobo::Crawler::Engine

Includes:

Plugin::Crawler::Constants

Defined in:

plugins/crawler/lib/engine.rb

Constant Summary

Constants included from Plugin::Crawler::Constants

Plugin::Crawler::Constants::CRAWL_NONE, Plugin::Crawler::Constants::CRAWL_PAUSED, Plugin::Crawler::Constants::CRAWL_RUNNING

Instance Method Summary

• #cancel ⇒ Object
• #clearEvents(event) ⇒ Object
• #get_page(url, opts = {}) ⇒ Object
• #initialize(opts = {}) ⇒ Engine constructor

  A new instance of Engine.

• #notify(event, *args) ⇒ Object
• #pause ⇒ Object
• #run(url, opts = {}) ⇒ Object
• #settings ⇒ Object
• #subscribe(event, &callback) ⇒ Object

Constructor Details

#initialize(opts = {}) ⇒ Engine

Returns a new instance of Engine.

# File 'plugins/crawler/lib/engine.rb', line 107

def initialize(opts={})
  @event_dispatcher_listeners = Hash.new
  @status_lock = Mutex.new
  
  @opts = {
    :submit_forms => true,
    :max_depth => 5,
    :max_repeat => 20,
    :max_threads => 4,
    :user_agent => "watobo-crawler",
    :proxy_host => '127.0.0.1',
    :proxy_port => Watobo::Conf::Interceptor.port,
    :delay => 0,
    :head_request_pattern => '(pdf|swf|doc|flv|jpg|png|gif)',
    :allowed_hosts => [], # regex's
    :allowed_urls => [], # regex's
    :excluded_urls => ["logout"], # regex's
    :excluded_fields => ["userid","username","password"], # regex's'
    :excluded_form_names => [], # regex's'
    :root_path => "",  # regex
    :username => "",
    :password => "",
    :auth_uri => nil,
    :auth_domain => "", # for ntlm auth
    :cookie_jar => nil
  }
  
  @opts.update opts
  @opts[:head_request_pattern] = '' if @opts[:head_request_pattern].nil? 
  
  if $DEBUG
  puts "* initializing crawler engine"
  puts @opts.to_yaml
  end
  @stats = {
    :total_requests => 0
  }
  
  @link_keys = Hash.new
  @link_counts = Hash.new
  
  @form_keys = Hash.new
  @form_counts = Hash.new
  
end

Dynamic Method Handling

This class handles dynamic methods through the method_missing method

#method_missing(name, *args, &block) ⇒ Object (private)

# File 'plugins/crawler/lib/engine.rb', line 502

def method_missing(name, *args, &block)
#   puts "* instance method missing (#{name})"
  if name =~ /(.*)=$/
  @opts.has_key? $1.to_sym || super
  @opts[$1.to_sym] = args[0]
  return @opts[$1.to_sym]
  else
    k = name.to_sym
  @opts.has_key? k || super
       #   puts "Value Found For #{k.to_yaml}"
  return @opts[k]

  end
end

Instance Method Details

#cancel ⇒ Object

# File 'plugins/crawler/lib/engine.rb', line 157

def cancel
  puts "[CRAWLER] - CANCEL!!"
 @status_lock.synchronize do
   @engine_status = CRAWL_NONE
 end
  @grabber_threads.each do |gt|
    puts "Killing Thread #{gt}"
    gt.kill
    gt.raise "CANCEL"
  end
  @grabber_threads.each{|t| t.join }
  
   @link_queue.clear
  @page_queue.clear
  @grabber_threads.clear
   @link_keys.clear
  @link_counts.clear
  
  @form_keys.clear
  @form_counts.clear
   
    notify( :update_status, current_status )
   puts "CANCELED - CANCELED"
  # exit
end

#clearEvents(event) ⇒ Object

# File 'plugins/crawler/lib/engine.rb', line 78

def clearEvents(event)
  @event_dispatcher_listeners[event] ||= []
  @event_dispatcher_listeners[event].clear
end

#get_page(url, opts = {}) ⇒ Object

# File 'plugins/crawler/lib/engine.rb', line 98

def get_page(url, opts={})  
  ro = {}.update @opts
  ro.update opts        
  agent = Crawler::Agent.new(ro)
  page = nil
    page = agent.get url
 return agent, page       
end

#notify(event, *args) ⇒ Object

# File 'plugins/crawler/lib/engine.rb', line 83

def notify(event, *args)
  if @event_dispatcher_listeners[event]
    # puts "NOTIFY: #{self}(:#{event}) [#{@event_dispatcher_listeners[event].length}]" if $DEBUG
    @event_dispatcher_listeners[event].each do |m|           
      m.call(*args) if m.respond_to? :call
    end
  end
end

#pause ⇒ Object

# File 'plugins/crawler/lib/engine.rb', line 153

def pause
false
end

#run(url, opts = {}) ⇒ Object

# File 'plugins/crawler/lib/engine.rb', line 183

def run(url, opts={})
  @opts.update opts
     @opts[:head_request_pattern] = '' if @opts[:head_request_pattern].nil? 

  @link_queue = Queue.new
  @page_queue = Queue.new
   @link_keys = Hash.new
  @link_counts = Hash.new
  
  @form_keys = Hash.new
  @form_counts = Hash.new
  
  @skipped_sites = Hash.new

  @grabber_threads = []
  start_link = URI.parse url
  return false if start_link.host.nil?

  allow_host(start_link)

  @link_queue.enq LinkBag.new(start_link, 0)

puts "[Crawler] Engine Settings:"
@opts.each do |k,v|  
    puts "#{k}: #{v}"
end
puts "---"
 notify(:log, "Crawling #{url} started ..." )

  @opts[:max_threads].times do |i|
    g = Grabber.new(@link_queue, @page_queue, @opts )
    @grabber_threads << g.run
  end

  @engine_status = CRAWL_RUNNING

  loop do
    pagebag = @page_queue.deq
   
    process_links(pagebag)
    
    process_forms(pagebag)
    @stats[:total_requests] += 1 unless pagebag.nil?
    
    puts "Links/Pages: #{@link_queue.size}/#{@page_queue.size}"
    notify( :update_status, current_status )
  # if @link_queue.empty? and @page_queue.empty?
    if @page_queue.empty?
     # if page_queue is empty wait for all grabber threads finishing the link_queue
      until @link_queue.num_waiting == @grabber_threads.length
        Thread.pass
      end
      # when the link_queue is finished check the page_queue. Crawling is finished if page_queue is empty too.
      if @page_queue.empty?
        @grabber_threads.each { |t| t.kill }
        puts "Finished Crawling"
        @status_lock.synchronize{ @engine_status = CRAWL_NONE }
        notify(:log, "Crawling finished")
        notify( :update_status, current_status )
        break
        
      end
    end

  end

end

#settings ⇒ Object

# File 'plugins/crawler/lib/engine.rb', line 92

def settings
  @opts
end

#subscribe(event, &callback) ⇒ Object

# File 'plugins/crawler/lib/engine.rb', line 74

def subscribe(event, &callback)
  (@event_dispatcher_listeners[event] ||= []) << callback
end