Class: Rack::DetectRobots

Inherits:
Object
  • Object
show all
Defined in:
lib/rack_detect_robots.rb

Constant Summary collapse

KNOWN_CRAWLERS =

this is fast for upto ca. 200 crawlers then use something different (trie…)

[
'yahoo! slurp',
'yahoo! de slurp',
'googlebot',
'ask jeeves',
'yanga worldsearch bot',
'gigaboti',
'ichiro',
'msnbot',
'crawler',
'ia_archiver',
'jobverifier',
'twiceler',
'eurobot',
'adsbot-google',
'speedy spider',
'yacybot',
'wget',
'findlinks',
'feedhub metadatafetcher',
'jobrobot.de',
'baiduspider']

Instance Method Summary collapse

Constructor Details

#initialize(app, crawler_regexp = nil) ⇒ DetectRobots

Returns a new instance of DetectRobots.



40
41
42
43
 
# File 'lib/rack_detect_robots.rb', line 40

def initialize(app, crawler_regexp=nil)
  @app=app
  @crawler_regexp = (crawler_regexp || Regexp.new(KNOWN_CRAWLERS.map{|bot| Regexp.escape(bot)}.join('|'),"i"))
end

Instance Method Details

#call(env) ⇒ Object 



45
46
47
48
 
# File 'lib/rack_detect_robots.rb', line 45

def call(env)
  env['rack_detect_robots']=test_for_robots(env)
  @app.call(env)
end