Method: Mechanize::HTTP::Agent#get_robots

Defined in:
lib/mechanize/http/agent.rb

#get_robots(uri) ⇒ Object

:nodoc:


1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
 
# File 'lib/mechanize/http/agent.rb', line 1086

def get_robots(uri) # :nodoc:
  robots_mutex.synchronize do
    Thread.current[RobotsKey] = true
    begin
      fetch(uri).body
    rescue Mechanize::ResponseCodeError => e
      case e.response_code
      when /\A4\d\d\z/
        ''
      else
        raise e
      end
    rescue Mechanize::RedirectLimitReachedError
      ''
    ensure
      Thread.current[RobotsKey] = false
    end
  end
end