Class: Robotstxt::Parser

Inherits:
Object
  • Object
show all
Defined in:
lib/robotstxt/parser.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(robot_id = nil) ⇒ Parser

Initializes a new Robots::Robotstxtistance with robot_id option.

client = Robotstxt::Robotstxtistance.new('my_robot_id')



29
30
31
32
33
34
35
36
 
# File 'lib/robotstxt/parser.rb', line 29

def initialize(robot_id = nil)
	
	@robot_id = '*'
	@rules = []
	@sitemaps = []
	@robot_id = robot_id.downcase if !robot_id.nil?
	
end

Instance Attribute Details

#body ⇒ Object (readonly)

Returns the value of attribute body.



23
24
25
 
# File 'lib/robotstxt/parser.rb', line 23

def body
  @body
end

#found ⇒ Object (readonly)

Returns the value of attribute found.



23
24
25
 
# File 'lib/robotstxt/parser.rb', line 23

def found
  @found
end

#robot_id ⇒ Object

Returns the value of attribute robot_id.



22
23
24
 
# File 'lib/robotstxt/parser.rb', line 22

def robot_id
  @robot_id
end

#rules ⇒ Object (readonly)

Returns the value of attribute rules.



23
24
25
 
# File 'lib/robotstxt/parser.rb', line 23

def rules
  @rules
end

#sitemaps ⇒ Object (readonly)

Analyze the robots.txt file to return an Array containing the list of XML Sitemaps URLs.

client = Robotstxt::Robotstxtistance.new('my_robot_id')
if client.get('http://www.simonerinzivillo.it')
  client.sitemaps.each{ |url|
  puts url
}
end



125
126
127
 
# File 'lib/robotstxt/parser.rb', line 125

def sitemaps
  @sitemaps
end

Instance Method Details

#allowed?(var) ⇒ Boolean

Check if the URL is allowed to be crawled from the current Robot_id.

client = Robotstxt::Robotstxtistance.new('my_robot_id')
if client.get('http://www.simonerinzivillo.it')
  client.allowed?('http://www.simonerinzivillo.it/no-dir/')
end

This method returns true if the robots.txt file does not block the access to the URL.

Returns:

  • (Boolean) 


94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/robotstxt/parser.rb', line 94

def allowed?(var)
	is_allow = true
	url = URI.parse(var)
	querystring = (!url.query.nil?) ? '?' + url.query : ''
	url_path = url.path + querystring
	
	@rules.each {|ua|
		
		if @robot_id == ua[0] || ua[0] == '*' 
			
			ua[1].each {|d|
				
				is_allow = false if url_path.match('^' + d ) || d == '/'
				
			}
			
		end
		
	}
	is_allow
end

#found? ⇒ Boolean

This method returns true if the Robots.txt parsing is gone.

Returns:

  • (Boolean) 


131
132
133
 
# File 'lib/robotstxt/parser.rb', line 131

def found?
	!!@found
end

#get(hostname) ⇒ Object

Requires and parses the Robots.txt file for the hostname.

client = Robotstxt::Robotstxtistance.new('my_robot_id')
client.get('http://www.simonerinzivillo.it')

This method returns true if the parsing is gone.



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/robotstxt/parser.rb', line 47

def get(hostname)
	
	@ehttp = true
	url = URI.parse(hostname)
	
	begin
		http = Net::HTTP.new(url.host, url.port)
		if url.scheme == 'https'
			http.verify_mode = OpenSSL::SSL::VERIFY_NONE
			http.use_ssl = true 
		end
		
		response =  http.request(Net::HTTP::Get.new('/robots.txt'))
		
		case response
			when Net::HTTPSuccess then
			@found = true
			@body = response.body
			parse()		
			
			else
			@found = false
		end 
		
		return @found
		
		rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET => e
		if @ehttp
			@ettp = false
			retry 
			else
			return nil
		end
	end
	
end