Module: Wmap::Utils

Extended by:
Utils
Includes:
DomainRoot, Logger, UrlMagic
Included in:
CidrTracker, DnsBruter, DomainTracker, DomainTracker::SubDomain, GeoIPTracker, GoogleSearchScraper, HostTracker, HostTracker::PrimaryHost, NetworkProfiler, PortScanner, SiteTracker, SiteTracker::DeactivatedSite, SiteTracker::WpTracker, UrlChecker, UrlCrawler, Utils, Whois
Defined in:
lib/wmap/utils/utils.rb,
lib/wmap/utils/logger.rb,
lib/wmap/utils/url_magic.rb,
lib/wmap/utils/wp_detect.rb,
lib/wmap/utils/domain_root.rb

Defined Under Namespace

Modules: DomainRoot, Logger, UrlMagic, WpDetect

Constant Summary

Constants included from UrlMagic

UrlMagic::Max_http_timeout, UrlMagic::User_agent

Constants included from DomainRoot

DomainRoot::File_ccsld, DomainRoot::File_cctld, DomainRoot::File_gtld, DomainRoot::File_tld

Instance Method Summary collapse

Methods included from Logger

#wlog

Methods included from UrlMagic

#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #landing_location, #make_absolute, #normalize_url, #open_page, #redirect_location, #response_code, #response_headers, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?

Methods included from DomainRoot

#get_domain_root, #get_domain_root_by_ccsld, #get_domain_root_by_cctld, #get_domain_root_by_tlds, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld

Instance Method Details

#cidr_2_ips(cidr) ⇒ Object

Convert a CIDR to a list of IPs: Input is a CIDR expression such as ‘192.168.1.1/30’, output is an array of IPs



315
316
317
318
319
320
321
322
323
324
325
326
 
# File 'lib/wmap/utils/utils.rb', line 315

def cidr_2_ips (cidr)
	puts "Method to convert a CIDR block into a list of IPs: #{cidr}" if @verbose
	begin
		cidr4 = NetAddr::CIDR.create(cidr)
		ips = cidr4.enumerate(:Limit => 0, :Bitstep => 1)
		#ips2 = ips.slice!(1, (ips.length-2))
		return ips
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
		return nil
	end
end

#file_2_hash(f, lc = true) ⇒ Object

Load entries from a text file and return a hash



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/wmap/utils/utils.rb', line 60

def file_2_hash(f,lc=true)
	puts "Loading records from file: #{f}" if @verbose
	begin
		hs=Hash.new
		file = File.open(f, "r")
		file.each_line do |line|
			line=line.chomp.strip
			next if line.nil?
			next if line.empty?
			line=line.downcase if lc==true
			next if line =~ /^\s*#/
			hs[line]=true unless hs.key?(line)
		end
		file.close
		return hs
	rescue => ee
		puts "Exception on method #{__method__} on #{f}: #{ee}" if @verbose
		return nil
	end
end

#file_2_list(f, lc = true) ⇒ Object

Load entries from a text file and return an array



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/wmap/utils/utils.rb', line 21

def file_2_list(f,lc=true)
	puts "Loading records from file: #{f}" if @verbose
	begin
		list=Array.new
		file = File.open(f, "r")
		file.each_line do |line|
			line=line.chomp.strip
			next if line.nil?
			next if line.empty?
			next if line =~ /^\s*#/
			line=line.downcase if lc==true
			list.push(line.chomp.strip)
		end
		file.close
		return list
	rescue => ee
		puts "Exception on method #{__method__} for file #{f}: #{ee}" if @verbose
		return nil
	end
end

#get_nameserver(host) ⇒ Object Also known as: get_ns

Retrieve the first name server from the Internet whois data repository for the host / subdomain / domain



281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/wmap/utils/utils.rb', line 281

def get_nameserver (host)
	puts "Retrieve the first authoritative name server for: #{host}" if @verbose
	begin
		domain=get_domain_root(host)
		w=Wmap::Whois.new
		ns = w.query(domain).nameservers.map! { |x| x.name }
		if ns.empty?
			puts "No name server found for domain root: #{domain}" if @verbose
			return nil
		else
			return ns.first
		end
	rescue => ee
		puts "Exception on method get_nameservers for #{host}: #{ee}" if @verbose
		return nil
	end
end

#get_nameservers(host) ⇒ Object

Retrieve a list of the authoritative name servers from the Internet whois data repository for the host / subdomain / domain



262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
# File 'lib/wmap/utils/utils.rb', line 262

def get_nameservers (host)
	puts "Retrieve a list of authoritative name server for: #{host}" if @verbose
	begin
		domain=get_domain_root(host)
		w=Wmap::Whois.new
		ns = w.query(domain).nameservers.map! { |x| x.name }
		if ns.empty?
			puts "No name server found for domain root: #{domain}" if @verbose
			return nil
		else
			return ns
		end
	rescue => ee
		puts "Exception on method get_nameservers for #{host}: #{ee}" if @verbose
		return nil
	end
end

#host_2_ip(hostname) ⇒ Object

Perform DNS query on a hostname. Return the first resolved IP as a string



238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
 
# File 'lib/wmap/utils/utils.rb', line 238

def host_2_ip (hostname)
	puts "Perform DNS query on host: #{hostname}" if @verbose
	begin
		ips=Array.new
		if is_ip?(hostname)
			puts "No change - same IP is returned. " if @verbose
			return hostname.strip
		else
			ips=Resolv.getaddresses(hostname)
			if (ips.empty?) then
				puts "Failed to resolve #{hostname}" if @verbose
				return nil
			else
				puts "IP found: #{ips.first}" if @verbose
				return ips.first.strip
			end
		end
	rescue => ee
		puts "Exception on method host_2_ip for host #{hostname}: #{ee}" if @verbose
		return nil
	end
end

#host_2_ips(hostname) ⇒ Object Also known as: ns_lookup

Perform the DNS query on a hostname over the Internet. Return the resolved IP(s) in an array



215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/wmap/utils/utils.rb', line 215

def host_2_ips (hostname)
	begin
		ips=Array.new
		if is_ip?(hostname)
			ips.push(hostname)
			return ips
		else
			ips = Resolv.getaddresses(hostname)
			if (ips.empty?) then
				puts "Failed to resolve #{hostname}" if @verbose
				return nil
			else
				return ips
			end
		end
	rescue => ee
		puts "Exception on method host_2_ips for host #{hostname}: #{ee}" if @verbose
		return nil
	end
end

#is_cidr?(cidr) ⇒ Boolean

Simple test to determine if the entry is in valid network cidr format

Returns:

  • (Boolean) 


178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
 
# File 'lib/wmap/utils/utils.rb', line 178

def is_cidr?(cidr)
	puts "Validate if the entry is valid CIDR format: #{cidr}" if @verbose
	begin
	cidr=cidr.strip
		if cidr =~  /^(\d+\.\d+\.\d+.\d+)\/(\d+)$/
			ip=$1
			mask=$2.to_i
			if is_ip?(ip)
				if mask >0 and mask <=32
					puts "confirmed as a valid CIDR entry: #{cidr}" if @verbose
					return true
				else
					return false
				end
			else
				return false
			end
		else
			return false
		end
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
		return false
	end
end

#is_fqdn?(host) ⇒ Boolean Also known as: is_host?

Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don’t be confused with another method ‘valid_dns_record?’, which is a stricter and time-consuming test on the DNS server for a resolvable internet host.

Returns:

  • (Boolean) 


160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/wmap/utils/utils.rb', line 160

def is_fqdn? (host)
	puts "Validate the host-name format is valid: #{host}" if @verbose
	return false if is_ip?(host) or is_url?(host)
	domain=get_domain_root(host)
	if domain.nil?
		return false
	elsif is_domain_root?(domain)
		return true
	else
		return false
	end
#	rescue => ee
#		puts "Exception on method is_fqdn? for #{host}: #{ee}" if @verbose
#		return false
end

#is_ip?(ip) ⇒ Boolean Also known as: is_valid_ip?

Test if it’s a legitimate IP4 address

Returns:

  • (Boolean) 


138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/wmap/utils/utils.rb', line 138

def is_ip? (ip)
	puts "Validate the IP format is valid: #{ip}" if @verbose
	begin
		ip=ip.strip
		raise "This is an URL: #{ip}" if is_url?(ip)
		if ip =~ /\d+\.\d+\.\d+.\d+/ and ip !~ /\/\d+/
			octs=ip.split('.')
			return false unless octs.size==4
			octs.map { |x| return false unless x.to_i >=0 and x.to_i <=255 }
		else
			return false
		end
		puts "Confirmed as a valid IP: #{ip}" if @verbose
		return true
	rescue => ee
		puts "Exception on method is_ip? for #{ip}: #{ee}" if @verbose
		return false
	end
end

#list_2_file(list, file) ⇒ Object

Save an array into a file



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/wmap/utils/utils.rb', line 43

def list_2_file (list,file)
	puts "Save list #{list} to plain file #{file}" if @verbose
	begin
		f = File.open(file, "w")
		list.map do |ent|
			#ent.strip!
			# Append the unix line break
			f.write("#{ent}\n")
		end
		f.close
	rescue => ee
		puts "Exception on method #{__method__} for file #{file}: #{ee}" if @verbose
		return nil
	end
end

#reverse_dns_lookup(ip) ⇒ Object Also known as: ip_2_host

Perform reverse dns lookup for an IP. Return the found ‘hostname’ if found, or the original IP if not



301
302
303
304
305
306
307
308
309
310
311
 
# File 'lib/wmap/utils/utils.rb', line 301

def reverse_dns_lookup (ip)
	puts "Retrieve the hostname by the reverse DNS lookup on IP: #{ip}"
	hostname = ip
	begin
		hostname = Socket.gethostbyaddr(ip.split('.').collect{ |x| x.to_i}.pack("CCCC"))[0]
		return hostname.downcase
	rescue => ee
		puts "Exception on method reverse_dns_lookup: #{ee}" if @verbose
		return hostname
	end
end

#sort_ips(ips) ⇒ Object

Sort an array of IPs in the ascendant order



205
206
207
208
209
210
211
212
 
# File 'lib/wmap/utils/utils.rb', line 205

def sort_ips (ips)
	begin
		"Sort the list of IP address in the ascendant order: #{ips}" if @verbose
		return NetAddr.sort(ips)
	rescue => ee
		puts "Exception on method sort_ips for IPs #{ips}: #{ee}" # if @verbose
	end
end

#valid_dns_record?(hostname) ⇒ Boolean Also known as: is_record?, a_record?

Query the name-server to see if the dns record is still valid

Returns:

  • (Boolean) 


82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/wmap/utils/utils.rb', line 82

def valid_dns_record? (hostname)
	puts "Validate the hostname record: #{hostname}" if @verbose
	begin
		ips=Resolv.getaddresses(hostname)
		if ips.empty?
			return false
		else
			puts "Found: #{hostname}" if @verbose
			return true
		end
	rescue => ee
		puts "Exception on method #{__method__} for host #{hostname}: #{ee}" if @verbose
		return false
	end
end

#zone_transferable?(domain) ⇒ Boolean

Test the DNS server if zone transfer is allowed. If allowed, save the found hosts into the class variable.

Returns:

  • (Boolean) 


101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/wmap/utils/utils.rb', line 101

def zone_transferable?(domain)
	puts "Check if the domain allows free zone transfer: #{domain}"
	begin
		transferable=false
		domain=domain.downcase
		nameservers = get_nameservers(domain)
		raise "Unable to determine the name servers for domain #{domain}" if nameservers.nil?
		puts "Retrieved name servers: #{nameservers}" if @verbose
		nameservers.each do |nssrv|
			begin
				puts "Attempt zone transfer on name server: #{nssrv}"
				if nssrv.nil?
					abort "Method input variable error: no name server found!" if @verbose
					next
				end
				zt = Dnsruby::ZoneTransfer.new
				zt.server=(nssrv) if nssrv!=""
				records = zt.transfer(domain)
				if records==nil
					puts "#{domain} zone transfer is not allowed on name server: #{nssrv}"
					next
				else
					puts "#{domain} zone transfer is allowed!"
					transferable=true
				end
			rescue Exception=>ee
				puts "Exception on method zone_transferable? for domain #{domain}: #{ee}"
			end
		end
		return transferable
	rescue Exception => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
		return false
	end
end