Class: Wmap::CidrTracker

Inherits:
Object
  • Object
show all
Includes:
Utils
Defined in:
lib/wmap/cidr_tracker.rb

Overview

Class to track host/IP to the known (trusted) network CIDR blocks

Constant Summary

Constants included from Utils::UrlMagic

Utils::UrlMagic::Max_http_timeout

Constants included from Utils::DomainRoot

Utils::DomainRoot::File_ccsld, Utils::DomainRoot::File_cctld, Utils::DomainRoot::File_gtld, Utils::DomainRoot::File_tld

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Utils

#cidr_2_ips, #file_2_hash, #file_2_list, #get_nameserver, #get_nameservers, #host_2_ip, #host_2_ips, #is_cidr?, #is_fqdn?, #is_ip?, #list_2_file, #reverse_dns_lookup, #sort_ips, #valid_dns_record?, #zone_transferable?

Methods included from Utils::Logger

#wlog

Methods included from Utils::UrlMagic

#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #landing_location, #make_absolute, #normalize_url, #open_page, #redirect_location, #response_code, #response_headers, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?

Methods included from Utils::DomainRoot

#get_domain_root, #get_domain_root_by_ccsld, #get_domain_root_by_cctld, #get_domain_root_by_tlds, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld

Constructor Details

#initialize(params = {}) ⇒ CidrTracker

Set class default variables



17
18
19
20
21
22
23
24
 
# File 'lib/wmap/cidr_tracker.rb', line 17

def initialize (params = {})
	@verbose=params.fetch(:verbose, false)
	@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
	Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
	@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + 'cidrs')
	File.write(@cidr_seeds, "") unless File.exist?(@cidr_seeds)
	load_cidr_blks_from_file(@cidr_seeds)
end

Instance Attribute Details

#cidr_seeds ⇒ Object

Returns the value of attribute cidr_seeds.



14
15
16
 
# File 'lib/wmap/cidr_tracker.rb', line 14

def cidr_seeds
  @cidr_seeds
end

#data_dir ⇒ Object

Returns the value of attribute data_dir.



14
15
16
 
# File 'lib/wmap/cidr_tracker.rb', line 14

def data_dir
  @data_dir
end

#known_cidr_blks ⇒ Object

Returns the value of attribute known_cidr_blks.



14
15
16
 
# File 'lib/wmap/cidr_tracker.rb', line 14

def known_cidr_blks
  @known_cidr_blks
end

#verbose ⇒ Object

Returns the value of attribute verbose.



14
15
16
 
# File 'lib/wmap/cidr_tracker.rb', line 14

def verbose
  @verbose
end

Instance Method Details

#add(cidr, ref = nil, netname = nil) ⇒ Object

‘setter’ to add an entry to CIDR store @known_cidr_blks



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/wmap/cidr_tracker.rb', line 75

def add (cidr,ref=nil,netname=nil)
	puts "Load the entry into the CIDR store: #{cidr}"
	raise "Unknown CIDR format: #{cidr}" unless is_cidr?(cidr)
	# Obtain the 'ref' and 'netname' value automatically in case not passed as method parameters
	if ref.nil? or netname.nil?
		whois = Wmap::Whois.new
		# Note 11/1/2014: Use IP instead of the CIDR to perform the query, as the current ruby-whois query does not support CIDR as query input
		ip=cidr.split("/")[0]
		ref=whois.get_net_desc(ip)
		netname=whois.get_netname(ip)
		whois=nil
	end
	@known_cidr_blks = Hash.new unless @known_cidr_blks
	if @known_cidr_blks.key?(cidr)
		puts "Skip! Entry is already exist: #{cidr}"
		return nil
	else
		@known_cidr_blks[cidr] = Hash.new
		@known_cidr_blks[cidr]['ref']=ref
		@known_cidr_blks[cidr]['netname']=netname
		puts "Entry loaded!"
	end
	# Re-sort the blocks in order for better performance
	#@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
	#@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
	@known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
	@known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
#rescue => ee
#	puts "Exception on method #{__method__}: #{ee}" # if @verbose
end

#cidr_known?(cidr) ⇒ Boolean Also known as: is_known?

Determine if a CIDR entry is already known

Returns:

  • (Boolean) 


197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/wmap/cidr_tracker.rb', line 197

def cidr_known? (cidr)
	puts "Determine if the CIDR is known: #{cidr}" if @verbose
	known=false
	cidr=cidr.strip unless cidr.nil?
	cidr=cidr+"/32" if is_ip?(cidr)
	raise "Invalid CIDR format: #{cidr}" unless is_cidr?(cidr)
	return false if @known_cidr_blks==nil
	return true if @known_cidr_blks.key?(cidr)
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return false
end

#cidr_lookup(ip) ⇒ Object Also known as: lookup, query

Return the matching CIDR block for a ip



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/wmap/cidr_tracker.rb', line 177

def cidr_lookup (ip)
	puts "Lookup the CIDR name from the known CIDR list for the IP: #{ip}" if @verbose
	return nil if @known_cidr_blks==nil
	puts "CIDR Lookup: #{ip} ..." if @verbose
	@known_cidr_blks_desc_index.each do |line|
		first_octet_ip = ip.split('.').first.to_i
		first_octet_blk = line.split('.').first.to_i
		next if first_octet_blk > first_octet_ip
		cidr4 = NetAddr::CIDR.create(line)
		known = cidr4.contains?(ip+'/32')
		return line if known
	end
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return nil
end

#cidr_trusted?(cidr) ⇒ Boolean Also known as: is_trusted?

Determine if a cidr is within the range of our known network CIDR blocks

Returns:

  • (Boolean) 


212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/wmap/cidr_tracker.rb', line 212

def cidr_trusted? (cidr)
	puts "Determine if the CIDR within our ranges: #{cidr}" if @verbose
	trusted=false
	cidr=cidr.strip unless cidr.nil?
	cidr=cidr+"/32" if is_ip?(cidr)
	raise "Invalid CIDR format: #{cidr}" unless is_cidr?(cidr)
	return false if @known_cidr_blks==nil
	return true if @known_cidr_blks.key?(cidr)
	@known_cidr_blks_asce_index.each do |line|
		cidr4 = NetAddr::CIDR.create(line)
		return true if cidr4.contains?(cidr)
	end
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return false
end

#cidr_worker(host) ⇒ Object Also known as: track

Main worker method to retrieve known network information for a host / ip



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/wmap/cidr_tracker.rb', line 27

def cidr_worker (host)
	puts "Starting tracking of known CIDR information for host: #{host}" if @verbose
	host=host.strip.downcase
	ip=host_2_ip(host)
	cidr=cidr_lookup(ip)
	ref=get_cidr_ref(cidr)
	netname=get_cidr_netname(cidr)
	# save the data
	tracker=Hash.new
	tracker['host']=host
	tracker['ip']=ip
	tracker['cidr']=cidr
	tracker['ref']=ref
	tracker['netname']=netname
	return tracker
rescue => ee
	puts "Exception on method #{__method__} for host #{host}: #{ee}" # if @verbose
	return nil
end

#count ⇒ Object

Count numbers of CIDR object entries in the CIDR cache table



129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/wmap/cidr_tracker.rb', line 129

def count
	puts "Counting number of entries in the CIDR cache table ..." if @verbose
	cnt=0
	@known_cidr_blks.keys.map do |key|
		if is_cidr?(key)
			cnt=cnt+1
		end
	end
	puts "Current number of CIDR object entries: #{cnt}" if @verbose
	return cnt
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#counts ⇒ Object

Count numbers of IPs within the trusted CIDR objects



144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/wmap/cidr_tracker.rb', line 144

def counts
	puts "Counting number of IPs within the CIDR store:" if @verbose
	cnt=0
	@known_cidr_blks.keys.map do |key|
		cnt=cnt+size(key)
	end
	puts "Total number of trusted IPs: #{cnt}" if @verbose
	return cnt
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#delete(cidr, ref = nil, netname = nil) ⇒ Object Also known as: del

‘setter’ to remove an entry to CIDR store @known_cidr_blks



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/wmap/cidr_tracker.rb', line 107

def delete (cidr,ref=nil,netname=nil)
	puts "Remove the entry from the CIDR store: #{cidr}"
	#cidr.strip!
	raise "Unknown CIDR format: #{cidr}" unless is_cidr?(cidr)
	if @known_cidr_blks.key?(cidr)
		puts "Deleting ..."
		@known_cidr_blks.delete(cidr)
		puts "Entry cleared!"
	else
		raise "Unknown CIDR entry: #{cidr}"
	end
	# Re-sort the blocks in order for better performance
	#@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
	#@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
	@known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
	@known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" # if @verbose
end

#get_cidr_netname(cidr) ⇒ Object

Retrieve the CIDR netname field for tracking purpose, if it’s a known CIDR entry



250
251
252
253
254
255
 
# File 'lib/wmap/cidr_tracker.rb', line 250

def get_cidr_netname (cidr)
	puts "Lookup CIDR block #{cidr} netname ..." if @verbose
	cidr=cidr.strip unless cidr.nil?
	return nil unless @known_cidr_blks.key?(cidr)
	return @known_cidr_blks[cidr]['netname']
end

#get_cidr_ref(cidr) ⇒ Object

Retrieve the CIDR reference text for tracking purpose, if it’s a known CIDR entry



242
243
244
245
246
247
 
# File 'lib/wmap/cidr_tracker.rb', line 242

def get_cidr_ref (cidr)
	puts "Lookup CIDR block #{cidr} reference text ..." if @verbose
	cidr=cidr.strip unless cidr.nil?
	return nil unless @known_cidr_blks.key?(cidr)
	return @known_cidr_blks[cidr]['ref']
end

#ip_trusted?(ip) ⇒ Boolean

Check if the specific IP within the range of a list of known CIDR blocks

Returns:

  • (Boolean) 


157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/wmap/cidr_tracker.rb', line 157

def ip_trusted? (ip)
	puts "Check if the IP within the range of the known CIDR blocks: #{ip}" if @verbose
	known = false
	return false if @known_cidr_blks==nil
	first_octet_ip = ip.split('.').first.to_i
	@known_cidr_blks_desc_index.each do |line|
		first_octet_blk = line.split('.').first.to_i
		next if first_octet_blk > first_octet_ip
		puts "line: #{line}" if @verbose
		cidr4 = NetAddr::CIDR.create(line)
		known = cidr4.contains?(ip+'/32')
		break if known
	end
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return false
end

#load_cidr_blks_from_file(file_cidrs = @cidr_seeds) ⇒ Object

‘setter’ to load the known CIDR blocks into an instance variable @known_cidr_blks



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/wmap/cidr_tracker.rb', line 49

def load_cidr_blks_from_file(file_cidrs=@cidr_seeds)
	puts "Load the known CIDR seed file: #{file_cidrs}" if @verbose
	f=File.open(file_cidrs, 'r')
	f.each do |line|
		entry=line.chomp.split(',')
		next unless is_cidr?(entry[0])
		puts "Loading: #{entry[0]}" if @verbose
		key=entry[0].strip
		@known_cidr_blks = Hash.new unless @known_cidr_blks
		@known_cidr_blks[key] = Hash.new if not @known_cidr_blks.key?(key)
		@known_cidr_blks[key]['ref']=entry[1].nil? ? nil : entry[1].strip
		@known_cidr_blks[key]['netname']=entry[2].nil? ? nil : entry[2].strip
	end
	f.close
	# Sort the blocks in order once for better performance. Update 10/29/2018 to support Netaddr 2.x syntax
	#@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
	#@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
	if @known_cidr_blks
		@known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
		@known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
	end
#rescue => ee
#	puts "Exception on method #{__method__}: #{ee}" # if @verbose
end

#print_known_cidr_blks ⇒ Object Also known as: inspect

Print summary report of a list of known CIDR blocks



277
278
279
280
281
282
283
284
285
286
 
# File 'lib/wmap/cidr_tracker.rb', line 277

def print_known_cidr_blks
	puts "Print the known CIDR Netblocks in ascendant order" if @verbose
	puts "Network CIDR, RIPE Reference Text, NETNAME"
	@known_cidr_blks_asce_index.map do |key|
		ref=@known_cidr_blks[key]['ref']
		netname=@known_cidr_blks[key]['netname']
		puts "#{key}, #{ref}, #{netname}"
	end
	puts "End of the summary"
end

#print_known_cidr_blks_asce ⇒ Object Also known as: print

Print summary report of a list of known CIDR blocks in the ascendant order



297
298
299
300
301
 
# File 'lib/wmap/cidr_tracker.rb', line 297

def print_known_cidr_blks_asce
	puts "\nIndex of known CIDR Net blocks in Ascending  Order:"
	puts @known_cidr_blks_asce_index
	puts "End of the Index"
end

#print_known_cidr_blks_desc ⇒ Object

Print summary report of a list of known CIDR blocks in the descendant order



290
291
292
293
294
 
# File 'lib/wmap/cidr_tracker.rb', line 290

def print_known_cidr_blks_desc
	puts "\nIndex of known CIDR Net blocks in Descendant Order:"
	puts @known_cidr_blks_desc_index
	puts "End of the Index"
end

#save_cidrs_to_file!(file_cidrs = @cidr_seeds) ⇒ Object Also known as: save!

Save the current cidr hash table into a file



258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
 
# File 'lib/wmap/cidr_tracker.rb', line 258

def save_cidrs_to_file!(file_cidrs=@cidr_seeds)
	puts "Saving the current cidrs cache table from memory to file: #{file_cidrs} ..." if @verbose
	timestamp=Time.now
	f=File.open(file_cidrs, 'w')
	f.write "# Local cidrs file created by Wmap::CidrTracker.save method at: #{timestamp}\n"
	f.write "Network CIDR, CIDR RIPE Reference Text, CIDR NETNAME\n"
	@known_cidr_blks_asce_index.map do |key|
		ref=get_cidr_ref(key)
		netname=get_cidr_netname(key)
		f.write "#{key},#{ref},#{netname}\n"
	end
	f.close
	puts "CIDR cache table is successfully saved: #{file_cidrs}"
#rescue => ee
#	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#size(cidr) ⇒ Object

NetAddr wrapper to determine number of IPs within the CIDR object.



231
232
233
234
235
236
237
238
239
 
# File 'lib/wmap/cidr_tracker.rb', line 231

def size (cidr)
	puts "Determine the size of CIDR object: #{cidr}" if @verbose
	raise "Invalid CIDR format: #{cidr}" unless is_cidr?(cidr)
	obj = NetAddr::CIDR.create(cidr)
	return obj.size.to_i
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return nil
end