Class: Wmap::DomainTracker::SubDomain

Inherits:
Wmap::DomainTracker show all
Includes:
Singleton, Utils
Defined in:
lib/wmap/domain_tracker/sub_domain.rb

Overview

Class to differentiate the sub-domain from the top domain for the enterprise. This is needed for better managing of the sub-domains and the associated entities

Constant Summary

Constants included from Utils::UrlMagic

Utils::UrlMagic::Max_http_timeout, Utils::UrlMagic::User_agent

Constants included from Utils::DomainRoot

Utils::DomainRoot::File_ccsld, Utils::DomainRoot::File_cctld, Utils::DomainRoot::File_gtld, Utils::DomainRoot::File_tld

Instance Attribute Summary collapse

Attributes inherited from Wmap::DomainTracker

#domains_file, #known_internet_domains

Instance Method Summary collapse

Methods included from Utils

#cidr_2_ips, #file_2_hash, #file_2_list, #get_nameserver, #get_nameservers, #host_2_ip, #host_2_ips, #is_cidr?, #is_fqdn?, #is_ip?, #list_2_file, #reverse_dns_lookup, #sort_ips, #valid_dns_record?, #zone_transferable?

Methods included from Utils::Logger

#wlog

Methods included from Utils::UrlMagic

#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #landing_location, #make_absolute, #normalize_url, #open_page, #redirect_location, #response_code, #response_headers, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?

Methods included from Utils::DomainRoot

#get_domain_root, #get_domain_root_by_ccsld, #get_domain_root_by_cctld, #get_domain_root_by_tlds, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld

Methods inherited from Wmap::DomainTracker

#bulk_delete, #count, #delete, #delete_all, #domain_known?, #file_add, #file_delete, #get_domains, #load_domains_from_file, #print_known_domains, #refresh, #save_domains_to_file!, #search

Constructor Details

#initialize(params = {}) ⇒ SubDomain

Set default instance variables



23
24
25
26
27
28
29
30
31
32
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 23

def initialize (params = {})
	@verbose=params.fetch(:verbose, false)
	@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
	Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
	@max_parallel=params.fetch(:max_parallel, 40)
	# Hash table to hold the trusted domains
	@sub_domains_file=params.fetch(:sub_domains_file, @data_dir + 'sub_domains')
	File.new(@sub_domains_file, "w") unless File.exist?(@sub_domains_file)
	@known_internet_sub_domains=load_domains_from_file(@sub_domains_file) #unless @known_internet_sub_domains.size>0
end

Instance Attribute Details

#data_dir ⇒ Object

Returns the value of attribute data_dir.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def data_dir
  @data_dir
end

#known_internet_sub_domains ⇒ Object

Returns the value of attribute known_internet_sub_domains.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def known_internet_sub_domains
  @known_internet_sub_domains
end

#max_parallel ⇒ Object

Returns the value of attribute max_parallel.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def max_parallel
  @max_parallel
end

#sub_domains_file ⇒ Object

Returns the value of attribute sub_domains_file.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def sub_domains_file
  @sub_domains_file
end

#verbose ⇒ Object

Returns the value of attribute verbose.



20
21
22
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 20

def verbose
  @verbose
end

Instance Method Details

#add(sub) ⇒ Object

‘setter’ to add sub-domain entry to the cache one at a time



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 35

def add(sub)
	puts "Add entry to the local sub domain cache table: #{sub}" if @verbose
	record=Hash.new
	sub=sub.strip.downcase
	if @known_internet_sub_domains.key?(sub)
		puts "Skip on known sub-domain: #{sub}" if @verbose
		return nil
	end
	if zone_transferable?(sub)
		record[sub]=true
	else
		record[sub]=false
	end
	puts "Adding new record into the data store: #{record}" if @verbose
	@known_internet_sub_domains.merge!(record)
	return record
rescue => ee
	puts "Exception on method #{__method__} for #{sub}: #{ee}" if @verbose
end

#bulk_add(list, num = @max_parallel) ⇒ Object Also known as: adds

‘setter’ to add domain entry to the cache in batch (from a list)



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 56

def bulk_add(list, num=@max_parallel)
	puts "Add entries to the local domains cache table from list: #{list}" if @verbose
	results=Hash.new
	domains=list
	if domains.size > 0
		Parallel.map(list, :in_processes => num) { |target|
			add(target)
		}.each do |process|
			if process.nil?
				next
			elsif process.empty?
				#do nothing
			else
				results.merge!(process)
			end
		end
		@known_internet_sub_domains.merge!(results)
		puts "Done loading sub_domain entries."
		return results
	else
		puts "Error: no entry is loaded. Please check your list and try again."
	end
	return results
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#print_known_sub_domains ⇒ Object

Print summary report on all known / trust domains in the domain cache table



129
130
131
132
133
134
135
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 129

def print_known_sub_domains
	puts "\nSummary of known Internet Sub-domains:"
	self.known_internet_sub_domains.keys.sort.each do |domain|
		puts domain
	end
	puts "End of the summary"
end

#save_sub_domains_to_file!(file_domains = @file_sub_domains, domains = @known_internet_sub_domains) ⇒ Object Also known as: save!

Save the current domain hash table into a file



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 108

def save_sub_domains_to_file!(file_domains=@file_sub_domains, domains=@known_internet_sub_domains)
	puts "Saving the current domains cache table from memory to file: #{file_domains} ..." if @verbose
	timestamp=Time.now
	f=File.open(file_domains, 'w')
	f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
	f.write "# domain name, free zone transfer detected?\n"
	domains.keys.sort.map do |key|
		if domains[key]
			f.write "#{key}, yes\n"
		else
			f.write "#{key}, no\n"
		end
	end
	f.close
	puts "Domain cache table is successfully saved: #{file_domains}"
rescue => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
end

#update_from_host_store! ⇒ Object Also known as: update!

Procedures to identify sub-domain from the hosts store



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/wmap/domain_tracker/sub_domain.rb', line 85

def update_from_host_store!
	puts "Invoke internal procedures to update the sub-domain list from the host store."
	# Step 1 - obtain the latest sub-domains
	my_tracker = Wmap::HostTracker.instance
	my_tracker.data_dir = @data_dir
	my_tracker.hosts_file = my_tracker.data_dir + "/" + "hosts"
	my_tracker.load_known_hosts_from_file(my_tracker.hosts_file)
	subs = my_tracker.dump_sub_domains - [nil,""]
	my_tracker = nil
	# Step 2 - update the sub-domain list
	unless subs.empty?
		#subs.map { |x| self.add(x) unless domain_known?(x) }
		self.bulk_add(subs,@max_parallel)
	end
	puts "Update discovered sub-domains into the store: #{@known_internet_sub_domains}"
	self.save!(file_domains=@file_sub_domains, domains=@known_internet_sub_domains)
rescue Exception => ee
	puts "Exception on method #{__method__}: #{ee}" if @verbose
	return nil
end