Class: Miteru::Feeds

Inherits:
Object
  • Object
show all
Defined in:
lib/miteru/feeds.rb,
lib/miteru/feeds/feed.rb,
lib/miteru/feeds/urlscan.rb,
lib/miteru/feeds/ayashige.rb,
lib/miteru/feeds/phishstats.rb,
lib/miteru/feeds/urlscan_pro.rb,
lib/miteru/feeds/phishing_database.rb

Defined Under Namespace

Classes: Ayashige, Feed, PhishStats, PhishingDatabase, UrlScan, UrlScanPro

Constant Summary collapse

IGNORE_EXTENSIONS = 
%w(.htm .html .php .asp .aspx .exe .txt).freeze
VALID_EXTENSIONS = 
[".zip", ".rar", ".7z", ".tar", ".gz"].freeze

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Feeds

Returns a new instance of Feeds.



15
16
17
18
19
20
21
22
23
 
# File 'lib/miteru/feeds.rb', line 15

def initialize
  @feeds = [
    PhishingDatabase.new,
    PhishStats.new,
    UrlScan.new(Miteru.configuration.size),
    UrlScanPro.new,
    Miteru.configuration.ayashige? ? Ayashige.new : nil
  ].compact
end

Instance Method Details

#breakdown(url) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/miteru/feeds.rb', line 39

def breakdown(url)
  begin
    uri = URI.parse(url)
  rescue URI::InvalidURIError => _e
    return []
  end

  base = "#{uri.scheme}://#{uri.hostname}"
  return [base] unless directory_traveling?

  segments = uri.path.split("/")
  return [base] if segments.length.zero?

  urls = (0...segments.length).map do |idx|
    breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
    breakdown = [breakdowned_url]
    if idx > 0 && idx < segments.length - 1
      VALID_EXTENSIONS.each do |ext|
        breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
      end
    end
    breakdown
  end.flatten

  urls.reject do |breakdowned_url|
    # Reject a url which ends with specific extension names
    invalid_extension? breakdowned_url
  end
end

#directory_traveling? ⇒ Boolean

Returns:

  • (Boolean) 


25
26
27
 
# File 'lib/miteru/feeds.rb', line 25

def directory_traveling?
  Miteru.configuration.directory_traveling?
end

#invalid_extension?(url) ⇒ Boolean

Returns:

  • (Boolean) 


69
70
71
 
# File 'lib/miteru/feeds.rb', line 69

def invalid_extension?(url)
  IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
end

#suspicious_urls ⇒ Object 



29
30
31
32
33
34
35
36
37
 
# File 'lib/miteru/feeds.rb', line 29

def suspicious_urls
  @suspicious_urls ||= [].tap do |arr|
    urls = @feeds.map do |feed|
      feed.urls.select { |url| url.start_with?("http://", "https://") }
    end.flatten.uniq

    urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
  end
end