Class: Miteru::Feeds

Inherits:
Object
  • Object
show all
Defined in:
lib/miteru/feeds.rb,
lib/miteru/feeds/feed.rb,
lib/miteru/feeds/urlscan.rb,
lib/miteru/feeds/ayashige.rb,
lib/miteru/feeds/phishstats.rb,
lib/miteru/feeds/urlscan_pro.rb,
lib/miteru/feeds/phishing_database.rb

Defined Under Namespace

Classes: Ayashige, Feed, PhishStats, PhishingDatabase, UrlScan, UrlScanPro

Constant Summary collapse

IGNORE_EXTENSIONS = 
%w[.htm .html .php .asp .aspx .exe .txt].freeze

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Feeds

Returns a new instance of Feeds.



14
15
16
17
18
19
20
21
22
 
# File 'lib/miteru/feeds.rb', line 14

def initialize
  @feeds = [
    PhishingDatabase.new,
    PhishStats.new,
    UrlScan.new(Miteru.configuration.size),
    UrlScanPro.new,
    Miteru.configuration.ayashige? ? Ayashige.new : nil
  ].compact
end

Instance Method Details

#breakdown(url) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/miteru/feeds.rb', line 38

def breakdown(url)
  begin
    uri = URI.parse(url)
  rescue URI::InvalidURIError => _e
    return []
  end

  base = "#{uri.scheme}://#{uri.hostname}"
  return [base] unless directory_traveling?

  segments = uri.path.split("/")
  return [base] if segments.length.zero?

  urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }

  urls.reject do |breakdowned_url|
    # Reject a url which ends with specific extension names
    invalid_extension? breakdowned_url
  end
end

#directory_traveling? ⇒ Boolean

Returns:

  • (Boolean) 


24
25
26
 
# File 'lib/miteru/feeds.rb', line 24

def directory_traveling?
  Miteru.configuration.directory_traveling?
end

#invalid_extension?(url) ⇒ Boolean

Returns:

  • (Boolean) 


59
60
61
 
# File 'lib/miteru/feeds.rb', line 59

def invalid_extension?(url)
  IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
end

#suspicious_urls ⇒ Object 



28
29
30
31
32
33
34
35
36
 
# File 'lib/miteru/feeds.rb', line 28

def suspicious_urls
  @suspicious_urls ||= [].tap do |arr|
    urls = @feeds.map do |feed|
      feed.urls.select { |url| url.start_with?("http://", "https://") }
    end.flatten.uniq

    urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
  end
end