Class: Miteru::Feeds

Inherits:
Object
  • Object
show all
Defined in:
lib/miteru/feeds.rb,
lib/miteru/feeds/feed.rb,
lib/miteru/feeds/urlscan.rb,
lib/miteru/feeds/ayashige.rb,
lib/miteru/feeds/urlscan_pro.rb

Defined Under Namespace

Classes: Ayashige, Feed, UrlScan, UrlScanPro

Constant Summary collapse

IGNORE_EXTENSIONS = 
%w(.htm .html .php .asp .aspx .exe .txt).freeze

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Feeds

Returns a new instance of Feeds.



12
13
14
15
16
17
18
 
# File 'lib/miteru/feeds.rb', line 12

def initialize
  @feeds = [
    UrlScan.new(Miteru.configuration.size),
    UrlScanPro.new,
    Miteru.configuration.ayashige? ? Ayashige.new : nil
  ].compact
end

Instance Method Details

#breakdown(url) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/miteru/feeds.rb', line 34

def breakdown(url)
  begin
    uri = URI.parse(url)
  rescue URI::InvalidURIError => _e
    return []
  end

  base = "#{uri.scheme}://#{uri.hostname}"
  return [base] unless directory_traveling?

  segments = uri.path.split("/")
  return [base] if segments.length.zero?

  urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
  urls.reject do |breakdowned_url|
    # Reject a url which ends with specific extension names
    invalid_extension? breakdowned_url
  end
end

#directory_traveling? ⇒ Boolean

Returns:

  • (Boolean) 


20
21
22
 
# File 'lib/miteru/feeds.rb', line 20

def directory_traveling?
  Miteru.configuration.directory_traveling?
end

#invalid_extension?(url) ⇒ Boolean

Returns:

  • (Boolean) 


54
55
56
 
# File 'lib/miteru/feeds.rb', line 54

def invalid_extension?(url)
  IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
end

#suspicious_urls ⇒ Object 



24
25
26
27
28
29
30
31
32
 
# File 'lib/miteru/feeds.rb', line 24

def suspicious_urls
  @suspicious_urls ||= [].tap do |arr|
    urls = @feeds.map do |feed|
      feed.urls.select { |url| url.start_with?("http://", "https://") }
    end.flatten.uniq

    urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
  end
end