Class: Recluse::Tasks::Find

Inherits:
Task
  • Object
show all
Defined in:
lib/recluse/tasks/find.rb

Overview

Find links matching glob patterns, starting from the roots. Overrides (but does not overwrite) internal_only behavior to true.

Instance Attribute Summary

Attributes inherited from Task

#queue, #results

Instance Method Summary collapse

Methods inherited from Task

#add, #run

Constructor Details

#initialize(profile, globs: [], quiet: false, results: nil) ⇒ Find

Create new find task.



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/recluse/tasks/find.rb', line 15

def initialize(profile, globs: [], quiet: false, results: nil)
  super(profile, queue_options: { redirect: profile.redirect }, results: results)
  addr_roots = profile.roots.map { |root| Addressable::URI.parse(root.url) }
  progress = ProgressBar.create(total: nil, format: '|%B|') unless quiet
  @queue.run_if do |link|
    match = link.match? globs
    if match
      @results.add link.absolute, link.parent
      progress.log "[#{profile.name.colorize(mode: :bold)}][#{'found'.colorize(color: :green, mode: :bold)}] #{link.parent} => #{link.absolute}" unless quiet
    end
    next false unless link.run?(profile.blacklist, profile.whitelist)
    internal = link.internal?(addr_roots)
    next false unless internal
    next false if @results.parent?(link.absolute)
    if profile.scheme_squash
      alt = link.address
      alt.scheme = alt.scheme == 'http' ? 'https' : 'http'
      next false if @results.parent?(alt.to_s)
    end
    @results.add_parent link.absolute
    true
  end
  @queue.on_complete do |link, response|
    result = Recluse::Result.new response.code.to_s, response.errors
    if response.success
      if profile.redirect
        result_link = Recluse::Link.new(response.page.uri.to_s, link.parent)
        next unless result_link.internal?(addr_roots)
      end
      @queue.add(response.page.links.map { |new_link| Link.new(new_link.uri.to_s, link.absolute) }) unless (response.page.class == Mechanize::File) || (response.page.class == Mechanize::Image)
    end
    progress.increment unless quiet
    unless quiet || (result.error == false)
      progress.log "[#{profile.name.colorize(mode: :bold)}][#{result.code.colorize(color: result.color, mode: :bold)}] #{link.absolute}"
      progress.log "\a^ #{'Error'.colorize(mode: :bold, color: :red)}: #{result.error}"
    end
  end
end