Class: Miteru::Orchestrator

Inherits:
Service
  • Object
show all
Defined in:
lib/miteru/orchestrator.rb

Instance Method Summary collapse

Methods inherited from Service

call, #result, result

Instance Method Details

#call ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/miteru/orchestrator.rb', line 5

def call
  Miteru.logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?

  if Miteru.sidekiq?
    non_cached_websites.each do |website|
      Jobs::CrawleJob.perform_async(website.url, website.source)
      Miteru.logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
    end
  else
    Miteru.logger.info("Use #{threads} thread(s).") if verbose?
    Parallel.each(non_cached_websites, in_threads: threads) do |website|
      Miteru.logger.info("Website:#{website.truncated_url} crawling started.") if verbose?

      result = Crawler.result(website)
      if result.success?
        Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
      else
        Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
      end
    end
  end
end

#feeds ⇒ Array<Miteru::Feeds::Base>

Returns:



55
56
57
 
# File 'lib/miteru/orchestrator.rb', line 55

def feeds
  Miteru.feeds.map(&:new)
end

#non_cached_websites ⇒ Object 



46
47
48
49
50
 
# File 'lib/miteru/orchestrator.rb', line 46

def non_cached_websites
  return websites unless cache?

  websites.reject { |website| cache.cached?(website.url) }
end

#websites ⇒ Array<Miteru::Website>

Returns:



31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/miteru/orchestrator.rb', line 31

def websites
  @websites ||= [].tap do |out|
    feeds.each do |feed|
      result = feed.result
      if result.success?
        websites = result.value!
        Miteru.logger.info("Feed:#{feed.source} has #{websites.length} websites.") if verbose?
        out << websites
      else
        Miteru.logger.warn("Feed:#{feed.source} failed - #{result.failure}")
      end
    end
  end.flatten.uniq(&:url)
end