Class: Miteru::Orchestrator

Inherits:
Service
  • Object
show all
Defined in:
lib/miteru/orchestrator.rb

Instance Method Summary collapse

Methods inherited from Service

call, #result, result

Instance Method Details

#call ⇒ Object 



5
6
7
8
9
10
11
12
13
 
# File 'lib/miteru/orchestrator.rb', line 5

def call
  logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?

  if sidekiq?
    sidekiq_call
  else
    parallel_call
  end
end

#feeds ⇒ Array<Miteru::Feeds::Base>

Returns:



67
68
69
 
# File 'lib/miteru/orchestrator.rb', line 67

def feeds
  @feeds ||= Miteru.feeds.map(&:new)
end

#non_cached_websites ⇒ Array<Miteru::Website>

Returns:



54
55
56
57
58
59
60
61
62
 
# File 'lib/miteru/orchestrator.rb', line 54

def non_cached_websites
  @non_cached_websites ||= [].tap do |out|
    out << if cache?
      websites.reject { |website| cache.cached?(website.url) }
    else
      websites
    end
  end.flatten.uniq(&:url)
end

#parallel_call ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/miteru/orchestrator.rb', line 20

def parallel_call
  logger.info("Use #{threads} thread(s).") if verbose?
  Parallel.each(non_cached_websites, in_threads: threads) do |website|
    logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
    result = Crawler.result(website)
    if result.success?
      logger.info("Crawler:#{website.truncated_url} succeeded.")
    else
      logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
    end
  end
end

#sidekiq_call ⇒ Object 



15
16
17
18
 
# File 'lib/miteru/orchestrator.rb', line 15

def sidekiq_call
  array_of_args = non_cached_websites.map { |website| [website.url, website.source] }
  Jobs::CrawleJob.perform_bulk(array_of_args, batch_size: Miteru.config.sidekiq_batch_size)
end

#websites ⇒ Array<Miteru::Website>

Returns:



36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/miteru/orchestrator.rb', line 36

def websites
  @websites ||= [].tap do |out|
    feeds.each do |feed|
      result = feed.result
      if result.success?
        websites = result.value!
        logger.info("Feed:#{feed.source} has #{websites.length} websites.") if verbose?
        out << websites
      else
        logger.warn("Feed:#{feed.source} failed - #{result.failure}")
      end
    end
  end.flatten.uniq(&:url)
end