Module: Cosmicrawler

Defined in:
lib/cosmicrawler.rb,
lib/cosmicrawler/em.rb,
lib/cosmicrawler/version.rb,
lib/cosmicrawler/em/crawler.rb,
lib/cosmicrawler/em/http_crawler.rb

Defined Under Namespace

Modules: Em

Constant Summary collapse

DEFAULT_CONCURRENCY = 
8
VERSION = 
"0.0.1"

Class Method Summary collapse

Class Method Details

.each(collection, concurrency = DEFAULT_CONCURRENCY, &block) ⇒ Object 



9
10
11
12
 
# File 'lib/cosmicrawler.rb', line 9

def each collection, concurrency = DEFAULT_CONCURRENCY, &block
  c = Em::Crawler.new concurrency
  c.each collection, &block
end

.http_crawl(urls, concurrency = DEFAULT_CONCURRENCY, &block) ⇒ Object 



14
15
16
17
 
# File 'lib/cosmicrawler.rb', line 14

def http_crawl urls, concurrency = DEFAULT_CONCURRENCY, &block
  c = Em::HttpCrawler.new concurrency
  c.crawl urls, &block
end