Class: Crawler

Inherits:
Object
  • Object
show all
Defined in:
lib/gsv_downloader/crawler.rb

Instance Method Summary collapse

Constructor Details

#initialize(area_validator, db) ⇒ Crawler

Returns a new instance of Crawler.



6
7
8
9
10
11
12
 
# File 'lib/gsv_downloader/crawler.rb', line 6

def initialize (area_validator, db)
  @metadata_downloader  = MetaDataDownloader.new
	@stats = Statistics.new
	@db = db
  @area_validator = area_validator
  @max = 200
end

Instance Method Details

#crawl(panoID) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/gsv_downloader/crawler.rb', line 21

def crawl(panoID)

  @db.mark_to_crawl(panoID)

  @metadata_downloader.download(panoID) do | response|

    json = extract_json(response)
    panoID = json["Location"]["panoId"]
    @stats.count

    unless @db.crawled?(panoID)
      @db.mark_as_crawled(panoID)

      # if inside the area
      if @area_validator.call(json)
        @db.add_pano(panoID, response) #TODO to save to save or whatever

        # for each valid and new link, crawl it
        extract_valid_links(json) do |link_id|
          crawl(link_id)
        end
      end
    end
  end
end

#start(pano_ids) ⇒ Object 



14
15
16
17
18
19
 
# File 'lib/gsv_downloader/crawler.rb', line 14

def start(pano_ids)
  pano_ids.each do |pano_id|
    crawl(pano_id)
  end
  @metadata_downloader.start()
end