Class: Toogle::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/toogle/scraper.rb

Instance Method Summary collapse

Instance Method Details

#get_page ⇒ Object 



3
4
5
 
# File 'lib/toogle/scraper.rb', line 3

def get_page
   Watir::Browser.new :chrome, headless: true, options: {options: {detach: true}}
end

#query_data(users_search, chosen_pages) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/toogle/scraper.rb', line 7

def query_data(users_search, chosen_pages)
    this = get_page
    total_pages = chosen_pages
    this.goto("http://google.com")
    this.input(name: 'q').send_keys(users_search, :return)

    results = {:titles => [], :links => [], :descriptions => []}

    next_page = 1
    while next_page <= total_pages.to_i
        this.elements(css: "div.srg div.rc").each do |ele|
            results[:titles] << ele.element(css: "h3.LC20lb").text
            results[:links] << ele.element(css: "div.r a:nth-of-type(1)").attribute('href')
            results[:descriptions] << ele.element(css: "div.s span.st").text
        end
        this.element(css: "table tbody tr td a#pnnext").click
        next_page += 1
    end
    results
end