Class: QuickTicker::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/scraper.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(cli) ⇒ Scraper

Returns a new instance of Scraper.



7
8
9
10
 
# File 'lib/scraper.rb', line 7

def initialize(cli)
	self.cli = cli
	self.browser = Watir::Browser.new(:phantomjs)
end

Instance Attribute Details

#browser ⇒ Object

Returns the value of attribute browser.



5
6
7
 
# File 'lib/scraper.rb', line 5

def browser
  @browser
end

#cli ⇒ Object

Returns the value of attribute cli.



5
6
7
 
# File 'lib/scraper.rb', line 5

def cli
  @cli
end

#gfs_noko_html ⇒ Object

Returns the value of attribute gfs_noko_html.



5
6
7
 
# File 'lib/scraper.rb', line 5

def gfs_noko_html
  @gfs_noko_html
end

#stock ⇒ Object

Returns the value of attribute stock.



5
6
7
 
# File 'lib/scraper.rb', line 5

def stock
  @stock
end

Instance Method Details

#create_stock(symbol) ⇒ Object 



31
32
33
34
 
# File 'lib/scraper.rb', line 31

def create_stock(symbol)
	data = package_stock(symbol)
	QuickTicker::Stock.new(data)
end

#gfs_url(symbol) ⇒ Object 



12
13
14
 
# File 'lib/scraper.rb', line 12

def gfs_url(symbol)
	"https://www.google.com/finance?q=" + symbol
end

#load_gfs(symbol, fixture_url = nil) ⇒ Object

Returns an array. Array is a stock if one was succesfully created and nil otherwise. Array indicates whether the symbol cooresponds to a mutual fund.



23
24
25
26
27
28
29
 
# File 'lib/scraper.rb', line 23

def load_gfs(symbol, fixture_url = nil)
	fixture_url.nil? ? load_gfs_noko_html(self.gfs_url(symbol)) : load_gfs_noko_html(fixture_url)
	return [nil, true] unless self.gfs_noko_html.text.match('\(MUTF:').nil?
	return [nil, false] if self.gfs_noko_html.css("span.pr").text.strip == "" # checks whether the page lists a price
#	return [nil, false] if self.gfs_noko_html.css("div.fjfe-content").text.include?("- produced no matches.")
	[self.create_stock(symbol), false]
end

#load_gfs_noko_html(url) ⇒ Object 



16
17
18
19
 
# File 'lib/scraper.rb', line 16

def load_gfs_noko_html(url)
	browser.goto(url)
	self.gfs_noko_html = Nokogiri::HTML(browser.html)
end

#nil_to_empty_str(data_hash) ⇒ Object

convert any nil values to empty strings to avoid exceptions



106
107
108
109
110
 
# File 'lib/scraper.rb', line 106

def nil_to_empty_str(data_hash)
	data_hash.each do |key, value|
		data_hash[key] = "" if data_hash[key].nil?
	end
end

#package_stock(symbol) ⇒ Object 



36
37
38
39
40
41
42
43
 
# File 'lib/scraper.rb', line 36

def package_stock(symbol)
	data = {}
	data[:stock] = self.scrape_stock(symbol)
	data[:quote] = self.scrape_stock_quote
	data[:description] = self.scrape_stock_description
	data[:related_companies] = self.scrape_stock_related_companies
	data
end

#scrape_stock(symbol) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/scraper.rb', line 45

def scrape_stock(symbol)
	data = {}
	data[:symbol] = symbol
	begin
		data[:name] = self.gfs_noko_html.css("div.g-first a").text.match('(?<=All news for )[\w,.)() ]*(?= »)')[0]
	rescue NoMethodError
		data[:name] = ""
	end
	data[:exchange] = self.gfs_noko_html.css("span.dis-large").text.split("\n")[0]
	nil_to_empty_str(data)
end

#scrape_stock_description ⇒ Object 



77
78
79
80
81
82
83
 
# File 'lib/scraper.rb', line 77

def scrape_stock_description
	data = {}
	data[:sector] = self.gfs_noko_html.css("a#sector").text
	data[:industry] = self.gfs_noko_html.css("a#sector+a").text
	data[:summary] = self.gfs_noko_html.css("div.companySummary").text.gsub("More from Reuters »", "").strip
	nil_to_empty_str(data)
end

#scrape_stock_quote ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/scraper.rb', line 57

def scrape_stock_quote
	data = {}
	data[:price] = self.gfs_noko_html.css("span.pr").text.strip
	data[:change] = self.gfs_noko_html.css("div.nwp span.bld").text.split("\n")[0]
	begin
		data[:change_pct] = self.gfs_noko_html.css("div.nwp span.bld").text.split("\n")[1].gsub(/[)(]/, '').chomp("%")
	rescue NoMethodError
		data[:change_pct] = ""
	end
	data[:range] = self.gfs_noko_html.css("td[data-snapfield='range']+td").text.strip
	data[:range_yr] = self.gfs_noko_html.css("td[data-snapfield='range_52week']+td").text.strip
	data[:open] = self.gfs_noko_html.css("td[data-snapfield='open']+td").text.strip
	data[:volume] = self.gfs_noko_html.css("td[data-snapfield='vol_and_avg']+td").text.strip.split("/")[0]
	data[:volume_avg] = self.gfs_noko_html.css("td[data-snapfield='vol_and_avg']+td").text.strip.split("/")[1]
	data[:mkt_cap] = self.gfs_noko_html.css("td[data-snapfield='market_cap']+td").text.strip
	data[:pe_ttm] = self.gfs_noko_html.css("td[data-snapfield='pe_ratio']+td").text.strip
	data[:div_yld] = self.gfs_noko_html.css("td[data-snapfield='latest_dividend-dividend_yield']+td").text.strip.split("/")[1]
	nil_to_empty_str(data)
end

#scrape_stock_related_companies ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/scraper.rb', line 85

def scrape_stock_related_companies 
	data = []
	for i in (0..10) do
		begin
			data << {
				symbol: self.gfs_noko_html.css("table#cc-table td.ctsymbol")[i].text,
				price: self.gfs_noko_html.css("table#cc-table td.ctsymbol+td+td")[i].text,
				change: self.gfs_noko_html.css("table#cc-table td.ctsymbol+td+td+td")[i].text,
				change_pct: self.gfs_noko_html.css("table#cc-table td.ctsymbol+td+td+td+td")[i].text.chomp("%"),
				mkt_cap: self.gfs_noko_html.css("table#cc-table td.ctsymbol+td+td+td+td+td+td")[i].text
			}
		rescue NoMethodError
			i = 11
		end
	end
	data.collect do |related_company_hash|
		nil_to_empty_str(related_company_hash)
	end
end