Class: Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/scraper.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(cli) ⇒ Scraper

Returns a new instance of Scraper.



5
6
7
 
# File 'lib/scraper.rb', line 5

def initialize(cli)
  self.cli = cli
end

Instance Attribute Details

#cli ⇒ Object

Returns the value of attribute cli.



3
4
5
 
# File 'lib/scraper.rb', line 3

def cli
  @cli
end

#gfs_noko_html ⇒ Object

Returns the value of attribute gfs_noko_html.



3
4
5
 
# File 'lib/scraper.rb', line 3

def gfs_noko_html
  @gfs_noko_html
end

#stock ⇒ Object

Returns the value of attribute stock.



3
4
5
 
# File 'lib/scraper.rb', line 3

def stock
  @stock
end

Instance Method Details

#create_stock(symbol) ⇒ Object 



27
28
29
30
 
# File 'lib/scraper.rb', line 27

def create_stock(symbol)
  data = scrape_stock(symbol)
  Stock.new(data)
end

#gfs_url(symbol) ⇒ Object 



9
10
11
 
# File 'lib/scraper.rb', line 9

def gfs_url(symbol)
  "https://www.google.com/finance?q=" + symbol
end

#load_gfs(symbol, fixture_url = nil) ⇒ Object

Returns an array. Array is a stock if one was succesfully created and nil otherwise. Array indicates whether the symbol cooresponds to a mutual fund.



19
20
21
22
23
24
25
 
# File 'lib/scraper.rb', line 19

def load_gfs(symbol, fixture_url = nil)
  fixture_url.nil? ? load_gfs_noko_html(self.gfs_url(symbol)) : load_gfs_noko_html(fixture_url)
  return [nil, true] unless self.gfs_noko_html.text.match('\(MUTF:').nil?
  return [nil, false] if self.gfs_noko_html.css("span.pr").text.strip == "" # checks whether the page lists a price
# return [nil, false] if self.gfs_noko_html.css("div.fjfe-content").text.include?("- produced no matches.")
  [self.create_stock(symbol), false]
end

#load_gfs_noko_html(url) ⇒ Object 



13
14
15
 
# File 'lib/scraper.rb', line 13

def load_gfs_noko_html(url)
  self.gfs_noko_html = Nokogiri::HTML(open(url))
end

#nil_to_empty_str(data_hash) ⇒ Object

convert any nil values to empty strings to avoid exceptions



76
77
78
79
80
 
# File 'lib/scraper.rb', line 76

def nil_to_empty_str(data_hash)
  data_hash.each do |key, value|
    data_hash[key] = "" if data_hash[key].nil?
  end
end

#scrape_desc ⇒ Object 



67
68
69
70
71
72
73
 
# File 'lib/scraper.rb', line 67

def scrape_desc
  data = {}
  data[:sector] = self.gfs_noko_html.css("a#sector").text
  data[:industry] = self.gfs_noko_html.css("a#sector+a").text
  data[:summary] = self.gfs_noko_html.css("div.companySummary").text.gsub("More from Reuters »", "").strip
  nil_to_empty_str(data)
end

#scrape_quote ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/scraper.rb', line 47

def scrape_quote
  data = {}
  data[:price] = self.gfs_noko_html.css("span.pr").text.strip
  data[:change] = self.gfs_noko_html.css("div.nwp span.bld").text.split("\n")[0]
  begin
    data[:change_pct] = self.gfs_noko_html.css("div.nwp span.bld").text.split("\n")[1].gsub(/[)(]/, '')
  rescue NoMethodError
    data[:change_pct] = ""
  end
  data[:range] = self.gfs_noko_html.css("td[data-snapfield='range']+td").text.strip
  data[:range_yr] = self.gfs_noko_html.css("td[data-snapfield='range_52week']+td").text.strip
  data[:open] = self.gfs_noko_html.css("td[data-snapfield='open']+td").text.strip
  data[:volume] = self.gfs_noko_html.css("td[data-snapfield='vol_and_avg']+td").text.strip.split("/")[0]
  data[:volume_avg] = self.gfs_noko_html.css("td[data-snapfield='vol_and_avg']+td").text.strip.split("/")[1]
  data[:mkt_cap] = self.gfs_noko_html.css("td[data-snapfield='market_cap']+td").text.strip
  data[:pe_ttm] = self.gfs_noko_html.css("td[data-snapfield='pe_ratio']+td").text.strip
  data[:div_yld] = self.gfs_noko_html.css("td[data-snapfield='latest_dividend-dividend_yield']+td").text.strip.split("/")[1]
  nil_to_empty_str(data)
end

#scrape_stock(symbol) ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/scraper.rb', line 32

def scrape_stock(symbol)
  data = { stock: {} }
  data[:stock][:symbol] = symbol
  begin
    data[:stock][:name] = self.gfs_noko_html.css("div.g-first a").text.match('(?<=All news for )[\w,.)() ]*(?= »)')[0]
  rescue NoMethodError
    data[:stock][:name] = ""
  end
  data[:stock][:exchange] = self.gfs_noko_html.css("span.dis-large").text.split("\n")[0]
  data[:stock] = self.nil_to_empty_str(data[:stock])
  data[:quote] = self.scrape_quote
  data[:desc] = self.scrape_desc
  data
end