Module: AngelList::Tools
- Defined in:
- lib/angel_list/tools.rb
Class Method Summary collapse
- .curb(url) ⇒ Object
- .doc(html) ⇒ Object
- .download(url) ⇒ Object
- .encode(data) ⇒ Object
- .hashify(source) ⇒ Object
- .last_effective_url(url) ⇒ Object
- .parse(data) ⇒ Object
- .post(url, options) ⇒ Object
- .sleep_time ⇒ Object
- .user_agent ⇒ Object
Class Method Details
.curb(url) ⇒ Object
24 25 26 27 28 29 30 31 32 33 34 |
# File 'lib/angel_list/tools.rb', line 24 def self.curb(url) c = Curl::Easy.new(url) do |curl| curl.headers["User-Agent"] = user_agent # curl.verbose = true curl.timeout = 120 curl.follow_location = true curl.max_redirects = 20 end c.perform c end |
.doc(html) ⇒ Object
71 72 73 |
# File 'lib/angel_list/tools.rb', line 71 def self.doc(html) Hpricot(html) end |
.download(url) ⇒ Object
10 11 12 13 |
# File 'lib/angel_list/tools.rb', line 10 def self.download(url) c = self.curb(url) c.body_str end |
.encode(data) ⇒ Object
40 41 42 |
# File 'lib/angel_list/tools.rb', line 40 def self.encode(data) Yajl::Encoder.encode(data) end |
.hashify(source) ⇒ Object
51 52 53 |
# File 'lib/angel_list/tools.rb', line 51 def self.hashify(source) Digest::SHA1.hexdigest(source) end |
.last_effective_url(url) ⇒ Object
19 20 21 22 |
# File 'lib/angel_list/tools.rb', line 19 def self.last_effective_url(url) c = self.curb(url) c.last_effective_url end |
.parse(data) ⇒ Object
36 37 38 |
# File 'lib/angel_list/tools.rb', line 36 def self.parse(data) Hashie::Mash.new Yajl::Parser.new.parse(data) end |
.post(url, options) ⇒ Object
15 16 17 |
# File 'lib/angel_list/tools.rb', line 15 def self.post(url, ) Curl.post(url, ) end |
.sleep_time ⇒ Object
45 46 47 48 49 |
# File 'lib/angel_list/tools.rb', line 45 def self.sleep_time array_of_numbers = [1.2, 2.35, 2.45, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.1, 1.4] random_number = array_of_numbers[rand(array_of_numbers.size)] random_number end |
.user_agent ⇒ Object
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
# File 'lib/angel_list/tools.rb', line 55 def self.user_agent array_of_agents = ["Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja; rv:1.9.2a1pre) Gecko/20090403 Firefox/3.6a1pre", "Opera/9.80 (Windows NT 6.0; U; fi) Presto/2.2.0 Version/10.00", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; zh-CN) AppleWebKit/530.19.2 (KHTML, like Gecko) Version/4.0.2 Safari/530.19", "ia_archiver (+http://www.alexa.com/site/help/webmasters; [email protected])", "Baiduspider+(+http://www.baidu.com/search/spider.htm)", 'msnbot-webmaster/1.0 (+http://search.msn.com/msnbot.htm)', 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; ar) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.18', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET CLR 1.1.4322; Tablet PC 2.0; OfficeLiveConnector.1.3; OfficeLivePatch.1.3; MS-RTC LM 8; InfoPath.3)', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506; .NET CLR 1.1.4322; InfoPath.2; .NET CLR 3.5.21022)'] random_number = array_of_agents[rand(array_of_agents.size)] random_number end |