Class: CachedWeb
- Inherits:
-
Object
- Object
- CachedWeb
- Defined in:
- lib/cached_web.rb,
lib/cached_web/version.rb
Constant Summary collapse
- VERSION =
"0.0.1"
- @@min_wait_times =
{}
- @@last_request_times =
{}
Class Method Summary collapse
Instance Method Summary collapse
- #escape_key(key) ⇒ Object
- #fetch_cache(params) ⇒ Object
- #get(params) ⇒ Object
- #get_cache(params) ⇒ Object
- #post(url, params) ⇒ Object
- #set_cache(params) ⇒ Object
Class Method Details
.fetch_cache(params, &block) ⇒ Object
54 55 56 |
# File 'lib/cached_web.rb', line 54 def self.fetch_cache(params, &block) CachedWeb.new.fetch_cache(params, &block) end |
.get(params) ⇒ Object
16 17 18 |
# File 'lib/cached_web.rb', line 16 def self.get(params) CachedWeb.new.get(params) end |
Instance Method Details
#escape_key(key) ⇒ Object
8 9 10 11 12 13 14 |
# File 'lib/cached_web.rb', line 8 def escape_key(key) if key.size > 140 Digest::MD5.hexdigest(key) else key.gsub(/[^a-z0-9]/i,"_") end end |
#fetch_cache(params) ⇒ Object
58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
# File 'lib/cached_web.rb', line 58 def fetch_cache(params) ret_val = nil begin ret_val = get_cache(params) rescue #puts "didn't find data in cache: #{$!}" end unless ret_val ret_val = yield self set_cache(params.merge(:page=>ret_val)) end return ret_val end |
#get(params) ⇒ Object
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
# File 'lib/cached_web.rb', line 20 def get(params) url = params[:url].gsub(" ","%20").gsub("%off", "%25off") page, redirect_url, headers = get_cache(:url=>url, :expires_in=>params[:expires_in], :details=>true) rescue [nil, nil, nil] if params[:cache_only] return [nil, nil, nil] if page.blank? return [page, redirect_url, headers] end if page.blank? uri = URI.parse(url) domain = uri.host.downcase #puts "Not in cache - getting: #{params[:url]}" # Don't hit a site more than once per 5 seconds time_to_wait = (@@min_wait_times[domain] || 5) - (Time.now - (@@last_request_times[domain] || Time.at(0))) if time_to_wait > 0 puts "Waiting for #{time_to_wait} on domain #{domain}" sleep time_to_wait end @agent = Mechanize.new unless @agent page = @agent.get(url) @@last_request_times[uri.host] content = page.body redirect_url = page.uri.to_s headers = page.header set_cache(:url=>url, :redirect_url=>redirect_url, :headers=>headers, :content=>content) [content, redirect_url, headers] else [page, redirect_url, headers] end end |
#get_cache(params) ⇒ Object
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
# File 'lib/cached_web.rb', line 116 def get_cache(params) key = escape_key(params[:url].present? ? params[:url] : params[:key]) details = params[:details] expires_in = params[:expires_in] = nil path = "/tmp/cachedweb/#{key}" = File.ctime(path) if expires_in and .nil? #puts "There is no timestamp, force a get of this URL so we can add a timestamp" throw "There is no timestamp, force a get of this URL so we can add a timestamp" end if expires_in and and < (Time.now - expires_in) #puts "Cache is older than desired, saved on #{timestamp}" throw "Cache is older than desired, saved on #{}" end h = YAML::load_file(path) content = h[:content] redirect_url = h[:redirect_url] if details [content, redirect_url, nil] else content end end |
#post(url, params) ⇒ Object
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 |
# File 'lib/cached_web.rb', line 73 def post(url, params) puts "Not in cache - POSTing to: #{url}" @agent = Mechanize.new unless @agent begin page = @agent.post(url, params) uri = URI.parse(url) @@last_request_times[uri.host] content = page.body redirect_url = page.uri.to_s headers = page.header set_cache(:url=>url, :redirect_url=>redirect_url, :headers=>headers, :page=>content) [content, redirect_url, headers] rescue Exception => e puts e puts e.backtrace Toadhopper(AIRBRAKE_KEY).post!(e) ["", url, {}] end end |
#set_cache(params) ⇒ Object
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
# File 'lib/cached_web.rb', line 93 def set_cache(params) key = escape_key(params[:url].present? ? params[:url] : params[:key]) content = params[:content] redirect_url = params[:redirect_url] h = {:content => content, :redirect_url=>redirect_url} path = "/tmp/cachedweb/#{key}" #puts "Saving file to #{path}" begin File.open(path, 'w') do |out| YAML.dump(h, out) end rescue Exception `mkdir -p /tmp/cachedweb` File.open(path, 'w') do |out| YAML.dump(h, out) end end true end |