18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
|
# File 'lib/pluto/update/site_fetcher.rb', line 18
def fetch( site_rec )
site_url = site_rec.url
site_key = site_rec.key
@worker.use_cache = true
@worker.cache[ site_url ] = {
'etag' => site_rec.http_etag,
'last-modified' => site_rec.http_last_modified
}
response = @worker.get( site_url )
@worker.use_cache = false
site_fetched = Time.now
site_text = response.body.to_s
site_text = site_text.force_encoding( Encoding::UTF_8 )
if response.code == '304'
puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
puts "no change; request returns not modified (304); skipping parsing site config"
return nil
elsif response.code != '200'
puts "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
site_attribs = {
http_code: response.code.to_i,
http_server: response.[ 'server' ],
http_etag: nil,
http_last_modified: nil,
fetched: site_fetched
}
site_rec.update!( site_attribs )
Activity.create!( text: "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}" )
return nil else
end
puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
site_attribs = {
http_code: response.code.to_i,
http_server: response.[ 'server' ],
http_etag: response.[ 'etag' ],
http_last_modified: response.[ 'last-modified' ], fetched: site_fetched
}
puts "http header - server: #{response.['server']} - #{response.['server'].class.name}"
puts "http header - etag: #{response.['etag']} - #{response.['etag'].class.name}"
puts "http header - last-modified: #{response.['last-modified']} - #{response.['last-modified'].class.name}"
site_rec.update!( site_attribs )
site_text
end
|