9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
# File 'lib/vnews/opml.rb', line 9
def self.import(opml)
sqlclient = Vnews.sql_client
doc = Nokogiri::XML.parse(opml)
feeds = []
doc.xpath('/opml/body/outline').each_slice(CONCURRENCY) do |xs|
pool = ThreadPool.new(10)
puts "Using thread pool size of 10"
xs.each do |n|
pool.process do
if n.attributes['xmlUrl']
feeds << Vnews::Feed.fetch_feed(n.attributes['xmlUrl'].to_s)
else
folder = n.attributes["title"].to_s
$stderr.print "Found folder: #{folder}\n"
n.xpath("outline[@xmlUrl]").each do |m|
feeds << Vnews::Feed.fetch_feed(m.attributes['xmlUrl'].to_s, folder)
end
end
end
end
pool.join
end
$stderr.puts "Making database records"
feeds.each do |x|
feed_url, f, folder = *x
folder ||= "Misc"
if f.nil?
$stderr.print "\nNo feed found for #{feed_url}\n"
else
Vnews::Feed.save_feed(feed_url, f, folder)
end
end
$stderr.puts "\nDone."
end
|