24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
# File 'lib/dumper/profiles/sankakucomplex.rb', line 24
def dump(url, path, from, to)
if url.include?('idol.sankakucomplex') || url.include?('chan.sankakucomplex')
to = 1 if to == -1
prefix = url.include?('idol.sankakucomplex') ? 'idol' : 'chan'
from.upto(to) { |page|
u = "#{url}&page=#{page}"
begin
op = open u
rescue Exception => e
sleep 1
retry
end
Nokogiri::HTML(op).xpath('//a/@href').each { |p|
next unless p.to_s.start_with? '/post/show'
errors = 0
@pool.process {
begin
img = Nokogiri::HTML(open("http://#{prefix}.sankakucomplex.com/#{p}")).at_xpath('//a[@itemprop="contentUrl"]/@href').to_s
Dumper.get path, img, { referer: u }
rescue Exception => e
if errors <= 3
sleep 3
errors += 1
retry
end
end
}
}
}
else
from -= 1
to -= 1 if to >= 1
[].tap { |urls|
Nokogiri::HTML(open(url)).xpath('//a[@target="_blank"]/@href').each { |u|
urls << u if u.to_s.start_with? 'http://images.sankakucomplex.com/wp-content/'
}
}[from..to].each { |p|
@pool.process {
Dumper.get path, p, { referer: url }
}
}
end
end
|