Class: ProxyList

Inherits:
Object
  • Object
show all
Defined in:
lib/sync_qiniu/getproxy.rb

Overview

Fetch proxy from: proxy.com.ru/

Instance Method Summary collapse

Constructor Details

#initialize ⇒ ProxyList

Returns a new instance of ProxyList.



9
10
11
12
13
14
15
 
# File 'lib/sync_qiniu/getproxy.rb', line 9

def initialize
  @proxylists = []
  @urls = []
  (1..2).each do |i|
    @urls << "http://proxy.com.ru/list_#{i}.html"
  end
end

Instance Method Details

#fetch_list ⇒ Object Also known as: start 



17
18
19
20
21
22
23
24
25
 
# File 'lib/sync_qiniu/getproxy.rb', line 17

def fetch_list
  @urls.each do |url|
puts proxys_in_url(url).class
    @proxylists += proxys_in_url(url)
  end
  @proxylists.uniq!
  puts @proxylists
  puts "Fetched proxys #{@proxylists.size}"
end

#get_proxylists ⇒ Object 



49
50
51
 
# File 'lib/sync_qiniu/getproxy.rb', line 49

def get_proxylists
 return @proxylists
end

#get_urls ⇒ Object 



45
46
47
 
# File 'lib/sync_qiniu/getproxy.rb', line 45

def get_urls
 return @urls
end

#proxys_in_url(url) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/sync_qiniu/getproxy.rb', line 29

def proxys_in_url(url)
  proxys = []
  agent = Mechanize.new
  page = agent.get(url)
	list = page.search("body font table tr td:last font table tr")
	list[1..-1].each do |tr|
		ip = tr.search("td")[1].text
		port = tr.search("td")[2].text
		proxy = { ip: ip, port: port }
		proxy = verify_proxy(proxy) # 比较费时间
		proxys << proxy unless proxy == false
		puts "add #{proxy} to proxys array..."
	end 
	return proxys
end

#save_all_proxylists ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/sync_qiniu/getproxy.rb', line 65

def save_all_proxylists
	pls = @proxylists.to_s
	puts pls
	if File.exist?("allproxylists.txt")
 File.rename("allproxylists.txt","allproxylists.txt.bak")
	end
	f = File.new("allproxylists.txt","w+")
	f.write(pls)
	f.close
	puts "Save all proxylists successfully."
end

#save_proxylists ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/sync_qiniu/getproxy.rb', line 53

def save_proxylists
	pls = @proxylists.to_s
	puts pls
	if File.exist?("proxylists.txt")
 File.rename("proxylists.txt","proxylists.txt.bak")
	end
	f = File.new("proxylists.txt","w+")
	f.write(pls)
	f.close
	puts "Save proxylists successfully."
end

#verify_proxy(proxy) ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/sync_qiniu/getproxy.rb', line 77

def verify_proxy(proxy)
	ip, port = proxy[:ip], proxy[:port]
	testagent = Mechanize.new
	testagent.set_proxy ip, port
	testagent.read_timeout = 10
	begin
page = testagent.get("http://www.baidu.com")	
if page.title == "百度一下，你就知道"
  return proxy
puts 'That a good proxy'
end
	rescue => e
puts e.message
puts 'That a bad proxy'
proxy = nil
return
	end
end