Class: Strawman::ProxyList

Inherits:
Object
  • Object
show all
Defined in:
lib/strawman/proxy_list.rb

Overview

Represents a group of proxy sources

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(verification_url) ⇒ ProxyList

verification_url

The url to use to verify that the proxy is valid. All it needs to do is return an HTTP status of 200.



12
13
14
15
16
 
# File 'lib/strawman/proxy_list.rb', line 12

def initialize(verification_url)
  @proxies = []
  @dead_proxies = []
  @verification_url = verification_url
end

Instance Attribute Details

#proxies ⇒ Object

Returns the value of attribute proxies.



6
7
8
 
# File 'lib/strawman/proxy_list.rb', line 6

def proxies
  @proxies
end

Instance Method Details

#add_sources(sources) ⇒ Object

Takes a list of sources and returns a deferrable which will complete once all sources have been fetched.



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/strawman/proxy_list.rb', line 22

def add_sources(sources)
  sources_ready = EventMachine::MultiRequest.new

  sources.each do |source|
    sources_ready.add(source)
  end

  sources_ready.callback do
    sources.each do |source|
      source.proxies.each do |proxy|
        @proxies << proxy unless @proxies.include? proxy
      end
    end
  end

  sources_ready
end

#load(filepath) ⇒ Object

Loads all proxies from the given file



81
82
83
 
# File 'lib/strawman/proxy_list.rb', line 81

def load(filepath)
  @proxies = YAML.load(File.read(filepath))
end

#proxy(deferrable = nil) ⇒ Object

Selects a random proxy from the list of available proxies and verifies it. If it isn’t valid it keeps trying all available proxies before returning nil.



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/strawman/proxy_list.rb', line 45

def proxy(deferrable=nil)
  deferrable ||= EventMachine::DefaultDeferrable.new

  proxy = @proxies.choice
  deferrable.fail unless proxy

  proxy_response = proxy.validate(@verification_url)
  proxy_response.callback do
    if proxy.valid?
      deferrable.succeed(proxy)
    else
      self.proxy(deferrable)
    end
  end

  proxy_response.errback do
    @proxies.delete(proxy)
    @dead_proxies.push(proxy)
  end

  deferrable
end

#save(filepath) ⇒ Object

Saves all proxies that were loaded into this instance, including proxies with errors.



72
73
74
75
76
 
# File 'lib/strawman/proxy_list.rb', line 72

def save(filepath)
  File.open(filepath, "w") do |f|
    f.write((@proxies + @dead_proxies).to_yaml)
  end
end