Module: ProxyRotator

Defined in:
lib/proxy_rotator.rb,
lib/proxy_rotator/rotator.rb,
lib/proxy_rotator/version.rb,
lib/proxy_rotator/configuration.rb

Defined Under Namespace

Classes: Configuration

Constant Summary collapse

VERSION = 
"0.1.2"

Class Method Summary collapse

Class Method Details

.base_url(config = {}) ⇒ Object 



51
52
53
54
55
56
 
# File 'lib/proxy_rotator/rotator.rb', line 51

def self.base_url(config={})
  url = ProxyRotator.configuration.base_url + "?apiKey=" + ProxyRotator.configuration.api_key
  url = url + "&" + config.to_a.map { |x| "#{x[0]}=#{x[1]}" }.join("&") unless config.empty?

  return url
end

.configuration ⇒ Object 



48
49
50
 
# File 'lib/proxy_rotator/configuration.rb', line 48

def self.configuration
  @configuration ||= initialize_configuration!
end

.configure {|configuration| ... } ⇒ Object

Yields:



52
53
54
 
# File 'lib/proxy_rotator/configuration.rb', line 52

def self.configure
  yield(configuration)
end

.describe_remote(config = {}) ⇒ Object 



68
69
70
71
 
# File 'lib/proxy_rotator/rotator.rb', line 68

def self.describe_remote(config={})
  url = self.base_url(config)
  self.get_proxyrotator_proxy(url)
end

.get_proxyrotator_proxy(url) ⇒ Object 



73
74
75
76
 
# File 'lib/proxy_rotator/rotator.rb', line 73

def self.get_proxyrotator_proxy(url)
  response = RestClient.get(url)
  JSON.parse(response.body)
end

.initialize_configuration! ⇒ Object 



56
57
58
 
# File 'lib/proxy_rotator/configuration.rb', line 56

def self.initialize_configuration!
  @configuration = Configuration.new
end

.load_file(filepath) ⇒ Object 



9
10
11
12
13
14
15
16
17
18
 
# File 'lib/proxy_rotator/rotator.rb', line 9

def self.load_file(filepath)
  @start_index=0
  File.open(filepath, "r") do |f|
    f.each_line do |line|
      unless validate_url(line).nil? || @proxies.include?(line)
        @proxies << line.strip
      end
    end
  end
end

.remote_first_then_rotate(check_first = false, config = {}) ⇒ Object 



85
86
87
88
89
90
 
# File 'lib/proxy_rotator/rotator.rb', line 85

def self.remote_first_then_rotate(check_first=false, config={})
  result = self.rotate_remote(config)
  result = self.rotate(check_first) unless !result.nil?

  result
end

.reset_rotate(check_first = false) ⇒ Object 



20
21
22
23
 
# File 'lib/proxy_rotator/rotator.rb', line 20

def self.reset_rotate(check_first=false)
  @start_index = 0
  self.rotate(check_first)
end

.rotate(check_first = false) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/proxy_rotator/rotator.rb', line 25

def self.rotate(check_first=false)
  return nil unless @proxies.count > 0
  proxy = @proxies[@start_index]
  @start_index=@start_index+1

  if check_first
    begin
      RestClient::Request.new(
          :method => :get,
          :url => ProxyRotator.configuration.default_test_url,
          :proxy => proxy,
          :timeout => ProxyRotator.configuration.default_timeout
      ).execute

      return proxy
    rescue Exception => e
      p "Proxy #{proxy} failed : #{e.message}"
      @failed_proxies << proxy unless @failed_proxies.include? proxy
      return nil unless @proxies.count > 1 || @proxies.count == @failed_proxies.count
      self.rotate(true)
    end
  else
    return proxy
  end
end

.rotate_first_then_remote(check_first = false, config = {}) ⇒ Object 



78
79
80
81
82
83
 
# File 'lib/proxy_rotator/rotator.rb', line 78

def self.rotate_first_then_remote(check_first=false, config={})
  result = self.rotate(check_first)
  result = self.rotate_remote(config) unless !result.nil?

  result
end

.rotate_remote(config = {}) ⇒ Object 



58
59
60
61
62
63
64
65
66
 
# File 'lib/proxy_rotator/rotator.rb', line 58

def self.rotate_remote(config={})
  url = self.base_url(config)

  json = self.get_proxyrotator_proxy(url)
  proxy = json["proxy"]
  return "http://#{proxy}" unless proxy.nil?

  return nil
end

.validate_url(url) ⇒ Object 



92
93
94
 
# File 'lib/proxy_rotator/rotator.rb', line 92

def self.validate_url(url)
  url =~ URI::regexp(%w(http https))
end