Class: OzonParser::Agent

Inherits:
Object
  • Object
show all
Includes:
RoundRobin
Defined in:
lib/ozon_parser/agent.rb

Constant Summary collapse

AGENT_ALIASES = 
['Mac Safari', 'Mac Firefox', 'Linux Firefox', 'Linux Mozilla',
'Windows IE 8', 'Windows IE 9', 'Windows Mozilla']

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from RoundRobin

#next!

Constructor Details

#initialize(options = {}) ⇒ Agent

Returns a new instance of Agent.



10
11
12
13
14
15
16
17
 
# File 'lib/ozon_parser/agent.rb', line 10

def initialize(options = {})
  @agent = Mechanize.new { |agent|
    agent.open_timeout = OzonParser.config.open_timeout if OzonParser.config.open_timeout
    agent.read_timeout = OzonParser.config.read_timeout if OzonParser.config.read_timeout
  }
  @options = options
  @proxy = Proxy.new(options[:proxy]) if options[:proxy]
end

Instance Attribute Details

#current_proxy ⇒ Object (readonly)

Returns the value of attribute current_proxy.



8
9
10
 
# File 'lib/ozon_parser/agent.rb', line 8

def current_proxy
  @current_proxy
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



8
9
10
 
# File 'lib/ozon_parser/agent.rb', line 8

def options
  @options
end

Instance Method Details

#create_temp(name, body, ext) ⇒ Object 



59
60
61
62
63
64
65
 
# File 'lib/ozon_parser/agent.rb', line 59

def create_temp(name, body, ext)
  temp_file = Tempfile.new([name, ".#{ext}"])
  temp_file.binmode
  temp_file.write body.string
  temp_file.flush
  temp_file
end

#get(url, parameters = [], referer = nil, headers = {}) ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/ozon_parser/agent.rb', line 23

def get(url, parameters = [], referer = nil, headers = {})
  @agent.reset
  @agent.follow_redirect = false

  set_proxy
  set_user_agent

  begin
    @page = @agent.get(url, [], referer)
    raise if @page.code == '302'
  rescue => e
    p e
    freeze_current_proxy
    raise e
  end

  @page.body
end

#get_file(url) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/ozon_parser/agent.rb', line 42

def get_file(url)
  begin
    file = @agent.get url
  rescue => e
    freeze_current_proxy
    raise e
  end
  
  ext = file.filename.split('.').last
  if file.body_io.instance_of?(Tempfile)
    body = StringIO.new(file.body_io.read)
    create_temp(file.filename, body, ext)
  else
    create_temp(file.filename, file.body_io, ext)
  end
end

#history ⇒ Object 



19
20
21
 
# File 'lib/ozon_parser/agent.rb', line 19

def history
  @agent.history
end

#set_proxy ⇒ Object 



67
68
69
70
71
72
73
74
 
# File 'lib/ozon_parser/agent.rb', line 67

def set_proxy
  return if @options[:proxy] === false

  return unless @proxy || OzonParser.config.use_proxy?

  @current_proxy = @proxy || OzonParser.config.proxy_list.next
  @agent.set_proxy current_proxy.host, current_proxy.port
end

#set_user_agent ⇒ Object 



76
77
78
 
# File 'lib/ozon_parser/agent.rb', line 76

def set_user_agent
  @agent.user_agent_alias = next! AGENT_ALIASES
end