Module: Spidr

Extended by:
Settings::Proxy, Settings::Timeouts, Settings::UserAgent
Defined in:
lib/spidr/page.rb,
lib/spidr/agent.rb,
lib/spidr/proxy.rb,
lib/spidr/rules.rb,
lib/spidr/spidr.rb,
lib/spidr/version.rb,
lib/spidr/page/html.rb,
lib/spidr/auth_store.rb,
lib/spidr/cookie_jar.rb,
lib/spidr/agent/events.rb,
lib/spidr/agent/robots.rb,
lib/spidr/page/cookies.rb,
lib/spidr/agent/actions.rb,
lib/spidr/agent/filters.rb,
lib/spidr/session_cache.rb,
lib/spidr/settings/proxy.rb,
lib/spidr/auth_credential.rb,
lib/spidr/agent/sanitizers.rb,
lib/spidr/page/status_codes.rb,
lib/spidr/settings/timeouts.rb,
lib/spidr/page/content_types.rb,
lib/spidr/settings/user_agent.rb

Defined Under Namespace

Modules: Settings Classes: Agent, AuthCredential, AuthStore, CookieJar, Page, Proxy, Rules, SessionCache

Constant Summary collapse

VERSION =

Spidr version

'0.7.1'

Instance Attribute Summary

Attributes included from Settings::Timeouts

#continue_timeout, #keep_alive_timeout, #open_timeout, #read_timeout, #ssl_timeout

Attributes included from Settings::UserAgent

#user_agent

Class Method Summary collapse

Methods included from Settings::Proxy

disable_proxy!, proxy, proxy=

Class Method Details

.domain(name, **kwargs, &block) ⇒ Object

See Also:

Since:

  • 0.7.0



57
58
59
 
# File 'lib/spidr/spidr.rb', line 57

def self.domain(name,**kwargs,&block)
  Agent.domain(name,**kwargs,&block)
end

.host(name, **kwargs, &block) ⇒ Object

See Also:



48
49
50
 
# File 'lib/spidr/spidr.rb', line 48

def self.host(name,**kwargs,&block)
  Agent.host(name,**kwargs,&block)
end

.robots ⇒ Object

This method is abstract. 


71
72
 
# File 'lib/spidr/spidr.rb', line 71

def self.robots
end

.robots=(mode) ⇒ Boolean

Enables or disables robots.txt globally.

Parameters:

  • mode (Boolean)

Returns:

  • (Boolean)

Since:

  • 0.5.0



34
35
36
 
# File 'lib/spidr/spidr.rb', line 34

def self.robots=(mode)
  @robots = mode
end

.robots? ⇒ Boolean

Specifies whether robots.txt should be honored globally.

Returns:

  • (Boolean)

Since:

  • 0.5.0



20
21
22
23
 
# File 'lib/spidr/spidr.rb', line 20

def self.robots?
  @robots ||= false
  @robots
end

.site(url, **kwargs, &block) ⇒ Object

See Also:



64
65
66
 
# File 'lib/spidr/spidr.rb', line 64

def self.site(url,**kwargs,&block)
  Agent.site(url,**kwargs,&block)
end

.start_at(url, **kwargs, &block) ⇒ Object

See Also:



41
42
43
 
# File 'lib/spidr/spidr.rb', line 41

def self.start_at(url,**kwargs,&block)
  Agent.start_at(url,**kwargs,&block)
end