Class: RecipeScrapers::Configuration

Inherits:
Object
  • Object
show all
Defined in:
lib/recipe_scrapers/configuration.rb

Overview

The settings for fetching pages and parsing fields. Read them through config and change them with configure. Changing a fetch setting rebuilds the connection on its next use, unless an application assigned its own with #connection=.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Configuration

Returns a new instance of Configuration.



74
75
76
77
78
79
 
# File 'lib/recipe_scrapers/configuration.rb', line 74

def initialize
  DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
  @parsers = Parsers::Chain.defaults
  @connection = nil
  @connection_assigned = false
end

Instance Attribute Details

#adapter ⇒ Symbol, Class

Returns the Faraday adapter, :recipe_scrapers_net_http (Http::Adapter) by default.

Returns:

  • (Symbol, Class) —

    the Faraday adapter, :recipe_scrapers_net_http (Http::Adapter) by default



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#allow_private_addresses ⇒ Boolean

Returns fetch hosts that resolve to private or loopback addresses, false by default.

Returns:

  • (Boolean) —

    fetch hosts that resolve to private or loopback addresses, false by default



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#error_tracker ⇒ #call

Returns gets the error of a failing parser, ErrorTracker by default.

Returns:

  • (#call) —

    gets the error of a failing parser, ErrorTracker by default



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#max_body_bytes ⇒ Integer

Returns the largest body accepted before ResponseTooLarge, 5 MiB by default.

Returns:

  • (Integer) —

    the largest body accepted before ResponseTooLarge, 5 MiB by default



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#max_redirects ⇒ Integer

Returns redirects to follow before TooManyRedirects, 3 by default.

Returns:



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#open_timeout ⇒ Integer

Returns seconds to wait for the connection to open, 5 by default.

Returns:

  • (Integer) —

    seconds to wait for the connection to open, 5 by default



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#parsers ⇒ Hash{Symbol => Array<#call>} (readonly)

The parsers of every field that takes them, tried in order. A parser answers call(text, language:) with a hash of the model's attributes, or nil for text it cannot read.

Examples:

Put a parser in front of the bundled one

RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)

Returns:

  • (Hash{Symbol => Array<#call>}) 


72
73
74
 
# File 'lib/recipe_scrapers/configuration.rb', line 72

def parsers
  @parsers
end

#resolver ⇒ #call?

Returns turns a host into IPAddr addresses, such as a DNS cache. Nil uses the system resolver.

Returns:

  • (#call, nil) —

    turns a host into IPAddr addresses, such as a DNS cache. Nil uses the system resolver



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#retry_options ⇒ Hash

Returns options for faraday-retry, used on timeouts, refused connections and 5xx.

Returns:

  • (Hash) —

    options for faraday-retry, used on timeouts, refused connections and 5xx



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#timeout ⇒ Integer

Returns seconds to wait for a whole response, 15 by default.

Returns:

  • (Integer) —

    seconds to wait for a whole response, 15 by default



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

#user_agent ⇒ String

Returns the User-Agent header every request sends.

Returns:

  • (String) —

    the User-Agent header every request sends



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/recipe_scrapers/configuration.rb', line 32

class Configuration
  DEFAULT_ADAPTER = :recipe_scrapers_net_http
  RETRIED = [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::ServerError].freeze

  SETTINGS = i[
    user_agent timeout open_timeout max_redirects max_body_bytes
    allow_private_addresses error_tracker retry_options adapter resolver
  ].freeze

  SETTINGS.each do |setting|
    attr_reader setting

    define_method(:"#{setting}=") do |value|
      instance_variable_set(:"@#{setting}", value)
      @connection = nil unless @connection_assigned
      value
    end
  end

  DEFAULTS = {
    user_agent: "recipe_scrapers/#{VERSION} (+https://github.com/davqasd/recipe_scrapers)",
    timeout: 15,
    open_timeout: 5,
    max_redirects: 3,
    max_body_bytes: 5 * 1024 * 1024,
    allow_private_addresses: false,
    error_tracker: ErrorTracker,
    retry_options: { max: 2, interval: 1, backoff_factor: 2 },
    adapter: DEFAULT_ADAPTER,
    resolver: nil
  }.freeze

  private_constant :DEFAULT_ADAPTER, :RETRIED, :SETTINGS, :DEFAULTS

  # The parsers of every field that takes them, tried in order. A parser answers
  # `call(text, language:)` with a hash of the model's attributes, or nil for text it cannot read.
  #
  # @example Put a parser in front of the bundled one
  #   RecipeScrapers.config.parsers[:ingredients].unshift(GramsOnly)
  # @return [Hash{Symbol => Array<#call>}]
  attr_reader :parsers

  def initialize
    DEFAULTS.each { |setting, value| instance_variable_set(:"@#{setting}", value.dup) }
    @parsers = Parsers::Chain.defaults
    @connection = nil
    @connection_assigned = false
  end

  # The Faraday connection {RecipeScrapers.scrape} fetches with, built from these settings.
  #
  # @return [Faraday::Connection]
  def connection
    @connection ||= build_connection
  end

  # Replaces the connection with one the application builds. Nil goes back to the built one.
  #
  # @param value [Faraday::Connection, nil]
  def connection=(value)
    @connection_assigned = !value.nil?
    @connection = value
  end

  # Builds a new connection with the retry, redirect, address guard, body limit and encoding
  # middleware, in that order.
  #
  # @return [Faraday::Connection]
  def build_connection
    Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
      install_middleware(faraday)
      faraday.options.timeout = timeout
      faraday.options.open_timeout = open_timeout
      faraday.adapter(adapter)
    end
  end

  private

  def install_middleware(faraday)
    faraday.request(:retry, methods: [:get], exceptions: RETRIED, **retry_options)
    faraday.use(:recipe_scrapers_follow_redirects, limit: max_redirects)
    faraday.use(:recipe_scrapers_address_guard, allow_private: allow_private_addresses, resolver: resolver)
    faraday.use(:recipe_scrapers_body_limit, bytes: max_body_bytes)
    faraday.response(:recipe_scrapers_encoding)
  end
end

Instance Method Details

#build_connection ⇒ Faraday::Connection

Builds a new connection with the retry, redirect, address guard, body limit and encoding middleware, in that order.

Returns:

  • (Faraday::Connection) 


100
101
102
103
104
105
106
107
 
# File 'lib/recipe_scrapers/configuration.rb', line 100

def build_connection
  Faraday.new(headers: { "User-Agent" => user_agent }) do |faraday|
    install_middleware(faraday)
    faraday.options.timeout = timeout
    faraday.options.open_timeout = open_timeout
    faraday.adapter(adapter)
  end
end

#connection ⇒ Faraday::Connection

The Faraday connection RecipeScrapers.scrape fetches with, built from these settings.

Returns:

  • (Faraday::Connection) 


84
85
86
 
# File 'lib/recipe_scrapers/configuration.rb', line 84

def connection
  @connection ||= build_connection
end

#connection=(value) ⇒ Object

Replaces the connection with one the application builds. Nil goes back to the built one.

Parameters:

  • value (Faraday::Connection, nil) 


91
92
93
94
 
# File 'lib/recipe_scrapers/configuration.rb', line 91

def connection=(value)
  @connection_assigned = !value.nil?
  @connection = value
end