Module: RecipeScrapers::Text

Defined in:
lib/recipe_scrapers/text.rb

Constant Summary collapse

MARKUP = 
/[&<]/
WHITESPACE = 
/[[:space:]]+/
DROPPED = 
["​", "‎", "‏", "⠀"].freeze
TEXTLESS = 
%w[script style template noscript].freeze
BLOCKS = 
%w[
  address article aside blockquote dd details div dl dt fieldset figcaption figure footer form
  h1 h2 h3 h4 h5 h6 header hr li main nav ol p pre section summary table tbody tfoot thead tr ul
].freeze
CELLS = 
%w[td th].freeze
LINE_BREAK = 
"\n"
LINE_BREAKS = 
%r{\r?\n|<br\s*/?>}i
CLOSING = 
/\A[\s.,;:!?)\]}%]/

Class Method Summary collapse

Class Method Details

.content(node) ⇒ Object 



21
22
23
24
25
26
27
 
# File 'lib/recipe_scrapers/text.rb', line 21

def content(node)
  return nil if node.nil? || TEXTLESS.include?(node.name)
  return node.text if node.text? || node.cdata?
  return nil unless node.element?

  joined(node.children)
end

.lines(node) ⇒ Object 



29
30
31
 
# File 'lib/recipe_scrapers/text.rb', line 29

def lines(node)
  split_lines(content(node)).filter_map { |line| normalize(line) }
end

.normalize(value) ⇒ Object 



37
38
39
40
41
42
43
44
 
# File 'lib/recipe_scrapers/text.rb', line 37

def normalize(value)
  return nil if value.nil?

  cleaned = collapse_parentheses(strip_markup(value.to_s))
  cleaned = DROPPED.reduce(cleaned) { |text, dropped| text.delete(dropped) }
  cleaned = cleaned.gsub(WHITESPACE, " ").strip
  cleaned.empty? ? nil : cleaned
end

.recode(html, encoding_name) ⇒ Object 



46
47
48
49
50
51
52
53
54
 
# File 'lib/recipe_scrapers/text.rb', line 46

def recode(html, encoding_name)
  return html if encoding_name.nil?
  return html if html.encoding == ::Encoding::UTF_8 && html.valid_encoding?

  bytes = html.dup.force_encoding(encoding_name)
  bytes.encode(::Encoding::UTF_8, invalid: :replace, undef: :replace)
rescue ArgumentError
  html
end

.split_lines(value) ⇒ Object 



33
34
35
 
# File 'lib/recipe_scrapers/text.rb', line 33

def split_lines(value)
  value.to_s.split(LINE_BREAKS)
end