Module: Puppet::Pops::Parser::LexerSupport

Included in:
HeredocSupport, Lexer2, SlurpSupport
Defined in:
lib/puppet/pops/parser/lexer_support.rb

Overview

This is an integral part of the Lexer. It is broken out into a separate module for maintainability of the code, and making the various parts of the lexer focused.

Defined Under Namespace

Classes: TokenValue

Constant Summary collapse

MM = 
Puppet::Util::MultiMatch
MM_ANY = 
MM::NOT_NIL
BOM_UTF_8 = 
MM.new(0xEF, 0xBB, 0xBF,        MM_ANY)
BOM_UTF_16_1 = 
MM.new(0xFE, 0xFF,              MM_ANY, MM_ANY)
BOM_UTF_16_2 = 
MM.new(0xFF, 0xFE,              MM_ANY, MM_ANY)
BOM_UTF_32_1 = 
MM.new(0x00, 0x00, 0xFE, 0xFF)
BOM_UTF_32_2 = 
MM.new(0xFF, 0xFE, 0x00, 0x00)
BOM_UTF_1 = 
MM.new(0xF7, 0x64, 0x4C, MM_ANY)
BOM_UTF_EBCDIC = 
MM.new(0xDD, 0x73, 0x66, 0x73)
BOM_SCSU = 
MM.new(0x0E, 0xFE, 0xFF,        MM_ANY)
BOM_BOCU = 
MM.new(0xFB, 0xEE, 0x28,        MM_ANY)
BOM_GB_18030 = 
MM.new(0x84, 0x31, 0x95, 0x33)
LONGEST_BOM = 
4

Instance Method Summary collapse

Instance Method Details

#assert_not_bom(content) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 168

def assert_not_bom(content)
  name, size =
    case bom = get_bom(content)

    when BOM_UTF_32_1, BOM_UTF_32_2
      ['UTF-32', 4]

    when BOM_GB_18030
      ['GB-18030', 4]

    when BOM_UTF_EBCDIC
      ['UTF-EBCDIC', 4]

    when BOM_SCSU
      ['SCSU', 3]

    when BOM_UTF_8
      ['UTF-8', 3]

    when BOM_UTF_1
      ['UTF-1', 3]

    when BOM_BOCU
      ['BOCU', 3]

    when BOM_UTF_16_1, BOM_UTF_16_2
      ['UTF-16', 2]

    else
      return
    end

  lex_error_without_pos(
    Puppet::Pops::Issues::ILLEGAL_BOM,
    { :format_name => name,
      :bytes => "[#{bom.values[0, size].map { |b| '%X' % b }.join(' ')}]" }
  )
end

#assert_numeric(value, pos) ⇒ Object

Asserts that the given string value is a float, or an integer in decimal, octal or hex form. An error is raised if the given value does not comply.



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 84

def assert_numeric(value, pos)
  case value
  when /^0[xX]/
    lex_error(Issues::INVALID_HEX_NUMBER, { :value => value }, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/

  when /^0[^.]/
    lex_error(Issues::INVALID_OCTAL_NUMBER, { :value => value }, pos)   unless value =~ /^0[0-7]+$/

  when /^\d+[eE.]/
    lex_error(Issues::INVALID_DECIMAL_NUMBER, { :value => value }, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/

  else
    lex_error(Issues::ILLEGAL_NUMBER, { :value => value }, pos) unless value =~ /^\d+$/
  end
end

#create_lex_error(issue, args = {}, pos = nil) ⇒ Puppet::ParseErrorWithIssue

Returns the created error.

Parameters:

  • issue (Issues::Issue) —

    the issue

  • args (Hash<Symbol,String>) (defaults to: {}) —

    Issue arguments

  • pos (Integer) (defaults to: nil)

Returns:



69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 69

def create_lex_error(issue, args = {}, pos = nil)
  Puppet::ParseErrorWithIssue.new(
    issue.format(args),
    filename,
    line(pos),
    position(pos),
    nil,
    issue.issue_code,
    args
  )
end

#filename ⇒ Object 



41
42
43
44
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 41

def filename
  file = @locator.file
  file.is_a?(String) && !file.empty? ? file : nil
end

#followed_by ⇒ Object

Returns “<eof>” if at end of input, else the following 5 characters with n r t escaped



12
13
14
15
16
17
18
19
20
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 12

def followed_by
  return "<eof>" if @scanner.eos?

  result = @scanner.rest[0, 5] + "..."
  result.gsub!("\t", '\t')
  result.gsub!("\n", '\n')
  result.gsub!("\r", '\r')
  result
end

#format_quote(q) ⇒ Object

Returns a quoted string using “ or ‘ depending on the given a strings’s content



23
24
25
26
27
28
29
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 23

def format_quote(q)
  if q == "'"
    '"\'"'
  else
    "'#{q}'"
  end
end

#get_bom(content) ⇒ Object 



207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 207

def get_bom(content)
  # get 5 bytes as efficiently as possible (none of the string methods works since a bom consists of
  # illegal characters on most platforms, and there is no get_bytes(n). Explicit calls are faster than
  # looping with a lambda. The get_byte returns nil if there are too few characters, and they
  # are changed to spaces
  MM.new(
    (content.getbyte(0) || ' '),
    (content.getbyte(1) || ' '),
    (content.getbyte(2) || ' '),
    (content.getbyte(3) || ' ')
  )
end

#lex_error(issue, args = {}, pos = nil) ⇒ Object

Raises a Puppet::ParserErrorWithIssue with the given issue and arguments



37
38
39
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 37

def lex_error(issue, args = {}, pos = nil)
  raise create_lex_error(issue, args, pos)
end

#lex_error_without_pos(issue, args = {}) ⇒ Object

Raises a Puppet::LexError with the given message

Raises:



32
33
34
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 32

def lex_error_without_pos(issue, args = {})
  raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code, args)
end

#lex_warning(issue, args = {}, pos = nil) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 54

def lex_warning(issue, args = {}, pos = nil)
  Puppet::Util::Log.create({
                             :level => :warning,
                             :message => issue.format(args),
                             :issue_code => issue.issue_code,
                             :file => filename,
                             :line => line(pos),
                             :pos => position(pos),
                           })
end

#line(pos) ⇒ Object 



46
47
48
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 46

def line(pos)
  @locator.line_for_offset(pos || @scanner.pos)
end

#position(pos) ⇒ Object 



50
51
52
 
# File 'lib/puppet/pops/parser/lexer_support.rb', line 50

def position(pos)
  @locator.pos_on_line(pos || @scanner.pos)
end