Class: InternetMessage::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/internet_message/tokenizer.rb

Constant Summary collapse

TOKEN_RE = 
/[0-9a-zA-Z\!\#\$\%\'\*\+\-\/\=\?\^\_\`\{\|\}\~\.]+/n

Instance Method Summary collapse

Constructor Details

#initialize(s, opt = {}) ⇒ Tokenizer

Returns a new instance of Tokenizer.



7
8
9
10
 
# File 'lib/internet_message/tokenizer.rb', line 7

def initialize(s, opt={})
  @ss = StringScanner.new(s.gsub(/\r?\n/, ''))
  @token_re = opt[:token_re] || TOKEN_RE
end

Instance Method Details

#scan_comment ⇒ Object 



36
37
38
39
40
41
42
43
44
 
# File 'lib/internet_message/tokenizer.rb', line 36

def scan_comment
  ret = []
  @ss.scan(/\(/) or return ret
  until @ss.scan(/\)/) or @ss.eos?
    s = @ss.scan(/(\\.|[^\\\(\)])*/) and ret.push s.gsub(/\\(.)/){$1}
    @ss.check(/\(/) and ret.push scan_comment
  end
  ret
end

#tokenize(opt = {}) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/internet_message/tokenizer.rb', line 12

def tokenize(opt={})
  ret = []
  until @ss.eos?
    case
    when s = @ss.scan(/[ \t]+/)
      ret.push Token.new(:WSP, s) unless opt[:skip_wsp]
    when s = @ss.scan(@token_re)
      ret.push Token.new(:TOKEN, s)
    when s = @ss.scan(/\"(\\.|[^\"])+\"/)
      ret.push Token.new(:QUOTED, s.gsub(/\A\"|\"\z/,'').gsub(/\\(.)/){$1})
    when @ss.check(/\(/)
      comment = scan_comment
      ret.push Token.new(:COMMENT, comment) unless opt[:skip_comment]
    else
      ret.push Token.new(:CHAR, @ss.scan(/./))
    end
  end
  ret
end

#tokenize2 ⇒ Object 



32
33
34
 
# File 'lib/internet_message/tokenizer.rb', line 32

def tokenize2
  tokenize(:skip_wsp=>true, :skip_comment=>true)
end