Class: TDPUtils::BasicStringTokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tdp/utils.rb

Direct Known Subclasses

StringTokenizer

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(rule, ignore = nil) ⇒ BasicStringTokenizer

Returns a new instance of BasicStringTokenizer.



31
32
33
34
35
36
 
# File 'lib/tdp/utils.rb', line 31

def initialize(rule, ignore=nil)
  require("strscan")
  @rule = rule
  @scan_pattern = Regexp.new(@rule.keys.join("|"))
  @ignore_pattern = ignore
end

Class Method Details

.[](rule, ignore = nil) ⇒ Object 



27
28
29
 
# File 'lib/tdp/utils.rb', line 27

def self.[](rule, ignore=nil)
  self.new(rule, ignore)
end

Instance Method Details

#generate(str) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/tdp/utils.rb', line 38

def generate(str)
  scanner = StringScanner.new(str)
  TDParser::TokenGenerator.new{|x|
    while(!scanner.empty?)
      if (@ignore_pattern)
        while(scanner.scan(@ignore_pattern))
        end
      end
      sstr = scanner.scan(@scan_pattern)
      if (sstr)
        @rule.each{|reg,kind|
          if (reg =~ sstr)
            x.yield(Token.new(kind, sstr))
            yielded = true
            break
          end
        }
      else
        c = scanner.scan(/./)
        x.yield(c)
      end
    end
  }
end