Class: NLP::TokenScanner

Inherits:
Object
  • Object
show all
Defined in:
lib/tagger/token_scanner.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text) ⇒ TokenScanner

Returns a new instance of TokenScanner.



6
7
8
9
10
 
# File 'lib/tagger/token_scanner.rb', line 6

def initialize(text)
  @text = text
  @pos = 0
  @tokens = @text.flatten
end

Instance Attribute Details

#text ⇒ Object (readonly)

Returns the value of attribute text.



4
5
6
 
# File 'lib/tagger/token_scanner.rb', line 4

def text
  @text
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



4
5
6
 
# File 'lib/tagger/token_scanner.rb', line 4

def tokens
  @tokens
end

Instance Method Details

#current ⇒ Object 



37
38
39
40
41
42
43
 
# File 'lib/tagger/token_scanner.rb', line 37

def current 
  if @pos == @tokens.size
    nil
  else
    @tokens[@pos]
  end
end

#end? ⇒ Boolean

Returns:

  • (Boolean) 


53
54
55
 
# File 'lib/tagger/token_scanner.rb', line 53

def end?
  @pos == tokens.size
end

#index ⇒ Object 



49
50
51
 
# File 'lib/tagger/token_scanner.rb', line 49

def index
  @pos
end

#next(type) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/tagger/token_scanner.rb', line 12

def next(type)
  @pos+=1

  case type
  when :word
    while @pos < @tokens.size and !@tokens[@pos].word?
      @pos+= 1
    end

  when :interp
    while @pos < @tokens.size and !@tokens[@pos].interp?
      @pos+= 1
    end

  when :number
    while @pos < @tokens.size and !@tokens[@pos].number?
      @pos+= 1
    end
  when :alphanum
    while @pos < @tokens.size and !@tokens[@pos].number? and !@tokens[@pos].word?
      @pos+= 1
    end
  end
end

#rewind ⇒ Object 



45
46
47
 
# File 'lib/tagger/token_scanner.rb', line 45

def rewind
  @pos = 0
end