Class: Gem::RequestSet::Lockfile::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/rubygems/request_set/lockfile/tokenizer.rb

Defined Under Namespace

Classes: Token

Constant Summary collapse

EOF = 
Token.new :EOF

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input, filename = nil, line = 0, pos = 0) ⇒ Tokenizer

Returns a new instance of Tokenizer.



14
15
16
17
18
19
20
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 14

def initialize(input, filename = nil, line = 0, pos = 0)
  @line     = line
  @line_pos = pos
  @tokens   = []
  @filename = filename
  tokenize input
end

Class Method Details

.from_file(file) ⇒ Object 



10
11
12
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 10

def self.from_file(file)
  new File.read(file), file
end

Instance Method Details

#empty? ⇒ Boolean

Returns:

  • (Boolean) 


42
43
44
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 42

def empty?
  @tokens.empty?
end

#make_parser(set, platforms) ⇒ Object 



22
23
24
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 22

def make_parser(set, platforms)
  Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename
end

#next_token ⇒ Object Also known as: shift 



50
51
52
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 50

def next_token
  @tokens.shift
end

#peek ⇒ Object 



55
56
57
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 55

def peek
  @tokens.first || EOF
end

#skip(type) ⇒ Object 



30
31
32
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 30

def skip(type)
  @tokens.shift while !@tokens.empty? && peek.type == type
end

#to_a ⇒ Object 



26
27
28
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 26

def to_a
  @tokens.map {|token| [token.type, token.value, token.column, token.line] }
end

#token_pos(byte_offset) ⇒ Object

Calculates the column (by byte) and the line of the current token based on byte_offset.



38
39
40
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 38

def token_pos(byte_offset) # :nodoc:
  [byte_offset - @line_pos, @line]
end

#unshift(token) ⇒ Object 



46
47
48
 
# File 'lib/rubygems/request_set/lockfile/tokenizer.rb', line 46

def unshift(token)
  @tokens.unshift token
end