Class: Tagfish::Tokeniser

Inherits:
Object
  • Object
show all
Defined in:
lib/tagfish/tokeniser.rb

Defined Under Namespace

Classes: Text, URI

Class Method Summary collapse

Class Method Details

.dump(tokens) ⇒ Object 



33
34
35
 
# File 'lib/tagfish/tokeniser.rb', line 33

def self.dump(tokens)
  tokens.join('')
end

.tokenise(rest) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/tagfish/tokeniser.rb', line 16

def self.tokenise(rest)
  tokens = []
  while true
    match = rest.match /[\w\/:.-]+\/[\w.-]+:[\w.-]+/
    if match.nil?
      tokens << Text.new(rest)
      break
    else
      tokens << Text.new(match.pre_match)
      tokens << URI.new(match.to_s)
      rest = match.post_match
    end
  end

  tokens
end