Class: Thot::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/thot/tokenizer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(string:) ⇒ Tokenizer

Returns a new instance of Tokenizer.



8
9
10
11
12
 
# File 'lib/thot/tokenizer.rb', line 8

def initialize(string: )
    @string = string
    @definitions = {}
    @tokens = []
end

Instance Attribute Details

#definitions ⇒ Object (readonly)

Returns the value of attribute definitions.



6
7
8
 
# File 'lib/thot/tokenizer.rb', line 6

def definitions
  @definitions
end

#string ⇒ Object (readonly)

Returns the value of attribute string.



4
5
6
 
# File 'lib/thot/tokenizer.rb', line 4

def string
  @string
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



5
6
7
 
# File 'lib/thot/tokenizer.rb', line 5

def tokens
  @tokens
end

Instance Method Details

#detect ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/thot/tokenizer.rb', line 14

def detect
    if @string.valid_encoding?
        detected = @string.force_encoding("UTF-8").scan(/%%[^\%]*%%/).concat @string.force_encoding("UTF-8").scan(/\{\{[^\}]*\}\}/)
        detected.each  do |token|
            key,*rest = token[2,(token.length - 4)].split('.')
            filters = []; default = nil
            if key =~ /(.*)\((.*)\)/ then
                key = $1
                default = $2
            end
            rest.each {|item|
                if item =~ /(.*)\((.*)\)/ then 
                    filters.push $1  
                        default = $2 

                else
                    filters.push item
                end
                }
            @tokens.push key unless @tokens.include? key
            @definitions[token] = {key: key, filters: filters , default: default }
        end
        @tokens.map!(&:downcase)
        @tokens.map!(&:to_sym)
    end
end