Module: ScopedSearch::QueryLanguage::Tokenizer

Included in:
Compiler
Defined in:
lib/scoped_search/query_language/tokenizer.rb

Constant Summary collapse

KEYWORDS = 
{ 'and' => :and, 'or' => :or, 'not' => :not, 'set?' => :notnull, 'null?' => :null }
OPERATORS = 
{ '&' => :and, '|' => :or, '&&' => :and, '||' => :or, '-'=> :not, '!' => :not, '~' => :like, '!~' => :unlike,
'=' => :eq, '==' => :eq, '!=' => :ne, '<>' => :ne, '>' => :gt, '<' => :lt, '>=' => :gte, '<=' => :lte }

Instance Method Summary collapse

Instance Method Details

#current_char ⇒ Object 



13
14
15
 
# File 'lib/scoped_search/query_language/tokenizer.rb', line 13

def current_char
  @current_char
end

#each_token(&block) ⇒ Object Also known as: each 



26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/scoped_search/query_language/tokenizer.rb', line 26

def each_token(&block)
  while next_char
    case current_char
    when /^\s?$/; # ignore
    when '(';  yield(:lparen)
    when ')';  yield(:rparen)
    when ',';  yield(:comma)
    when /\&|\||=|<|>|!|~|-/;  tokenize_operator(&block)
    when '"';                  tokenize_quoted_keyword(&block)
    else;                      tokenize_keyword(&block)
    end
  end
end

#next_char ⇒ Object 



21
22
23
24
 
# File 'lib/scoped_search/query_language/tokenizer.rb', line 21

def next_char
  @current_char_pos += 1
  @current_char = @str[@current_char_pos, 1]
end

#peek_char(amount = 1) ⇒ Object 



17
18
19
 
# File 'lib/scoped_search/query_language/tokenizer.rb', line 17

def peek_char(amount = 1)
  @str[@current_char_pos + amount, 1]
end

#tokenize ⇒ Object 



8
9
10
11
 
# File 'lib/scoped_search/query_language/tokenizer.rb', line 8

def tokenize
  @current_char_pos = -1
  to_a
end

#tokenize_keyword(&block) ⇒ Object 



46
47
48
49
50
 
# File 'lib/scoped_search/query_language/tokenizer.rb', line 46

def tokenize_keyword(&block)
  keyword = current_char
  keyword << next_char while /[^=<>\s\&\|\)\(,]/ =~ peek_char      
  KEYWORDS.has_key?(keyword.downcase) ? yield(KEYWORDS[keyword.downcase]) : yield(keyword)
end

#tokenize_operator {|OPERATORS[operator]| ... } ⇒ Object

Yields:



40
41
42
43
44
 
# File 'lib/scoped_search/query_language/tokenizer.rb', line 40

def tokenize_operator(&block)
  operator = current_char
  operator << next_char if OPERATORS.has_key?(operator + peek_char)
  yield(OPERATORS[operator])
end

#tokenize_quoted_keyword {|keyword| ... } ⇒ Object

Yields:

  • (keyword) 


52
53
54
55
56
57
58
 
# File 'lib/scoped_search/query_language/tokenizer.rb', line 52

def tokenize_quoted_keyword(&block)
  keyword = ""
  until next_char.nil? || current_char == '"'
    keyword << (current_char == "\\" ? next_char : current_char)
  end
  yield(keyword)      
end