Class: Liquid::C::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
ext/liquid_c/tokenizer.c

Instance Method Summary collapse

Constructor Details

#initialize(source, line_numbers) ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'ext/liquid_c/tokenizer.c', line 41

static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE line_numbers)
{
    tokenizer_t *tokenizer;

    Check_Type(source, T_STRING);
    Tokenizer_Get_Struct(self, tokenizer);
    source = rb_str_dup_frozen(source);
    tokenizer->source = source;
    tokenizer->cursor = RSTRING_PTR(source);
    tokenizer->length = RSTRING_LEN(source);
    tokenizer->lstrip_flag = 0;
    // tokenizer->line_number keeps track of the current line number or it is 0
    // to indicate that line numbers aren't being calculated
    tokenizer->line_number = RTEST(line_numbers) ? 1 : 0;
    return Qnil;
}

Instance Method Details

#line_number ⇒ Object 



165
166
167
168
169
170
171
172
173
174
 
# File 'ext/liquid_c/tokenizer.c', line 165

static VALUE tokenizer_line_number_method(VALUE self)
{
    tokenizer_t *tokenizer;
    Tokenizer_Get_Struct(self, tokenizer);

    if (tokenizer->line_number == 0)
        return Qnil;

    return UINT2NUM(tokenizer->line_number);
}

#shift ⇒ Object 



152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'ext/liquid_c/tokenizer.c', line 152

static VALUE tokenizer_shift_method(VALUE self)
{
    tokenizer_t *tokenizer;
    Tokenizer_Get_Struct(self, tokenizer);

    token_t token;
    tokenizer_next(tokenizer, &token);
    if (!token.type)
        return Qnil;

    return rb_enc_str_new(token.str, token.length, utf8_encoding);
}