Class: FerretTokenizer

Inherits:
Object
  • Object
show all
Defined in:
ext/ferret_tokenizer.c

Instance Method Summary collapse

Constructor Details

#initialize(str) ⇒ Object 



52
53
54
55
56
57
58
 
# File 'ext/ferret_tokenizer.c', line 52

VALUE method_initialize(VALUE self, VALUE str) {
	char* s = RSTRING(str)->ptr;
	Tokenizer *t;
  Data_Get_Struct(self, Tokenizer, t);
  t->data = s;
  return self;
}

Instance Method Details

#next ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'ext/ferret_tokenizer.c', line 33

VALUE method_next(VALUE self) {
	const char *start = NULL;
  const char *end = NULL;
  int len;
  
  Tokenizer *t;
  Data_Get_Struct(self, Tokenizer, t);

  char buffer[1024];
  
  frt_std_scan(t->data, buffer, sizeof(buffer) - 1,
                         &start, &end, &len);
  if (len == 0)
    return Qnil;
  
  t->data = end;
  return rb_str_new2(buffer);
}