TokyoTyrant Ruby Client

This is a c extension for Ruby to access TokyoTyrant databases. It currently supports key/value, table databases and table queries.

Install

# install tokyocabinet (1.4.30) and tokyotyrant (requires 1.1.33)
# after installing tc and tt on linux I had to /sbin/ldconfig (as root)
gem sources -a http://gems.github.com
sudo gem install actsasflinn-ruby-tokyotyrant

Performance

This is not in production but the initial benchmarks are very interesting. Results look closer to the memcached gem than any other tyrant client I’ve seen for Ruby.

Example

Hash DB

# start tyrant like so:
# ttserver example.tch

require 'tokyo_tyrant'
db = TokyoTyrant::DB.new('127.0.0.1', 1978)

db['foo'] = 'Bar' # => "Bar"
db['foo']         # => "Bar"

db.each{ |k,v| puts [k, v].inspect }
# ["foo", "Bar"]
# => nil

db.mput("1"=>"number_1", "2"=>"number_2", "3"=>"number_3", "4"=>"number_4", "5"=>"number_5")
db.mget(1..3) # => {"1"=>"number_1", "2"=>"number_2", "3"=>"number_3"}

Table DB

# start tyrant like so:
# ttserver example.tct

require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

t['bar'] = { :baz => 'box' } # => { :baz => 'box' }
t['bar']                     # => { :baz => 'box' }

t.each{ |k,v| puts [k, v].inspect }
# ["bar", {:baz=>"box"}]
# => nil

# bulk operations
h = {}
100.times do |i|
  h[i] = { :name => 'Pat', :sex => i % 2 > 0 ? 'male' : 'female' }
end
t.mput(h)
t.mget(0..3)
# => {"0"=>{:name=>"Pat", :sex=>"female"}, "1"=>{:name=>"Pat", :sex=>"male"}, "2"=>{:name=>"Pat", :sex=>"female"}, "3"=>{:name=>"Pat", :sex=>"male"}}

Table Query

require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

100.times do |i|
  t[i] = { 'name' => "Pat #{i}", 'sex' => i % 2 > 0 ? 'male' : 'female' }
end

q = t.query
q.condition('sex', :streq, 'male')
q.limit(5)
# Get a list of IDs
ids = q.search
# => ["1", "3", "5", "7", "9"]
q.order_by(:name, :strdesc)
ids = q.search
# => ["99", "97", "95", "93", "91"]
# Get the actual records
q.get
# => [{:__id=>"99", :sex=>"male", :name=>"Pat 99"}, {:__id=>"97", :sex=>"male", :name=>"Pat 97"}, {:__id=>"95", :sex=>"male", :name=>"Pat 95"}, {:__id=>"93", :sex=>"male", :name=>"Pat 93"}, {:__id=>"91", :sex=>"male", :name=>"Pat 91"}]

# Alternative Syntax (better)

# Query using a block
t.query{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => ["1", "3", "5", "7", "9"]

# Get records for a query
t.find{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

# Prepare but don't run a search, return a prepared query object
q = t.prepare_query{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => #<TokyoTyrant::Query:0x247c14 @rdb=#<Object:0x2800a0>, @rdbquery=#<Object:0x247c00>>
q.search
# => ["1", "3", "5", "7", "9"]
q.get
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

Full Text Search

require 'tokyo_tyrant'
require 'nokogiri'
require 'open-uri'

t = TokyoTyrant::Table.new('127.0.0.1', 1978)

(1..13).each do |n|
  doc = Nokogiri::HTML(open("http://www.sacred-texts.com/chr/herm/hermes#{n}.htm"))
  chapter = doc.css('h2').last.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
  doc.css('p').each_with_index do |paragraph, i|
    paragraph = paragraph.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
    key = "chapter:#{n}:paragraph:#{i+1}"
    t[key] = { :chapter => chapter, :paragraph => paragraph }
  end
end

# full-text search with the phrase of
t.query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "chapter:13:paragraph:69", "chapter:13:paragraph:125"]

# full-text search with all tokens in
t.query{ |q| q.condition(:paragraph, :ftsand, 'logos word') }
# => ["chapter:1:paragraph:12", "chapter:1:paragraph:14", "chapter:1:paragraph:17", "chapter:1:paragraph:19", "chapter:1:paragraph:24", "chapter:1:paragraph:27", "chapter:1:paragraph:43", "chapter:1:paragraph:53", "... lots more ..."]

# full-text search with at least one token in
t.query{ |q| q.condition(:paragraph, :ftsor, 'sermon key') }
# => ["chapter:5:paragraph:1", "chapter:9:paragraph:3", "chapter:10:paragraph:1", "chapter:10:paragraph:4", "chapter:10:paragraph:28", "chapter:11:paragraph:3", "chapter:11:paragraph:66", "chapter:11:paragraph:69", "... lots more ..."]

# negated full-text search with at least one token in
t.query{ |q| q.condition(:paragraph, '!ftsor', 'the god he and I that said') }
# => ["chapter:1:paragraph:95", "chapter:1:paragraph:96", "chapter:1:paragraph:97", "chapter:1:paragraph:98", "chapter:1:paragraph:99", "chapter:2:paragraph:3", "chapter:2:paragraph:5", "chapter:2:paragraph:6", "... lots more ..."]

Meta Search (Multi Query)

query1 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
query2 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'logos') }

# Get the union of two query sets (OR)
t.search(:union, query1, query2)
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "... lots more ..."]

# Get the intersection of two query sets (AND)
t.search(:intersection, query1, query2)
# => ["chapter:13:paragraph:5", "chapter:13:paragraph:44", "chapter:13:paragraph:69"]

# Get the difference of two query sets (ANDNOT)
t.search(:diff, query1, query2)
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:57", "chapter:13:paragraph:125"]

Lua Extension

# ttserver -ext spec/ext.lua 
require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

t.run(:echo, 'hello', 'world')
# => "hello\tworld"

Contributors

  • Flinn Mueller (actsasflinn) author/maintainer

  • Justin Reagor (cheapRoc) specs

  • Seth Yates (sethyates) run method (lua ext)

  • John Mettraux (jmettraux) inspiration (rufus-tokyo)