TokyoTyrant Ruby Client

This is a c extension for Ruby to access TokyoTyrant databases. It currently supports key/value, table databases and table queries.

Install

# install tokyocabinet (1.4.47) and tokyotyrant (requires 1.1.41)
# after installing tc and tt on linux I had to /sbin/ldconfig (as root)
sudo gem install --no-ri --no-rdoc ruby-tokyotyrant

Performance

This is not in production but the initial benchmarks are very interesting. Results look closer to the memcached gem than any other tyrant client I’ve seen for Ruby.

Example

Hash DB

# start tyrant like so:
# ttserver example.tch

require 'tokyo_tyrant'
db = TokyoTyrant::DB.new('127.0.0.1', 1978)

db['foo'] = 'Bar' # => "Bar"
db['foo']         # => "Bar"

db.each{ |k,v| puts [k, v].inspect }
# ["foo", "Bar"]
# => nil

db.mput("1"=>"number_1", "2"=>"number_2", "3"=>"number_3", "4"=>"number_4", "5"=>"number_5")
db.mget(1..3) # => {"1"=>"number_1", "2"=>"number_2", "3"=>"number_3"}

B+ Tree DB

# start tyrant like so:
# ttserver example.tcb

require 'tokyo_tyrant'
bdb = TokyoTyrant::BDB.new('127.0.0.1', 1978)

bdb.putdup('foo', 'bar')
# => true

bdb.putlist({ 'foo' => ['baz', 'bat']})
# => []

bdb.getlist('foo')
# => {"foo"=>["bar", "baz", "bat"]}

bdb.each{ |k,v| puts [k, v].inspect }
# ["foo", "bar"]
# ["foo", "baz"]
# ["foo", "bat"]

Table DB

# start tyrant like so:
# ttserver example.tct

require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

t['bar'] = { :baz => 'box' } # => { :baz => 'box' }
t['bar']                     # => { :baz => 'box' }

t.each{ |k,v| puts [k, v].inspect }
# ["bar", {:baz=>"box"}]
# => nil

# bulk operations
h = {}
100.times do |i|
  h[i] = { :name => 'Pat', :sex => i % 2 > 0 ? 'male' : 'female' }
end
t.mput(h)
t.mget(0..3)
# => {"0"=>{:name=>"Pat", :sex=>"female"}, "1"=>{:name=>"Pat", :sex=>"male"}, "2"=>{:name=>"Pat", :sex=>"female"}, "3"=>{:name=>"Pat", :sex=>"male"}}

Table Query

require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

100.times do |i|
  t[i] = { 'name' => "Pat #{i}", 'sex' => i % 2 > 0 ? 'male' : 'female' }
end

q = t.query
q.condition('sex', :streq, 'male')
q.limit(5)
# Get a list of IDs
ids = q.search
# => ["1", "3", "5", "7", "9"]
q.order_by(:name, :strdesc)
ids = q.search
# => ["99", "97", "95", "93", "91"]
# Get the actual records
q.get
# => [{:__id=>"99", :sex=>"male", :name=>"Pat 99"}, {:__id=>"97", :sex=>"male", :name=>"Pat 97"}, {:__id=>"95", :sex=>"male", :name=>"Pat 95"}, {:__id=>"93", :sex=>"male", :name=>"Pat 93"}, {:__id=>"91", :sex=>"male", :name=>"Pat 91"}]

# Alternative Syntax (better)

# Query using a block
t.query{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => ["1", "3", "5", "7", "9"]

# Get records for a query
t.find{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

# Prepare but don't run a search, return a prepared query object
q = t.prepare_query{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => #<TokyoTyrant::Query:0x247c14 @rdb=#<Object:0x2800a0>, @rdbquery=#<Object:0x247c00>>
q.search
# => ["1", "3", "5", "7", "9"]
q.get
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

Full Text Search

require 'tokyo_tyrant'
require 'nokogiri'
require 'open-uri'

t = TokyoTyrant::Table.new('127.0.0.1', 1978)

(1..13).each do |n|
  doc = Nokogiri::HTML(open("http://www.sacred-texts.com/chr/herm/hermes#{n}.htm"))
  chapter = doc.css('h2').last.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
  doc.css('p').each_with_index do |paragraph, i|
    paragraph = paragraph.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
    key = "chapter:#{n}:paragraph:#{i+1}"
    t[key] = { :chapter => chapter, :paragraph => paragraph }
  end
end

# full-text search with the phrase of
t.query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "chapter:13:paragraph:69", "chapter:13:paragraph:125"]

# full-text search with all tokens in
t.query{ |q| q.condition(:paragraph, :ftsand, 'logos word') }
# => ["chapter:1:paragraph:12", "chapter:1:paragraph:14", "chapter:1:paragraph:17", "chapter:1:paragraph:19", "chapter:1:paragraph:24", "chapter:1:paragraph:27", "chapter:1:paragraph:43", "chapter:1:paragraph:53", "... lots more ..."]

# full-text search with at least one token in
t.query{ |q| q.condition(:paragraph, :ftsor, 'sermon key') }
# => ["chapter:5:paragraph:1", "chapter:9:paragraph:3", "chapter:10:paragraph:1", "chapter:10:paragraph:4", "chapter:10:paragraph:28", "chapter:11:paragraph:3", "chapter:11:paragraph:66", "chapter:11:paragraph:69", "... lots more ..."]

# negated full-text search with at least one token in
t.query{ |q| q.condition(:paragraph, '!ftsor', 'the god he and I that said') }
# => ["chapter:1:paragraph:95", "chapter:1:paragraph:96", "chapter:1:paragraph:97", "chapter:1:paragraph:98", "chapter:1:paragraph:99", "chapter:2:paragraph:3", "chapter:2:paragraph:5", "chapter:2:paragraph:6", "... lots more ..."]

Meta Search (Multi Query)

query1 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
query2 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'logos') }

# Get the union of two query sets (OR)
t.search(:union, query1, query2)
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "... lots more ..."]

# Get the intersection of two query sets (AND)
t.search(:intersection, query1, query2)
# => ["chapter:13:paragraph:5", "chapter:13:paragraph:44", "chapter:13:paragraph:69"]

# Get the difference of two query sets (ANDNOT)
t.search(:diff, query1, query2)
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:57", "chapter:13:paragraph:125"]

Parallel Querying (Take that scalability!)

require 'tokyo_tyrant'
require 'faker'
tyrants = { 1 => TokyoTyrant::Table.new('127.0.0.1', 1978),
            2 => TokyoTyrant::Table.new('127.0.0.1', 1979) }

# dummy data
tyrants.each{ |, table|
  table.clear
  20.times do |i|
    table["#{}/#{i}"] = { # consistent hashing would be good here
      :name => Faker::Company.name,
      :plan => rand(3),
    }
  end
}

queries = tyrants.collect{ |, table|
  table.prepare_query{ |q| q.condition 'plan', :numlt, '1' }
}

TokyoTyrant::Query.parallel_search(*queries).collect{ |r| {r[""] => r["name"]} }
# => [{"1/3"=>"Zemlak-Jerde"}, {"1/6"=>"O'Conner-Batz"}, {"1/9"=>"Kutch, Erdman and Aufderhar"}, {"1/11"=>"Bartoletti, Armstrong and Barrows"}, {"1/12"=>"Ferry-Dicki"}, {"2/0"=>"Schultz-O'Hara"}, {"2/1"=>"Emmerich, Feest and Huels"}, {"2/2"=>"Borer and Sons"}, {"2/3"=>"D'Amore Inc"}, {"2/5"=>"Koch and Sons"}, {"2/8"=>"Schaefer Group"}, {"2/11"=>"Stroman, Toy and Abernathy"}, {"2/19"=>"Gaylord, Reinger and White"}]

Lua Extension

# ttserver -ext spec/ext.lua 
require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

t.run(:echo, 'hello', 'world')
# => "hello\tworld"

Balanced Nodes with Consistent Hashing

# usage is similar to single node
require 'tokyo_tyrant/balancer'

servers = ['127.0.0.1:1978',
           '127.0.0.1:1979',
           '127.0.0.1:1980',
           '127.0.0.1:1981']

tb = TokyoTyrant::Balancer::Table.new(servers)

# store server is determined by key which is consistent
tb[:foo] = { 'foo' => 'bar' }
tb[:bar] = { 'bar' => 'baz' }

# retrieval server is determined by key which is consistent
tb[:foo]
# => { 'foo' => 'bar' }

# aggregate from all nodes
tb.size

# parallel_search based querying across all nodes
tb.find{ |q| q.condition(:foo, :streq, 'bar') }

Contributors

  • Flinn Mueller (actsasflinn) author/maintainer

  • ??? (gottlike) ruby 1.9.2 compatibility

  • Justin Reagor (cheapRoc) specs

  • Seth Yates (sethyates) run method (lua ext)

  • John Mettraux (jmettraux) inspiration (rufus-tokyo)