Class: Embulk::Input::Redis
- Inherits:
-
InputPlugin
- Object
- InputPlugin
- Embulk::Input::Redis
- Defined in:
- lib/embulk/input/rediskeys.rb
Class Method Summary collapse
Instance Method Summary collapse
- #deserialize_element(name, x) ⇒ Object
- #expect_value_is_json ⇒ Object
-
#init ⇒ Object
TODO def self.guess(config) sample_records = [ “column”=>1, “value”=>0.1, “column”=>2, “value”=>0.2, ] columns = Guess::SchemaGuess.from_hash_records(sample_records) return => columns end.
- #match_key_as_key ⇒ Object
- #run ⇒ Object
Class Method Details
.transaction(config, &control) ⇒ Object
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
# File 'lib/embulk/input/rediskeys.rb', line 10 def self.transaction(config, &control) # configuration code: task = { 'host' => config.param('host', :string, :default => 'localhost'), 'port' => config.param('port', :integer, :default => 6379), 'db' => config.param('db', :integer, :default => 0), 'key_prefix' => config.param('key_prefix', :string, :default => ''), 'match_key_as_key' => config.param('match_key_as_key', :bool, :default => true), 'encode' => config.param('encode', :string, :default => 'json') } redis = ::Redis.new(:host => task['host'], :port => task['port'], :db => task['db']) keys = redis.keys("#{task['key_prefix']}*").inject([]){|col, k| col.push({'name' => k, 'type' => 'string'}) col } puts "keys:#{keys}" task['columns'] = config.param('columns', :array, :default => keys).inject({}){|a, col| a[col['name']] = col['type'].to_sym a } columns = task['columns'].map.with_index{|(name, type), i| Column.new(i, name, type) } #resume(task, columns, 1, &control) puts "Redis input started." task_reports = yield(task, columns, 1) puts "Redis input finished. Commit reports = #{task_reports.to_json}" return {} end |
Instance Method Details
#deserialize_element(name, x) ⇒ Object
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 |
# File 'lib/embulk/input/rediskeys.rb', line 70 def deserialize_element(name, x) begin type = nil @task['columns'].each do |key, value| if key == name type = value break end end val = x case type.to_sym # Converted to String implicitly? when :boolean if val.is_a?(TrueClass) || val.is_a?(FalseClass) val else downcased_val = val.downcase case downcased_val when 'true' then true when 'false' then false else nil end end when :long Integer(val) when :double Float(val) when :string val when :timestamp Time.parse(val) else raise "Shouldn't reach here: val:#{val}, col_name:#{name}, col_type:#{type}" end rescue => e STDERR.puts "Failed to deserialize: val:#{val}, col_name:#{name}, col_type:#{type}, error:#{e.inspect}" end end |
#expect_value_is_json ⇒ Object
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
# File 'lib/embulk/input/rediskeys.rb', line 127 def expect_value_is_json @redis.keys("#{@task['key_prefix']}*").each do |k| case @task['encode'] when 'json' @page_builder.add(JSON.parse(@redis.get(k)).each_with_object({}) {|(key, value), new_hash| new_hash[key] = deserialize_element(key, value) }.values) @rows += 1 when 'list' @redis.lrange(k, 0, -1).each do |r| @page_builder.add(JSON.parse(r).each_with_object({}) {|(key, value), new_hash| new_hash[key] = deserialize_element(key, value) }.values) @rows += 1 end when 'hash' @page_builder.add(JSON.parse(@redis.hgetall(k).to_json).each_with_object({}) {|(key, value), new_hash| new_hash[key] = deserialize_element(key, value) }.values) @rows += 1 end end end |
#init ⇒ Object
TODO def self.guess(config)
sample_records = [
{"example"=>"a", "column"=>1, "value"=>0.1},
{"example"=>"a", "column"=>2, "value"=>0.2},
]
columns = Guess::SchemaGuess.from_hash_records(sample_records)
return {"columns" => columns}
end
62 63 64 65 66 67 68 |
# File 'lib/embulk/input/rediskeys.rb', line 62 def init # initialization code: puts "Redis input thread #{index}..." super @rows = 0 @redis = ::Redis.new(:host => task['host'], :port => task['port'], :db => task['db']) end |
#match_key_as_key ⇒ Object
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 |
# File 'lib/embulk/input/rediskeys.rb', line 108 def match_key_as_key records = [] @redis.keys("#{@task['key_prefix']}*").each do |k| case @task['encode'] when 'json' v = @redis.get(k) when 'list' v = @redis.lrange(k, 0, -1) when 'hash' v = @redis.hgetall(k).to_json end v = "{\"#{k}\":#{v}}" x = JSON.parse(v) records.push(deserialize_element(k, x)) @rows += 1 end @page_builder.add(records) end |
#run ⇒ Object
151 152 153 154 155 156 157 158 159 160 161 162 163 |
# File 'lib/embulk/input/rediskeys.rb', line 151 def run if @task['match_key_as_key'] then match_key_as_key() else expect_value_is_json() end @page_builder.finish # don't forget to call finish :-) task_report = { "rows" => @rows } return task_report end |