Class: Redis::Dump
- Inherits:
-
Object
- Object
- Redis::Dump
- Extended by:
- ClassMethods
- Defined in:
- lib/redis/dump.rb
Defined Under Namespace
Modules: ClassMethods, VERSION Classes: Problem
Constant Summary collapse
- VALID_TYPES =
['string', 'set', 'list', 'zset', 'hash', 'none'].freeze
Class Attribute Summary collapse
-
.chunk_size ⇒ Object
Returns the value of attribute chunk_size.
-
.debug ⇒ Object
Returns the value of attribute debug.
-
.encoder ⇒ Object
Returns the value of attribute encoder.
-
.host ⇒ Object
Returns the value of attribute host.
-
.parser ⇒ Object
Returns the value of attribute parser.
-
.port ⇒ Object
Returns the value of attribute port.
-
.safe ⇒ Object
Returns the value of attribute safe.
-
.with_optimizations ⇒ Object
Returns the value of attribute with_optimizations.
Instance Attribute Summary collapse
-
#dbs ⇒ Object
Returns the value of attribute dbs.
-
#redis_connections ⇒ Object
readonly
Returns the value of attribute redis_connections.
-
#uri ⇒ Object
Returns the value of attribute uri.
Class Method Summary collapse
Instance Method Summary collapse
- #connect(this_uri) ⇒ Object
-
#dump(filter = nil) ⇒ Object
See each_key.
- #each_database ⇒ Object
-
#initialize(dbs = nil, uri = "redis://#{Redis::Dump.host}:#{Redis::Dump.port}") ⇒ Dump
constructor
A new instance of Dump.
- #load(string_or_stream, &each_record) ⇒ Object
- #redis(db) ⇒ Object
- #report(filter = '*') ⇒ Object
Methods included from ClassMethods
dump_strings, set_value, set_value_hash, set_value_list, set_value_none, set_value_set, set_value_string, set_value_zset, stringify, stringify_hash, stringify_list, stringify_none, stringify_set, stringify_string, stringify_zset, type, value, value_hash, value_list, value_none, value_set, value_string, value_zset
Constructor Details
#initialize(dbs = nil, uri = "redis://#{Redis::Dump.host}:#{Redis::Dump.port}") ⇒ Dump
Returns a new instance of Dump.
40 41 42 43 44 45 46 47 48 |
# File 'lib/redis/dump.rb', line 40 def initialize(dbs=nil,uri="redis://#{Redis::Dump.host}:#{Redis::Dump.port}") @redis_connections = {} @uri = uri unless dbs.nil? @dbs = Range === dbs ? dbs : (dbs..dbs) @dbs = (@dbs.first.to_i..@dbs.last.to_i) # enforce integers @dbs.to_a.each { |db| redis(db) } # open_all_connections end end |
Class Attribute Details
.chunk_size ⇒ Object
Returns the value of attribute chunk_size.
22 23 24 |
# File 'lib/redis/dump.rb', line 22 def chunk_size @chunk_size end |
.debug ⇒ Object
Returns the value of attribute debug.
22 23 24 |
# File 'lib/redis/dump.rb', line 22 def debug @debug end |
.encoder ⇒ Object
Returns the value of attribute encoder.
22 23 24 |
# File 'lib/redis/dump.rb', line 22 def encoder @encoder end |
.host ⇒ Object
Returns the value of attribute host.
22 23 24 |
# File 'lib/redis/dump.rb', line 22 def host @host end |
.parser ⇒ Object
Returns the value of attribute parser.
22 23 24 |
# File 'lib/redis/dump.rb', line 22 def parser @parser end |
.port ⇒ Object
Returns the value of attribute port.
22 23 24 |
# File 'lib/redis/dump.rb', line 22 def port @port end |
.safe ⇒ Object
Returns the value of attribute safe.
22 23 24 |
# File 'lib/redis/dump.rb', line 22 def safe @safe end |
.with_optimizations ⇒ Object
Returns the value of attribute with_optimizations.
22 23 24 |
# File 'lib/redis/dump.rb', line 22 def with_optimizations @with_optimizations end |
Instance Attribute Details
#dbs ⇒ Object
Returns the value of attribute dbs.
38 39 40 |
# File 'lib/redis/dump.rb', line 38 def dbs @dbs end |
#redis_connections ⇒ Object (readonly)
Returns the value of attribute redis_connections.
39 40 41 |
# File 'lib/redis/dump.rb', line 39 def redis_connections @redis_connections end |
#uri ⇒ Object
Returns the value of attribute uri.
38 39 40 |
# File 'lib/redis/dump.rb', line 38 def uri @uri end |
Class Method Details
.check_utf8=(check) ⇒ Object
32 33 34 35 36 |
# File 'lib/redis/dump.rb', line 32 def check_utf8=(check) if check == false @parser = Yajl::Parser.new(:check_utf8 => false) end end |
.ld(msg) ⇒ Object
26 27 28 |
# File 'lib/redis/dump.rb', line 26 def ld(msg) STDERR.puts "#%.4f: %s" % [Time.now.utc.to_f, msg] if debug end |
.le(msg) ⇒ Object
23 24 25 |
# File 'lib/redis/dump.rb', line 23 def le(msg) STDERR.puts "#%.4f: %s" % [Time.now.utc.to_f, msg] end |
.memory_usage ⇒ Object
29 30 31 |
# File 'lib/redis/dump.rb', line 29 def memory_usage `ps -o rss= -p #{Process.pid}`.to_i # in kb end |
Instance Method Details
#connect(this_uri) ⇒ Object
52 53 54 55 |
# File 'lib/redis/dump.rb', line 52 def connect(this_uri) #self.class.ld 'CONNECT: ' << this_uri Redis.connect :url => this_uri end |
#dump(filter = nil) ⇒ Object
See each_key
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
# File 'lib/redis/dump.rb', line 64 def dump filter=nil filter ||= '*' entries = [] each_database do |redis| chunk_entries = [] dump_keys = redis.keys(filter) dump_keys_size = dump_keys.size Redis::Dump.ld "Memory after loading keys: #{Redis::Dump.memory_usage}kb" dump_keys.each_with_index do |key,idx| entry, idxplus = key, idx+1 #self.class.ld " #{key} (#{key_dump['type']}): #{key_dump['size'].to_bytes}" #entry_enc = self.class.encoder.encode entry if block_given? chunk_entries << entry process_chunk idx, dump_keys_size do |count| Redis::Dump.ld " dumping #{chunk_entries.size} (#{count}) from #{redis.client.id}" output_buffer = [] chunk_entries.select! do |key| type = Redis::Dump.type(redis, key) if self.class.with_optimizations && type == 'string' true else output_buffer.push self.class.encoder.encode(Redis::Dump.dump(redis, key, type)) false end end unless output_buffer.empty? yield output_buffer end unless chunk_entries.empty? yield Redis::Dump.dump_strings(redis, chunk_entries) { |obj| self.class.encoder.encode(obj) } end output_buffer.clear chunk_entries.clear end else entries << self.class.encoder.encode(Redis::Dump.dump(redis, entry)) end end end entries end |
#each_database ⇒ Object
57 58 59 60 61 |
# File 'lib/redis/dump.rb', line 57 def each_database @redis_connections.keys.sort.each do |redis_uri| yield redis_connections[redis_uri] end end |
#load(string_or_stream, &each_record) ⇒ Object
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 |
# File 'lib/redis/dump.rb', line 140 def load(string_or_stream, &each_record) count = 0 Redis::Dump.ld " LOAD SOURCE: #{string_or_stream}" Redis::Dump.parser.parse string_or_stream do |obj| unless @dbs.member?(obj["db"].to_i) Redis::Dump.ld "db out of range: #{obj["db"]}" next end this_redis = redis(obj["db"]) #Redis::Dump.ld "load[#{this_redis.hash}, #{obj}]" if each_record.nil? if Redis::Dump.safe && this_redis.exists(obj['key']) #Redis::Dump.ld " record exists (no change)" next end begin Redis::Dump.set_value this_redis, obj['key'], obj['type'], obj['value'], obj['ttl'] rescue => ex Redis::Dump.le '(key: %s) %s' % [obj['key'], ex.] end else each_record.call obj end count += 1 end count end |
#redis(db) ⇒ Object
49 50 51 |
# File 'lib/redis/dump.rb', line 49 def redis(db) redis_connections["#{uri}/#{db}"] ||= connect("#{uri}/#{db}") end |
#report(filter = '*') ⇒ Object
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
# File 'lib/redis/dump.rb', line 113 def report filter='*' values = [] total_size, dbs = 0, {} each_database do |redis| chunk_entries = [] dump_keys = redis.keys(filter) dump_keys_size = dump_keys.size dump_keys.each_with_index do |key,idx| entry, idxplus = Redis::Dump.report(redis, key), idx+1 chunk_entries << entry process_chunk idx, dump_keys_size do |count| Redis::Dump.ld " reporting on #{chunk_entries.size} (#{idxplus}) from #{redis.client.id}" chunk_entries.each do |e| #puts record if obj.global.verbose >= 1 dbs[e['db']] ||= 0 dbs[e['db']] += e['size'] total_size += e['size'] end chunk_entries.clear end end end puts dbs.keys.sort.collect { |db| " db#{db}: #{dbs[db].to_bytes}" } puts "total: #{total_size.to_bytes}" values end |