Class: MogileFS::MogileFS

Inherits:
Client
  • Object
show all
Includes:
Bigfile, Util
Defined in:
lib/mogilefs/mogilefs.rb

Overview

MogileFS File manipulation client.

Constant Summary

Constants included from Bigfile

Bigfile::GZIP_HEADER

Constants included from Util

Util::CHUNK_SIZE

Instance Attribute Summary collapse

Attributes inherited from Client

#backend, #hosts

Instance Method Summary collapse

Methods included from Bigfile

#bigfile_stat, #bigfile_write

Methods included from Util

#sysread_full, #sysrwloop, #syswrite_full

Methods inherited from Client

#err, #errstr, #readonly?, #reload

Constructor Details

#initialize(args = {}) ⇒ MogileFS

Creates a new MogileFS::MogileFS instance. args must include a key :domain specifying the domain of this client.

Raises:

  • (ArgumentError) 


26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/mogilefs/mogilefs.rb', line 26

def initialize(args = {})
  @domain = args[:domain]

  @get_file_data_timeout = 5

  raise ArgumentError, "you must specify a domain" unless @domain

  if @backend = args[:db_backend]
    @readonly = true
  else
    super
  end
end

Instance Attribute Details

#domain ⇒ Object (readonly)

The domain of keys for this MogileFS client.



15
16
17
 
# File 'lib/mogilefs/mogilefs.rb', line 15

def domain
  @domain
end

#get_file_data_timeout ⇒ Object

The timeout for get_file_data. Defaults to five seconds.



20
21
22
 
# File 'lib/mogilefs/mogilefs.rb', line 20

def get_file_data_timeout
  @get_file_data_timeout
end

Instance Method Details

#delete(key) ⇒ Object

Removes key.

Raises:



175
176
177
178
179
 
# File 'lib/mogilefs/mogilefs.rb', line 175

def delete(key)
  raise MogileFS::ReadOnlyError if readonly?

  @backend.delete :domain => @domain, :key => key
end

#each_key(prefix) ⇒ Object

Enumerates keys starting with key.



43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/mogilefs/mogilefs.rb', line 43

def each_key(prefix)
  after = nil

  keys, after = list_keys prefix

  until keys.nil? or keys.empty? do
    keys.each { |k| yield k }
    keys, after = list_keys prefix, after
  end

  nil
end

#get_file_data(key, &block) ⇒ Object

Retrieves the contents of key.



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/mogilefs/mogilefs.rb', line 59

def get_file_data(key, &block)
  paths = get_paths(key) or return nil
  paths.each do |path|
    begin
      sock = http_read_sock(URI.parse(path))
      begin
        return yield(sock) if block_given?
        return sysread_full(sock, sock.mogilefs_size, @get_file_data_timeout)
      ensure
        sock.close rescue nil
      end
    rescue MogileFS::Timeout, MogileFS::InvalidResponseError,
           Errno::ECONNREFUSED, EOFError, SystemCallError
    end
  end
  nil
end

#get_paths(key, noverify = true, zone = nil) ⇒ Object

Get the paths for key.



80
81
82
83
84
85
86
 
# File 'lib/mogilefs/mogilefs.rb', line 80

def get_paths(key, noverify = true, zone = nil)
  opts = { :domain => @domain, :key => key,
           :noverify => noverify ? 1 : 0, :zone => zone }
  @backend.respond_to?(:_get_paths) and return @backend._get_paths(opts)
  res = @backend.get_paths(opts)
  (1..res['paths'].to_i).map { |i| res["path#{i}"] }.compact
end

#get_uris(key, noverify = true, zone = nil) ⇒ Object

Get the URIs for key.



91
92
93
 
# File 'lib/mogilefs/mogilefs.rb', line 91

def get_uris(key, noverify = true, zone = nil)
  get_paths(key, noverify, zone).map { |path| URI.parse(path) }
end

#list_keys(prefix, after = nil, limit = 1000, &block) ⇒ Object

Lists keys starting with prefix follwing after up to limit. If after is nil the list starts at the beginning.



222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/mogilefs/mogilefs.rb', line 222

def list_keys(prefix, after = nil, limit = 1000, &block)
  if @backend.respond_to?(:_list_keys)
    return @backend._list_keys(domain, prefix, after, limit, &block)
  end

  res = begin
    @backend.list_keys(:domain => domain, :prefix => prefix,
                       :after => after, :limit => limit)
  rescue MogileFS::Backend::NoneMatchError
    return nil
  end

  keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
  if block_given?
    # emulate the MogileFS::Mysql interface, slowly...
    keys.each do |key|
      paths = get_paths(key) or next
      length = paths_size(paths) or next
      yield key, length, paths.size
    end
  end

  [ keys, res['next_after'] ]
end

#new_file(key, klass = nil, bytes = 0, &block) ⇒ Object

Creates a new file key in klass. bytes is currently unused.

The block operates like File.open.

Raises:



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
 
# File 'lib/mogilefs/mogilefs.rb', line 100

def new_file(key, klass = nil, bytes = 0, &block) # :yields: file
  raise MogileFS::ReadOnlyError if readonly?
  opts = { :domain => @domain, :key => key, :multi_dest => 1 }
  opts[:class] = klass if klass
  res = @backend.create_open(opts)

  dests = if dev_count = res['dev_count'] # multi_dest succeeded
    (1..dev_count.to_i).map do |i|
      [res["devid_#{i}"], res["path_#{i}"]]
    end
  else # single destination returned
    # 0x0040:  d0e4 4f4b 2064 6576 6964 3d31 2666 6964  ..OK.devid=1&fid
    # 0x0050:  3d33 2670 6174 683d 6874 7470 3a2f 2f31  =3&path=http://1
    # 0x0060:  3932 2e31 3638 2e31 2e37 323a 3735 3030  92.168.1.72:7500
    # 0x0070:  2f64 6576 312f 302f 3030 302f 3030 302f  /dev1/0/000/000/
    # 0x0080:  3030 3030 3030 3030 3033 2e66 6964 0d0a  0000000003.fid..

    [[res['devid'], res['path']]]
  end

  case (dests[0][1] rescue nil)
  when nil, '' then
    raise MogileFS::EmptyPathError
  when /^http:\/\// then
    MogileFS::HTTPFile.open(self, res['fid'], klass, key,
                            dests, bytes, &block)
  else
    raise MogileFS::UnsupportedPathError,
          "paths '#{dests.inspect}' returned by backend is not supported"
  end
end

#paths_size(paths) ⇒ Object 



206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/mogilefs/mogilefs.rb', line 206

def paths_size(paths)
  paths.each do |path|
    begin
      return http_read_sock(URI.parse(path), "HEAD").mogilefs_size
    rescue MogileFS::InvalidResponseError, MogileFS::Timeout,
           Errno::ECONNREFUSED, EOFError, SystemCallError => err
      next
    end
  end
  nil
end

#rename(from, to) ⇒ Object

Renames a key from to key to.

Raises:



191
192
193
194
195
196
 
# File 'lib/mogilefs/mogilefs.rb', line 191

def rename(from, to)
  raise MogileFS::ReadOnlyError if readonly?

  @backend.rename :domain => @domain, :from_key => from, :to_key => to
  nil
end

#size(key) ⇒ Object

Returns the size of key.



200
201
202
203
204
 
# File 'lib/mogilefs/mogilefs.rb', line 200

def size(key)
  @backend.respond_to?(:_size) and return @backend._size(domain, key)
  paths = get_paths(key) or return nil
  paths_size(paths)
end

#sleep(duration) ⇒ Object

Sleeps duration.



184
185
186
 
# File 'lib/mogilefs/mogilefs.rb', line 184

def sleep(duration)
  @backend.sleep :duration => duration
end

#store_content(key, klass, content) ⇒ Object

Stores content into key in class klass.

Raises:



158
159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/mogilefs/mogilefs.rb', line 158

def store_content(key, klass, content)
  raise MogileFS::ReadOnlyError if readonly?

  new_file key, klass do |mfp|
    if content.is_a?(MogileFS::Util::StoreContent)
      mfp.streaming_io = content
    else
      mfp << content
    end
  end

  content.length
end

#store_file(key, klass, file) ⇒ Object

Copies the contents of file into key in class klass. file can be either a file name or an object that responds to #sysread. Returns size of file stored

Raises:



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/mogilefs/mogilefs.rb', line 137

def store_file(key, klass, file)
  raise MogileFS::ReadOnlyError if readonly?

  new_file key, klass do |mfp|
    if file.respond_to? :sysread then
      sysrwloop(file, mfp)
    else
      size = File.size(file)
      if size > 0x10000 # Bigass file, handle differently
        mfp.big_io = file
        size
      else
        File.open(file, "rb") { |fp| sysrwloop(fp, mfp) }
      end
    end
  end
end