Class: MogileFS::MogileFS

Inherits:
Client
  • Object
show all
Includes:
Util
Defined in:
lib/mogilefs/mogilefs.rb

Overview

MogileFS File manipulation client.

Constant Summary

Constants included from Util

Util::CHUNK_SIZE

Instance Attribute Summary collapse

Attributes inherited from Client

#backend, #hosts

Instance Method Summary collapse

Methods included from Util

#sysrwloop

Methods inherited from Client

#err, #errstr, #readonly?, #reload

Constructor Details

#initialize(args = {}) ⇒ MogileFS

Creates a new MogileFS::MogileFS instance. args must include a key :domain specifying the domain of this client. A key :root will be used to specify the root of the NFS file system.

Raises:

  • (ArgumentError) 


41
42
43
44
45
46
47
48
49
50
 
# File 'lib/mogilefs/mogilefs.rb', line 41

def initialize(args = {})
  @domain = args[:domain]
  @root = args[:root]

  @get_file_data_timeout = 5

  raise ArgumentError, "you must specify a domain" unless @domain

  super
end

Instance Attribute Details

#domain ⇒ Object (readonly)

The domain of keys for this MogileFS client.



29
30
31
 
# File 'lib/mogilefs/mogilefs.rb', line 29

def domain
  @domain
end

#get_file_data_timeout ⇒ Object

The timeout for get_file_data. Defaults to five seconds.



34
35
36
 
# File 'lib/mogilefs/mogilefs.rb', line 34

def get_file_data_timeout
  @get_file_data_timeout
end

#root ⇒ Object (readonly)

The path to the local MogileFS mount point if you are using NFS mode.



24
25
26
 
# File 'lib/mogilefs/mogilefs.rb', line 24

def root
  @root
end

Instance Method Details

#delete(key) ⇒ Object

Removes key.



204
205
206
207
208
209
210
211
212
 
# File 'lib/mogilefs/mogilefs.rb', line 204

def delete(key)
  raise 'readonly mogilefs' if readonly?

  res = @backend.delete :domain => @domain, :key => key

  if res.nil? and @backend.lasterr != 'unknown_key' then
    raise "unable to delete #{key}: #{@backend.lasterr}"
  end
end

#each_key(prefix) ⇒ Object

Enumerates keys starting with key.



55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/mogilefs/mogilefs.rb', line 55

def each_key(prefix)
  after = nil

  keys, after = list_keys prefix

  until keys.nil? or keys.empty? do
    keys.each { |k| yield k }
    keys, after = list_keys prefix, after
  end

  return nil
end

#get_file_data(key, &block) ⇒ Object

Retrieves the contents of key.



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/mogilefs/mogilefs.rb', line 71

def get_file_data(key, &block)
  paths = get_paths key

  return nil unless paths

  paths.each do |path|
    next unless path
    case path
    when /^http:\/\// then
      begin
        path = URI.parse path

        if block_given?
          sock = nil
          timeout @get_file_data_timeout, MogileFS::Timeout do
            sock = TCPSocket.new(path.host, path.port)
            sock.sync = true
            sock.syswrite("GET #{path.request_uri} HTTP/1.0\r\n\r\n")
            buf = sock.recv(4096, Socket::MSG_PEEK)
            head, body = buf.split(/\r\n\r\n/, 2)
            head = sock.recv(head.size + 4)
          end
          return yield(sock)
        else
          return path.read
        end
      rescue MogileFS::Timeout
        next
      end
    else
      next unless File.exist? path
      return File.read(path)
    end
  end

  return nil
end

#get_paths(key, noverify = true, zone = nil) ⇒ Object

Get the paths for key.



112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/mogilefs/mogilefs.rb', line 112

def get_paths(key, noverify = true, zone = nil)
  noverify = noverify ? 1 : 0
  res = @backend.get_paths(:domain => @domain, :key => key,
                           :noverify => noverify, :zone => zone)
  
  return nil if res.nil? and @backend.lasterr == 'unknown_key'
  paths = (1..res['paths'].to_i).map { |i| res["path#{i}"] }
  return paths if paths.empty?
  return paths if paths.first =~ /^http:\/\//
  return paths.map { |path| File.join @root, path }
end

#list_keys(prefix, after = nil, limit = 1000) ⇒ Object

Lists keys starting with prefix follwing after up to limit. If after is nil the list starts at the beginning.



273
274
275
276
277
278
279
280
281
282
 
# File 'lib/mogilefs/mogilefs.rb', line 273

def list_keys(prefix, after = nil, limit = 1000)
  res = @backend.list_keys(:domain => domain, :prefix => prefix,
                           :after => after, :limit => limit)

  return nil if res.nil?

  keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }

  return keys, res['next_after']
end

#new_file(key, klass, bytes = 0, &block) ⇒ Object

Creates a new file key in klass. bytes is currently unused.

The block operates like File.open.



129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/mogilefs/mogilefs.rb', line 129

def new_file(key, klass, bytes = 0, &block) # :yields: file
  raise 'readonly mogilefs' if readonly?

  res = @backend.create_open(:domain => @domain, :class => klass,
                             :key => key, :multi_dest => 1)

  raise "#{@backend.lasterr}: #{@backend.lasterrstr}" if res.nil? || res == {} # HACK

  dests = nil

  if res.include? 'dev_count' then # HACK HUH?
    dests = (1..res['dev_count'].to_i).map do |i|
      [res["devid_#{i}"], res["path_#{i}"]]
    end
  else
    # 0x0040:  d0e4 4f4b 2064 6576 6964 3d31 2666 6964  ..OK.devid=1&fid
    # 0x0050:  3d33 2670 6174 683d 6874 7470 3a2f 2f31  =3&path=http://1
    # 0x0060:  3932 2e31 3638 2e31 2e37 323a 3735 3030  92.168.1.72:7500
    # 0x0070:  2f64 6576 312f 302f 3030 302f 3030 302f  /dev1/0/000/000/
    # 0x0080:  3030 3030 3030 3030 3033 2e66 6964 0d0a  0000000003.fid..

    dests = [[res['devid'], res['path']]]
  end

  dest = dests.first
  devid, path = dest

  case path
  when nil, '' then
    raise 'Empty path for mogile upload'
  when /^http:\/\// then
    MogileFS::HTTPFile.open(self, res['fid'], path, devid, klass, key,
                            dests, bytes, &block)
  else
    MogileFS::NFSFile.open(self, res['fid'], path, devid, klass, key, &block)
  end
end

#rename(from, to) ⇒ Object

Renames a key from to key to.



224
225
226
227
228
229
230
231
232
 
# File 'lib/mogilefs/mogilefs.rb', line 224

def rename(from, to)
  raise 'readonly mogilefs' if readonly?

  res = @backend.rename :domain => @domain, :from_key => from, :to_key => to

  if res.nil? and @backend.lasterr != 'unknown_key' then
    raise "unable to rename #{from} to #{to}: #{@backend.lasterr}"
  end
end

#size(key) ⇒ Object

Returns the size of key.



236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/mogilefs/mogilefs.rb', line 236

def size(key)
  paths = get_paths key

  return nil unless paths

  paths.each do |path|
    next unless path
    case path
    when /^http:\/\// then
      begin
        url = URI.parse path

        req = Net::HTTP::Head.new url.request_uri

        res = timeout @get_file_data_timeout, MogileFS::Timeout do
          Net::HTTP.start url.host, url.port do |http|
            http.request req
          end
        end

        return res['Content-Length'].to_i
      rescue MogileFS::Timeout
        next
      end
    else
      next unless File.exist? path
      return File.size(path)
    end
  end

  return nil
end

#sleep(duration) ⇒ Object

Sleeps duration.



217
218
219
 
# File 'lib/mogilefs/mogilefs.rb', line 217

def sleep(duration)
  @backend.sleep :duration => duration
end

#store_content(key, klass, content) ⇒ Object

Stores content into key in class klass.



191
192
193
194
195
196
197
198
199
 
# File 'lib/mogilefs/mogilefs.rb', line 191

def store_content(key, klass, content)
  raise 'readonly mogilefs' if readonly?

  new_file key, klass do |mfp|
    mfp << content
  end

  return content.length
end

#store_file(key, klass, file) ⇒ Object

Copies the contents of file into key in class klass. file can be either a file name or an object that responds to #read.



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/mogilefs/mogilefs.rb', line 171

def store_file(key, klass, file)
  raise 'readonly mogilefs' if readonly?

  new_file key, klass do |mfp|
    if file.respond_to? :sysread then
      return sysrwloop(file, mfp)
    else
	if File.size(file) > 0x10000 # Bigass file, handle differently
 mfp.bigfile = file
 return mfp.close
	else
        return File.open(file) { |fp| sysrwloop(fp, mfp) }
      end
    end
  end
end