Class: MogileFS::HTTPFile

Inherits:
StringIO
  • Object
show all
Includes:
Util
Defined in:
lib/mogilefs/httpfile.rb

Overview

HTTPFile wraps up the new file operations for storing files onto an HTTP storage node.

You really don’t want to create an HTTPFile by hand. Instead you want to create a new file using MogileFS::MogileFS.new_file.

– TODO dup’d content in MogileFS::NFSFile

Defined Under Namespace

Classes: BadResponseError, EmptyResponseError, NoStorageNodesError, UnparseableResponseError

Constant Summary

Constants included from Util

Util::CHUNK_SIZE

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Util

#sysread_full, #sysrwloop, #syswrite_full

Constructor Details

#initialize(mg, fid, klass, key, dests, content_length) ⇒ HTTPFile

Creates a new HTTPFile with MogileFS-specific data. Use MogileFS::MogileFS#new_file instead of this method.



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/mogilefs/httpfile.rb', line 70

def initialize(mg, fid, klass, key, dests, content_length)
  super ''
  @mg = mg
  @fid = fid
  @uri = @devid = nil
  @klass = klass
  @key = key
  @big_io = nil
  @streaming_io = nil

  @dests = dests
  @tried = {}

  @socket = nil
end

Instance Attribute Details

#big_io ⇒ Object

The big_io name in case we have file > 256M



45
46
47
 
# File 'lib/mogilefs/httpfile.rb', line 45

def big_io
  @big_io
end

#class ⇒ Object (readonly)

The class of this file.



40
41
42
 
# File 'lib/mogilefs/httpfile.rb', line 40

def class
  @class
end

#key ⇒ Object (readonly)

The key for this file. This key won’t represent a real file until you’ve called #close.



35
36
37
 
# File 'lib/mogilefs/httpfile.rb', line 35

def key
  @key
end

#streaming_io ⇒ Object

Returns the value of attribute streaming_io.



47
48
49
 
# File 'lib/mogilefs/httpfile.rb', line 47

def streaming_io
  @streaming_io
end

#uri ⇒ Object (readonly)

The URI this file will be stored to.



29
30
31
 
# File 'lib/mogilefs/httpfile.rb', line 29

def uri
  @uri
end

Class Method Details

.open(*args) ⇒ Object

Works like File.open. Use MogileFS::MogileFS#new_file instead of this method.



53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/mogilefs/httpfile.rb', line 53

def self.open(*args)
  fp = new(*args)
  fp.set_encoding(Encoding::BINARY) if fp.respond_to?(:set_encoding)

  return fp unless block_given?

  begin
    yield fp
  ensure
    fp.close
  end
end

Instance Method Details

#close ⇒ Object

def upload



135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/mogilefs/httpfile.rb', line 135

def close
  try_dests = @dests.dup
  last_err = nil

  loop do
    devid, url = try_dests.shift
    devid && url or break

    uri = URI.parse(url)
    begin
      bytes = upload(devid, uri)
      @devid, @uri = devid, uri
      return bytes
    rescue SystemCallError, Errno::ECONNREFUSED, MogileFS::Timeout,
           EmptyResponseError, BadResponseError,
           UnparseableResponseError => err
      last_err = @tried[url] = err
    end
  end

  raise last_err ? last_err : NoStorageNodesError
end

#upload(devid, uri) ⇒ Object

Writes an HTTP PUT request to sock to upload the file and returns file size if the socket finished writing



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/mogilefs/httpfile.rb', line 89

def upload(devid, uri)
  file_size = length
  sock = Socket.mogilefs_new(uri.host, uri.port)
  sock.mogilefs_tcp_cork = true

  if @streaming_io
    file_size = @streaming_io.length
    syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
                        "Content-Length: #{file_size}\r\n\r\n")
    @streaming_io.call(Proc.new do |data_to_write|
      syswrite_full(sock, data_to_write)
    end)
  elsif @big_io
    # Don't try to run out of memory
    File.open(@big_io, "rb") do |fp|
      file_size = fp.stat.size
      fp.sync = true
      syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
                          "Content-Length: #{file_size}\r\n\r\n")
      sysrwloop(fp, sock)
    end
  else
    syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
                        "Content-Length: #{length}\r\n\r\n#{string}")
  end
  sock.mogilefs_tcp_cork = false

  line = sock.gets or
    raise EmptyResponseError, 'Unable to read response line from server'

  if line =~ %r%^HTTP/\d+\.\d+\s+(\d+)% then
    case $1.to_i
    when 200..299 then # success!
    else
      raise BadResponseError, "HTTP response status from upload: #{$1}"
    end
  else
    raise UnparseableResponseError, "Response line not understood: #{line}"
  end

  @mg.backend.create_close(:fid => @fid, :devid => devid,
                           :domain => @mg.domain, :key => @key,
                           :path => uri.to_s, :size => file_size)
  file_size
end