Class: SdbS3Interface::FilesMgr

Inherits:
Object
  • Object
show all
Includes:
AWS
Defined in:
lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb

Constant Summary collapse

AccessKey = 
ENV["AMAZON_ACCESS_KEY_ID"]
SecretKey = 
ENV["AMAZON_SECRET_ACCESS_KEY"]
BucketNamespacePrefix = 
'forforf'
@@s3_connection = 
S3::Base.establish_connection!(:access_key_id => AccessKey,
:secret_access_key => SecretKey,
:persistent => false)

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(glue_env, node_key_value) ⇒ FilesMgr

Returns a new instance of FilesMgr.



33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 33

def initialize(glue_env, node_key_value)
  #@s3_connection = S3::Base.establish_connection!(:access_key_id => AccessKey,
  #                                                                    :secret_access_key => SecretKey)
  @bucket_name = "#{BucketNamespacePrefix}_#{glue_env.user_datastore_location}"
  #@attachment_bucket = use_bucket(@bucket_name)  #This can be stale!!!
  #verify bucket is ready
  #puts "Previous Response: #{S3::Service.response}"
  #puts "#{__LINE__} - #{ S3::Service.buckets(true).map{|b| b.name} }"
  #puts "This Bucket: #{@attachment_bucket.name}"
  #puts "Last Response: #{S3::Service.response}"
  #size = @attachment_bucket.size
end

Instance Attribute Details

#bucket_name ⇒ Object

Returns the value of attribute bucket_name.



31
32
33
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 31

def bucket_name
  @bucket_name
end

Instance Method Details

#add(node, file_datas) ⇒ Object

TODO: Move common file management functions from base node to here



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 47

def add(node, file_datas)
  filenames = []
  file_datas.each do |file_data|
    filenames << file_data[:src_filename]
  end

  filenames.each do |filename|
    basename = File.basename(filename)
    esc_basename = TkEscape.escape(basename)
    begin
      S3::S3Object.store(esc_basename, open(filename), @bucket_name)
    rescue AWS::S3::NoSuchBucket
      puts "Rescued while adding files, retrying"
      retry_request { S3::S3Object.store(esc_basename, open(filename), @bucket_name) }
    end
  end
  #verify files are there
  files = self.list_attachments

  filenames.each do |f|
    bname = TkEscape.escape(File.basename(f))
    esc_bname = TkEscape.escape(bname)
    retry_request { S3::S3Object.store(esc_bname, open(f), @bucket_name) } unless files.include?(esc_bname)
  end
  
  #update the metadata (have to wait until they're uploaded *sigh*
  filenames.each do |f|
    basename = File.basename(f)
    attach_name = TkEscape.escape(basename)
    begin
      s3_obj = S3::S3Object.find(attach_name, @bucket_name)
    rescue AWS::S3::NoSuchBucket
      puts "Rescued while finding bucket, retrying"
      retry_request { S3::S3Object.find(attach_name, @bucket_name) }
    end  
    modified_at = File.mtime(f).to_s
    s3_obj.metadata[:modified_at] = modified_at
    s3_obj.store
  end
  
  filenames.map {|f| TkEscape.escape(File.basename(f))} #return basenames
end

#add_raw_data(node, file_name, content_type, raw_data, file_modified_at = nil) ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 90

def add_raw_data(node, file_name, content_type, raw_data, file_modified_at = nil)
  
  attach_name = TkEscape.escape(file_name)
  
  options = {:content_type => content_type}
#=begin
  begin
    resp = S3::S3Object.store(attach_name, raw_data, @bucket_name, options)
  rescue AWS::S3::NoSuchBucket
    puts "Rescued while adding raw data, retrying"
    retry_request { S3::S3Object.store(attach_name, raw_data, @bucket_name, options) }
  end

    obj = S3::S3Object.find(attach_name, @bucket_name)
    obj.metadata[:modified_at] = "2011-01-13" #CGI.escape(file_modified_at)
    obj.store

  #verify files are there
  max_wait_time = 20
  now = Time.now
  until (S3::S3Object.exists?(attach_name, @bucket_name) ) || (Time.now > (now + max_wait_time))
    puts "Retrying store for add raw data"
    sleep 2
    S3::S3Object.store(attach_name, raw_data, @bucket_name, options)
  end
  
  [attach_name]
end

#destroy_file_container ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 203

def destroy_file_container
  this_bucket = use_bucket(@bucket_name)
  begin
    this_bucket.delete(:force => true)
  rescue AWS::S3::NoSuchBucket
    puts "Running sanity check"
    buckets = S3::Service.buckets(true).map{|b| b.name}
    if buckets.include?(@bucket_name)
      puts "AWS temporarily lost bucket before finding it so it can be deleted"
      retry_request { this_bucket.delete(:force => true) }
    end
  end
end

#get_attachments_metadata(node) ⇒ Object

todo change name to get_files_metadata



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 158

def get_attachments_metadata(node)
  files_md = {}
  begin
    this_bucket = use_bucket(@bucket_name)
    objects = this_bucket.objects
  rescue  AWS::S3::NoSuchBucket
    puts "rescued while getting objects from bucket to check metadata"
    objects = retry_request{ this_bucket.objects }
  end
  objects.each do |object|
    begin
      obj_md = object.about.merge(object.metadata)
    rescue AWS::S3::NoSuchBucket 
      puts "Rescued while getting metadata from object"
      obj_md = retry_request{object.about}
    end
    time_str = obj_md["x-amz-meta-modified-at"]||Time.parse(obj_md["last_modified"]).to_s
    obj_md_file_modified = time_str
    obj_md_content_type = obj_md["content-type"]
    new_md = {:content_type => obj_md_content_type, :file_modified => obj_md_file_modified}

    new_md.merge(obj_md)  #where does the original metadata go?
    files_md[object.key] = new_md
  end
  files_md
end

#get_raw_data(node, basename) ⇒ Object 



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 137

def get_raw_data(node, basename)
  rtn = nil
  
  attach_name = TkEscape.escape(basename)

  begin
    rtn = S3::S3Object.value(attach_name, @bucket_name)
  rescue AWS::S3::NoSuchBucket
    puts "Rescued while getting raw data, bucket name: #{@bucket_name}"
    begin
      rtn = retry_request(attach_name, @bucket_name){|obj, buck| puts "sdbs3: #{obj.inspect} - #{buck.inspect}"; S3::S3Object.value(obj, buck)}
    rescue AWS::S3::NoSuchKey
      rtn = nil
    end
  rescue AWS::S3::NoSuchKey
    rtn = nil
  end
  rtn
end

#list(node) ⇒ Object 



119
120
121
122
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 119

def list(node)
  #conforming to base file mgr
  list_attachments
end

#list_attachments ⇒ Object 



197
198
199
200
201
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 197

def list_attachments
  objs = list_objects
  atts = objs.map{|o| o.key} if objs
  atts || []
end

#list_objects ⇒ Object

def



185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 185

def list_objects
  list = nil
  this_bucket = use_bucket(@bucket_name)
  begin
    list = this_bucket.objects
  rescue AWS::S3::NoSuchBucket
    puts "Rescued while listing attachments"
    list = retry_request{this_bucket.objects}
  end
  list
end

#retry_request(*args, &block) ⇒ Object 



255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 255

def retry_request(*args, &block)
  puts "RETRYING Request with block: #{block.inspect}"
  wait_time = 2
  backoff_delay = 0.5
  max_retries = 10
  
  resp = nil

  1.upto(max_retries) do |i|
    puts "Wating #{wait_time} secs to try again"
    sleep wait_time
    begin
      resp = yield *args
      raise TypeError, "Response was Nil, retrying" unless resp
      break
    rescue AWS::S3::NoSuchKey => e
      raise e  #we want to raise this one"
    rescue AWS::S3::ResponseError => e 
      puts "rescued #{e.inspect}"
      backoff_delay += backoff_delay# * i
      wait_time += backoff_delay
      if (wait_time > 3) && (e.class == AWS::S3::NoSuchBucket)
        puts "Attempting to reset bucket"
        @attachment_bucket = use_bucket(@bucket_name)
      end
      next
    end#begin-rescue
  end#upto
  
  
end

#subtract(node, file_basenames) ⇒ Object 



124
125
126
127
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 124

def subtract(node, file_basenames)
  #conforming to base file mgr
  subtract_files(node, file_basenames)
end

#subtract_all ⇒ Object 



224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 224

def subtract_all
  #Changed behavior to leave bucket (this is different than other FileMgrs) 
  this_bucket = use_bucket(@bucket_name)
  begin  
    this_bucket.delete_all
  rescue AWS::S3::NoSuchBucket
    puts "Bucket not found while deleting all. Maybe it's already been deleted?"
    return nil
    #aws_names = retry_request{@attachment_bucket.objects}
  end
  
  max_wait_time = 20
  now = Time.now
  
  while Time.now < (now + max_wait_time)
    begin
      this_bucket.delete
    rescue AWS::S3::BucketNotEmpty
      sleep 1
      puts "Bucket not empty yet, trying again"
      this_bucket.delete_all
      next
    end
    break
  end
  
  use_bucket(@bucket_name)
  #file_basenames = aws_names.map{|o| o.key} if aws_names
  #self.subtract_some(file_basenames) if file_basenames
end

#subtract_files(node, file_basenames) ⇒ Object 



129
130
131
132
133
134
135
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 129

def subtract_files(node, file_basenames)
  if file_basenames == :all
    subtract_all
  else
    subtract_some(file_basenames)
  end
end

#subtract_some(file_basenames) ⇒ Object 



217
218
219
220
221
222
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 217

def subtract_some(file_basenames)
  file_basenames.each do |basename|
    attach_name = TkEscape.escape(basename)
    S3::S3Object.delete(attach_name, @bucket_name)
  end
end

#use_bucket(bucket_name) ⇒ Object

def

Raises:



287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
 
# File 'lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb', line 287

def use_bucket(bucket_name)
  begin
    bucket = S3::Bucket.find(bucket_name)
  rescue (AWS::S3::NoSuchBucket||NilBucketError) => e
    begin
      puts "Rescued error in use_bucket: #{e.inspect}"
      S3::Bucket.create(bucket_name)
      bucket = S3::Bucket.find(bucket_name)
    rescue AWS::S3::NoSuchBucket #we just made it!!
      bucket = retry_request(bucket_name){|buck_name| S3::Bucket.find(buck_name)}
    end#inner begin-rescue
  end#outer begin-rescue
  
  #verify bucket exists
  found_buckets = S3::Service.buckets(true).map{|b| b.name}
  unless found_buckets.include?(bucket_name)
    #bucket = retry(:retry_block, bucket_name){|buck_name| S3::Bucket.find(buck_name)}
  end#unless
  unless bucket
    puts "NIL Bucket cannot be returned"
    retry_request(bucket_name){|buck_name| S3::Bucket.find(buck_name)}
  end
  raise(NilBucketError, "NIL Bucket cannot be returned",nil) unless bucket
  return bucket
end