Class: SdbDal::Storage

Inherits:
Object
  • Object
show all
Defined in:
lib/sdb_dal/storage.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(aws_key_id, aws_secret_key, memcache_servers, tokens = [], options = {}) ⇒ Storage

Returns a new instance of Storage.



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/sdb_dal/storage.rb', line 19

def initialize( aws_key_id,
    aws_secret_key,
    memcache_servers,
    tokens=[] ,
    options={})
  @memcache_servers=memcache_servers
   if options.has_key?(:fail_fast)
    self.fail_fast=options[:fail_fast]
   end
  if options.has_key?(:memory_only) and options[:memory_only]
    self.memory_only=true
  else
    @aws_key_id=aws_key_id
    @aws_secret_key=aws_secret_key
    @tokens=tokens

  end


  if @memcache_servers and @memcache_servers.length>0
   @cache= MemCache.new @memcache_servers, :namespace => 'my_namespace'
  end
end

Instance Attribute Details

#fail_fast ⇒ Object

Returns the value of attribute fail_fast.



18
19
20
 
# File 'lib/sdb_dal/storage.rb', line 18

def fail_fast
  @fail_fast
end

#memory_only ⇒ Object

Returns the value of attribute memory_only.



17
18
19
 
# File 'lib/sdb_dal/storage.rb', line 17

def memory_only
  @memory_only
end

#tokens ⇒ Object

Returns the value of attribute tokens.



16
17
18
 
# File 'lib/sdb_dal/storage.rb', line 16

def tokens
  @tokens
end

Instance Method Details

#create_bucket(bucket, headers = {}) ⇒ Object 



99
100
101
 
# File 'lib/sdb_dal/storage.rb', line 99

def create_bucket(bucket,headers={})
  real_s3.create_bucket(bucket,headers)
end

#create_direct_url(bucket, key, time_to_live_minutes = 60) ⇒ Object 



196
197
198
199
200
201
 
# File 'lib/sdb_dal/storage.rb', line 196

def create_direct_url(bucket,key,time_to_live_minutes=60) 
   real_s3_query_auth.expires_in=time_to_live_minutes*60
   real_s3_query_auth.get(bucket,key)
     

end

#create_list_bucket_url(bucket, time_to_live_minutes = 60) ⇒ Object 



208
209
210
211
212
213
 
# File 'lib/sdb_dal/storage.rb', line 208

def create_list_bucket_url(bucket,time_to_live_minutes=60) 
   real_s3_query_auth.expires_in=time_to_live_minutes*60
   real_s3_query_auth.list_bucket(bucket)
     

end

#create_public_url(bucket, key) ⇒ Object 



193
194
195
 
# File 'lib/sdb_dal/storage.rb', line 193

def create_public_url(bucket,key) 
      return "http://s3.amazonaws.com/"+bucket+"/"+key
end

#delete(bucket, key) ⇒ Object 



126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/sdb_dal/storage.rb', line 126

def delete(bucket,key)
  if @cache
    begin
      @cache[encode_key(bucket,key)]=nil
    rescue
      #memcache might be down
    end
  end
  real_s3.delete(bucket,key)
  
end

#encode_key(bucket, key) ⇒ Object 



63
64
65
66
67
68
69
 
# File 'lib/sdb_dal/storage.rb', line 63

def encode_key(bucket,key)
  val=CGI.escape(bucket+'$@#$%'+key)
  if val.length>250#250 is max key length in memcached
    val=val.hash.to_s
  end
  return val
end

#get(bucket, key) ⇒ Object 



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/sdb_dal/storage.rb', line 137

def get(bucket,key)
  value   =nil
  if@cache
    begin
      value=@cache[encode_key(bucket,key)]
    rescue=>e
                 s= "#{e.message}"

      puts("error on  /#{bucket}/#{key} from cache.\n#{s}\n")
    end
  end
     
  if !value
   value=real_s3_get(bucket,key)
    if @cache
      begin
        @cache[encode_key(bucket,key)]=value
      rescue
        #might be too large or memcache might be down
      end
          
    end
  
  end
  return value 
  
end

#list_bucket(bucket, prefix = nil) ⇒ Object 



187
188
189
190
191
192
 
# File 'lib/sdb_dal/storage.rb', line 187

def list_bucket(bucket,prefix=nil) 
    options={}
    options[:prefix]=prefix if prefix
   real_s3.list_bucket(bucket,options)

end

#put(bucket, key, object, attributes = {}) ⇒ Object 



164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/sdb_dal/storage.rb', line 164

def put(bucket,key,object,attributes={})

  real_s3_put(bucket,key,object,attributes)
  
  #cache in memcache if not media file
  if   memory_only ||
      !attributes ||
      !attributes.has_key?('Content-Type') || 
      (attributes['Content-Type'].index('image')!=0 && attributes['Content-Type'].index('audio')!=0  && attributes['Content-Type'].index('video')!=0   ) 
    if @cache
      begin
        @cache[encode_key(bucket,key)]=object

      rescue=>e
                 s= "#{e.message}\n"

        puts("ERROR when putting /#{bucket}/#{key}into  cache.\n#{s}\n-----------------------\n")
        #TODO try to whack any old value
      end
    end
  end
end

#real_s3 ⇒ Object 



43
44
45
46
47
48
49
50
51
 
# File 'lib/sdb_dal/storage.rb', line 43

def real_s3
  if self.memory_only
    raise "this is a memcache only storage.  there is no s3"
  end
  unless @conn
    @conn = S3::AWSAuthConnection.new(@aws_key_id, @aws_secret_key,@tokens,false)
  end 
  return @conn
end

#real_s3_get(bucket, key) ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/sdb_dal/storage.rb', line 70

def real_s3_get(bucket,key)

  if self.memory_only
    return nil
  end
  20.times do |i|
    begin
      response=real_s3.get(bucket,key)
      if response and response.http_response.code=='404'
        return nil
      end
      if response and response.http_response.code=='200'
        return response.object.data

      end
                
      renew_s3_connection

    rescue=> e

               s= "#{e.message}\n#{e.backtrace}"
      puts(s)
      puts "retrying s3 get #{i.to_s}"
      raise e if self.fail_fast
      sleep(i*i)
    end
  end
  return nil
end

#real_s3_put(bucket, key, object, attributes) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/sdb_dal/storage.rb', line 102

def real_s3_put(bucket,key,object,attributes)

  return if self.memory_only
x=nil
  20.times do |i|
    begin
    x= real_s3.put(bucket,key,S3::S3Object.new(object),attributes)
    

   break
    rescue =>e
      raise e if self.fail_fast
               s= "#{e.message}\n#{e.backtrace}"
      puts(s)
      puts "retrying s3 put #{i.to_s}"
      sleep(i*i)
      #try again
    end
  end
  if x.http_response.code!="200"

       raise "bucket #{bucket} key #{key} response #{x.http_response.code}"
    end
end

#real_s3_query_auth ⇒ Object 



52
53
54
55
56
57
 
# File 'lib/sdb_dal/storage.rb', line 52

def real_s3_query_auth
  unless @query_conn
    @query_conn = S3::QueryStringAuthGenerator.new(@aws_key_id, @aws_secret_key,@tokens,false,S3::DEFAULT_HOST, 80,S3::CallingFormat::SUBDOMAIN)
  end 
  return @query_conn
end

#renew_s3_connection ⇒ Object 



58
59
60
61
 
# File 'lib/sdb_dal/storage.rb', line 58

def renew_s3_connection
  @conn=nil
  real_s3
end