Class: Swineherd::S3FileSystem

Inherits:
Object
  • Object
show all
Defined in:
lib/swineherd-fs/s3filesystem.rb

Overview

Methods for interacting with Amazon’s Simple Store Service (S3).

Defined Under Namespace

Classes: S3File

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ S3FileSystem

Returns a new instance of S3FileSystem.



10
11
12
13
14
15
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 10

def initialize options={}
  aws_access_key = options[:aws_access_key] || (Swineherd.config[:aws] && Swineherd.config[:aws][:access_key])
  aws_secret_key = options[:aws_secret_key] || (Swineherd.config[:aws] && Swineherd.config[:aws][:secret_key])
  raise "Missing AWS keys" unless aws_access_key && aws_secret_key
  @s3 = RightAws::S3.new(aws_access_key, aws_secret_key,:logger => Logger.new(nil)) #FIXME: Just wanted it to shut up
end

Instance Attribute Details

#s3 ⇒ Object

Returns the value of attribute s3.



8
9
10
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 8

def s3
  @s3
end

Instance Method Details

#bucket(path) ⇒ Object

alias :get :copy_to_local



213
214
215
216
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 213

def bucket path
  #URI.parse(path).path.split('/').reject{|x| x.empty?}.first
  split_path(path).first
end

#copy_from_local(srcpath, destpath) ⇒ Object

FIXME: Not implemented for directories @srcpath@ is assumed to be on the local filesystem



188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 188

def copy_from_local srcpath, destpath
  bucket,key = split_path(destpath)
  if File.exists?(srcpath)
    if File.directory?(srcpath)
      raise "NotYetImplemented"
    else
      @s3.interface.put(bucket, key, File.open(srcpath))
    end
  else
    raise Errno::ENOENT, "No such file or directory - #{srcpath}"
  end
end

#copy_to_local(srcpath, dstpath) ⇒ Object

FIXME: Not implemented for directories



203
204
205
206
207
208
209
210
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 203

def copy_to_local srcpath, dstpath
  src_bucket,src_key_path = split_path(srcpath)
  dstfile = File.new(dstpath, 'w')
  @s3.interface.get(src_bucket, src_key_path) do |chunk|
    dstfile.write(chunk)
  end
  dstfile.close
end

#cp(srcpath, dstpath) ⇒ Object 



124
125
126
127
128
129
130
131
132
133
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 124

def cp srcpath, dstpath
  src_bucket,src_key_path = split_path(srcpath)
  dst_bucket,dst_key_path = split_path(dstpath)
  mkdir_p(dstpath) unless exists?(dstpath)
  if src_key_path.empty? || directory?(srcpath)
    raise Errno::EISDIR,"#{srcpath} is a directory or bucket, use cp_r"
  else
    @s3.interface.copy(src_bucket, src_key_path, dst_bucket, dst_key_path)
  end
end

#cp_r(srcpath, dstpath) ⇒ Object

mv is just a special case of cp_r…this is a waste



136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 136

def cp_r srcpath, dstpath
  src_bucket,src_key_path = split_path(srcpath)
  dst_bucket,dst_key_path = split_path(dstpath)
  mkdir_p(dstpath) unless exists?(dstpath)
  if directory? srcpath
    paths_to_copy = ls_r(srcpath)
    common_dir    = common_directory(paths_to_copy)
    paths_to_copy.each do |path|
      bkt,key = split_path(path)
      src_key = key
      dst_key = File.join(dst_key_path, path.gsub(common_dir, ''))
      @s3.interface.copy(src_bucket, src_key, dst_bucket, dst_key)
    end
  else
    @s3.interface.copy(src_bucket, src_key_path, dst_bucket, dst_key_path)
  end
end

#directory?(path) ⇒ Boolean

Returns:

  • (Boolean) 


85
86
87
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 85

def directory? path
  exists?(path) && !file?(path)
end

#exists?(path) ⇒ Boolean

Returns:

  • (Boolean) 


59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 59

def exists? path
  bucket,key = split_path(path)
  begin
    if key.empty? #only a bucket was passed in, check if it exists
      #FIXME: there may be a better way to test, relying on error to be raised here
      @s3.interface.bucket_location(bucket) && true
    elsif file?(path) #simply test for existence of the file
      true
    else #treat as directory and see if there are files beneath it
      #if it's not a file, it is harmless to add '/'.
      #the prefix search may return files with the same root extension,
      #ie. foo.txt and foo.txt.bak, if we leave off the trailing slash
      key+="/" unless key =~ /\/$/
      @s3.interface.list_bucket(bucket,:prefix => key).size > 0
    end
  rescue RightAws::AwsError => error
    if error.message =~ /nosuchbucket/i
      false
    elsif error.message =~ /not found/i
      false
    else
      raise
    end
  end
end

#file?(path) ⇒ Boolean

Returns:

  • (Boolean) 


89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 89

def file? path
  bucket,key = split_path(path)
  begin
    return false if (key.nil? || key.empty?) #buckets are not files
    #FIXME: there may be a better way to test, relying on error to be raised
    @s3.interface.head(bucket,key) && true
  rescue RightAws::AwsError => error
    if error.message =~ /nosuchbucket/i
      false
    elsif  error.message =~ /not found/i
      false
    else
      raise
    end
  end
end

#key_for(path) ⇒ Object 



218
219
220
221
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 218

def key_for path
  #File.join(URI.parse(path).path.split('/').reject{|x| x.empty?}[1..-1])
  split_path(path).last
end

#ls(path) ⇒ Object 



163
164
165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 163

def ls path
  if exists?(path)
    bkt,prefix = split_path(path)
    prefix += '/' if directory?(path) && !(prefix =~ /\/$/) && !prefix.empty?
    contents = []
    @s3.interface.incrementally_list_bucket(bkt, {'prefix' => prefix,:delimiter => '/'}) do |res|
      contents += res[:common_prefixes].map{|c| File.join(bkt,c)}
      contents += res[:contents].map{|c| File.join(bkt, c[:key])}
    end
    contents
  else
    raise Errno::ENOENT, "No such file or directory - #{path}"
  end
end

#ls_r(path) ⇒ Object 



178
179
180
181
182
183
184
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 178

def ls_r path
  if(file?(path))
    [path]
  else
    ls(path).inject([]){|paths,path| paths << path if directory?(path);paths << ls_r(path)}.flatten
  end
end

#mkdir_p(path) ⇒ Object

This is a bit funny, there’s actually no need to create a ‘path’ since s3 is nothing more than a glorified key-value store. When you create a ‘file’ (key) the ‘path’ will be created for you. All we do here is create the bucket unless it already exists.



158
159
160
161
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 158

def mkdir_p path
  bkt,key = split_path(path)
  @s3.interface.create_bucket(bkt) unless exists? path
end

#mv(srcpath, dstpath) ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 106

def mv srcpath, dstpath
  src_bucket,src_key_path = split_path(srcpath)
  dst_bucket,dst_key_path = split_path(dstpath)
  mkdir_p(dstpath) unless exists?(dstpath)
  if directory? srcpath
    paths_to_copy = ls_r(srcpath)
    common_dir    = common_directory(paths_to_copy)
    paths_to_copy.each do |path|
      bkt,key = split_path(path)
      src_key = key
      dst_key = File.join(dst_key_path, path.gsub(common_dir, ''))
      @s3.interface.move(src_bucket, src_key, dst_bucket, dst_key)
    end
  else
    @s3.interface.move(src_bucket, src_key_path, dst_bucket, dst_key_path)
  end
end

#open(path, mode = "r", &blk) ⇒ Object 



17
18
19
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 17

def open path, mode="r", &blk
  S3File.new(path,mode,self,&blk)
end

#rm(path) ⇒ Object 



29
30
31
32
33
34
35
36
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 29

def rm path
  bkt,key = split_path(path)
  if key.empty? || directory?(path)
    raise Errno::EISDIR,"#{path} is a directory or bucket, use rm_r or rm_bucket"
  else
    @s3.interface.delete(bkt, key)
  end
end

#rm_bucket(bucket_name) ⇒ Object 



55
56
57
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 55

def rm_bucket bucket_name
  @s3.interface.force_delete_bucket(bucket_name)
end

#rm_r(path) ⇒ Object

rm_r - Remove recursively. Does not delete buckets, use rm_bucket params: @path@ - Path of file or folder to delete returns: Array - Array of paths which were deleted



41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 41

def rm_r path
  bkt,key = split_path(path)
  if key.empty?
    # only the bucket was passed in
  else
    if directory?(path)
      @s3.interface.delete_folder(bkt,key).flatten
    else
      @s3.interface.delete(bkt, key)
      [path]
    end
  end
end

#size(path) ⇒ Object 



21
22
23
24
25
26
27
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 21

def size path
  if directory?(path)
    ls_r(path).inject(0){|sum,file| sum += filesize(file)}
  else
    filesize(path)
  end
end

#split_path(path) ⇒ Object 



223
224
225
226
227
228
229
230
 
# File 'lib/swineherd-fs/s3filesystem.rb', line 223

def split_path path
  uri = URI.parse(path)
  base_uri = ""
  base_uri << uri.host if uri.scheme
  base_uri << uri.path
  path = base_uri.split('/').reject{|x| x.empty?}
  [path[0],path[1..-1].join("/")]
end