Class: S3File

Inherits:
Object
  • Object
show all
Defined in:
lib/s3file.rb,
lib/s3file/cp.rb,
lib/s3file/ls.rb,
lib/s3file/mv.rb,
lib/s3file/rm.rb,
lib/s3file/bucket.rb,
lib/s3file/errors.rb

Defined Under Namespace

Classes: AuthorizationError, ConnectionError, InitializationError, PathError, S3CommandError, S3PathError, UnknownError

Constant Summary collapse

@@config_file = 
File.join(File.expand_path("~/"), ".s3file.cfg")

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(keys = {}) ⇒ S3File

Intialize the object to the bucket given the access_key and secret_access_key You need to have the following set *:access_key and *:secret_access_key s3cmd requires a config file to be created. Based upon the values provided - a .s3file.cfg is created in the users home directory. If the keys are not provided - it will raise an InitializationError



22
23
24
25
26
27
 
# File 'lib/s3file.rb', line 22

def initialize(keys = {})
  if keys.nil? || keys[:access_key].nil? || keys[:secret_access_key].nil?
    raise(InitializationError, "Keys not set when initializing S3 connection")
  end
  S3File.create_config_file(keys)
end

Class Method Details

.config_file_exists? ⇒ Boolean

Returns true or false if s3cmd config file exists

Returns:

  • (Boolean) 


47
48
49
 
# File 'lib/s3file.rb', line 47

def config_file_exists?
  File.exists?(@@config_file)
end

.cp(source, destination) ⇒ Object

Copy either from local to S3 or the other way around. Used for copying files and not folders. Raises a S3CommandError in case attempt to copy a folder.



5
6
7
8
9
10
11
12
 
# File 'lib/s3file/cp.rb', line 5

def cp(source, destination)
  if s3?(source)
    raise(S3CommandError, "Attempting to copy folder using cp. Use cp_r instead.") if source.match(/\/\z/)
    run_command("s3cmd get #{source} #{destination} -c #{@@config_file}")
  elsif local?(source)
    run_command("s3cmd put #{source} #{destination} -c #{@@config_file}")
  end
end

.cp_f(source, destination) ⇒ Object

Force copy from local to S3 or the other way around. This will over write local files. Raises a S3CommandError in case attempt to copy a folder.



16
17
18
19
20
21
22
23
 
# File 'lib/s3file/cp.rb', line 16

def cp_f(source, destination)
  if s3?(source)
    raise(S3CommandError, "Attempting to copy folder using cp. Use cp_rf instead.") if source.match(/\/\z/)
    run_command("s3cmd get #{source} #{destination} -c #{@@config_file} --force")
  elsif local?(source)
    run_command("s3cmd put #{source} #{destination} -c #{@@config_file} --force")
  end
end

.cp_r(source, destination) ⇒ Object

Recursively copy from local to S3 or the other way around. This is used for copying of folders.



26
27
28
29
30
31
32
 
# File 'lib/s3file/cp.rb', line 26

def cp_r(source, destination)
  if s3?(source)
    run_command("s3cmd get #{source} #{destination} -c #{@@config_file} --recursive")
  elsif local?(source)
    run_command("s3cmd put #{source} #{destination} -c #{@@config_file} --recursive")
  end
end

.cp_rf(source, destination) ⇒ Object

Recursively and forcefully copy a folder between S3 and local or the other way around.



35
36
37
38
39
40
41
 
# File 'lib/s3file/cp.rb', line 35

def cp_rf(source, destination)
  if s3?(source)
    run_command("s3cmd get #{source} #{destination} -c #{@@config_file} --recursive --force")
  elsif local?(source)
    run_command("s3cmd put #{source} #{destination} -c #{@@config_file} --recursive --force")
  end
end

.create_config_file(keys = {}) ⇒ Object

Creates the config_file



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/s3file.rb', line 52

def create_config_file(keys={})
  # Data to be put into the config file
  config_data = %Q{[default]
access_key = #{keys[:access_key].to_s}
acl_public = False
bucket_location = US
cloudfront_host = cloudfront.amazonaws.com
cloudfront_resource = /2008-06-30/distribution
default_mime_type = binary/octet-stream
delete_removed = False
dry_run = False
encoding = UTF-8
encrypt = False
force = False
get_continue = False
gpg_command = /usr/bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase = 
guess_mime_type = True
host_base = s3.amazonaws.com
host_bucket = %(bucket)s.s3.amazonaws.com
human_readable_sizes = False
list_md5 = False
preserve_attrs = True
progress_meter = True
proxy_host = 
proxy_port = 0
recursive = False
recv_chunk = 4096
secret_key = #{keys[:secret_access_key]}
send_chunk = 4096
simpledb_host = sdb.amazonaws.com
skip_existing = False
urlencoding_mode = normal
use_https = False
verbosity = WARNING}

  File.delete(@@config_file) if File.exists?(@@config_file)
  file = File.new(@@config_file, "w")
  file.puts config_data
  file.close
end

.delete_config_file! ⇒ Object

Deletes the config file that was created.



31
32
33
 
# File 'lib/s3file.rb', line 31

def delete_config_file!
  FileUtils.rm_f(@@config_file) if S3File.config_file_exists?
end

.disk_usage(bucket) ⇒ Object

Gets the disk usage of the bucket. Raises a S3PathError in case bucket specified does not start with s3://

Raises:



19
20
21
22
 
# File 'lib/s3file/bucket.rb', line 19

def disk_usage(bucket)
  raise(S3PathError, "Invalid S3 path") unless s3?(bucket)
  run_command("s3cmd du #{bucket} -c #{@@config_file}")
end

.fix_bucket(bucket) ⇒ Object

Fixes any invalid file names in the bucket. Raises S3PathError in case bucket specified does not start with s3://

Raises:



26
27
28
29
 
# File 'lib/s3file/bucket.rb', line 26

def fix_bucket(bucket)
  raise(S3PathError, "Invalid S3 path") unless s3?(bucket)
  run_command("s3cmd fixbucket #{bucket} -c #{@@config_file}")
end

.list(location = nil, display = nil) ⇒ Object

Gets a list of all the files in the given location in the bucket

Raises:



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/s3file/ls.rb', line 21

def list(location = nil, display = nil)
  return [] if location.nil?
  raise(PathError, "Not S3 Path") unless s3?(location)
  # display can take 3 possible values "all", "dir" or "file"
  # if display is nil or any other value other than dir or file then
  # display = "all"
  # if display is 'dir' or 'file' then
  # display takes that value
  if display.nil? || !display.match(/dir|file/)
    display = "all"
  end

  directories = []
  files = []
  location += "/" if location == s3_bucket(location)
  entries = run_command("s3cmd ls #{location} -c #{@@config_file}")

  directory = s3_directory(location)
  entries.split("\n").each do |entry|
    if display.match(/all|dir/) && entry.strip.match(/DIR/)
      dir = entry.strip.sub(/DIR/, '').strip.sub("#{directory}", "")
      if dir == "/"
        directories << (location + "/")
      else
        directories << dir
      end
    elsif display.match(/all|file/) && !entry.strip.match(/DIR/)
      file = entry.strip.sub(/\d{4}-\d{2}-\d{2}\s*\d{2}:\d{2}\s*/, '').split(" ")[1].sub("#{directory}","")
      if file == ""
        files << location
      else
        files << file
      end
    end
  end
  directories + files
end

.local?(location) ⇒ Boolean

Checks if the location is on local or not Checking that the location does not start with s3:// Short cut - use the opposite of s3? to get the solution

Returns:

  • (Boolean) 


107
108
109
110
111
112
113
 
# File 'lib/s3file.rb', line 107

def local?(location)
  if s3?(location)
    return false
  else
    return File.exists?(File.expand_path(location))
  end
end

.ls(location = nil) ⇒ Object

Gets a list of all the entries in a given directory



4
5
6
 
# File 'lib/s3file/ls.rb', line 4

def ls(location=nil)
  list(location, 'all')
end

.ls_directories(location = nil) ⇒ Object

Gets a list of all the directories in the given location in the bucket



9
10
11
 
# File 'lib/s3file/ls.rb', line 9

def ls_directories(location=nil)
  list(location, 'dir')
end

.ls_files(location = nil) ⇒ Object

Gets a list of all the files in the given location in the bucket



14
15
16
 
# File 'lib/s3file/ls.rb', line 14

def ls_files(location=nil)
  list(location, 'file')
end

.make_bucket(bucket) ⇒ Object

Makes a bucket in S3. Raises a S3PathError in case bucket specified does not start with s3://

Raises:



5
6
7
8
 
# File 'lib/s3file/bucket.rb', line 5

def make_bucket(bucket)
  raise(S3PathError, "Invalid S3 path") unless s3?(bucket)
  run_command("s3cmd mb #{bucket} -c #{@@config_file}")
end

.md5_hash(location) ⇒ Object

Gets the md5 hash of the s3 file. This will be used to confirm if the copy occurred successfully or not.

Raises:



37
38
39
40
 
# File 'lib/s3file.rb', line 37

def md5_hash(location)
  raise(S3PathError, "Not an S3 location") unless s3?(location)
  run_command("s3cmd --list-md5 ls #{location} -c #{@@config_file}").split(" ")[3]
end

.mv(source, destination) ⇒ Object

Moves a file from a bucket to another. Requires that both source and destination be s3 location - raises S3PathError otherwise.

Raises:



5
6
7
8
 
# File 'lib/s3file/mv.rb', line 5

def mv(source, destination)
  raise(S3PathError, "Either source or destination (or both) not S3") unless (s3?(source) && s3?(destination))
  run_command("s3cmd mv #{source} #{destination} -c #{@@config_file}")
end

.remove_bucket(bucket) ⇒ Object

Removes a bucket from S3. Raises a S3PathError in case bucket specified does not start with s3://

Raises:



12
13
14
15
 
# File 'lib/s3file/bucket.rb', line 12

def remove_bucket(bucket)
  raise(S3PathError, "Invalid S3 path") unless s3?(bucket)
  run_command("s3cmd rb #{bucket} -c #{@@config_file}")
end

.rm(location) ⇒ Object

Deletes the file from S3. Raises an S3CommandError in case of directory deletion. Use s3_r instead. Raises a S3PathError in case the location provided is not S3



6
7
8
9
10
11
12
13
 
# File 'lib/s3file/rm.rb', line 6

def rm(location)
  if s3?(location)
    raise(S3CommandError, "Attempting to delete folder using rm. Use rm_r instead.") if source.match(/\/\z/)
    run_command("s3cmd del #{location} -c #{@@config_file}")
  else
    raise(S3PathError, "Incorrect S3 path")
  end
end

.rm_r(location) ⇒ Object

Recursively deletes a folder from S3 Raises a S3PathError in case the location provided is not S3



17
18
19
20
21
22
23
 
# File 'lib/s3file/rm.rb', line 17

def rm_r(location)
  if s3?(location)
    run_command("s3cmd del #{location} -c #{@@config_file} --recursive")
  else
    raise(S3PathError, "Incorrect S3 path")
  end
end

.rm_rf(location) ⇒ Object

Recursively and forcefully deletes a folder from S3 Raises S3PathError in case the location provided is not an S3 location



27
28
29
30
31
32
33
 
# File 'lib/s3file/rm.rb', line 27

def rm_rf(location)
  if s3?(location)
    run_command("s3cmd del #{location} -c #{@@config_file} --recursive --force")
  else
    raise(S3PathError, "Incorrect S3 path")
  end
end

.run_command(command) ⇒ Object 



126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/s3file.rb', line 126

def run_command(command)
  output = status = err = nil
  status =
          Open4::popen4(command) do |pid, stdin, stdout, stderr|
            output = stdout.read
            err = stderr.read
          end

  if err.empty?
    raise(PathError, "File not found") if output.empty?
    return output
  else
    puts "Error #{err}"
    raise(AuthorizationError, "Authorization keys failed") if err.match("S3 error: 403")
    raise(S3PathError, "S3 file does not exist") if err.match("S3 error: 404")
    raise(S3PathError, "S3 bucket does not exist") if err.match("does not exist")
    raise(S3PathError, "S3 bucket does not exist") if err.match("was denied")
    raise(PathError, "Local file already exists") if err.match("already exists")
    raise(ConnectionError, "Check your connection and try again") if err.match("Errno -2")
    raise(UnknownError, "Unknown Error caught - #{err}")
  end
end

.s3?(location) ⇒ Boolean

Checks if the location is on s3 or not Checking if the location starts with s3://

Returns:

  • (Boolean) 


98
99
100
101
 
# File 'lib/s3file.rb', line 98

def s3?(location)
  return true if location.match(/s3:\/\//)
  return false
end

.s3_bucket(location) ⇒ Object

Gets the s3 bucket given a location



121
122
123
 
# File 'lib/s3file.rb', line 121

def s3_bucket(location)
  location.match("s3://[^\/]*").to_s
end

.s3_directory(location) ⇒ Object

Gets the directory of the file given the location



116
117
118
 
# File 'lib/s3file.rb', line 116

def s3_directory(location)
  location.sub(%r{#{location.match(/[^\/]*\z/)}\z}, "")
end