Class: Mortar::S3::S3
Instance Method Summary
collapse
-
#check_bucket(s3, bucket) ⇒ Object
checks s3 bucket to see if bucket and key exists for given aws keys.
-
#do_download(bucket, key, output_path, concat = nil) ⇒ Object
Implementation of the download command.
-
#download_s3(s3, bucket, key, output_path, concat = nil) ⇒ Object
-
#get_bucket_and_key(s3_path) ⇒ Object
returns bucket and key from s3 path.
-
#get_file_name(key_str) ⇒ Object
-
#get_s3 ⇒ Object
-
#get_s3_objects(s3, bucket, key) ⇒ Object
gets s3 object, where each item is a file in bucket and key.
-
#is_file(path, key) ⇒ Object
-
#is_hidden_file(str) ⇒ Object
-
#is_not_folder(path, key) ⇒ Object
-
#is_valid_s3_path(uri) ⇒ Object
checks if string is a valid s3 path.
-
#remove_file(file_path) ⇒ Object
-
#remove_slash(str) ⇒ Object
-
#write_s3_to_file(s3_object, file_name, write_mode) ⇒ Object
Methods included from Helpers
#action, #ask, #confirm, #copy_if_not_present_at_dest, #default_host, #deprecate, #display, #display_header, #display_object, #display_row, #display_table, #display_with_indent, #download_to_file, #ensure_dir_exists, #error, error_with_failure, error_with_failure=, extended, extended_into, #format_bytes, #format_date, #format_with_bang, #full_host, #get_terminal_environment, #home_directory, #host, #hprint, #hputs, included, included_into, #installed_with_omnibus?, #json_decode, #json_encode, #line_formatter, #longest, #output_with_bang, #pending_github_team_state_message, #quantify, #redisplay, #retry_on_exception, #running_on_a_mac?, #running_on_windows?, #set_buffer, #shell, #spinner, #status, #string_distance, #styled_array, #styled_error, #styled_hash, #styled_header, #suggestion, #test_name, #ticking, #time_ago, #truncate, #warning, #with_tty, #write_to_file
Instance Method Details
#check_bucket(s3, bucket) ⇒ Object
checks s3 bucket to see if bucket and key exists for given aws keys
107
108
109
|
# File 'lib/mortar/s3.rb', line 107
def check_bucket(s3, bucket)
s3.buckets[bucket].exists?
end
|
#do_download(bucket, key, output_path, concat = nil) ⇒ Object
Implementation of the download command
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
# File 'lib/mortar/s3.rb', line 160
def do_download(bucket, key, output_path, concat=nil)
s3 = get_s3
unless check_bucket(s3, bucket)
error("Requested S3 bucket #{bucket} in path, s3://#{bucket}/#{key}, does not exist.")
end
unless File.directory?(output_path)
FileUtils.mkdir_p(output_path)
end
download_s3(s3, bucket, key, output_path, concat)
display "All done! "
end
|
#download_s3(s3, bucket, key, output_path, concat = nil) ⇒ Object
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
# File 'lib/mortar/s3.rb', line 139
def download_s3(s3, bucket, key, output_path, concat = nil)
s3_objects = get_s3_objects(s3, bucket,key)
concat_file_name = "output"
concat_target = "#{output_path}/#{concat_file_name}"
if concat
remove_file(concat_target)
end
if !s3_objects.empty?
s3_objects.each do |s3_obj|
key_str = s3_obj.key
out_file = concat ? concat_file_name : get_file_name(key_str)
output_target = "#{output_path}/#{out_file}"
display "Writing s3://#{bucket}/#{key_str} to #{output_target}"
write_s3_to_file(s3_obj, output_target, concat ? "a" : "w+")
end
else
error("No contents were found at path s3://#{bucket}/#{key}. Please specify again.")
end
end
|
#get_bucket_and_key(s3_path) ⇒ Object
returns bucket and key from s3 path
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
# File 'lib/mortar/s3.rb', line 42
def get_bucket_and_key(s3_path)
uri = ''
begin
uri = URI.parse(s3_path)
rescue URI::InvalidURIError => msg
error("#{msg}.\nIt is strongly suggested that your bucket name does not contain an underscore character.\nPlease see http://blog.mortardata.com/post/58920122308/s3-hadoop-performance at tip #4.")
end
unless uri != '' && is_valid_s3_path(uri)
error("Requested S3 path, #{s3_path}, is invalid. Please ensure that your S3 path begins with 's3://'. Example: s3://my-bucket/my-key.")
end
return uri.host,uri.path[1, uri.path.length]
end
|
#get_file_name(key_str) ⇒ Object
71
72
73
74
75
76
77
|
# File 'lib/mortar/s3.rb', line 71
def get_file_name(key_str)
if !key_str.rindex("/")
return key_str
else
return Pathname.new(key_str).basename.to_s
end
end
|
#get_s3 ⇒ Object
33
34
35
36
37
38
39
|
# File 'lib/mortar/s3.rb', line 33
def get_s3
ctrl = Mortar::Local::Controller.new
ctrl.require_aws_keys
return AWS::S3.new(
:access_key_id => ENV['AWS_ACCESS_KEY'],
:secret_access_key => ENV['AWS_SECRET_KEY'])
end
|
#get_s3_objects(s3, bucket, key) ⇒ Object
gets s3 object, where each item is a file in bucket and key
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
# File 'lib/mortar/s3.rb', line 113
def get_s3_objects(s3, bucket, key)
buck = s3.buckets[bucket]
key = remove_slash(key)
if buck.objects[key].exists?
[buck.objects[key]]
else
valid_items = Array.new
buck.objects.with_prefix(key).each do |obj|
if is_file(obj.key, key)
valid_items.push(obj)
end
end
return valid_items
end
end
|
#is_file(path, key) ⇒ Object
79
80
81
82
83
84
85
86
87
|
# File 'lib/mortar/s3.rb', line 79
def is_file(path, key)
if is_not_folder(path,key)
res = Pathname.new(path).basename.to_s
if !is_hidden_file(res)
return true
end
end
return false
end
|
#is_hidden_file(str) ⇒ Object
96
97
98
|
# File 'lib/mortar/s3.rb', line 96
def is_hidden_file(str)
str[0,1] == "." || str[0,1] == "_"
end
|
#is_not_folder(path, key) ⇒ Object
89
90
91
92
93
94
|
# File 'lib/mortar/s3.rb', line 89
def is_not_folder(path, key)
suffix = ["/", "_$folder$"]
path[-1,1] != suffix[0] && !path.index(suffix[1]) and !path[key.length+1, path.length].index("/")
end
|
#is_valid_s3_path(uri) ⇒ Object
checks if string is a valid s3 path
59
60
61
|
# File 'lib/mortar/s3.rb', line 59
def is_valid_s3_path(uri)
uri.scheme == 's3' && uri.host && !uri.path[1, uri.path.length].to_s.empty?
end
|
#remove_file(file_path) ⇒ Object
100
101
102
103
104
|
# File 'lib/mortar/s3.rb', line 100
def remove_file(file_path)
if File.file?(file_path)
FileUtils.remove(file_path)
end
end
|
#remove_slash(str) ⇒ Object
64
65
66
67
68
69
|
# File 'lib/mortar/s3.rb', line 64
def remove_slash(str)
if str[-1, 1] == "/"
str = str[0, str.length-1]
end
return str
end
|
#write_s3_to_file(s3_object, file_name, write_mode) ⇒ Object
130
131
132
133
134
135
136
|
# File 'lib/mortar/s3.rb', line 130
def write_s3_to_file(s3_object, file_name, write_mode)
File.open(file_name, write_mode) do |file|
s3_object.read do |chunk|
file.write(chunk)
end
end
end
|