Module: S3sync

Includes:
S3Config
Defined in:
lib/s3sync/s3cmd.rb,
lib/s3sync/s3try.rb,
lib/s3sync/s3sync.rb,
lib/s3sync/HTTPStreaming.rb

Overview

module

Defined Under Namespace

Classes: LocalNode, Node, ProgressStream, S3Node

Class Method Summary collapse

Class Method Details

.hashPairs(ar) ⇒ Object

turn an array into a hash of pairs



228
229
230
231
232
233
234
235
236
 
# File 'lib/s3sync/s3cmd.rb', line 228

def S3sync.hashPairs(ar)
	ret = Hash.new
	ar.each do |item|
		name = (/^(.*?):/.match(item))[1]
		item = (/^.*?:(.*)$/.match(item))[1]
		ret[name] = item
	end if ar
	ret
end

.main ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
 
# File 'lib/s3sync/s3sync.rb', line 52

def S3sync.main 	
	# ---------- OPTIONS PROCESSING ---------- #	

	$S3syncOptions = Hash.new
	optionsParser = GetoptLong.new(
		  [ '--help',    '-h',	GetoptLong::NO_ARGUMENT ],
		  [ '--ssl',     '-s',	GetoptLong::NO_ARGUMENT ],
		  [ '--recursive','-r',	GetoptLong::NO_ARGUMENT ],
		  [ '--public-read','-p', GetoptLong::NO_ARGUMENT ],
		  [ '--delete',			GetoptLong::NO_ARGUMENT ],
		  [ '--verbose', '-v',	GetoptLong::NO_ARGUMENT ],
		  [ '--dryrun',  '-n',	GetoptLong::NO_ARGUMENT ], 
		  [ '--debug',   '-d',	GetoptLong::NO_ARGUMENT ],
		  [ '--memory',   '-m',	GetoptLong::NO_ARGUMENT ],
		  [ '--progress',	GetoptLong::NO_ARGUMENT ],
             [ '--expires',        GetoptLong::REQUIRED_ARGUMENT ],
             [ '--cache-control',  GetoptLong::REQUIRED_ARGUMENT ],
      [ '--exclude',        GetoptLong::REQUIRED_ARGUMENT ],
		  [ '--make-dirs',	GetoptLong::NO_ARGUMENT ]
		  )
		  
	def S3sync.usage(message = nil)
		$stderr.puts message if message
		name = $0.split('/').last
		$stderr.puts <<"ENDUSAGE"
#{name} [options] <source> <destination>\t\tversion #{$S3SYNC_VERSION}
 --help    -h          --verbose     -v     --dryrun    -n	
 --ssl     -s          --recursive   -r     --delete
 --public-read -p      --expires="<exp>"    --cache-control="<cc>"
 --exclude="<regexp>"  --progress           --debug   -d
 --make-dirs
One of <source> or <destination> must be of S3 format, the other a local path.
Reminders:
* An S3 formatted item with bucket 'mybucket' and prefix 'mypre' looks like:
   mybucket:mypre/some/key/name
* Local paths should always use forward slashes '/' even on Windows
* Whether you use a trailing slash on the source path makes a difference.
* For examples see README.
ENDUSAGE
	exit
	end #usage
	
	begin
		optionsParser.each {|opt, arg| $S3syncOptions[opt] = (arg || true)}
	rescue StandardError
		usage # the parser already printed an error message
	end
	usage if $S3syncOptions['--help']
	$S3syncOptions['--verbose'] = true if $S3syncOptions['--dryrun'] or $S3syncOptions['--debug'] or $S3syncOptions['--progress']
	$S3syncOptions['--ssl'] = true if $S3syncOptions['--ssl'] # change from "" to true to appease s3 port chooser

	
	# ---------- CONNECT ---------- #
	S3sync::s3trySetup 

	# ---------- PREFIX PROCESSING ---------- #

	def S3sync.s3Prefix?(pre)
		# allow for dos-like things e.g. C:\ to be treated as local even with colon
		pre.include?(':') and not pre.match('^[A-Za-z]:[\\\\/]')
	end
	sourcePrefix, destinationPrefix = ARGV
	usage("You didn't set up your environment variables; see README.txt") if not($AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY) 
	usage('Need a source and a destination') if sourcePrefix == nil or destinationPrefix == nil
	usage('Both arguments can\'t be on S3') if s3Prefix?(sourcePrefix) and s3Prefix?(destinationPrefix)
	usage('One argument must be on S3') if !s3Prefix?(sourcePrefix) and !s3Prefix?(destinationPrefix)

	# so we can modify them
	sourcePrefix, destinationPrefix = sourcePrefix.dup, destinationPrefix.dup

	# handle trailing slash for source properly
	if(sourcePrefix !~ %r{/$})
		# no slash on end of source means we need to append the last src dir to dst prefix
		# testing for empty isn't good enough here.. needs to be "empty apart from potentially having 'bucket:'"
		slash = (destinationPrefix.empty? or destinationPrefix.match(%r{:$}))? "" : "/"
		# not good enough.. sometimes this coughs up the bucket as a prefix destinationPrefix.replace(destinationPrefix + slash + sourcePrefix.split(/(?:\/|:)/).last)
		# take everything at the end after a slash or colon
		destinationPrefix.replace(destinationPrefix + slash + %r{([^/:]*)$}.match(sourcePrefix)[1])
	end
	# no trailing slash on dest, ever.
	destinationPrefix.sub!(%r{/$}, "")
	
	# don't repeat slashes
	sourcePrefix.squeeze!('/')
	destinationPrefix.squeeze!('/')
	
	# here's where we find out what direction we're going
	sourceIsS3 = s3Prefix?(sourcePrefix)
	# alias these variables to the other strings (in ruby = does not make copies of strings)
	s3Prefix = sourceIsS3 ? sourcePrefix : destinationPrefix
	localPrefix = sourceIsS3 ? destinationPrefix : sourcePrefix
	
	# canonicalize the S3 stuff
	s3Bucket = (/^(.*?):/.match(s3Prefix))[1]
	s3Prefix.replace((/:(.*)$/.match(s3Prefix))[1])
	debug("s3Prefix #{s3Prefix}")
	$S3SyncOriginalS3Prefix = s3Prefix.dup
	
	# canonicalize the local stuff
	# but that can kill a trailing slash, which we need to preserve long enough to know whether we mean "the dir" or "its contents"
	# it will get re-stripped by the local generator after expressing this knowledge
	localTrailingSlash = localPrefix.match(%r{/$}) 
	localPrefix.replace(File.expand_path(localPrefix))
	localPrefix += '/' if localTrailingSlash
	debug("localPrefix #{localPrefix}")
	# used for exclusion parsing
	$S3SyncOriginalLocalPrefix = localPrefix.dup
	
	# exclude preparation
	# we don't want to build then throw away this regexp for each node in the universe; do it once globally
	$S3SyncExclude = Regexp.new($S3syncOptions['--exclude']) if $S3syncOptions['--exclude']
	
	
	# ---------- GENERATORS ---------- #

	
	# a generator that will return the files/dirs of the local tree one by one
	# sorted and decorated for easy comparison with the S3 tree
	localTree = Generator.new do |g|
		def S3sync.localTreeRecurse(g, prefix, path)
			debug("localTreeRecurse #{prefix} #{path}")
			#if $S3syncOptions['--memory']
			#	$stderr.puts "Starting local recurse"
			#	stats = ostats stats 
			#end
			d = nil
			begin
				slash = prefix.empty? ? "" : "/"
				d = Dir.new(prefix + slash + path)
			rescue Errno::ENOENT
				# ok the dir doesn't exist at all (this only really occurs for the root i.e. first dir)
				return nil
			rescue Errno::EACCES
				# vista won't even let us touch some stuff in our own profile
				return nil
			end
			# do some pre-processing
			# the following sleight of hand is to make the recursion match the way s3 sorts
			# take for example the directory 'foo' and the file 'foo.bar'
			# when we encounter the dir we would want to recurse into it
			# but S3 would just say 'period < slash' and sort 'foo.bar' between the dir node 
			# and the contents in that 'dir'
			#
			# so the solution is to not recurse into the directory until the point where
			# it would come up "next" in the S3 list
			# We have to do these hoops on the local side, because we have very little control
			# over how S3 will return its results
			toAdd = Array.new
			d.each do |name|
				slash = path.empty? ? "" : "/"
				partialPath = path + slash + name
				slash = prefix.empty? ? "" : "/"
				fullPath = prefix + slash + partialPath
				if name == "." or name == ".."
					# skip
				else
					# add a dir node if appropriate
					debug("Test #{fullPath}")
					if ((not FileTest.symlink?(fullPath)) and FileTest.directory?(fullPath)) and $S3syncOptions['--recursive']
						debug("Adding it as a dir node")
						toAdd.push(name + '/') # always trail slash here for sorting purposes (removed below with rindex test)
					end
				end
			end
			dItems = d.collect + toAdd
			d.close
			d = toAdd = nil
			dItems.sort! #aws says we will get alpha sorted results but ruby doesn't
			dItems.each do |name|
				isDirNode = false
				if name.rindex('/') == name.length-1
					name = name.slice(0...name.length-1)
					isDirNode = true
					debug("#{name} is a dir node")
				end
				slash = path.empty? ? "" : "/"
				partialPath = path + slash + name
				slash = prefix.empty? ? "" : "/"
				fullPath = prefix + slash + partialPath
				excludePath = fullPath.slice($S3SyncOriginalLocalPrefix.length...fullPath.length)
				if name == "." or name == ".."
					# skip
				elsif $S3SyncExclude and $S3SyncExclude.match(excludePath)
					debug("skipping local item #{excludePath} because of --exclude")
				elsif isDirNode
					localTreeRecurse(g, prefix, partialPath)
				else
					# a normal looking node we should try to process
					debug("local item #{fullPath}")
					g.yield(LocalNode.new(prefix, partialPath))
				end
			end
			#if $S3syncOptions['--memory']
			#	$stderr.puts "Ending local recurse"
			#	stats = ostats stats 
			#end
		end
		# a bit of a special case for local, since "foo/" and "foo" are essentially treated the same by file systems
		# so we need to think harder about what the user really meant in the command line.
		localPrefixTrim = localPrefix
		if localPrefix !~ %r{/$}
			# no trailing slash, so yield the root itself first, then recurse if appropriate
			# gork this is still not quite good enough.. if local is the dest then we don't know whether s3 will have a root dir node yielded a priori, so we can't know whether to do this.  only matters for --erase though
			g.yield(LocalNode.new(localPrefixTrim, "")) # technically we should check this for exclusion, but excluding the root node is kind of senseless.. and that would be a pain to set up here
			localTreeRecurse(g, localPrefixTrim, "") if $S3syncOptions['--recursive']
		else
			# trailing slash, so ignore the root itself, and just go into the first level
			localPrefixTrim.sub!(%r{/$}, "") # strip the slash because of how we do local node slash accounting in the recurse above
			localTreeRecurse(g, localPrefixTrim, "") 
		end
	end
	
	# a generator that will return the nodes in the S3 tree one by one
	# sorted and decorated for easy comparison with the local tree
	s3Tree = Generator.new do |g|
		def S3sync.s3TreeRecurse(g, bucket, prefix, path)
			if $S3syncOptions['--memory']
				$stderr.puts "Starting S3 recurse"
				GC.start
				stats = ostats stats 
			end
			$stderr.puts "s3TreeRecurse #{bucket} #{prefix} #{path}" if $S3syncOptions['--debug']
			nextPage = true
			marker = ''
			while nextPage do
				fullPrefix = prefix + path
				debug("nextPage: #{marker}") if marker != ''
				options = {}
				options['prefix'] = fullPrefix # start at the right depth
				options['delimiter'] = '/' # only one dir at a time please
				options['max-keys'] = '200' # use manageable chunks
				options['marker'] = marker unless marker == ''
				d = S3sync.S3try(:list_bucket, bucket, options)
				$stderr.puts "S3 ERROR: #{d.http_response}" unless d.http_response.is_a? Net::HTTPSuccess
				# the 'directories' and leaf nodes are in two separate collections
				# because a dir will never have the same name as a node, we can just shove them together and sort
				# it's important to evaluate them alphabetically for efficient comparison to the local tree
				tItems = d.entries + d.common_prefix_entries
				tItems.sort! do |a,b|
					aName = a.respond_to?('key') ? a.key : a.prefix
					bName = b.respond_to?('key') ? b.key : b.prefix
					# the full path will be returned, efficient to ignore the part we know will be in common
					aName.slice(fullPrefix.length..aName.length) <=> bName.slice(fullPrefix.length..bName.length)
				end
				# get rid of the big s3 objects asap, just save light-weight nodes and strings
				items = tItems.collect do |item|
					if item.respond_to?('key')
						key = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.key).join
						Node.new(key, item.size, item.etag)
					else
						Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.prefix).join
					end
				end
				nextPage = d.properties.is_truncated
				marker = (d.properties.next_marker)? d.properties.next_marker : ((d.entries.length > 0)? d.entries.last.key : '')
				# get this into native char set (because when we feed it back to s3 that's what it will expect)
				marker = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", marker).join
				tItems = nil
				d = nil # get rid of this before recursing; it's big
				item = nil
				GC.start # not sure but I think yielding before doing this is causing evil closure bloat
				items.each do |item|
					if not (item.kind_of? String)
						# this is an item
						excludePath = item.name.slice($S3SyncOriginalS3Prefix.length...item.name.length)
						if $S3SyncExclude and $S3SyncExclude.match(excludePath)
							debug("skipping S3 item #{excludePath} due to --exclude")
						else
							debug("S3 item #{item.name}")
							g.yield(S3Node.new(bucket, prefix, item))
						end
					else
						# it's a prefix (i.e. there are sub keys)
						partialPath = item.slice(prefix.length..item.length) # will have trailing slash
						excludePath = item.slice($S3SyncOriginalS3Prefix.length...item.length)
						# recurse
						if $S3SyncExclude and $S3SyncExclude.match(excludePath)
							debug("skipping prefix #{excludePath} due to --exclude")
						else
							debug("prefix found: #{partialPath}")
							s3TreeRecurse(g, bucket, prefix, partialPath) if $S3syncOptions['--recursive'] 
						end
					end
				end
				items = nil
			end # of while nextPage
			if $S3syncOptions['--memory']
				$stderr.puts "Ending S3 recurse"
				GC.start
				stats = ostats stats 
			end
		end
		# this will yield the root node first and then recurse
		s3TreeRecurse(g, s3Bucket, s3Prefix, "")
		
	end
	
	# alias the tree objects so we don't care below which direction the transfer is going
	if sourceIsS3
		sourceTree, destinationTree = s3Tree, localTree
	else
		sourceTree, destinationTree = localTree, s3Tree
	end
	
	
	# ---------- COMPARATOR ---------- #
	
	# run the comparison engine and act according to what we find for each check
	nodesToDelete = Array.new # a stack. have to delete in reverse order of normal create/update processing
	
	sourceNode = sourceTree.next? ? sourceTree.next : nil
	destinationNode = destinationTree.next? ? destinationTree.next : nil
	while sourceNode or destinationNode do
		debug("source: #{sourceNode.name}") if sourceNode
		debug("dest: #{destinationNode.name}") if destinationNode
		if (!destinationNode) or (sourceNode and (sourceNode.name < destinationNode.name))
			dNode = 
			if sourceNode.kind_of? LocalNode
				S3Node.new(s3Bucket, s3Prefix, sourceNode.name)
			else
				LocalNode.new(localPrefix, sourceNode.name)
			end
			puts "Create node #{sourceNode.name}" if $S3syncOptions['--verbose']
			dNode.updateFrom(sourceNode) unless $S3syncOptions['--dryrun']
			sourceNode = sourceTree.next? ? sourceTree.next : nil
		elsif (!sourceNode) or (destinationNode and (sourceNode.name > destinationNode.name))
			$stderr.puts "Source does not have #{destinationNode.name}" if $S3syncOptions['--debug']
			if $S3syncOptions['--delete']
				if destinationNode.directory?
					# have to wait
					nodesToDelete.push(destinationNode) 
				else
					puts "Remove node #{destinationNode.name}" if $S3syncOptions['--verbose']
					destinationNode.delete unless $S3syncOptions['--dryrun']
				end
			end
			destinationNode = destinationTree.next? ? destinationTree.next : nil
		elsif sourceNode.name == destinationNode.name
			if (sourceNode.size != destinationNode.size) or (sourceNode.tag != destinationNode.tag)
				puts "Update node #{sourceNode.name}" if $S3syncOptions['--verbose']
				destinationNode.updateFrom(sourceNode) unless $S3syncOptions['--dryrun']
			elsif $S3syncOptions['--debug']
				$stderr.puts "Node #{sourceNode.name} unchanged" 
			end
			sourceNode = sourceTree.next? ? sourceTree.next : nil
			destinationNode = destinationTree.next? ? destinationTree.next : nil
		end					
	end
	
	# get rid of the (now empty, except for other directories) directories
	nodesToDelete.reverse_each do |node|
		puts "Remove node #{node.name}" if $S3syncOptions['--verbose']
		node.delete unless $S3syncOptions['--dryrun']
	end
	
end

.s3cmdList(bucket, path, max = nil, delim = nil, marker = nil, headers = {}) ⇒ Object

main



217
218
219
220
221
222
223
224
225
 
# File 'lib/s3sync/s3cmd.rb', line 217

def S3sync.s3cmdList(bucket, path, max=nil, delim=nil, marker=nil, headers={})
	debug(max)
	options = Hash.new
	options['prefix'] = path # start at the right depth
	options['max-keys'] = max ? max.to_s : 100
	options['delimiter'] = delim if delim
	options['marker'] = marker if marker
	S3try(:list_bucket, bucket, options, headers)
end

.s3cmdMain ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/s3sync/s3cmd.rb', line 26

def S3sync.s3cmdMain 	
	# ---------- OPTIONS PROCESSING ---------- #	

	$S3syncOptions = Hash.new
	optionsParser = GetoptLong.new(
		  [ '--help',    '-h',	GetoptLong::NO_ARGUMENT ],
		  [ '--ssl',     '-s',	GetoptLong::NO_ARGUMENT ],
		  [ '--verbose', '-v',	GetoptLong::NO_ARGUMENT ],
		  [ '--dryrun',  '-n',	GetoptLong::NO_ARGUMENT ], 
		  [ '--debug',   '-d',	GetoptLong::NO_ARGUMENT ],
		  [ '--progress',       GetoptLong::NO_ARGUMENT ],
          [ '--expires-in', GetoptLong::REQUIRED_ARGUMENT ]
		  )
		  
	def S3sync.s3cmdUsage(message = nil)
		$stderr.puts message if message
		name = $0.split('/').last
		$stderr.puts <<"ENDUSAGE"
#{name} [options] <command> [arg(s)]\t\tversion #{$S3CMD_VERSION}
 --help    -h        --verbose     -v     --dryrun    -n	
 --ssl     -s        --debug       -d     --progress
 --expires-in=( <# of seconds> | [#d|#h|#m|#s] )
 
Commands:
#{name}  listbuckets  [headers]
#{name}  createbucket  <bucket>  [constraint (i.e. EU)]
#{name}  deletebucket  <bucket>  [headers]
#{name}  list  <bucket>[:prefix]  [max/page]  [delimiter]  [headers]
#{name}  location  <bucket> [headers]
#{name}  delete  <bucket>:key  [headers]
#{name}  deleteall  <bucket>[:prefix]  [headers]
#{name}  get|put  <bucket>:key  <file>  [headers]
ENDUSAGE
	exit
	end #usage
	
	begin
		optionsParser.each {|opt, arg| $S3syncOptions[opt] = (arg || true)}
	rescue StandardError
		s3cmdUsage # the parser already printed an error message
	end
	s3cmdUsage if $S3syncOptions['--help']
	$S3syncOptions['--verbose'] = true if $S3syncOptions['--dryrun'] or $S3syncOptions['--debug'] or $S3syncOptions['--progress']
	$S3syncOptions['--ssl'] = true if $S3syncOptions['--ssl'] # change from "" to true to appease s3 port chooser
     
     if $S3syncOptions['--expires-in'] =~ /d|h|m|s/
        e = $S3syncOptions['--expires-in']
        days = (e =~ /(\d+)d/)? (/(\d+)d/.match(e))[1].to_i : 0
        hours = (e =~ /(\d+)h/)? (/(\d+)h/.match(e))[1].to_i : 0
        minutes = (e =~ /(\d+)m/)? (/(\d+)m/.match(e))[1].to_i : 0
        seconds = (e =~ /(\d+)s/)? (/(\d+)s/.match(e))[1].to_i : 0
        $S3syncOptions['--expires-in'] = seconds + 60 * ( minutes + 60 * ( hours + 24 * ( days ) ) )
     end

	# ---------- CONNECT ---------- #
	S3sync::s3trySetup 
	# ---------- COMMAND PROCESSING ---------- #		

	command, path, file = ARGV
	
	s3cmdUsage("You didn't set up your environment variables; see README.txt") if not($AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY) 
	s3cmdUsage("Need a command (etc)") if not command

	path = '' unless path
	path = path.dup # modifiable
	path += ':' unless path.match(':')
	bucket = (/^(.*?):/.match(path))[1]
	path.replace((/:(.*)$/.match(path))[1])

	case command
		when "delete"
			s3cmdUsage("Need a bucket") if bucket == ''
			s3cmdUsage("Need a key") if path == ''
			headers = hashPairs(ARGV[2...ARGV.length])
			$stderr.puts "delete #{bucket}:#{path} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
			S3try(:delete, bucket, path) unless $S3syncOptions['--dryrun']
		when "deleteall"
			s3cmdUsage("Need a bucket") if bucket == ''
			headers = hashPairs(ARGV[2...ARGV.length])
			$stderr.puts "delete ALL entries in #{bucket}:#{path} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
			more = true
			marker = nil
			while more do
				res = s3cmdList(bucket, path, nil, nil, marker)
				res.entries.each do |item|
					# the s3 commands (with my modified UTF-8 conversion) expect native char encoding input
					key = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.key).join
					$stderr.puts "delete #{bucket}:#{key} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
					S3try(:delete, bucket, key) unless $S3syncOptions['--dryrun']		
				end
				more = res.properties.is_truncated
				marker = (res.properties.next_marker)? res.properties.next_marker : ((res.entries.length > 0) ? res.entries.last.key : nil)
				# get this into local charset; when we pass it to s3 that is what's expected
				marker = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", marker).join if marker
			end
		when "list"
			s3cmdUsage("Need a bucket") if bucket == ''
			max, delim = ARGV[2..3]
			headers = hashPairs(ARGV[4...ARGV.length])
			$stderr.puts "list #{bucket}:#{path} #{max} #{delim} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
			puts "--------------------"
			
			more = true
			marker = nil
			while more do
				res = s3cmdList(bucket, path, max, delim, marker, headers)
				if delim
					res.common_prefix_entries.each do |item|
						
						puts "dir: " + Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.prefix).join
					end
					puts "--------------------"
				end
				res.entries.each do |item|
					puts Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.key).join
				end
				if res.properties.is_truncated
					printf "More? Y/n: "
					more = (STDIN.gets.match('^[Yy]?$'))
					marker = (res.properties.next_marker)? res.properties.next_marker : ((res.entries.length > 0) ? res.entries.last.key : nil)
					# get this into local charset; when we pass it to s3 that is what's expected
					marker = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", marker).join if marker
					
				else
					more = false
				end
			end # more
		when "listbuckets"
			headers = hashPairs(ARGV[1...ARGV.length])
			$stderr.puts "list all buckets #{headers.inspect if headers}" if $S3syncOptions['--verbose']
           if $S3syncOptions['--expires-in']
              $stdout.puts S3url(:list_all_my_buckets, headers)
           else
              res = S3try(:list_all_my_buckets, headers)
              res.entries.each do |item|
				   puts item.name
              end
           end
		when "createbucket"
			s3cmdUsage("Need a bucket") if bucket == ''
           lc = ''
           if(ARGV.length > 2)
              lc = '<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01"><LocationConstraint>' + ARGV[2] + '</LocationConstraint></CreateBucketConfiguration>'
           end
			$stderr.puts "create bucket #{bucket} #{lc}" if $S3syncOptions['--verbose']
			S3try(:create_bucket, bucket, lc) unless $S3syncOptions['--dryrun']
		when "deletebucket"
			s3cmdUsage("Need a bucket") if bucket == ''
			headers = hashPairs(ARGV[2...ARGV.length])
			$stderr.puts "delete bucket #{bucket} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
			S3try(:delete_bucket, bucket, headers) unless $S3syncOptions['--dryrun']
		when "location"
			s3cmdUsage("Need a bucket") if bucket == ''
			headers = hashPairs(ARGV[2...ARGV.length])
           query = Hash.new
           query['location'] = 'location'
			$stderr.puts "location request bucket #{bucket} #{query.inspect} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
			S3try(:get_query_stream, bucket, '', query, headers, $stdout) unless $S3syncOptions['--dryrun']
		when "get"
			s3cmdUsage("Need a bucket") if bucket == ''
			s3cmdUsage("Need a key") if path == ''
			s3cmdUsage("Need a file") if file == ''
			headers = hashPairs(ARGV[3...ARGV.length])
			$stderr.puts "get from key #{bucket}:#{path} into #{file} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
			unless $S3syncOptions['--dryrun']
              if $S3syncOptions['--expires-in']
                 $stdout.puts S3url(:get, bucket, path, headers)
              else
                 outStream = File.open(file, 'wb')
                 outStream = ProgressStream.new(outStream) if $S3syncOptions['--progress']
                 S3try(:get_stream, bucket, path, headers, outStream)
                 outStream.close
              end
			end
		when "put"
			s3cmdUsage("Need a bucket") if bucket == ''
			s3cmdUsage("Need a key") if path == ''
			s3cmdUsage("Need a file") if file == ''
			headers = hashPairs(ARGV[3...ARGV.length])
			stream = File.open(file, 'rb')
			stream = ProgressStream.new(stream, File.stat(file).size) if $S3syncOptions['--progress']
			s3o = S3::S3Object.new(stream, {}) # support meta later?
			headers['Content-Length'] = FileTest.size(file).to_s
			$stderr.puts "put to key #{bucket}:#{path} from #{file} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
			S3try(:put, bucket, path, s3o, headers) unless $S3syncOptions['--dryrun']
			stream.close
		else
			s3cmdUsage
	end

end

.s3cmdUsage(message = nil) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/s3sync/s3cmd.rb', line 40

def S3sync.s3cmdUsage(message = nil)
	$stderr.puts message if message
	name = $0.split('/').last
	$stderr.puts <<"ENDUSAGE"
#{name} [options] <command> [arg(s)]\t\tversion #{$S3CMD_VERSION}
--help    -h        --verbose     -v     --dryrun    -n	
--ssl     -s        --debug       -d     --progress
--expires-in=( <# of seconds> | [#d|#h|#m|#s] )

Commands:
#{name}  listbuckets  [headers]
#{name}  createbucket  <bucket>  [constraint (i.e. EU)]
#{name}  deletebucket  <bucket>  [headers]
#{name}  list  <bucket>[:prefix]  [max/page]  [delimiter]  [headers]
#{name}  location  <bucket> [headers]
#{name}  delete  <bucket>:key  [headers]
#{name}  deleteall  <bucket>[:prefix]  [headers]
#{name}  get|put  <bucket>:key  <file>  [headers]
ENDUSAGE
exit
end

.s3Prefix?(pre) ⇒ Boolean

———- PREFIX PROCESSING ———- #

Returns:

  • (Boolean) 


109
110
111
112
 
# File 'lib/s3sync/s3sync.rb', line 109

def S3sync.s3Prefix?(pre)
	# allow for dos-like things e.g. C:\ to be treated as local even with colon
	pre.include?(':') and not pre.match('^[A-Za-z]:[\\\\/]')
end

.S3try(command, bucket, *args) ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/s3sync/s3try.rb', line 65

def S3sync.S3try(command, bucket, *args)
     if(not $S3syncHttp or (bucket != $S3syncLastBucket))
        $stderr.puts "Creating new connection" if $S3syncOptions['--debug']
        $S3syncLastBucket = bucket
        S3sync.S3tryConnect(bucket)
     end
     
	result = nil
	delim = $,
	$,=' '
	while $S3syncRetriesLeft > 0 do
        $stderr.puts "Trying command #{command} #{bucket} #{args} with #{$S3syncRetriesLeft} retries left" if $S3syncOptions['--debug']
		forceRetry = false
        now = false
        hush = false
		begin
			result = $S3syncConnection.send(command, bucket, *args)
		rescue Errno::EPIPE => e
			forceRetry = true
			$stderr.puts "Broken pipe: #{e}" 
		rescue Errno::ECONNRESET => e
			forceRetry = true
			$stderr.puts "Connection reset: #{e}" 
		rescue Errno::ECONNABORTED => e
			forceRetry = true
			$stderr.puts "Connection aborted: #{e}" 
		rescue Errno::ETIMEDOUT => e
			forceRetry = true
			$stderr.puts "Connection timed out: #{e}"
		rescue Timeout::Error => e
			forceRetry = true
			$stderr.puts "Connection timed out: #{e}" 
		rescue EOFError => e
			# i THINK this is happening like a connection reset
			forceRetry = true
			$stderr.puts "EOF error: #{e}"
		rescue OpenSSL::SSL::SSLError => e
			forceRetry = true
			$stderr.puts "SSL Error: #{e}"
		rescue NoMethodError
			# we get this when using --progress, and the local item is something unreadable
			$stderr.puts "Null stream error: #{e}"
			break
		end
        if forceRetry
			# kill and reset connection when we receive a non-50x yet retry-able error
			S3sync.S3tryConnect(bucket)
        end
		begin
			debug("Response code: #{result.http_response.code}")
			break if ((200...300).include? result.http_response.code.to_i) and (not forceRetry)
           if result.http_response.code.to_i == 301
              $stderr.puts "Permanent redirect received. Try setting AWS_CALLING_FORMAT to SUBDOMAIN"
           elsif result.http_response.code.to_i == 307
              # move to the other host
              host = %r{https?://([^/]+)}.match(result.http_response['Location'])[1]
              $stderr.puts("Temporary Redirect to: " + host)
              debug("Host: " + host)
              S3sync.S3tryConnect(bucket, host)
              $S3syncRetriesLeft = $S3syncRetriesLeft+1 # don't use one up below
              forceRetry = true
              now = true
              hush = true
           else
              $stderr.puts "S3 command failed:\n#{command} #{args}"
              $stderr.puts "With result #{result.http_response.code} #{result.http_response.message}\n"
              debug(result.http_response.body)
           end
			# only retry 500's, per amazon
			break unless ((500...600).include? result.http_response.code.to_i) or forceRetry
		rescue NoMethodError
			debug("No result available")
		end
		$S3syncRetriesLeft -= 1
		$stderr.puts "#{$S3syncRetriesLeft} retries left" unless hush
		Kernel.sleep $S3SYNC_WAITONERROR unless now
	end
     if $S3syncRetriesLeft <= 0
        $stderr.puts "Ran out of retries; operations did not complete!"
     end
	$, = delim
	result
end

.S3tryConnect(bucket, host = '') ⇒ Object 



61
62
63
 
# File 'lib/s3sync/s3try.rb', line 61

def S3sync.S3tryConnect(bucket, host='')
      $S3syncHttp = $S3syncConnection.make_http(bucket, host, $HTTP_PROXY_HOST, $HTTP_PROXY_PORT, $HTTP_PROXY_USER, $HTTP_PROXY_PASSWORD)
end

.s3trySetup ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/s3sync/s3try.rb', line 39

def S3sync.s3trySetup 	
	
	# ---------- CONNECT ---------- #

	$S3syncConnection = S3::AWSAuthConnection.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
     $S3syncConnection.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
	if $S3syncOptions['--ssl']
		if $SSL_CERT_DIR
			$S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
			$S3syncConnection.ca_path = $SSL_CERT_DIR
		elsif $SSL_CERT_FILE
			$S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
			$S3syncConnection.ca_file = $SSL_CERT_FILE
		end
	end
end

.S3url(command, bucket, *args) ⇒ Object 



149
150
151
152
153
154
155
156
157
158
 
# File 'lib/s3sync/s3try.rb', line 149

def S3sync.S3url(command, bucket, *args)
     S3sync.s3urlSetup() unless $S3syncGenerator
	result = nil
	delim = $,
	$,=' '
     $stderr.puts "Calling command #{command} #{bucket} #{args}" if $S3syncOptions['--debug']
     result = $S3syncGenerator.send(command, bucket, *args)
	$, = delim
	result
end

.s3urlSetup ⇒ Object 



55
56
57
58
59
 
# File 'lib/s3sync/s3try.rb', line 55

def S3sync.s3urlSetup 	
	$S3syncGenerator = S3::QueryStringAuthGenerator.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
     $S3syncGenerator.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
     $S3syncGenerator.expires_in = $S3syncOptions['--expires-in']
end

.usage(message = nil) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/s3sync/s3sync.rb', line 73

def S3sync.usage(message = nil)
	$stderr.puts message if message
	name = $0.split('/').last
	$stderr.puts <<"ENDUSAGE"
#{name} [options] <source> <destination>\t\tversion #{$S3SYNC_VERSION}
--help    -h          --verbose     -v     --dryrun    -n	
--ssl     -s          --recursive   -r     --delete
--public-read -p      --expires="<exp>"    --cache-control="<cc>"
--exclude="<regexp>"  --progress           --debug   -d
--make-dirs
One of <source> or <destination> must be of S3 format, the other a local path.
Reminders:
* An S3 formatted item with bucket 'mybucket' and prefix 'mypre' looks like:
  mybucket:mypre/some/key/name
* Local paths should always use forward slashes '/' even on Windows
* Whether you use a trailing slash on the source path makes a difference.
* For examples see README.
ENDUSAGE
exit
end