Module: UploadUtils

Defined in:
lib/folio/uploadutils.rb

Overview

UploadUtils

Upload files to host. These means of uploading are current supported: ftp, sftp, scp and rsync.

user       Username for host.
host       Host server's domain name.
root       Document root path on host.
copy       Directory of files to publish, or
           Files to publish using from and to.

dryrun     If true only pretend to upload.
quiet      Supress all output.
verbose    Provide extra details.

The copy parameter allows you to simply specify a file or directory which will be published to host’s document root location.

If you need more control over which files to publish where, you can use the copy parameter instead. Provide an array of pattern strings in the form of “from to”. If the desitination is the host’s document root you do not need to specify the to part. For example:

copy = [ 'web/*', 'doc/api/* doc/api' ]

The first copies the files under your project’s web directory to the host’s document root. The second copies your projects doc/api files to the doc/api location on the host.

The internal template used for the outbound destination is ‘username@host:root/’.

TODOs

  • Needs general improvements.

  • Reduce shelling-out.

  • Incorporate password into scp and ftp ?

  • rsync needs –delete option

Class Method Summary collapse

Class Method Details

.files(dir, copy) ⇒ Object

Put together the list of files to copy.



270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
 
# File 'lib/folio/uploadutils.rb', line 270

def files( dir, copy )
  Dir.chdir(dir) do
    del, add = copy.partition{ |f| /^[-]/ =~ f }

    # remove - and + prefixes
    del.collect!{ |f| f.sub(/^[-]/,'') }
    add.collect!{ |f| f.sub(/^[+]/,'') }

    #del.concat(must_exclude)

    files = []
    add.each{ |g| files += Dir.glob(g) }
    del.each{ |g| files -= Dir.glob(g) }

    files.collect!{ |f| f.sub(/^\//,'') }

    return files
  end
end

.ftp(keys) ⇒ Object

Use ftp to upload files.



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/folio/uploadutils.rb', line 63

def ftp( keys )
  keys = upload_parameters(keys)

  # set transfer rules
  if keys.stage
    trans = stage_transfer(keys.stage)
  else
    files(keys.dir, keys.copy).each do |from|
      trans << [from,from]
    end
  end

  # append location of publication dir to from
  dir = keys.dir
  trans.collect!{ |from,to| [File.join(dir,from), to] }

  if keys.dryrun
    puts "ftp open #{keys.user}@#{keys.host}:#{keys.root}/"
    keys.trans.each do |f, t|
      puts "ftp put #{f} #{t}"
    end
  else
    require 'net/ftp'
    Net::FTP.open(keys.host) do |ftp|
      ftp.(keys.user) #password?
      ftp.chdir(keys.root)
      keys.trans.each do |f, t|
        puts "ftp #{f} #{t}" unless keys.quiet
        ftp.putbinaryfile( f, t, 1024 )
      end
    end
  end
end

.rsync(keys) ⇒ Object

Use rsync to upload files.

NOTE: This shells out to the commandline.



185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/folio/uploadutils.rb', line 185

def rsync( keys )
  keys = upload_parameters(keys)

  flags = []
  flags << "-n" if keys.dryrun
  flags << "-q" if keys.quiet
  flags << "-v" if keys.verbose
  flags << "--progress" unless keys.quiet
  flags = flags.join(' ').strip
  flags = ' ' + flags unless flags.empty?

  manfile = 'Publish.txt'

  if keys.stage
    dir = stage_linkdir(keys.dir, keys.stage)
    Dir.chdir(dir) do
      cpy = files(keys.copy)
    end
    manifest = File.join(dir,manfile)
    cmd = %{rsync#{flags} -L -arz --files-from='#{manifest}' #{dir} #{keys.user}@#{keys.host}:/#{keys.root}}
  else
    dir = keys.dir
    cpy = files(dir, keys.copy)
    manifest = File.join(dir,manfile)
    cmd = %{rsync#{flags} -arz --files-from='#{manifest}' #{dir} #{keys.user}@#{keys.host}:/#{keys.root}}
  end

  #Dir.chdir(keys.dir) do
    begin
      File.open(manifest, 'w'){ |f| f << cpy.join("\n") }
      ENV['RSYNC_PASSWORD'] = keys.pass if keys.pass
      puts cmd unless keys.quiet
      system cmd
    ensure
      ENV.delete('RSYNC_PASSWORD') if keys.pass
    end
  #end
end

.scp(keys) ⇒ Object

Use scp to upload files.



135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/folio/uploadutils.rb', line 135

def scp( keys )
  keys = upload_parameters(keys)

  # append location of publication dir to from
  dir = keys.dir
  trans.collect!{ |from,to| [File.join(dir,from), to] }

  flags = []
  flags << "-v" if keys.verbose
  flags << "-q" if keys.quiet
  flags = flags.join(' ').strip
  flags = ' ' + flags unless flags.empty?

  if keys.stage
    dir = stage_linkdir(keys.dir, keys.stage)
    Dir.chdir(dir) do
      cpy = files(keys.copy)
    end
    manifest = File.join(dir,manfile)
    #cmd = %{rsync#{flags} -L -arz --files-from='#{manifest}' #{dir} #{keys.user}@#{keys.host}:/#{keys.root}}
    cmd = "scp -r#{flags} * #{keys.user}@#{keys.host}:/#{keys.root}"
  else
    dir = keys.dir
    cpy = files(dir, keys.copy)
    manifest = File.join(dir,manfile)
    #cmd = %{rsync#{flags} -arz --files-from='#{manifest}' #{dir} #{keys.user}@#{keys.host}:/#{keys.root}}
    cmd = "scp -r#{flags} * #{keys.user}@#{keys.host}:/#{keys.root}"
  end

  Dir.chdir(keys.dir) do
    begin
      File.open(manifest, 'w'){ |f| f << cpy.join("\n") }
      #ENV['RSYNC_PASSWORD'] = keys.pass if keys.pass
      puts cmd unless keys.quiet
      system cmd
    ensure
      #ENV.delete('RSYNC_PASSWORD') if keys.pass
    end
  end

  #upload_stage(keys) do #|tmpdir|
  #  puts cmd unless keys.quiet
  #  system cmd unless keys.dryrun
  #end
end

.sftp(keys) ⇒ Object

Use sftp to upload files.



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/folio/uploadutils.rb', line 99

def sftp( keys )
  keys = upload_parameters(keys)

  # set transfer rules
  if keys.stage
    trans = stage_transfer(keys.stage)
  else
    files(keys.dir, keys.copy).each do |from|
      trans << [from,from]
    end
  end

  # append location of publication dir to from
  dir = keys.dir
  trans.collect!{ |from,to| [File.join(dir,from), to] }

  if keys.dryrun
    puts "sftp open #{keys.user}@#{keys.host}:#{keys.root}/"
    keys.trans.each do |f,t|
      puts "sftp put #{f} #{t}"
    end
  else
    require 'net/sftp'
    Net::SFTP.start(keys.host, keys.user, keys.pass) do |sftp|
      #sftp.login( user )
      sftp.chdir(keys.root)
      keys.trans.each do |f,t|
        puts "sftp #{f} #{t}" unless keys.quiet
        sftp.put_file(f,t) #, 1024 )
      end
    end
  end
end

.stage_linkdir(dir, list) ⇒ Object

When using stage options this will create temporary linked location.



314
315
316
317
318
319
320
321
322
323
324
325
326
 
# File 'lib/folio/uploadutils.rb', line 314

def stage_linkdir( dir, list )
  folder = File.join(Dir.tmpdir, 'ratchets', 'project', object_id.abs.to_s)
  FileUtils.mkdir_p(folder)

  Dir.chdir(dir) do
    stage_transfer(list).each do |file, to|
      link = File.join(folder,to)
      FileUtils.ln_s(link,file)
    end
  end

  return folder
end

.stage_transfer(list) ⇒ Object

Combine three part stage list into a two part from->to list.

Using the stage list of three space separated fields.

fromdir file todir

This is used to generate a from -> to list of the form:

fromdir/file todir/file



300
301
302
303
304
305
306
307
308
309
310
311
 
# File 'lib/folio/uploadutils.rb', line 300

def stage_transfer( list )
  trans = []
  list.each do |line|
    trans << Shellwords.shellwords(line)
  end

  trans.collect! do |from, base, to|
    file = File.join(from,base)
    to = File.join(to,base)
    [from, to]
  end
end

.upload(protocol, opts) ⇒ Object

Upload via given protocol.



57
58
59
 
# File 'lib/folio/uploadutils.rb', line 57

def upload(protocol, opts)
  __send__(protocol.to_s.downcase,opts)
end

.upload_parameters(opts) ⇒ Object

parse publishing options.

Raises:

  • (ArgumentError) 


227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/folio/uploadutils.rb', line 227

def upload_parameters(opts)
  keys = OpenStruct===opts ? opts : OpenStruct.new(opts)

  keys.copy = keys.copy || '**/*'
  keys.host = keys.host || keys.domain
  keys.user = keys.user || keys.username
  keys.root = keys.root || '/'
  #keys.pass = keys.pass || keys.password

  # validate
  raise ArgumentError, "missing publish parameter -- dir"  unless keys.dir
  raise ArgumentError, "missing publish parameter -- host" unless keys.host
  raise ArgumentError, "missing publish parameter -- user" unless keys.user
  #raise ArgumentError, "missing publish parameter -- copy" unless keys.copy
  #raise ArgumentError, "missing publish parameter -- root" unless keys.root

  keys.root = '' if keys.root.nil?
  keys.root.sub!(/^\//,'')

  if String===keys.copy and File.directory?(keys.copy)
    copy = File.join(keys.copy, '*')
  end
  keys.copy = [keys.copy].flatten.compact

#     trans = []
#     keys.copy.each do |from|
#       #from, to = *Shellwords.shellwords(c)
#       #to = from if to.nil?
#       #to = to[1..-1] if to[0,1] == '/'
#       from.sub('*','**/*') unless from =~ /\*\*/
#       files = Dir.glob(from)
#       files.each do |f|
#         #t = File.join(to,File.basename(f))
#         #t = t[1..-1] if t[0,1] == '/'
#         trans << [f,f]
#       end
#     end
#     keys.trans = trans

  return keys
end