Class: Deb::S3::CLI

Inherits:
Thor
  • Object
show all
Defined in:
lib/deb/s3/cli.rb

Instance Method Summary collapse

Instance Method Details

#clean ⇒ Object 



607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
 
# File 'lib/deb/s3/cli.rb', line 607

def clean
  configure_s3_client

  begin
    if options[:lock]
      log("Checking for existing lock file")
      log("Locking repository for updates")
      Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
      @lock_acquired = true
    end

    log("Retrieving existing manifests")

    # Enumerate objects under the dists/<codename>/ prefix to find any
    # Packages files and load them....

    req = Deb::S3::Utils.s3.list_objects_v2({
      :bucket => Deb::S3::Utils.bucket,
      :prefix => Deb::S3::Utils.s3_path("dists/#{ options[:codename] }/"),
    })

    manifests = []
    req.contents.each do |object|
      if match = object.key.match(/dists\/([^\/]+)\/([^\/]+)\/binary-([^\/]+)\/Packages$/)
        codename, component, arch = match.captures
        manifests.push(Deb::S3::Manifest.retrieve(codename, component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload]))
      end
    end

    # Iterate over the packages in each manifest and build a Set of all the
    # referenced URLs (relative to bucket root)...

    refd_urls = Set[]
    manifests.each do |manifest|
      manifest.packages.each do |package|
        refd_urls.add(Deb::S3::Utils.s3_path(package.url_filename(manifest.codename)))
      end
    end

    log("Searching for unreferenced packages")

    # Enumerate objects under the pools/<codename> prefix and delete any that
    # arent referenced by any of the manifests.

    continuation_token = nil
    while true
      req = Deb::S3::Utils.s3.list_objects_v2({
        :bucket => Deb::S3::Utils.bucket,
        :prefix => Deb::S3::Utils.s3_path("pool/#{ options[:codename] }/"),
        :continuation_token => continuation_token,
      })

      req.contents.each do |object|
        if not refd_urls.include?(object.key)
          sublog("Deleting #{ object.key }")

          Deb::S3::Utils.s3.delete_object({
            :bucket => Deb::S3::Utils.bucket,
            :key => object.key,
          })
        end
      end

      if req.is_truncated
        continuation_token = req.next_continuation_token
      else
        break
      end
    end
  ensure
    if options[:lock] && @lock_acquired
      Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
      log("Lock released.")
    end
  end
end

#copy(package_name, to_codename, to_component) ⇒ Object 



376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
 
# File 'lib/deb/s3/cli.rb', line 376

def copy(package_name, to_codename, to_component)
  if package_name.nil?
    error "You must specify a package name."
  end
  if to_codename.nil?
    error "You must specify a codename to copy to."
  end
  if to_component.nil?
    error "You must specify a component to copy to."
  end

  arch = options[:arch]
  if arch.nil?
    error "You must specify the architecture of the package to copy."
  end

  versions = options[:versions]
  if versions.nil?
    warn "===> WARNING: Copying all versions of #{package_name}"
  else
    log "Versions to copy: #{versions.join(', ')}"
  end

  configure_s3_client

  begin
    if options[:lock]
      log("Checking for existing lock file")
      log("Locking repository for updates")
      Deb::S3::Lock.lock(options[:codename], to_component, options[:arch], options[:cache_control])
      @lock_acquired = true
    end

    # retrieve the existing manifests
    log "Retrieving existing manifests"
    from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
                                               component, arch,
                                               options[:cache_control],
                                               false, options[:skip_package_upload])
    to_release = Deb::S3::Release.retrieve(to_codename)
    to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
                                             options[:cache_control],
                                             options[:fail_if_exists],
                                             options[:skip_package_upload])
    packages = from_manifest.packages.select { |p|
      p.name == package_name &&
        (versions.nil? || versions.include?(p.full_version))
    }
    if packages.size == 0
      error "No packages found in repository."
    end

    packages.each do |package|
      begin
        to_manifest.add package, options[:preserve_versions], false
      rescue Deb::S3::Utils::AlreadyExistsError => e
        error("Preparing manifest failed because: #{e}")
      end
    end

    begin
      to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
    rescue Deb::S3::Utils::AlreadyExistsError => e
      error("Copying manifest failed because: #{e}")
    end
    to_release.update_manifest(to_manifest)
    to_release.write_to_s3 { |f| sublog("Transferring #{f}") }

    log "Copy complete."
  ensure
    if options[:lock] && @lock_acquired
      Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
      log("Lock released.")
    end
  end
end

#delete(package) ⇒ Object 



477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
 
# File 'lib/deb/s3/cli.rb', line 477

def delete(package)
  if package.nil?
    error("You must specify a package name.")
  end

  versions = options[:versions]
  if versions.nil?
    warn("===> WARNING: Deleting all versions of #{package}")
  else
    log("Versions to delete: #{versions.join(', ')}")
  end

  arch = options[:arch]
  if arch.nil?
    error("You must specify the architecture of the package to remove.")
  end

  configure_s3_client

  begin
    if options[:lock]
      log("Checking for existing lock file")
      log("Locking repository for updates")
      Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
      @lock_acquired = true
    end

    # retrieve the existing manifests
    log("Retrieving existing manifests")
    release  = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
    if arch == 'all'
      selected_arch = release.architectures
    else
      selected_arch = [arch]
    end
    all_found = 0
    selected_arch.each { |ar|
      manifest = Deb::S3::Manifest.retrieve(options[:codename], component, ar, options[:cache_control], false, options[:skip_package_upload])

      deleted = manifest.delete_package(package, versions)
      all_found += deleted.length
      if deleted.length == 0
          if versions.nil?
              sublog("No packages were deleted. #{package} not found in arch #{ar}.")
              next
          else
              sublog("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found in arch #{ar}.")
              next
          end
      else
          deleted.each { |p|
              sublog("Deleting #{p.name} version #{p.full_version} from arch #{ar}")
          }
      end

      log("Uploading new manifests to S3")
      manifest.write_to_s3 {|f| sublog("Transferring #{f}") }
      release.update_manifest(manifest)
      release.write_to_s3 {|f| sublog("Transferring #{f}") }

      log("Update complete.")
    }
    if all_found == 0
      if versions.nil?
        error("No packages were deleted. #{package} not found.")
      else
        error("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found.")
      end
    end
  ensure
    if options[:lock] && @lock_acquired
      Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
      log("Lock released.")
    end
  end
end

#list ⇒ Object 



275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
 
# File 'lib/deb/s3/cli.rb', line 275

def list
  configure_s3_client

  release = Deb::S3::Release.retrieve(options[:codename])
  archs = release.architectures
  archs &= [options[:arch]] if options[:arch] && options[:arch] != "all"
  widths = [0, 0]
  rows = archs.map { |arch|
    manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
                                          arch, options[:cache_control],
                                          false, false)
    manifest.packages.map do |package|
      if options[:long]
        package.generate(options[:codename])
      else
        [package.name, package.full_version, package.architecture].tap do |row|
          row.each_with_index do |col, i|
            widths[i] = [widths[i], col.size].max if widths[i]
          end
        end
      end
    end
  }.flatten(1)

  if options[:long]
    $stdout.puts rows.join("\n")
  else
    rows.each do |row|
      $stdout.puts "% -#{widths[0]}s  % -#{widths[1]}s  %s" % row
    end
  end
end

#show(package_name, version, arch) ⇒ Object 



310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
 
# File 'lib/deb/s3/cli.rb', line 310

def show(package_name, version, arch)
  if version.nil?
    error "You must specify the name of the package to show."
  end
  if version.nil?
    error "You must specify the version of the package to show."
  end
  if arch.nil?
    error "You must specify the architecture of the package to show."
  end

  configure_s3_client

  # retrieve the existing manifests
  manifest = Deb::S3::Manifest.retrieve(options[:codename], component, arch,
                                        options[:cache_control], false, false)
  package = manifest.packages.detect { |p|
    p.name == package_name && p.full_version == version
  }
  if package.nil?
    error "No such package found."
  end

  puts package.generate(options[:codename])
end

#upload(*files) ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
 
# File 'lib/deb/s3/cli.rb', line 151

def upload(*files)
  if files.nil? || files.empty?
    error("You must specify at least one file to upload")
  end

  # make sure all the files exists
  if missing_file = files.find { |pattern| Dir.glob(pattern).empty? }
    error("File '#{missing_file}' doesn't exist")
  end

  # configure AWS::S3
  configure_s3_client

  begin
    if options[:lock]
      log("Checking for existing lock file")
      log("Locking repository for updates")
      Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
      @lock_acquired = true
    end

    # retrieve the existing manifests
    log("Retrieving existing manifests")
    release  = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
    manifests = {}
    release.architectures.each do |arch|
      manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
    end

    packages_arch_all = []

    # examine all the files
    files.collect { |f| Dir.glob(f) }.flatten.each do |file|
      log("Examining package file #{File.basename(file)}")
      pkg = Deb::S3::Package.parse_file(file)

      # copy over some options if they weren't given
      arch = options[:arch] || pkg.architecture

      # If they've specified an arch type that doesn't match the package let them know
      if options.key?("arch") && options[:arch] != pkg.architecture
        warn("You specified architecture #{options[:arch]} but package #{pkg.name} has architecture type of #{pkg.architecture}")
      end

      # validate we have them
      error("No architcture given and unable to determine one for #{file}. " +
            "Please specify one with --arch [i386|amd64|armhf].") unless arch

      # If the arch is all and the list of existing manifests is none, then
      # throw an error. This is mainly the case when initializing a brand new
      # repository. With "all", we won't know which architectures they're using.
      if arch == "all" && manifests.count == 0
        manifests['amd64'] = Deb::S3::Manifest.retrieve(options[:codename], component,'amd64', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
        manifests['i386'] = Deb::S3::Manifest.retrieve(options[:codename], component,'i386', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
        manifests['armhf'] = Deb::S3::Manifest.retrieve(options[:codename], component,'armhf', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])

       # error("Package #{File.basename(file)} had architecture \"all\", " +
       #       "however noexisting package lists exist. This can often happen " +
       #       "if the first package you are add to a new repository is an " +
       #       "\"all\" architecture file. Please use --arch [i386|amd64|armhf] or " +
       #       "another platform type to upload the file.")
      end

      # retrieve the manifest for the arch if we don't have it already
      manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])

      # add package in manifests
      begin
        manifests[arch].add(pkg, options[:preserve_versions])
      rescue Deb::S3::Utils::AlreadyExistsError => e
        error("Preparing manifest failed because: #{e}")
      end

      # If arch is all, we must add this package in all arch available
      if arch == 'all'
        packages_arch_all << pkg
      end
    end

    manifests.each do |arch, manifest|
      next if arch == 'all'
      packages_arch_all.each do |pkg|
        begin
          manifest.add(pkg, options[:preserve_versions], false)
        rescue Deb::S3::Utils::AlreadyExistsError => e
          error("Preparing manifest failed because: #{e}")
        end
      end
    end

    # upload the manifest
    log("Uploading packages and new manifests to S3")
    manifests.each_value do |manifest|
      begin
        manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
      rescue Deb::S3::Utils::AlreadyExistsError => e
        error("Uploading manifest failed because: #{e}")
      end
      release.update_manifest(manifest)
    end
    release.write_to_s3 { |f| sublog("Transferring #{f}") }

    log("Update complete.")
  ensure
    if options[:lock] && @lock_acquired
      Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
      log("Lock released.")
    end
  end
end

#verify ⇒ Object 



563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
 
# File 'lib/deb/s3/cli.rb', line 563

def verify
  configure_s3_client

  log("Retrieving existing manifests")
  release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])

  release.architectures.each do |arch|
    log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
    manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
                                          arch, options[:cache_control], false,
                                          options[:skip_package_upload])
    missing_packages = []

    manifest.packages.each do |p|
      unless Deb::S3::Utils.s3_exists? p.url_filename(options[:codename])
        sublog("The following packages are missing:\n\n") if missing_packages.empty?
        puts(p.generate(options[:codename]))
        puts("")

        missing_packages << p
      end
    end

    if options[:sign] || (options[:fix_manifests] && !missing_packages.empty?)
      log("Removing #{missing_packages.length} package(s) from the manifest...")
      missing_packages.each { |p| manifest.packages.delete(p) }
      manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
      release.update_manifest(manifest)
      release.write_to_s3 { |f| sublog("Transferring #{f}") }

      log("Update complete.")
    end
  end
end