Class: Autobuild::ArchiveImporter

Inherits:
Importer
  • Object
show all
Defined in:
lib/autobuild/import/archive.rb

Constant Summary collapse

Plain =

rubocop:disable Naming/ConstantName The tarball is not compressed

0
Gzip =

The tarball is compressed with gzip

1
Bzip =

The tarball is compressed using bzip

2
Zip =

Not a tarball but a zip

3
TAR_OPTION =

rubocop:enable Naming/ConstantName

{
    Plain => '',
    Gzip => 'z',
    Bzip => 'j'
}.freeze
VALID_URI_SCHEMES =

Known URI schemes for url

%w[file http https ftp].freeze
WINDOWS_VALID_URI_SCHEMES =

Known URI schemes for url on windows

%w[file http https].freeze

Class Attribute Summary collapse

Instance Attribute Summary collapse

Attributes inherited from Importer

#interactive, #options, #post_hooks, #repository_id, #source_id

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Importer

#add_post_hook, add_post_hook, #apply, cache_dirs, #call_patch, #currently_applied_patches, default_cache_dirs, default_cache_dirs=, #each_post_hook, each_post_hook, #execute_post_hooks, #fallback, fallback, #fingerprint, #import, #interactive?, #parse_patch_list, #patch, #patchdir, #patches, #patches_fingerprint, #patchlist, #perform_checkout, #perform_update, #retry_count, #retry_count=, #save_patch_state, set_cache_dirs, #supports_relocation?, #unapply, unset_cache_dirs, #update_retry_count

Constructor Details

#initialize(url, options = Hash.new) ⇒ ArchiveImporter

Creates a new importer which downloads url in cachedir and unpacks it. The following options are allowed:

:cachedir

the cache directory. Defaults to “#Autobuild.prefix/cache”

:archive_dir

the directory contained in the archive file. If set,

the importer will rename that directory to make it match
Package#srcdir
:no_subdirectory

the archive does not have the custom archive

subdirectory.
:retries

The number of retries for downloading

:timeout

The timeout (in seconds) used during downloading, it

defaults to 10s
:filename

Rename the archive to this filename (in cache) – will be

also used to infer the mode
:mode

The unpack mode: one of Zip, Bzip, Gzip or Plain, this is

usually automatically inferred from the filename



387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
 
# File 'lib/autobuild/import/archive.rb', line 387

def initialize(url, options = Hash.new)
    sourceopts, options = Kernel.filter_options(
        options,
        :source_id, :repository_id, :filename, :mode, :update_cached_file,
        :user, :password, :expected_digest
    )
    super(options)

    @filename = nil
    @update_cached_file = false
    @cachedir = @options[:cachedir] || ArchiveImporter.cachedir
    @retries  = @options[:retries] || ArchiveImporter.retries
    @timeout  = @options[:timeout] || ArchiveImporter.timeout
    relocate(url, sourceopts)
end

Class Attribute Details

.cachedir ⇒ Object

The directory in which downloaded files are saved

It defaults, if set, to the value returned by Importer.cache_dirs and falls back #prefix/cache



38
39
40
41
42
43
44
45
 
# File 'lib/autobuild/import/archive.rb', line 38

def cachedir
    if @cachedir then @cachedir
    elsif (cache_dirs = Importer.cache_dirs('archives'))
        @cachedir = cache_dirs.first
    else
        "#{Autobuild.prefix}/cache"
    end
end

.retries ⇒ Object

The number of time we should retry downloading if the underlying tool supports it (wget does).

It defaults to 1 as autobuild has its own retry mechanism



62
63
64
 
# File 'lib/autobuild/import/archive.rb', line 62

def retries
  @retries
end

.timeout ⇒ Object

The timeout (in seconds) used during downloading.

With wget, it is the timeout used for DNS resolution, connection and idle time (time without receiving data)

It defaults to 10s



56
57
58
 
# File 'lib/autobuild/import/archive.rb', line 56

def timeout
  @timeout
end

Instance Attribute Details

#cachedir ⇒ Object

The directory in which remote files are cached

Defaults to ArchiveImporter.cachedir



319
320
321
 
# File 'lib/autobuild/import/archive.rb', line 319

def cachedir
  @cachedir
end

#cachefile ⇒ Object (readonly)

The local file (either a downloaded file if url is not local, or url itself)



308
309
310
 
# File 'lib/autobuild/import/archive.rb', line 308

def cachefile
  @cachefile
end

#cachefile_digest ⇒ String (readonly)

The SHA1 digest of the current cachefile. It is updated only once the cachefile has been downloaded

Returns:

  • (String) —

    hexadecimal SHA1 digest of the file



313
314
315
 
# File 'lib/autobuild/import/archive.rb', line 313

def cachefile_digest
  @cachefile_digest
end

#filename ⇒ Object (readonly)

The filename that should be used locally (for remote files)

This is usually inferred by using the URL’s basename, but some download URLs do not allow this (for instance bitbucket tarballs)

Change it by calling #relocate



352
353
354
 
# File 'lib/autobuild/import/archive.rb', line 352

def filename
  @filename
end

#mode ⇒ Object (readonly)

The unpack mode. One of Zip, Bzip, Gzip or Plain



315
316
317
 
# File 'lib/autobuild/import/archive.rb', line 315

def mode
  @mode
end

#retries ⇒ Object

The number of time we should retry downloading if the underlying tool supports it (wget does).

It defaults to the global ArchiveImporter.retries



342
343
344
 
# File 'lib/autobuild/import/archive.rb', line 342

def retries
  @retries
end

#timeout ⇒ Object

The timeout (in seconds) used during downloading.

With wget, it is the timeout used for DNS resolution, connection and idle time (time without receiving data)

It defaults to the global ArchiveImporter.timeout



360
361
362
 
# File 'lib/autobuild/import/archive.rb', line 360

def timeout
  @timeout
end

#update_cached_file=(value) ⇒ Object (writeonly)

Sets the attribute update_cached_file

Parameters:

  • value —

    the value to set the attribute update_cached_file to.



108
109
110
 
# File 'lib/autobuild/import/archive.rb', line 108

def update_cached_file=(value)
  @update_cached_file = value
end

#url ⇒ Object (readonly)

The source URL



306
307
308
 
# File 'lib/autobuild/import/archive.rb', line 306

def url
  @url
end

Class Method Details

.auto_update=(flag) ⇒ Object 



103
104
105
 
# File 'lib/autobuild/import/archive.rb', line 103

def self.auto_update=(flag)
    @auto_update = flag
end

.auto_update? ⇒ Boolean

Tells the importer that the checkout should be automatically deleted on update, without asking the user

Returns:

  • (Boolean) —

    true if the archive importer should automatically delete the current checkout when the archive changed, false otherwise. The default is to set it to true if the AUTOBUILD_ARCHIVE_AUTOUPDATE environment variable is set to 1, and to false in all other cases



99
100
101
 
# File 'lib/autobuild/import/archive.rb', line 99

def self.auto_update?
    @auto_update
end

.filename_to_mode(filename) ⇒ Object

Returns the unpack mode from the file name



83
84
85
86
87
88
89
90
 
# File 'lib/autobuild/import/archive.rb', line 83

def self.filename_to_mode(filename)
    if (mode = find_mode_from_filename(filename))
        mode
    else
        raise "cannot infer the archive type from '#{filename}', "\
            "provide it explicitely with the mode: option"
    end
end

.find_mode_from_filename(filename) ⇒ Integer?

Returns the unpack mode from the file name

Returns:

  • (Integer, nil) —

    either one of the pack constants (Zip, Plain, …) or nil if it cannot be inferred

See Also:



73
74
75
76
77
78
79
80
 
# File 'lib/autobuild/import/archive.rb', line 73

def self.find_mode_from_filename(filename)
    case filename
    when /\.zip$/ then Zip
    when /\.tar$/ then Plain
    when /\.tar\.gz$|\.tgz$/ then Gzip
    when /\.bz2$/ then Bzip
    end
end

Instance Method Details

#archive_changed?(package) ⇒ Boolean

Returns true if the archive that has been used to checkout this package is different from the one we are supposed to checkout now

Returns:

  • (Boolean) 


482
483
484
485
 
# File 'lib/autobuild/import/archive.rb', line 482

def archive_changed?(package)
    checkout_digest = File.read(checkout_digest_stamp(package)).strip
    checkout_digest != cachefile_digest
end

#archive_dir ⇒ Object

The directory contained in the archive. If not set, we assume that it is the same than the source dir



334
335
336
 
# File 'lib/autobuild/import/archive.rb', line 334

def archive_dir
    @options[:archive_dir] || tardir
end

#checkout(package, options = Hash.new) ⇒ Object

:nodoc:



487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
 
# File 'lib/autobuild/import/archive.rb', line 487

def checkout(package, options = Hash.new) # :nodoc:
    options = Kernel.validate_options options,
                                      allow_interactive: true

    update_cache(package)

    # Check whether the archive file changed, and if that is the case
    # then ask the user about deleting the folder
    if File.file?(checkout_digest_stamp(package)) && archive_changed?(package)
        if ArchiveImporter.auto_update?
            response = 'yes'
        elsif options[:allow_interactive]
            package.progress_done
            package.message "The archive #{@url} is different from "\
                "the one currently checked out at #{package.srcdir}", :bold
            package.message "I will have to delete the current folder to go on "\
                "with the update"
            response = TTY::Prompt.new.ask "  Continue (yes or no) ? "\
                "If no, this update will be ignored, "\
                "which can lead to build problems.", convert: :bool
        else
            raise Autobuild::InteractionRequired, "importing #{package.name} "\
                "would have required user interaction and "\
                "allow_interactive is false"
        end

        if !response
            package.message "not updating #{package.srcdir}"
            package.progress_done
            return false
        else
            package.message "deleting #{package.srcdir} to update to new archive"
            FileUtils.rm_rf package.srcdir
            package.progress "checking out %s"
        end
    end

    # Un-apply any existing patch so that, when the files get
    # overwritten by the new checkout, the patch are re-applied
    patch(package, [])

    base_dir = File.dirname(package.srcdir)

    if mode == Zip
        main_dir = if @options[:no_subdirectory] then package.srcdir
                   else
                       base_dir
                   end

        FileUtils.mkdir_p base_dir
        cmd = ['-o', cachefile, '-d', main_dir]
        package.run(:import, Autobuild.tool('unzip'), *cmd)

        archive_dir = (self.archive_dir || File.basename(package.name))
        if archive_dir != File.basename(package.srcdir)
            FileUtils.rm_rf File.join(package.srcdir)
            FileUtils.mv File.join(base_dir, archive_dir), package.srcdir
        elsif !File.directory?(package.srcdir)
            raise Autobuild::Exception, "#{cachefile} does not contain "\
                "directory called #{File.basename(package.srcdir)}. "\
                "Did you forget to use the archive_dir option ?"
        end
    else
        FileUtils.mkdir_p package.srcdir
        cmd = ["x#{TAR_OPTION[mode]}f", cachefile, '-C', package.srcdir]
        cmd << '--strip-components=1' unless @options[:no_subdirectory]

        if Autobuild.windows?
            io = if mode == Plain
                     File.open(cachefile, 'r')
                 else
                     Zlib::GzipReader.open(cachefile)
                 end
            extract_tar_gz(io, package.srcdir)
        else
            package.run(:import, Autobuild.tool('tar'), *cmd)
        end
    end
    write_checkout_digest_stamp(package)
    true
rescue SubcommandFailed
    FileUtils.rm_f(cachefile) if cachefile != url.path
    raise
end

#checkout_digest_stamp(package) ⇒ Object 



470
471
472
 
# File 'lib/autobuild/import/archive.rb', line 470

def checkout_digest_stamp(package)
    File.join(package.srcdir, "archive-autobuild-stamp")
end

#download_from_url(package) ⇒ Object 



226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
 
# File 'lib/autobuild/import/archive.rb', line 226

def download_from_url(package)
    FileUtils.mkdir_p(cachedir)
    begin
        if %w[http https].include?(@url.scheme)
            if File.file?(cachefile)
                return false unless update_cached_file?

                cached_mtime = File.lstat(cachefile).mtime
            end

            updated = download_http(package, @url, "#{cachefile}.partial",
                                    user: @user, password: @password,
                                    current_time: cached_mtime)
            return false unless updated
        elsif Autobuild.bsd?
            return false unless update_needed?(package)

            package.run(:import, Autobuild.tool('curl'),
                        '-Lso', "#{cachefile}.partial", @url)
        else
            return false unless update_needed?(package)

            additional_options = []
            if (timeout = self.timeout)
                additional_options << "--timeout" << timeout
            end
            if (retries = self.retries)
                additional_options << "--tries" << retries
            end
            package.run(:import, Autobuild.tool('wget'), '-q', '-P', cachedir,
                        *additional_options, @url, '-O', "#{cachefile}.partial",
                        retry: true)
        end
    rescue Exception
        FileUtils.rm_f "#{cachefile}.partial"
        raise
    end
    FileUtils.mv "#{cachefile}.partial", cachefile
    true
end

#download_http(package, uri, filename, user: nil, password: nil, current_time: nil) ⇒ Object

rubocop:disable Metrics/ParameterLists



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/autobuild/import/archive.rb', line 114

def download_http(package, uri, filename, # rubocop:disable Metrics/ParameterLists
        user: nil, password: nil, current_time: nil)
    request = Net::HTTP::Get.new(uri)
    request['If-Modified-Since'] = current_time.rfc2822 if current_time
    request.basic_auth(user, password) if user

    Net::HTTP.start(
        uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
        http.request(request) do |resp|
            case resp
            when Net::HTTPNotModified
                return false
            when Net::HTTPSuccess
                if current_time && (last_modified = resp['last-modified']) &&
                   (current_time >= Time.rfc2822(last_modified))
                   return false
                end

                if (length = resp['Content-Length'])
                    length = Integer(length)
                    expected_size = "/#{Autobuild.human_readable_size(length)}"
                end

                File.open(filename, 'wb') do |io|
                    size = 0
                    next_update = Time.now
                    resp.read_body do |chunk|
                        io.write chunk
                        size += chunk.size
                        if size != 0 && (Time.now > next_update)
                            formatted_size = Autobuild.human_readable_size(size)
                            package.progress "downloading %s "\
                                "(#{formatted_size}#{expected_size})"
                            next_update = Time.now + 1
                        end
                    end
                    formatted_size = Autobuild.human_readable_size(size)
                    package.progress "downloaded %s "\
                        "(#{formatted_size}#{expected_size})"
                end
            when Net::HTTPRedirection
                if (location = resp['location']).start_with?('/')
                    redirect_uri = uri.dup
                    redirect_uri.path = resp['location']
                else
                    redirect_uri = location
                end

                return download_http(package, URI(redirect_uri), filename,
                                     user: user, password: password,
                                     current_time: current_time)
            else
                raise PackageException.new(package, 'import'),
                      "failed download of #{package.name} from #{uri}: "\
                      "#{resp.class}"
            end
        end
    end
    true
end

#extract_tar_gz(io, target) ⇒ Object 



175
176
177
178
179
180
181
182
183
184
185
186
187
188
 
# File 'lib/autobuild/import/archive.rb', line 175

def extract_tar_gz(io, target)
    Gem::Package::TarReader.new(io).each do |entry|
        newname = File.join(
            target, File.basename(entry.full_name))
        FileUtils.mkdir_p(newname) if entry.directory?
        if entry.file?
            dir = File.dirname(newname)
            FileUtils.mkdir_p(dir) unless File.directory?(dir)
            File.open(newname, "wb") do |file|
                file.write(entry.read)
            end
        end
    end
end

#has_subdirectory? ⇒ Boolean

Tests whether the archive’s content is stored within a subdirectory or not

If it has a subdirectory, its name is assumed to be the package’s basename, or the value returned by #archive_dir if the archive_dir option was given to #initialize

Returns:

  • (Boolean) 


368
369
370
 
# File 'lib/autobuild/import/archive.rb', line 368

def has_subdirectory?
    !@options[:no_subdirectory]
end

#read_cachefile_digest ⇒ Object 



284
285
286
 
# File 'lib/autobuild/import/archive.rb', line 284

def read_cachefile_digest
    Digest::SHA1.hexdigest File.read(cachefile)
end

#relocate(url, options = Hash.new) ⇒ Object

Changes the URL from which we should pick the archive



404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
 
# File 'lib/autobuild/import/archive.rb', line 404

def relocate(url, options = Hash.new)
    parsed_url = URI.parse(url).normalize
    @url = parsed_url
    if !VALID_URI_SCHEMES.include?(@url.scheme)
        raise ConfigException, "invalid URL #{@url} (local files "\
            "must be prefixed with file://)"
    elsif Autobuild.windows?
        unless WINDOWS_VALID_URI_SCHEMES.include?(@url.scheme)
            raise ConfigException, "downloading from a #{@url.scheme} URL "\
                "is not supported on windows"
        end
    end

    @repository_id = options[:repository_id] || parsed_url.to_s
    @source_id     = options[:source_id] || parsed_url.to_s
    @expected_digest = options[:expected_digest]

    @filename =
        options[:filename] ||
        @filename ||
        File.basename(url).gsub(/\?.*/, '')
    @update_cached_file = options[:update_cached_file]

    @mode =
        options[:mode] ||
        ArchiveImporter.find_mode_from_filename(filename) ||
        @mode

    if Autobuild.windows? && (mode != Gzip)
        raise ConfigException, "only gzipped tar archives "\
            "are supported on Windows"
    end
    @user = options[:user]
    @password = options[:password]
    if @user && !%w[http https].include?(@url.scheme)
        raise ConfigException, "authentication is only supported for "\
            "http and https URIs"
    end

    @cachefile =
        if @url.scheme == 'file'
            @url.path
        else
            File.join(cachedir, filename)
        end
end

#tardir ⇒ Object

Deprecated.

use #archive_dir instead



328
329
330
 
# File 'lib/autobuild/import/archive.rb', line 328

def tardir
    @options[:tardir]
end

#update(package, options = Hash.new) ⇒ Object

:nodoc:



451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'lib/autobuild/import/archive.rb', line 451

def update(package, options = Hash.new) # :nodoc:
    if options[:only_local]
        package.warn "%s: the archive importer does not support local updates, "\
            "skipping"
        return false
    end
    needs_update = update_cache(package)

    unless File.file?(checkout_digest_stamp(package))
        write_checkout_digest_stamp(package)
    end

    if needs_update || archive_changed?(package)
        checkout(package, allow_interactive: options[:allow_interactive])
    else
        false
    end
end

#update_cache(package) ⇒ Boolean

Updates the downloaded file in cache only if it is needed

Returns:

  • (Boolean) —

    true if a new file was downloaded, false otherwise



272
273
274
275
276
277
278
279
280
281
282
 
# File 'lib/autobuild/import/archive.rb', line 272

def update_cache(package)
    updated = download_from_url(package)
    @cachefile_digest = read_cachefile_digest

    if @expected_digest && @expected_digest != @cachefile_digest
        raise ConfigException,
              "The archive #{@url} does not match the digest provided"
    end

    updated
end

#update_cached_file? ⇒ Boolean

Returns:

  • (Boolean) 


110
111
112
 
# File 'lib/autobuild/import/archive.rb', line 110

def update_cached_file?
    @update_cached_file
end

#update_needed?(package) ⇒ Boolean

Returns:

  • (Boolean) 


190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/autobuild/import/archive.rb', line 190

def update_needed?(package)
    return true  unless File.file?(cachefile)
    return false unless update_cached_file?

    cached_size = File.lstat(cachefile).size
    cached_mtime = File.lstat(cachefile).mtime

    size, mtime = nil
    if @url.scheme == "file"
        size  = File.stat(@url.path).size
        mtime = File.stat(@url.path).mtime
    else
        # rubocop:disable Security/Open
        open @url, :content_length_proc => ->(v) { size = v } do |file|
            mtime = file.last_modified
        end
        # rubocop:enable Security/Open
    end

    if mtime && size
        size != cached_size || mtime > cached_mtime
    elsif mtime
        package.warn "%s: archive size is not available for #{@url}, "\
            "relying on modification time"
        mtime > cached_mtime
    elsif size
        package.warn "%s: archive modification time "\
            "is not available for #{@url}, relying on size"
        size != cached_size
    else
        package.warn "%s: neither the archive size nor its modification time "\
            "are available for #{@url}, will always update"
        true
    end
end

#vcs_fingerprint(_package) ⇒ Object

Fingerprint for archive importer, we are using its digest whether is calculated or expected



291
292
293
294
295
296
297
298
299
300
301
302
303
 
# File 'lib/autobuild/import/archive.rb', line 291

def vcs_fingerprint(_package)
    if @cachefile_digest
        @cachefile_digest
    elsif File.file?(cachefile)
        read_cachefile_digest
    elsif @expected_digest
        @expected_digest
    else
        raise ConfigException,
              "There is no digest for archive #{@url}, make sure "\
              "cache directories are configured correctly"
    end
end

#write_checkout_digest_stamp(package) ⇒ Object 



474
475
476
477
478
 
# File 'lib/autobuild/import/archive.rb', line 474

def write_checkout_digest_stamp(package)
    File.open(checkout_digest_stamp(package), 'w') do |io|
        io.write cachefile_digest
    end
end