Class: CiCd::Builder::Repo::Artifactory

Inherits:
Base
  • Object
show all
Defined in:
lib/cicd/builder/mixlib/repo/artifactory.rb

Instance Attribute Summary

Attributes inherited from Base

#builder

Instance Method Summary collapse

Methods inherited from Base

#getKey, #initInventory, #tempArtifactFile

Constructor Details

#initialize(builder) ⇒ Artifactory 



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 67

def initialize(builder)
  # Check for the necessary environment variables
  map_keys = {}

  %w[ARTIFACTORY_ENDPOINT ARTIFACTORY_USERNAME ARTIFACTORY_PASSWORD ARTIFACTORY_REPO].each { |k|
    map_keys[k]= (not ENV.has_key?(k) or ENV[k].empty?)
  }
  missing = map_keys.keys.select{ |k| map_keys[k] }

  if missing.count() > 0
    raise("Need these environment variables: #{missing.ai}")
  end

  super(builder)

  # ::Artifactory.configure do |config|
  #   # The endpoint for the Artifactory server. If you are running the "default"
  #   # Artifactory installation using tomcat, don't forget to include the
  #   # +/artifactoy+ part of the URL.
  #   config.endpoint = artifactory_endpoint()
  #
  #   # The basic authentication information. Since this uses HTTP Basic Auth, it
  #   # is highly recommended that you run Artifactory over SSL.
  #   config.username = ENV['ARTIFACTORY_USERNAME']
  #   config.password = ENV['ARTIFACTORY_PASSWORD']
  #
  #   # Speaking of SSL, you can specify the path to a pem file with your custom
  #   # certificates and the gem will wire it all up for you (NOTE: it must be a
  #   # valid PEM file).
  #   # config.ssl_pem_file = '/path/to/my.pem'
  #
  #   # Or if you are feelying frisky, you can always disable SSL verification
  #   # config.ssl_verify = false
  #
  #   # You can specify any proxy information, including any authentication
  #   # information in the URL.
  #   # config.proxy_username = 'user'
  #   # config.proxy_password = 'password'
  #   # config.proxy_address  = 'my.proxy.server'
  #   # config.proxy_port     = '8080'
  # end
  if ENV['ARTIFACTORY_READ_TIMEOUT']
    # [2015-04-29 Christo] Sometimes you just have to shake your head ...
    # a) The Artifactory object does it's setup and read ENV variables during require phase ...
    # b) They do not check if the passed value is valid ( in range or even a number at all ) and they don't convert it to an int
    # c) ENV does not allow one to do this: ENV['ARTIFACTORY_READ_TIMEOUT'] = ENV['ARTIFACTORY_READ_TIMEOUT'].to_s.to_i
    # d) ::Artifactory.setup and ::Artifactory.reset! does not reread those options!!!! OMG!
    # Only resort: Open ::Artifactory class and override the method with the code it should have had ... %)
    ::Artifactory.setup
  end

  @client = ::Artifactory::Client.new()
end

Dynamic Method Handling

This class handles dynamic methods through the method_missing method

#method_missing(name, *args) ⇒ Object 



122
123
124
125
126
127
128
129
130
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 122

def method_missing(name, *args)
  if name =~ %r'^artifactory_'
    key = name.to_s.upcase
    raise "ENV has no key #{key}" unless ENV.has_key?(key)
    ENV[key]
  else
    super
  end
end

Instance Method Details

#cleanupTempFiles ⇒ Object 



502
503
504
505
506
507
508
509
510
511
512
513
514
515
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 502

def cleanupTempFiles
  @vars[:artifacts].each do |art|
    if art[:data][:temp].is_a?(FalseClass)
      if File.exists?(art[:data][:file])
        File.unlink(art[:data][:file]) if File.exists?(art[:data][:file])
        art[:data].delete(:file)
        art[:data].delete(:temp)
      else
        @logger.warn "Temporary file disappeared: #{data.ai}"
        @vars[:return_code] = Errors::TEMP_FILE_MISSING
      end
    end
  end
end

#copyArtifact(artifact_module, artifact_version, artifact_path, artifact, repo = nil) ⇒ Object 



430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 430

def copyArtifact(artifact_module, artifact_version, artifact_path, artifact, repo=nil)
  begin
    if artifact.attributes[:uri].eql?(File.join(artifactory_endpoint, repo || artifactory_repo, artifact_path))
      @logger.info "Not copying (identical artifact): #{artifact_path}"
    else
      @logger.info "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S %z')}] Start copy #{artifact_path} = #{artifact.attributes[:size]} bytes"
      copied = false
      3.times{
        copied = false
        monitor(30){
          result = artifact.copy("#{repo || artifactory_repo()}/#{artifact_path}")
          @logger.info "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S %z')}] Copied: #{result.ai}"
          copied = true
        }
        break if copied
      }
      raise "Failed to copy #{artifact_path}" unless copied
    end
    objects = maybeArtifactoryObject(artifact_module, artifact_version, false, repo)
    unless objects.size > 0
      sleep 10
      objects = maybeArtifactoryObject(artifact_module, artifact_version, false, repo)
      raise "Failed to copy '#{artifact_path}'" unless objects.size > 0
    end
    objects
  rescue Exception => e
    @logger.error "Failed to copy #{artifact_path}: #{e.class.name} #{e.message}"
    raise e
  end
end

#get_artifact_file_name_ext(data) ⇒ Object 



173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 173

def get_artifact_file_name_ext(data)
  file_name = File.basename(data[:file])
  if file_name =~ %r'^#{data[:name]}'
    file_name.gsub!(%r'^#{data[:name]}\.*', '')
  end
  file_name.gsub!(%r'\.\.+','.')
  file_name.gsub!(%r'\.*-*#{data[:version]}', '')
  file_name.gsub!(%r'\.*-*#{data[:build]}-*', '')
  file_ext = file_name.dup
  file_ext.gsub!(%r'^.*?\.*(tar\.gz|tgz|tar\.bzip2|bzip2|tar\.bz2|bz2|zip|jar|war|groovy)$', '\1')
  unless file_ext.empty?
    file_name.gsub!(%r'\.*#{file_ext}$', '')
  end
  file_name.gsub!(%r'(\.\d+)+$', '')
  return file_name, file_ext
end

#getArtifactName(name, file_name, artifact_version, file_ext) ⇒ Object 



304
305
306
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 304

def getArtifactName(name, file_name, artifact_version, file_ext)
  artifact_name = "#{name}#{artifact_version.empty? ? '' : "-#{artifact_version}"}.#{file_ext}" # #{file_name}
end

#getArtifactPath(artifact_module, artifact_version, artifact_name) ⇒ Object 



300
301
302
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 300

def getArtifactPath(artifact_module, artifact_version, artifact_name)
  artifact_path = "#{artifactory_org_path()}/#{artifact_module}/#{artifact_version}/#{artifact_name}"
end

#latestArtifactoryVersion(artifact_name, repo = nil) ⇒ Object 



333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 333

def latestArtifactoryVersion(artifact_name, repo=nil)
  begin
    # Get a list of matching artifacts in this repository
    @logger.info "Artifactory latest_version g=#{artifactory_org_path()},a=#{artifact_name},r=#{repo || artifactory_repo()}"
    @arti_search_result     = []
    monitor(@vars[:upload_timer], 'artifact_latest_version'){
      @arti_search_result = ::Artifactory::Resource::Artifact.latest_version(client: @client, group: artifactory_org_path(), name: artifact_name, repos: [repo || artifactory_repo()])
    }
    # noinspection RubyScope
    if @arti_search_result and @arti_search_result.size > 0
      @logger.info "\tresult: #{@arti_search_result}"
    end
    @arti_search_result
  rescue Exception => e
    @logger.error "Artifactory error: #{e.class.name} #{e.message}"
    raise e
  end
end

#matchArtifactoryObjects(artifact_path, data, objects) ⇒ Object 



288
289
290
291
292
293
294
295
296
297
298
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 288

def matchArtifactoryObjects(artifact_path, data, objects)
  # matched = false
  objects.select do |artifact|
    @logger.debug "\tChecking: #{artifact.attributes.ai} for #{artifact_path}"
    # if artifact.uri.match(%r'#{artifact_path}$')
    #   @logger.info "\tMatched: #{artifact.attributes.select { |k, _| k != :client }.ai}"
    # end
    matched = (artifact.md5.eql?(data[:md5]) or artifact.sha1.eql?(data[:sha1]))
    matched
  end
end

#maybeArtifactoryObject(artifact_name, artifact_version, wide = true, repo = nil) ⇒ Object 



309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 309

def maybeArtifactoryObject(artifact_name,artifact_version,wide=true,repo=nil)
  begin
    # Get a list of matching artifacts in this repository
    @logger.info "Artifactory gavc_search g=#{artifactory_org_path()},a=#{artifact_name},v=#{artifact_version},r=#{repo || artifactory_repo()}"
    @arti_search_result     = []
    monitor(@vars[:upload_timer], 'artifact_gavc_search'){
      @arti_search_result = @client.artifact_gavc_search(group: artifactory_org_path(), name: artifact_name, version: "#{artifact_version}", repos: [repo || artifactory_repo()])
    }
    # noinspection RubyScope
    if @arti_search_result.size > 0
      @logger.info "\tresult: #{@arti_search_result}"
    elsif wide
      @logger.warn 'GAVC search came up empty!'
      @arti_search_result = @client.artifact_search(name: artifact_name, repos: [artifactory_repo()])
      @logger.info "Artifactory search match a=#{artifact_name},r=#{artifactory_repo()}: #{@arti_search_result}"
    end
    @arti_search_result
  rescue Exception => e
    @logger.error "Artifactory error: #{e.class.name} #{e.message}"
    raise e
  end
end

#maybeUploadArtifactoryObject(args) ⇒ Object Also known as: cicd_maybeUploadArtifactoryObject 



191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 191

def maybeUploadArtifactoryObject(args)
  data             = args[:data]
  artifact_module  = args[:artifact_module]
  artifact_version = args[:artifact_version]
  file_ext         = args[:file_ext]
  file_name        = args[:file_name]
  make_copy        = (args[:copy].nil? or args[:copy])

  artifact_name = getArtifactName(data[:name], file_name, artifact_version, file_ext) # artifact_path = "#{artifactory_org_path()}/#{data[:name]}/#{data[:version]}-#{@vars[:variant]}/#{artifact_name}"
  artifact_path = getArtifactPath(artifact_module, artifact_version, artifact_name)
  objects = maybeArtifactoryObject(artifact_module, artifact_version, false, args[:repo])
  upload = false
  matched = []
  if objects.nil? or objects.size == 0
    upload = true
  else
    @logger.info "#{artifactory_endpoint()}/#{args[:repo] || artifactory_repo()}/#{artifact_path} exists - #{objects.size} results"
    @logger.info "\t#{objects.map{|o| o.attributes[:uri]}.join("\n\t")}"
    matched = matchArtifactoryObjects(artifact_path, data, objects)
    upload ||= (matched.size == 0)
  end
  properties_matrix = {}
  data.select{|k,_| not k.to_s.match(%r'^file')}.each do |k,v|
    properties_matrix["product.#{k}"] = v
  end
  data[:properties] = properties_matrix.merge(@properties_matrix)
  if upload
    objects = uploadArtifact(artifact_module, artifact_version, artifact_path, data, args[:repo])
    matched = matchArtifactoryObjects(artifact_path, data, objects)
    properties = matched.map{ |artifact|
      artifact.properties
    }
    @logger.info "Matched artifacts properties: #{properties.ai}"
    matched.each { |match|
      artifact = ::Artifactory::Resource::Artifact.from_url(match.uri, client: @client)
      unless artifact.uri =~ %r'#{data[:properties]['product.version']}/'
        @logger.error "Artifact uploaded #{artifact_module}/#{artifact_version} to #{match.uri} does not contain #{data[:properties]['product.version']}/"
        @vars[:return_code] = Errors::ARTIFACT_NOT_UPLOADED
      end
    }
  else
    @logger.info "Keep existing #{matched.map{|o| o.attributes[:uri]}.join("\t")}"
  end
  if data[:temp]
    if File.exists?(data[:file])
      File.unlink(data[:file]) if File.exists?(data[:file])
      data.delete(:file)
      data.delete(:temp)
    else
      @logger.warn "Temporary file disappeared: #{data.ai}"
    end
  end
  @vars[:return_code] = Errors::ARTIFACT_NOT_UPLOADED unless matched.size > 0
  if @vars[:return_code] == 0 and make_copy
    artifact_version += "-#{data[:build] || @vars[:build_num]}"
    artifact_name = getArtifactName(data[:name], file_name, artifact_version, file_ext, )
    artifact_path = getArtifactPath(artifact_module, artifact_version, artifact_name)
    copies  = maybeArtifactoryObject(artifact_module, artifact_version, false, args[:repo])
    matched = matchArtifactoryObjects(artifact_path, data, copies)
    upload  = (matched.size == 0)
    if upload
      copies = []
      objects.each do |artifact|
        copied = copyArtifact(artifact_module, artifact_version, artifact_path, artifact, args[:repo])
        unless copied.size > 0
          @vars[:return_code] = Errors::ARTIFACT_NOT_COPIED
          break
        end
        copies << copied
      end
      copies.flatten!
      matched = matchArtifactoryObjects(artifact_path, data, copies)
      properties = matched.map{ |artifact|
        artifact.properties
      }
      @logger.info "Matched artifacts properties: #{properties.ai}"
      matched.each { |match|
        artifact = ::Artifactory::Resource::Artifact.from_url(match.uri, client: @client)
        unless artifact.uri =~ %r'#{data[:properties]['product.version']}-#{data[:properties]['product.build']}/'
          @logger.error "Artifact uploaded #{artifact_module}/#{artifact_version} to #{match.uri} does not contain #{data[:properties]['product.version']}-#{data[:properties]['product.version']}/"
          @vars[:return_code] = Errors::ARTIFACT_NOT_UPLOADED
        end
      }
    else
      @logger.info "Keep existing #{matched.map{|o| o.attributes[:uri]}.join("\t")}"
    end
  end
  args[:data]             = data
  args[:artifact_module]  = artifact_module
  args[:artifact_version] = artifact_version
  args[:file_ext]         = file_ext
  args[:file_name]        = file_name

  @vars[:return_code]
end

#monitor(limit, title = 'Time consumed') ⇒ Object 



352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 352

def monitor(limit,title='Time consumed')
  raise 'Must have a block' unless block_given?
  thread = Thread.new(){
    yield
  }
  progressbar = ::ProgressBar.create({title: title, format: '%a <%B> %p%% %t', progress_mark: '=', starting_at: 0, total: limit, remainder_mark: '.'}) if @logger.info?
  # progressbar = ::ProgressBar.create({title: title, format: '%a <%B> %p%% %t', progress_mark: '=', starting_at: 0, total: limit, length: limit, remainder_mark: '.', throttle_rate: 0.5}) if @logger.info?
  #progressbar = ::ProgressBar.create({title: title, format: '%a %bᗧ%i %p%%', progress_mark: ' ', remainder_mark: '=', starting_at: 0, throttle_rate: 0.5}) if @logger.info?
  limit.times do |i|
    res = thread.join(1)
    if @logger.info?
      progressbar.increment
      # progressbar.total = limit
      # progressbar.log i
    end
    unless thread.alive? #or thread.stop?
      puts '' if @logger.info?
      break
    end
  end
  thread.kill if thread.alive? or thread.stop?
end

#uploadArtifact(artifact_module, artifact_version, artifact_path, data, repo = nil) ⇒ Object 



375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 375

def uploadArtifact(artifact_module, artifact_version, artifact_path, data, repo=nil)
  data[:size] = File.size(data[:file])
  artifact = ::Artifactory::Resource::Artifact.new(local_path: data[:file], client: @client)
  # noinspection RubyStringKeysInHashInspection
  artifact.checksums =  {
                        'md5'   => data[:md5],
                        'sha1'  => data[:sha1],
                        }
  artifact.size = data[:size]
  @logger.info "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S %z')}] Start upload #{artifact_path} = #{data[:size]} bytes"
  monitor(@vars[:upload_timer], 'upload') {
    @arti_upload_result = artifact.upload(repo || artifactory_repo(), "#{artifact_path}", data[:properties] || {})
  }
  raise "Failed to upload artifact for #{artifact_path}" unless @arti_upload_result
  @logger.info "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S %z')}] Uploaded: #{@arti_upload_result.attributes.select { |k, _| k != :client }.ai}"
  3.times{
    @arti_upload_checksum = false
    monitor(@vars[:upload_timer], 'upload_checksum') {
      begin
        artifact.upload_checksum(repo || artifactory_repo(), "#{artifact_path}", :sha1, data[:sha1])
        @arti_upload_checksum = true
      rescue Exception => e
        @logger.fatal "Failed to upload #{artifact_path}: #{e.class.name} #{e.message}"
        raise e
      end
    }
    break if @arti_upload_checksum
  }
  raise "Failed to upload SHA1 for #{artifact_path}" unless @arti_upload_checksum
  3.times{
    @arti_upload_checksum = false
    monitor(@vars[:upload_timer], 'upload_checksum') {
      begin
        artifact.upload_checksum(repo || artifactory_repo(), "#{artifact_path}", :md5,  data[:md5])
        @arti_upload_checksum = true
      rescue Exception => e
        @logger.fatal "Failed to upload #{artifact_path}: #{e.class.name} #{e.message}"
        raise e
      end
    }
    break if @arti_upload_checksum
  }
  raise "Failed to upload MD5 for #{artifact_path}" unless @arti_upload_checksum
  attempt = 0
  objects = []
  while attempt < 3
    objects = maybeArtifactoryObject(artifact_module, artifact_version, false, repo)
    break if objects.size > 0
    sleep 2
    attempt += 1
  end
  raise "Failed to upload '#{artifact_path}'" unless objects.size > 0
  objects
end

#uploadBuildArtifacts ⇒ Object 



462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 462

def uploadBuildArtifacts()
  @logger.step CLASS+'::'+__method__.to_s
  if @vars.has_key?(:build_dir) and @vars.has_key?(:build_pkg)
    begin
      artifacts = @vars[:artifacts] rescue []

      key = getKey()
      if File.exists?(@vars[:build_pkg])
        # Store the assembly - be sure to inherit possible overrides in pkg name and ext but dictate the drawer!
        artifacts << {
            key:        "#{File.join(File.dirname(key),File.basename(@vars[:build_pkg]))}",
            data:       {:file => @vars[:build_pkg]},
            public_url: :build_url,
            label:      'Package URL'
        }
      else
        @logger.warn "Skipping upload of missing artifact: '#{@vars[:build_pkg]}'"
      end

      # Store the metadata
      manifest = manifestMetadata()
      hash     = JSON.parse(manifest)

      @vars[:return_code] = uploadToRepo(artifacts)
      # if 0 == @vars[:return_code]
      #   @vars[:return_code] = takeInventory()
      # end
      @vars[:return_code]
    rescue => e
      @logger.error "#{e.class.name} #{e.message}\n#{e.backtrace.ai}"
      @vars[:return_code] = Errors::ARTIFACT_UPLOAD_EXCEPTION
      raise e
    end
  else
    @vars[:return_code] = Errors::NO_ARTIFACTS
  end
  @vars[:return_code]
end

#uploadToRepo(artifacts) ⇒ Object Also known as: cicd_uploadToRepo 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/cicd/builder/mixlib/repo/artifactory.rb', line 133

def uploadToRepo(artifacts)
  # Set a few build properties on the endpoint URL
  @properties_matrix = {}
  @vars[:build_mdd].each do |k,v|
    @properties_matrix["build.#{k.downcase}"] = v
  end
  # noinspection RubyStringKeysInHashInspection
  @properties_matrix.merge!({
    'build.name'   =>  @vars[:build_mdd][:Project],
    'build.number' =>  @vars[:build_mdd][:Build],
    'build.branch' =>  @vars[:branch],
    'vcs.revision' =>  @vars[:build_mdd][:Commit],
    'vcs.branch'   =>  @vars[:build_mdd][:Branch],
  })
  # @client.endpoint += ";#{matrix}"
  artifacts.each{|art|
    data = art[:data]
    if data.has_key?(:data)
      tempArtifactFile(data[:name], data)
    end
    if data.has_key?(:file)
      data[:sha1] = Digest::SHA1.file(data[:file]).hexdigest
      data[:md5]  = Digest::MD5.file(data[:file]).hexdigest
    else
      raise 'Artifact does not have file or data?'
    end
    file_name, file_ext = (data[:file_name] and data[:file_ext]) ? [data[:file_name], data[:file_ext]] : get_artifact_file_name_ext(data)
    if file_name =~ %r'\.+'
      raise "Unable to parse out file name in #{data[:file]}"
    end
    unless file_name.empty?
      file_name = '_'+file_name.gsub(%r'^(\.|-|)(\w)', '\2').gsub(%r'(\.|-)+', '_')
    end
    maybeUploadArtifactoryObject(data: data, artifact_module: data[:module], artifact_version: data[:version] || @vars[:version], file_name: file_name, file_ext: file_ext) # -#{@vars[:variant]
    break unless @vars[:return_code] == 0
  }
  @vars[:return_code]
end