Class: Bosh::Director::Jobs::UpdateRelease
- Includes:
- DownloadHelper, LockHelper
- Defined in:
- lib/bosh/director/jobs/update_release.rb
Instance Attribute Summary collapse
-
#release_model ⇒ Object
Returns the value of attribute release_model.
Attributes inherited from BaseJob
Class Method Summary collapse
Instance Method Summary collapse
-
#backfill_source_for_packages(packages, release_dir) ⇒ boolean
True if sources were added to at least one package; false if the call had no effect.
- #compiled_release ⇒ Object
- #create_compiled_package(package, stemcell, release_dir) ⇒ Object
-
#create_compiled_packages(all_compiled_packages, release_dir) ⇒ boolean
True if at least one job was created; false if the call had no effect.
- #create_job(job_meta, release_dir) ⇒ Object
-
#create_jobs(jobs, release_dir) ⇒ boolean
True if at least one job was created; false if the call had no effect.
-
#create_package(package_meta, release_dir) ⇒ void
Creates package in DB according to given metadata.
-
#create_packages(package_metas, release_dir) ⇒ Array<Hash>, boolean
Creates packages using provided metadata.
- #download_remote_release ⇒ Object
-
#extract_release ⇒ void
Extracts release tarball.
-
#initialize(release_path, options = {}) ⇒ UpdateRelease
constructor
A new instance of UpdateRelease.
-
#normalize_manifest ⇒ void
Normalizes release manifest, so all names, versions, and checksums are Strings.
-
#perform ⇒ void
Extracts release tarball, verifies release manifest and saves release in DB.
-
#process_jobs(release_dir) ⇒ void
Finds job template definitions in release manifest and sorts them into two buckets: new and existing job templates, then creates new job template records in the database and points release version to existing ones.
-
#process_packages(release_dir) ⇒ void
Finds all package definitions in the manifest and sorts them into two buckets: new and existing packages, then creates new packages and points current release version to the existing packages.
-
#process_release(release_dir) ⇒ void
Processes uploaded release, creates jobs and packages in DB if needed.
-
#register_package(package) ⇒ void
Marks package model as used by release version model.
-
#resolve_package_dependencies(packages) ⇒ void
Resolves package dependencies, makes sure there are no cycles and all dependencies are present.
-
#save_package_source_blob(package, package_meta, release_dir) ⇒ boolean
True if a new blob was created; false otherwise.
- #source_release ⇒ Object
- #stemcells_used_by_package(package_meta) ⇒ Object
-
#use_existing_jobs(jobs) ⇒ boolean
True if at least one job was tied to the release version; false if the call had no effect.
-
#use_existing_packages(packages, release_dir) ⇒ Array<Hash>
Points release DB model to existing packages described by given metadata.
- #validate_tgz(tgz, desc) ⇒ Object
- #verify_manifest(release_dir) ⇒ void
Methods included from DownloadHelper
Methods included from LockHelper
#with_compile_lock, #with_deployment_lock, #with_release_lock, #with_release_locks, #with_stemcell_lock
Methods inherited from BaseJob
#begin_stage, #event_log, #logger, perform, #result_file, #single_step_stage, #task_cancelled?, #task_checkpoint, #track_and_log
Constructor Details
#initialize(release_path, options = {}) ⇒ UpdateRelease
Returns a new instance of UpdateRelease.
21 22 23 24 25 26 27 28 29 30 31 32 33 34 |
# File 'lib/bosh/director/jobs/update_release.rb', line 21 def initialize(release_path, = {}) if ['remote'] # file will be downloaded to the release_path @release_path = File.join(Dir.tmpdir, "release-#{SecureRandom.uuid}") @release_url = release_path else # file already exists at the release_path @release_path = release_path end @release_model, @release_version_model, @manifest, @name, @version = nil, nil, nil, nil, nil @rebase = !!['rebase'] end |
Instance Attribute Details
#release_model ⇒ Object
Returns the value of attribute release_model.
13 14 15 |
# File 'lib/bosh/director/jobs/update_release.rb', line 13 def release_model @release_model end |
Class Method Details
.job_type ⇒ Object
15 16 17 |
# File 'lib/bosh/director/jobs/update_release.rb', line 15 def self.job_type :update_release end |
Instance Method Details
#backfill_source_for_packages(packages, release_dir) ⇒ boolean
Returns true if sources were added to at least one package; false if the call had no effect.
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 |
# File 'lib/bosh/director/jobs/update_release.rb', line 278 def backfill_source_for_packages(packages, release_dir) return false if packages.empty? had_effect = false single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do packages.each do |package, | package_desc = "#{package.name}/#{package.version}" logger.info("Adding source for package `#{package_desc}'") had_effect |= save_package_source_blob(package, , release_dir) package.save end end had_effect end |
#compiled_release ⇒ Object
107 108 109 110 |
# File 'lib/bosh/director/jobs/update_release.rb', line 107 def compiled_release raise "Don't know what kind of release we have until verify_release is called" unless @manifest @compiled_release end |
#create_compiled_package(package, stemcell, release_dir) ⇒ Object
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 |
# File 'lib/bosh/director/jobs/update_release.rb', line 408 def create_compiled_package(package, stemcell, release_dir) tgz = File.join(release_dir, 'compiled_packages', "#{package.name}.tgz") validate_tgz(tgz, "#{package.name}.tgz") compiled_package = Models::CompiledPackage.new compiled_package.blobstore_id = BlobUtil.create_blob(tgz) compiled_package.sha1 = Digest::SHA1.file(tgz).hexdigest transitive_dependencies = @release_version_model.transitive_dependencies(package) compiled_package.dependency_key = Models::CompiledPackage.create_dependency_key(transitive_dependencies) compiled_package.build = Models::CompiledPackage.generate_build_number(package, stemcell) compiled_package.package_id = package.id compiled_package.stemcell_id = stemcell.id compiled_package.save end |
#create_compiled_packages(all_compiled_packages, release_dir) ⇒ boolean
Returns true if at least one job was created; false if the call had no effect.
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 |
# File 'lib/bosh/director/jobs/update_release.rb', line 363 def create_compiled_packages(all_compiled_packages, release_dir) return false if all_compiled_packages.nil? event_log.begin_stage('Creating new compiled packages', all_compiled_packages.size) had_effect = false all_compiled_packages.each do |compiled_package_spec| package = compiled_package_spec[:package] stemcell = compiled_package_spec[:stemcell] existing_compiled_package = Models::CompiledPackage.where( :package_id => package.id, :stemcell_id => stemcell.id) if existing_compiled_package.empty? package_desc = "#{package.name}/#{package.version} for #{stemcell.name}/#{stemcell.version}" event_log.track(package_desc) do create_compiled_package(package, stemcell, release_dir) had_effect = true end end end had_effect end |
#create_job(job_meta, release_dir) ⇒ Object
548 549 550 551 |
# File 'lib/bosh/director/jobs/update_release.rb', line 548 def create_job(, release_dir) release_job = ReleaseJob.new(, @release_model, release_dir, @packages, logger) release_job.create end |
#create_jobs(jobs, release_dir) ⇒ boolean
Returns true if at least one job was created; false if the call had no effect.
532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 |
# File 'lib/bosh/director/jobs/update_release.rb', line 532 def create_jobs(jobs, release_dir) return false if jobs.empty? event_log.begin_stage("Creating new jobs", jobs.size) jobs.each do || job_desc = "#{["name"]}/#{["version"]}" event_log.track(job_desc) do logger.info("Creating new template `#{job_desc}'") template = create_job(, release_dir) register_template(template) end end true end |
#create_package(package_meta, release_dir) ⇒ void
This method returns an undefined value.
Creates package in DB according to given metadata
431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 |
# File 'lib/bosh/director/jobs/update_release.rb', line 431 def create_package(, release_dir) name, version = ['name'], ['version'] package_attrs = { :release => @release_model, :name => name, :sha1 => nil, :blobstore_id => nil, :fingerprint => ['fingerprint'], :version => version } package = Models::Package.new(package_attrs) package.dependency_set = ['dependencies'] save_package_source_blob(package, , release_dir) unless @compiled_release package.save end |
#create_packages(package_metas, release_dir) ⇒ Array<Hash>, boolean
Creates packages using provided metadata
332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 |
# File 'lib/bosh/director/jobs/update_release.rb', line 332 def create_packages(, release_dir) if .empty? return [] end package_stemcell_hashes = [] event_log.begin_stage("Creating new packages", .size) .each do || package_desc = "#{["name"]}/#{["version"]}" package = nil event_log.track(package_desc) do logger.info("Creating new package `#{package_desc}'") package = create_package(, release_dir) register_package(package) end if @compiled_release stemcells = stemcells_used_by_package() stemcells.each do |stemcell| hash = { package: package, stemcell: stemcell} package_stemcell_hashes << hash end end end return package_stemcell_hashes end |
#download_remote_release ⇒ Object
61 62 63 |
# File 'lib/bosh/director/jobs/update_release.rb', line 61 def download_remote_release download_remote_file('release', @release_url, @release_path) end |
#extract_release ⇒ void
This method returns an undefined value.
Extracts release tarball
67 68 69 70 71 72 73 74 75 76 77 78 |
# File 'lib/bosh/director/jobs/update_release.rb', line 67 def extract_release release_dir = Dir.mktmpdir result = Bosh::Exec.sh("tar -C #{release_dir} -xzf #{@release_path} 2>&1", :on_error => :return) if result.failed? logger.error("Failed to extract release archive '#{@release_path}' into dir '#{release_dir}', tar returned #{result.exit_status}, output: #{result.output})") FileUtils.rm_rf(release_dir) raise ReleaseInvalidArchive, "Extracting release archive failed. Check task debug log for details." end release_dir end |
#normalize_manifest ⇒ void
This method returns an undefined value.
Normalizes release manifest, so all names, versions, and checksums are Strings.
153 154 155 156 157 158 |
# File 'lib/bosh/director/jobs/update_release.rb', line 153 def normalize_manifest Bosh::Director.hash_string_vals(@manifest, 'name', 'version') @manifest[@packages_folder].each { |p| Bosh::Director.hash_string_vals(p, 'name', 'version', 'sha1') } @manifest['jobs'].each { |j| Bosh::Director.hash_string_vals(j, 'name', 'version', 'sha1') } end |
#perform ⇒ void
This method returns an undefined value.
Extracts release tarball, verifies release manifest and saves release in DB
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
# File 'lib/bosh/director/jobs/update_release.rb', line 38 def perform logger.info("Processing update release") logger.info("Release rebase will be performed") if @rebase single_step_stage("Downloading remote release") { download_remote_release } if @release_url release_dir = nil single_step_stage("Extracting release") { release_dir = extract_release } single_step_stage("Verifying manifest") { verify_manifest(release_dir) } with_release_lock(@name) { process_release(release_dir) } "Created release `#{@name}/#{@version}'" rescue Exception => e raise e ensure FileUtils.rm_rf(release_dir) if release_dir FileUtils.rm_rf(@release_path) if @release_path end |
#process_jobs(release_dir) ⇒ void
This method returns an undefined value.
Finds job template definitions in release manifest and sorts them into two buckets: new and existing job templates, then creates new job template records in the database and points release version to existing ones.
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 |
# File 'lib/bosh/director/jobs/update_release.rb', line 496 def process_jobs(release_dir) logger.info("Checking for new jobs in release") new_jobs = [] existing_jobs = [] @manifest["jobs"].each do || # Checking whether we might have the same bits somewhere @release_version_model.templates.select { |t| t.name == ["name"] }.each do |tmpl| if tmpl.fingerprint != ["fingerprint"] raise ReleaseExistingJobFingerprintMismatch, "job `#{["name"]}' had different fingerprint in previously uploaded release `#{@name}/#{@version}'" end end jobs = Models::Template.where(fingerprint: ["fingerprint"]).all template = jobs.find do |job| job.release_id == @release_model.id && job.name == ["name"] && job.version == ["version"] end if template.nil? new_jobs << else existing_jobs << [template, ] end end did_something = create_jobs(new_jobs, release_dir) did_something |= use_existing_jobs(existing_jobs) did_something end |
#process_packages(release_dir) ⇒ void
This method returns an undefined value.
Finds all package definitions in the manifest and sorts them into two buckets: new and existing packages, then creates new packages and points current release version to the existing packages.
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 |
# File 'lib/bosh/director/jobs/update_release.rb', line 189 def process_packages(release_dir) logger.info("Checking for new packages in release") new_packages = [] existing_packages = [] registered_packages = [] @manifest[@packages_folder].each do || # Checking whether we might have the same bits somewhere (in any release, not just the one being uploaded) @release_version_model.packages.select { |pv| pv.name == ['name'] }.each do |package| if package.fingerprint != ['fingerprint'] raise ReleaseInvalidPackage, "package `#{['name']}' had different fingerprint in previously uploaded release `#{@name}/#{@version}'" end end packages = Models::Package.where(fingerprint: ["fingerprint"]).all if packages.empty? new_packages << next end existing_package = packages.find do |package| package.release_id == @release_model.id && package.name == ["name"] && package.version == ["version"] end if existing_package # clean up 'broken' dependency_set (a bug was including transitives) # dependency ordering impacts fingerprint # TODO: The following code can be removed after some reasonable time period (added 2014.10.06) if existing_package.dependency_set != ['dependencies'] existing_package.dependency_set = ['dependencies'] existing_package.save end if existing_package.release_versions.include? @release_version_model if existing_package.blobstore_id.nil? packages.each do |package| unless package.blobstore_id.nil? ["blobstore_id"] = package.blobstore_id ["sha1"] = package.sha1 break end end end registered_packages << [existing_package, ] else existing_packages << [existing_package, ] end else # We found a package with the same fingerprint but different # (release, name, version) tuple, so we need to make a copy # of the package blob and create a new db entry for it packages.each do |package| unless package.blobstore_id.nil? ["blobstore_id"] = package.blobstore_id ["sha1"] = package.sha1 break end end new_packages << end end package_stemcell_hashes1 = create_packages(new_packages, release_dir) package_stemcell_hashes2 = use_existing_packages(existing_packages, release_dir) if @compiled_release compatible_stemcell_combos = registered_packages.flat_map do |pkg, | stemcells_used_by_package().map do |stemcell| { package: pkg, stemcell: stemcell } end end consolidated_package_stemcell_hashes = Array(package_stemcell_hashes1) | Array(package_stemcell_hashes2) | compatible_stemcell_combos create_compiled_packages(consolidated_package_stemcell_hashes, release_dir) else backfill_source_for_packages(registered_packages, release_dir) end end |
#process_release(release_dir) ⇒ void
This method returns an undefined value.
Processes uploaded release, creates jobs and packages in DB if needed
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
# File 'lib/bosh/director/jobs/update_release.rb', line 119 def process_release(release_dir) @release_model = Models::Release.find_or_create(:name => @name) @version = next_release_version if @rebase version_attrs = { :release => @release_model, :version => @version.to_s } release_is_new = false @release_version_model = Models::ReleaseVersion.find_or_create(version_attrs){ release_is_new = true } if release_is_new @release_version_model.uncommitted_changes = @uncommitted_changes if @uncommitted_changes @release_version_model.commit_hash = @commit_hash if @commit_hash @release_version_model.save else if @release_version_model.commit_hash != @commit_hash || @release_version_model.uncommitted_changes != @uncommitted_changes raise ReleaseVersionCommitHashMismatch, "release `#{@name}/#{@version}' has already been uploaded with commit_hash as `#{@commit_hash}' and uncommitted_changes as `#{@uncommitted_changes}'" end end single_step_stage("Resolving package dependencies") do resolve_package_dependencies(@manifest[@packages_folder]) end @packages = {} process_packages(release_dir) process_jobs(release_dir) event_log.begin_stage(@compiled_release ? "Compiled Release has been created" : "Release has been created", 1) event_log.track("#{@name}/#{@version}") {} end |
#register_package(package) ⇒ void
This method returns an undefined value.
Marks package model as used by release version model
486 487 488 489 |
# File 'lib/bosh/director/jobs/update_release.rb', line 486 def register_package(package) @packages[package.name] = package @release_version_model.add_package(package) end |
#resolve_package_dependencies(packages) ⇒ void
This method returns an undefined value.
Resolves package dependencies, makes sure there are no cycles and all dependencies are present
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 |
# File 'lib/bosh/director/jobs/update_release.rb', line 163 def resolve_package_dependencies(packages) packages_by_name = {} packages.each do |package| packages_by_name[package["name"]] = package package["dependencies"] ||= [] end logger.info("Resolving package dependencies for #{packages_by_name.keys.inspect}") dependency_lookup = lambda do |package_name| packages_by_name[package_name]["dependencies"] end result = Bosh::Director::CycleHelper.check_for_cycle(packages_by_name.keys, :connected_vertices => true, &dependency_lookup) packages.each do |package| name = package["name"] dependencies = package["dependencies"] all_dependencies = result[:connected_vertices][name] logger.info("Resolved package dependencies for `#{name}': #{dependencies.pretty_inspect} => #{all_dependencies.pretty_inspect}") end end |
#save_package_source_blob(package, package_meta, release_dir) ⇒ boolean
Returns true if a new blob was created; false otherwise.
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 |
# File 'lib/bosh/director/jobs/update_release.rb', line 452 def save_package_source_blob(package, , release_dir) return false unless package.blobstore_id.nil? name, version, existing_blob = ['name'], ['version'], ['blobstore_id'] desc = "package '#{name}/#{version}'" package.sha1 = ['sha1'] if existing_blob logger.info("Creating #{desc} from existing blob #{existing_blob}") package.blobstore_id = BlobUtil.copy_blob(existing_blob) elsif package logger.info("Creating #{desc} from provided bits") package_tgz = File.join(release_dir, 'packages', "#{name}.tgz") validate_tgz(package_tgz, desc) package.blobstore_id = BlobUtil.create_blob(package_tgz) end true end |
#source_release ⇒ Object
112 113 114 |
# File 'lib/bosh/director/jobs/update_release.rb', line 112 def source_release !compiled_release end |
#stemcells_used_by_package(package_meta) ⇒ Object
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 |
# File 'lib/bosh/director/jobs/update_release.rb', line 388 def stemcells_used_by_package() if ['stemcell'].nil? raise 'stemcell informatiom(operating system/version) should be listed for each package of a compiled tarball' end values = ['stemcell'].split('/', 2) = values[0] stemcell_version = values[1] unless && stemcell_version raise 'stemcell informatiom(operating system/version) should be listed for each package of a compiled tarball' end stemcells = Models::Stemcell.where(:operating_system => , :version => stemcell_version) if stemcells.empty? raise "No stemcells matching OS #{} version #{stemcell_version}" end stemcells end |
#use_existing_jobs(jobs) ⇒ boolean
Returns true if at least one job was tied to the release version; false if the call had no effect.
555 556 557 558 559 560 561 562 563 564 565 566 567 |
# File 'lib/bosh/director/jobs/update_release.rb', line 555 def use_existing_jobs(jobs) return false if jobs.empty? single_step_stage("Processing #{jobs.size} existing job#{"s" if jobs.size > 1}") do jobs.each do |template, _| job_desc = "#{template.name}/#{template.version}" logger.info("Using existing job `#{job_desc}'") register_template(template) unless template.release_versions.include? @release_version_model end end true end |
#use_existing_packages(packages, release_dir) ⇒ Array<Hash>
Points release DB model to existing packages described by given metadata
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 |
# File 'lib/bosh/director/jobs/update_release.rb', line 297 def use_existing_packages(packages, release_dir) if packages.empty? return [] end package_stemcell_hashes = [] single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do packages.each do |package, | package_desc = "#{package.name}/#{package.version}" logger.info("Using existing package `#{package_desc}'") register_package(package) if compiled_release stemcells = stemcells_used_by_package() stemcells.each do |stemcell| hash = { package: package, stemcell: stemcell} package_stemcell_hashes << hash end end if source_release && package.blobstore_id.nil? save_package_source_blob(package, , release_dir) package.save end end end return package_stemcell_hashes end |
#validate_tgz(tgz, desc) ⇒ Object
475 476 477 478 479 480 481 |
# File 'lib/bosh/director/jobs/update_release.rb', line 475 def validate_tgz(tgz, desc) result = Bosh::Exec.sh("tar -tzf #{tgz} 2>&1", :on_error => :return) if result.failed? logger.error("Extracting #{desc} archive failed, tar returned #{result.exit_status}, output: #{result.output}") raise PackageInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details." end end |
#verify_manifest(release_dir) ⇒ void
This method returns an undefined value.
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
# File 'lib/bosh/director/jobs/update_release.rb', line 82 def verify_manifest(release_dir) manifest_file = File.join(release_dir, "release.MF") raise ReleaseManifestNotFound, "Release manifest not found" unless File.file?(manifest_file) @manifest = Psych.load_file(manifest_file) #handle compiled_release case @compiled_release = !!@manifest["compiled_packages"] @packages_folder = @compiled_release ? "compiled_packages" : "packages" normalize_manifest @name = @manifest["name"] begin @version = Bosh::Common::Version::ReleaseVersion.parse(@manifest["version"]) logger.info("Formatted version '#{@manifest["version"]}' => '#{@version}'") unless @version == @manifest["version"] rescue SemiSemantic::ParseError raise ReleaseVersionInvalid, "Release version invalid: #{@manifest["version"]}" end @commit_hash = @manifest.fetch("commit_hash", nil) @uncommitted_changes = @manifest.fetch("uncommitted_changes", nil) end |