Class: Bosh::Director::Jobs::UpdateRelease

Inherits:
BaseJob show all
Includes:
DownloadHelper, LockHelper
Defined in:
lib/bosh/director/jobs/update_release.rb

Instance Attribute Summary collapse

Attributes inherited from BaseJob

#task_id

Class Method Summary collapse

Instance Method Summary collapse

Methods included from DownloadHelper

#download_remote_file

Methods included from LockHelper

#with_compile_lock, #with_deployment_lock, #with_release_lock, #with_release_locks, #with_stemcell_lock

Methods inherited from BaseJob

#begin_stage, #event_log, #logger, perform, #result_file, #single_step_stage, #task_cancelled?, #task_checkpoint, #track_and_log

Constructor Details

#initialize(release_path, options = {}) ⇒ UpdateRelease

Returns a new instance of UpdateRelease.

Parameters:

  • release_path (String) —

    local path or remote url of the release archive

  • options (Hash) (defaults to: {}) —

    Release update options



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/bosh/director/jobs/update_release.rb', line 20

def initialize(release_path, options = {})
  if options['remote']
    # file will be downloaded to the release_path
    @release_path = File.join(Dir.tmpdir, "release-#{SecureRandom.uuid}")
    @release_url = release_path
  else
    # file already exists at the release_path
    @release_path = release_path
  end

  @release_model = nil
  @release_version_model = nil

  @rebase = !!options['rebase']
  @skip_if_exists = !!options['skip_if_exists']

  @manifest = nil
  @name = nil
  @version = nil

  @packages_unchanged = false
  @jobs_unchanged = false
end

Instance Attribute Details

#release_model ⇒ Object

Returns the value of attribute release_model.



12
13
14
 
# File 'lib/bosh/director/jobs/update_release.rb', line 12

def release_model
  @release_model
end

Class Method Details

.job_type ⇒ Object 



14
15
16
 
# File 'lib/bosh/director/jobs/update_release.rb', line 14

def self.job_type
  :update_release
end

Instance Method Details

#create_job(job_meta, release_dir) ⇒ Object 



384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
 
# File 'lib/bosh/director/jobs/update_release.rb', line 384

def create_job(job_meta, release_dir)
  name, version = job_meta["name"], job_meta["version"]

  template_attrs = {
    :release => @release_model,
    :name => name,
    :sha1 => job_meta["sha1"],
    :fingerprint => job_meta["fingerprint"],
    :version => version
  }

  logger.info("Creating job template `#{name}/#{version}' " +
              "from provided bits")
  template = Models::Template.new(template_attrs)

  job_tgz = File.join(release_dir, "jobs", "#{name}.tgz")
  job_dir = File.join(release_dir, "jobs", "#{name}")

  FileUtils.mkdir_p(job_dir)

  desc = "job `#{name}/#{version}'"
  result = Bosh::Exec.sh("tar -C #{job_dir} -xzf #{job_tgz} 2>&1", :on_error => :return)
  if result.failed?
    logger.error("Extracting #{desc} archive failed in dir #{job_dir}, " +
                 "tar returned #{result.exit_status}, " +
                 "output: #{result.output}")
    raise JobInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
  end

  manifest_file = File.join(job_dir, "job.MF")
  unless File.file?(manifest_file)
    raise JobMissingManifest,
          "Missing job manifest for `#{template.name}'"
  end

  job_manifest = Psych.load_file(manifest_file)

  if job_manifest["templates"]
    job_manifest["templates"].each_key do |relative_path|
      path = File.join(job_dir, "templates", relative_path)
      unless File.file?(path)
        raise JobMissingTemplateFile,
              "Missing template file `#{relative_path}' for job `#{template.name}'"
      end
    end
  end

  main_monit_file = File.join(job_dir, "monit")
  aux_monit_files = Dir.glob(File.join(job_dir, "*.monit"))

  unless File.exists?(main_monit_file) || aux_monit_files.size > 0
    raise JobMissingMonit, "Job `#{template.name}' is missing monit file"
  end

  template.blobstore_id = BlobUtil.create_blob(job_tgz)

  package_names = []
  if job_manifest["packages"]
    unless job_manifest["packages"].is_a?(Array)
      raise JobInvalidPackageSpec,
            "Job `#{template.name}' has invalid package spec format"
    end

    job_manifest["packages"].each do |package_name|
      package = @packages[package_name]
      if package.nil?
        raise JobMissingPackage,
          "Job `#{template.name}' is referencing " +
            "a missing package `#{package_name}'"
      end
      package_names << package.name
    end
  end
  template.package_names = package_names

  if job_manifest["logs"]
    unless job_manifest["logs"].is_a?(Hash)
      raise JobInvalidLogSpec,
            "Job `#{template.name}' has invalid logs spec format"
    end

    template.logs = job_manifest["logs"]
  end

  if job_manifest["properties"]
    unless job_manifest["properties"].is_a?(Hash)
      raise JobInvalidPropertySpec,
            "Job `#{template.name}' has invalid properties spec format"
    end

    template.properties = job_manifest["properties"]
  end

  template.save
end

#create_jobs(jobs, release_dir) ⇒ Object 



367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
 
# File 'lib/bosh/director/jobs/update_release.rb', line 367

def create_jobs(jobs, release_dir)
  if jobs.empty?
    @jobs_unchanged = true
    return
  end

  event_log.begin_stage("Creating new jobs", jobs.size)
  jobs.each do |job_meta|
    job_desc = "#{job_meta["name"]}/#{job_meta["version"]}"
    event_log.track(job_desc) do
      logger.info("Creating new template `#{job_desc}'")
      template = create_job(job_meta, release_dir)
      register_template(template)
    end
  end
end

#create_package(package_meta, release_dir) ⇒ void

This method returns an undefined value.

Creates package in DB according to given metadata

Parameters:

  • package_meta (Hash) —

    Package metadata

  • release_dir (String) —

    local path to the unpacked release



289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
 
# File 'lib/bosh/director/jobs/update_release.rb', line 289

def create_package(package_meta, release_dir)
  name, version = package_meta["name"], package_meta["version"]

  package_attrs = {
    :release => @release_model,
    :name => name,
    :sha1 => package_meta["sha1"],
    :fingerprint => package_meta["fingerprint"],
    :version => version
  }

  package = Models::Package.new(package_attrs)
  package.dependency_set = package_meta["dependencies"]

  existing_blob = package_meta["blobstore_id"]
  desc = "package `#{name}/#{version}'"

  if existing_blob
    logger.info("Creating #{desc} from existing blob #{existing_blob}")
    package.blobstore_id = BlobUtil.copy_blob(existing_blob)
  else
    logger.info("Creating #{desc} from provided bits")

    package_tgz = File.join(release_dir, "packages", "#{name}.tgz")
    result = Bosh::Exec.sh("tar -tzf #{package_tgz} 2>&1", :on_error => :return)
    if result.failed?
      logger.error("Extracting #{desc} archive failed, " +
                   "tar returned #{result.exit_status}, " +
                   "output: #{result.output}")
      raise PackageInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
    end

    package.blobstore_id = BlobUtil.create_blob(package_tgz)
  end

  package.save
end

#create_packages(packages, release_dir) ⇒ void

This method returns an undefined value.

Creates packages using provided metadata

Parameters:

  • packages (Array<Hash>) —

    Packages metadata

  • release_dir (String) —

    local path to the unpacked release



254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
 
# File 'lib/bosh/director/jobs/update_release.rb', line 254

def create_packages(packages, release_dir)
  if packages.empty?
    @packages_unchanged = true
    return
  end

  event_log.begin_stage("Creating new packages", packages.size)
  packages.each do |package_meta|
    package_desc = "#{package_meta["name"]}/#{package_meta["version"]}"
    event_log.track(package_desc) do
      logger.info("Creating new package `#{package_desc}'")
      package = create_package(package_meta, release_dir)
      register_package(package)
    end
  end
end

#download_remote_release ⇒ Object 



74
75
76
 
# File 'lib/bosh/director/jobs/update_release.rb', line 74

def download_remote_release
  download_remote_file('release', @release_url, @release_path)
end

#extract_release ⇒ void

This method returns an undefined value.

Extracts release tarball



80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/bosh/director/jobs/update_release.rb', line 80

def extract_release
  release_dir = Dir.mktmpdir

  result = Bosh::Exec.sh("tar -C #{release_dir} -xzf #{@release_path} 2>&1", :on_error => :return)
  if result.failed?
    logger.error("Failed to extract release archive '#{@release_path}' into dir '#{release_dir}', " +
                 "tar returned #{result.exit_status}, " +
                 "output: #{result.output}")
    raise ReleaseInvalidArchive, "Extracting release archive failed. Check task debug log for details."
  end

  release_dir
end

#normalize_manifest ⇒ void

This method returns an undefined value.

Normalizes release manifest, so all names, versions, and checksums are Strings.



168
169
170
171
172
173
 
# File 'lib/bosh/director/jobs/update_release.rb', line 168

def normalize_manifest
  Bosh::Director.hash_string_vals(@manifest, 'name', 'version')

  @manifest['packages'].each { |p| Bosh::Director.hash_string_vals(p, 'name', 'version', 'sha1') }
  @manifest['jobs'].each { |j| Bosh::Director.hash_string_vals(j, 'name', 'version', 'sha1') }
end

#perform ⇒ void

This method returns an undefined value.

Extracts release tarball, verifies release manifest and saves release in DB



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/bosh/director/jobs/update_release.rb', line 46

def perform
  logger.info("Processing update release")
  logger.info("Release rebase will be performed") if @rebase

  single_step_stage("Downloading remote release") { download_remote_release } if @release_url

  release_dir = nil
  single_step_stage("Extracting release") { release_dir = extract_release }

  single_step_stage("Verifying manifest") { verify_manifest(release_dir) }

  with_release_lock(@name) { process_release(release_dir) }

  if @rebase && @packages_unchanged && @jobs_unchanged
    raise DirectorError, "Rebase is attempted without any job or package changes"
  end

  "Created release `#{@name}/#{@version}'"

rescue Exception => e
  remove_release_version_model
  raise e

ensure
  FileUtils.rm_rf(release_dir) if release_dir
  FileUtils.rm_rf(@release_path) if @release_path
end

#process_jobs(release_dir) ⇒ void

This method returns an undefined value.

Finds job template definitions in release manifest and sorts them into two buckets: new and existing job templates, then creates new job template records in the database and points release version to existing ones.

Parameters:

  • release_dir (String) —

    local path to the unpacked release



340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
 
# File 'lib/bosh/director/jobs/update_release.rb', line 340

def process_jobs(release_dir)
  logger.info("Checking for new jobs in release")

  new_jobs = []
  existing_jobs = []

  @manifest["jobs"].each do |job_meta|
    # Checking whether we might have the same bits somewhere
    jobs = Models::Template.where(fingerprint: job_meta["fingerprint"]).all

    template = jobs.find do |job|
      job.release_id == @release_model.id &&
      job.name == job_meta["name"] &&
      job.version == job_meta["version"]
    end

    if template.nil?
      new_jobs << job_meta
    else
      existing_jobs << [template, job_meta]
    end
  end

  create_jobs(new_jobs, release_dir)
  use_existing_jobs(existing_jobs)
end

#process_packages(release_dir) ⇒ void

This method returns an undefined value.

Finds all package definitions in the manifest and sorts them into two buckets: new and existing packages, then creates new packages and points current release version to the existing packages.

Parameters:

  • release_dir (String) —

    local path to the unpacked release



204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
 
# File 'lib/bosh/director/jobs/update_release.rb', line 204

def process_packages(release_dir)
  logger.info("Checking for new packages in release")

  new_packages = []
  existing_packages = []

  @manifest["packages"].each do |package_meta|
    # Checking whether we might have the same bits somewhere
    packages = Models::Package.where(fingerprint: package_meta["fingerprint"]).all

    if packages.empty?
      new_packages << package_meta
      next
    end

    existing_package = packages.find do |package|
      package.release_id == @release_model.id &&
      package.name == package_meta["name"] &&
      package.version == package_meta["version"]
    end

    if existing_package
      # clean up 'broken' dependency_set (a bug was including transitives)
      # dependency ordering impacts fingerprint
      # TODO: The following code can be removed after some reasonable time period (added 2014.10.06)
      if existing_package.dependency_set != package_meta['dependencies']
        existing_package.dependency_set = package_meta['dependencies']
        existing_package.save
      end

      existing_packages << [existing_package, package_meta]
    else
      # We found a package with the same fingerprint but different
      # (release, name, version) tuple, so we need to make a copy
      # of the package blob and create a new db entry for it
      package = packages.first
      package_meta["blobstore_id"] = package.blobstore_id
      package_meta["sha1"] = package.sha1
      new_packages << package_meta
    end
  end

  create_packages(new_packages, release_dir)
  use_existing_packages(existing_packages)
end

#process_release(release_dir) ⇒ void

This method returns an undefined value.

Processes uploaded release, creates jobs and packages in DB if needed

Parameters:

  • release_dir (String) —

    local path to the unpacked release



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/bosh/director/jobs/update_release.rb', line 123

def process_release(release_dir)
  @release_model = Models::Release.find_or_create(:name => @name)

  if @rebase
    @version = next_release_version
  end

  version_attrs = {
    :release => @release_model,
    :version => @version.to_s
  }
  version_attrs[:uncommitted_changes] = @uncommitted_changes if @uncommitted_changes
  version_attrs[:commit_hash] = @commit_hash if @commit_hash

  @release_version_model = Models::ReleaseVersion.new(version_attrs)
  unless @release_version_model.valid?
    if @release_version_model.errors[:version] == [:format]
      raise ReleaseVersionInvalid,
        "Release version invalid `#{@name}/#{@version}'"
    elsif @skip_if_exists
      event_log.begin_stage("Release already exists", 1)
      event_log.track("#{@name}/#{@version}") {}
      return
    else
      raise ReleaseAlreadyExists,
        "Release `#{@name}/#{@version}' already exists"
    end
  end

  @release_version_model.save

  single_step_stage("Resolving package dependencies") do
    resolve_package_dependencies(@manifest["packages"])
  end

  @packages = {}
  process_packages(release_dir)
  process_jobs(release_dir)

  event_log.begin_stage("Release has been created", 1)
  event_log.track("#{@name}/#{@version}") {}
end

#register_package(package) ⇒ void

This method returns an undefined value.

Marks package model as used by release version model

Parameters:



330
331
332
333
 
# File 'lib/bosh/director/jobs/update_release.rb', line 330

def register_package(package)
  @packages[package.name] = package
  @release_version_model.add_package(package)
end

#register_template(template) ⇒ void

This method returns an undefined value.

Marks job template model as being used by release version

Parameters:



497
498
499
 
# File 'lib/bosh/director/jobs/update_release.rb', line 497

def register_template(template)
  @release_version_model.add_template(template)
end

#resolve_package_dependencies(packages) ⇒ void

This method returns an undefined value.

Resolves package dependencies, makes sure there are no cycles and all dependencies are present



178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/bosh/director/jobs/update_release.rb', line 178

def resolve_package_dependencies(packages)
  packages_by_name = {}
  packages.each do |package|
    packages_by_name[package["name"]] = package
    package["dependencies"] ||= []
  end
  logger.info("Resolving package dependencies for #{packages_by_name.keys.inspect}")

  dependency_lookup = lambda do |package_name|
    packages_by_name[package_name]["dependencies"]
  end
  result = Bosh::Director::CycleHelper.check_for_cycle(packages_by_name.keys, :connected_vertices => true, &dependency_lookup)

  packages.each do |package|
    name = package["name"]
    dependencies = package["dependencies"]
    all_dependencies = result[:connected_vertices][name]
    logger.info("Resolved package dependencies for `#{name}': #{dependencies.pretty_inspect} => #{all_dependencies.pretty_inspect}")
  end
end

#use_existing_jobs(jobs) ⇒ void

This method returns an undefined value.

Parameters:



482
483
484
485
486
487
488
489
490
491
492
 
# File 'lib/bosh/director/jobs/update_release.rb', line 482

def use_existing_jobs(jobs)
  return if jobs.empty?

  single_step_stage("Processing #{jobs.size} existing job#{"s" if jobs.size > 1}") do
    jobs.each do |template, _|
      job_desc = "#{template.name}/#{template.version}"
      logger.info("Using existing job `#{job_desc}'")
      register_template(template)
    end
  end
end

#use_existing_packages(packages) ⇒ Object

Points release DB model to existing packages described by given metadata

Parameters:

  • packages (Array<Array>) —

    Existing packages metadata



273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/bosh/director/jobs/update_release.rb', line 273

def use_existing_packages(packages)
  return if packages.empty?

  single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do
    packages.each do |package, _|
      package_desc = "#{package.name}/#{package.version}"
      logger.info("Using existing package `#{package_desc}'")
      register_package(package)
    end
  end
end

#verify_manifest(release_dir) ⇒ void

This method returns an undefined value.

Parameters:

  • release_dir (String) —

    local path to the unpacked release



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/bosh/director/jobs/update_release.rb', line 96

def verify_manifest(release_dir)
  manifest_file = File.join(release_dir, "release.MF")
  unless File.file?(manifest_file)
    raise ReleaseManifestNotFound, "Release manifest not found"
  end

  @manifest = Psych.load_file(manifest_file)
  normalize_manifest

  @name = @manifest["name"]

  begin
    @version = Bosh::Common::Version::ReleaseVersion.parse(@manifest["version"])
    unless @version == @manifest["version"]
      logger.info("Formatted version '#{@manifest["version"]}' => '#{@version}'")
    end
  rescue SemiSemantic::ParseError
    raise ReleaseVersionInvalid, "Release version invalid: #{@manifest["version"]}"
  end

  @commit_hash = @manifest.fetch("commit_hash", nil)
  @uncommitted_changes = @manifest.fetch("uncommitted_changes", nil)
end