Class: Bosh::Agent::Message::CompilePackage
- Defined in:
- lib/bosh_agent/message/compile_package.rb
Overview
This agent message “compile_package” fetches the source package from the blobstore, fetches the dependency compiled packages, compiles the source package, packs it into a tgz blob, and uploads it to the same blobstore. It returns the uploaded blob’s blobstore_id & sha1.
This message has the following uses:
-
the bosh-release (within the stemcell_builder) to compile packages to the microbosh’s local blobstore
-
the director requests packages be compiled during deployment if they are not yet compiled and available in the blobstore
Source packages must have a ‘packaging` executable script. It can find the unpackaged source files in the directory it is run in (which is also provided as $BOSH_COMPILE_TARGET variable). The packaging script MUST place all compiled assets within the single folder specified as $BOSH_INSTALL_TARGET. The packaging script is also provided the environment variables $BOSH_PACKAGE_NAME and $BOSH_PACKAGE_VERSION
Instance Attribute Summary collapse
-
#blobstore_client ⇒ Object
readonly
Returns the value of attribute blobstore_client.
-
#blobstore_id ⇒ Object
Returns the value of attribute blobstore_id.
-
#compile_base ⇒ Object
Returns the value of attribute compile_base.
-
#install_base ⇒ Object
Returns the value of attribute install_base.
-
#package_name ⇒ Object
Returns the value of attribute package_name.
-
#package_sha1 ⇒ Object
Returns the value of attribute package_sha1.
-
#package_version ⇒ Object
Returns the value of attribute package_version.
Class Method Summary collapse
Instance Method Summary collapse
-
#clear_log_file(log_file) ⇒ Object
Clears the log file after a compilation runs.
- #compile ⇒ Object
- #compile_dir ⇒ Object
- #compiled_package ⇒ Object
-
#delete_tmp_files ⇒ Object
Delete the leftover compilation files after a compilation is done.
- #disk_info_as_array(path) ⇒ Object
-
#disk_total(path) ⇒ Integer
Get the amount of total disk (in KBytes).
-
#disk_used(path) ⇒ Integer
Get the amount of disk being used (in KBytes).
- #get_source_package ⇒ Object
-
#initialize(args) ⇒ CompilePackage
constructor
A new instance of CompilePackage.
- #install_dependencies ⇒ Object
- #install_dir ⇒ Object
- #pack ⇒ Object
-
#pct_disk_used(path) ⇒ Float
Get the percentage of disk that is used on the compilation VM.
- #start ⇒ Object
- #unpack_source_package ⇒ Object
- #upload ⇒ Object
Constructor Details
#initialize(args) ⇒ CompilePackage
Returns a new instance of CompilePackage.
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
# File 'lib/bosh_agent/message/compile_package.rb', line 35 def initialize(args) bsc_provider = Bosh::Agent::Config.blobstore_provider = Bosh::Agent::Config. @blobstore_client = Bosh::Blobstore::Client.safe_create(bsc_provider, ) @blobstore_id, @sha1, @package_name, @package_version, @dependencies = args @base_dir = Bosh::Agent::Config.base_dir # The maximum amount of disk percentage that can be used during # compilation before an error will be thrown. This is to prevent # package compilation throwing arbitrary errors when disk space runs # out. # @attr [Integer] The max percentage of disk that can be used in # compilation. @max_disk_usage_pct = 90 FileUtils.mkdir_p(File.join(@base_dir, 'data', 'tmp')) @log_file = "#{@base_dir}/data/tmp/#{Bosh::Agent::Config.agent_id}" @logger = Logger.new(@log_file) @logger.level = Logger::DEBUG @compile_base = "#{@base_dir}/data/compile" @install_base = "#{@base_dir}/data/packages" end |
Instance Attribute Details
#blobstore_client ⇒ Object (readonly)
Returns the value of attribute blobstore_client.
28 29 30 |
# File 'lib/bosh_agent/message/compile_package.rb', line 28 def blobstore_client @blobstore_client end |
#blobstore_id ⇒ Object
Returns the value of attribute blobstore_id.
26 27 28 |
# File 'lib/bosh_agent/message/compile_package.rb', line 26 def blobstore_id @blobstore_id end |
#compile_base ⇒ Object
Returns the value of attribute compile_base.
27 28 29 |
# File 'lib/bosh_agent/message/compile_package.rb', line 27 def compile_base @compile_base end |
#install_base ⇒ Object
Returns the value of attribute install_base.
27 28 29 |
# File 'lib/bosh_agent/message/compile_package.rb', line 27 def install_base @install_base end |
#package_name ⇒ Object
Returns the value of attribute package_name.
26 27 28 |
# File 'lib/bosh_agent/message/compile_package.rb', line 26 def package_name @package_name end |
#package_sha1 ⇒ Object
Returns the value of attribute package_sha1.
26 27 28 |
# File 'lib/bosh_agent/message/compile_package.rb', line 26 def package_sha1 @package_sha1 end |
#package_version ⇒ Object
Returns the value of attribute package_version.
26 27 28 |
# File 'lib/bosh_agent/message/compile_package.rb', line 26 def package_version @package_version end |
Class Method Details
.long_running? ⇒ Boolean
33 |
# File 'lib/bosh_agent/message/compile_package.rb', line 33 def self.long_running?; true; end |
.process(args) ⇒ Object
30 31 32 |
# File 'lib/bosh_agent/message/compile_package.rb', line 30 def self.process(args) self.new(args).start end |
Instance Method Details
#clear_log_file(log_file) ⇒ Object
Clears the log file after a compilation runs. This is needed because if reuse_compilation_vms is being used then without clearing the log then the log from each subsequent compilation will include the previous compilation’s output.
228 229 230 231 |
# File 'lib/bosh_agent/message/compile_package.rb', line 228 def clear_log_file(log_file) File.delete(log_file) if File.exists?(log_file) @logger = Logger.new(log_file) end |
#compile ⇒ Object
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 |
# File 'lib/bosh_agent/message/compile_package.rb', line 177 def compile FileUtils.rm_rf install_dir if File.directory?(install_dir) FileUtils.mkdir_p install_dir pkg_link_dst = File.join(@base_dir, 'packages', @package_name) FileUtils.ln_sf(install_dir, pkg_link_dst) pct_space_used = pct_disk_used(@compile_base) if pct_space_used >= @max_disk_usage_pct raise Bosh::Agent::MessageHandlerError, "Compile Package Failure. Greater than #{@max_disk_usage_pct}% " + "is used (#{pct_space_used}%." end Dir.chdir(compile_dir) do # Prevent these from getting inhereted from the agent %w{GEM_HOME BUNDLE_GEMFILE RUBYOPT}.each { |key| ENV.delete(key) } ENV['BOSH_COMPILE_TARGET'] = compile_dir ENV['BOSH_INSTALL_TARGET'] = pkg_link_dst ENV['BOSH_PACKAGE_NAME'] = @package_name.to_s ENV['BOSH_PACKAGE_VERSION'] = @package_version.to_s if File.exist?('packaging') @logger.info("Compiling #{@package_name} #{@package_version}") output = `bash -x packaging 2>&1` # stick the output in the blobstore @logger.fatal(output) unless $?.exitstatus == 0 raise Bosh::Agent::MessageHandlerError.new( "Compile Package Failure (exit code: #{$?.exitstatus})", output) end end end end |
#compile_dir ⇒ Object
115 116 117 |
# File 'lib/bosh_agent/message/compile_package.rb', line 115 def compile_dir @compile_dir ||= File.join(@compile_base, @package_name) end |
#compiled_package ⇒ Object
212 213 214 |
# File 'lib/bosh_agent/message/compile_package.rb', line 212 def compiled_package File.join(@source_file + ".compiled") end |
#delete_tmp_files ⇒ Object
Delete the leftover compilation files after a compilation is done. This is done so that the reuse_compilation_vms option does not fill up a VM.
80 81 82 83 84 85 86 |
# File 'lib/bosh_agent/message/compile_package.rb', line 80 def delete_tmp_files [@compile_base, @install_base].each do |dir| if Dir.exists?(dir) FileUtils.rm_rf(dir) end end end |
#disk_info_as_array(path) ⇒ Object
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 |
# File 'lib/bosh_agent/message/compile_package.rb', line 140 def disk_info_as_array(path) # "Filesystem 1024-blocks Used Available Capacity Mounted on\n # /dev/disk0s2 195312496 92743676 102312820 48% /\n" df_out = `df -Pk #{path} 2>&1` unless $?.exitstatus == 0 raise Bosh::Agent::MessageHandlerError, "Command 'df -Pk #{path}' " + "on compilation VM failed with:\n#{df_out}\nexit code: " + "#{$?.exitstatus}" end # "/dev/disk0s2 195312496 92743568 102312928 48% /\n" df_out = df_out.sub(/[^\/]*/, "") # ["/dev/disk0s2", "195312496", "92743568", "102312928", "48%", "/\n"] df_out.split(/[ ]+/) end |
#disk_total(path) ⇒ Integer
Get the amount of total disk (in KBytes).
159 160 161 |
# File 'lib/bosh_agent/message/compile_package.rb', line 159 def disk_total(path) disk_info_as_array(path)[1].to_i end |
#disk_used(path) ⇒ Integer
Get the amount of disk being used (in KBytes).
166 167 168 |
# File 'lib/bosh_agent/message/compile_package.rb', line 166 def disk_used(path) disk_info_as_array(path)[2].to_i end |
#get_source_package ⇒ Object
104 105 106 107 108 109 110 111 112 113 |
# File 'lib/bosh_agent/message/compile_package.rb', line 104 def get_source_package compile_tmp = File.join(@compile_base, 'tmp') FileUtils.mkdir_p compile_tmp @source_file = File.join(compile_tmp, @blobstore_id) FileUtils.rm @source_file if File.exist?(@source_file) File.open(@source_file, 'w') do |f| @blobstore_client.get(@blobstore_id, f) end end |
#install_dependencies ⇒ Object
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 |
# File 'lib/bosh_agent/message/compile_package.rb', line 88 def install_dependencies @logger.info("Installing Dependencies") @dependencies.each do |pkg_name, pkg| @logger.info("Installing depdendency: #{pkg_name} #{pkg.inspect}") blobstore_id = pkg['blobstore_id'] sha1 = pkg['sha1'] install_dir = File.join(@install_base, pkg_name, pkg['version']) Util.unpack_blob(blobstore_id, sha1, install_dir) pkg_link_dst = File.join(@base_dir, 'packages', pkg_name) FileUtils.ln_sf(install_dir, pkg_link_dst) end end |
#install_dir ⇒ Object
119 120 121 |
# File 'lib/bosh_agent/message/compile_package.rb', line 119 def install_dir @install_dir ||= File.join(@install_base, @package_name, @package_version.to_s) end |
#pack ⇒ Object
216 217 218 219 220 221 |
# File 'lib/bosh_agent/message/compile_package.rb', line 216 def pack @logger.info("Packing #{@package_name} #{@package_version}") Dir.chdir(install_dir) do `tar -zcf #{compiled_package} .` end end |
#pct_disk_used(path) ⇒ Float
Get the percentage of disk that is used on the compilation VM.
173 174 175 |
# File 'lib/bosh_agent/message/compile_package.rb', line 173 def pct_disk_used(path) 100 * disk_used(path).to_f / disk_total(path).to_f end |
#start ⇒ Object
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
# File 'lib/bosh_agent/message/compile_package.rb', line 59 def start begin install_dependencies get_source_package unpack_source_package compile pack result = upload clear_log_file(@log_file) return { "result" => result } rescue RuntimeError => e @logger.warn("%s\n%s" % [e., e.backtrace.join("\n")]) raise Bosh::Agent::MessageHandlerError, e ensure delete_tmp_files end end |
#unpack_source_package ⇒ Object
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
# File 'lib/bosh_agent/message/compile_package.rb', line 123 def unpack_source_package FileUtils.rm_rf compile_dir if File.directory?(compile_dir) FileUtils.mkdir_p compile_dir Dir.chdir(compile_dir) do output = `tar -zxf #{@source_file} 2>&1` @logger.info(output) # stick the output in the blobstore unless $?.exitstatus == 0 STDOUT.puts(output) raise Bosh::Agent::MessageHandlerError.new( "Compile Package Unpack Source Failure (exit code: #{$?.exitstatus})", output) end end end |
#upload ⇒ Object
233 234 235 236 237 238 239 240 241 242 243 244 245 246 |
# File 'lib/bosh_agent/message/compile_package.rb', line 233 def upload compiled_blobstore_id = nil File.open(compiled_package, 'r') do |f| compiled_blobstore_id = @blobstore_client.create(f) end compiled_sha1 = Digest::SHA1.file(compiled_package).hexdigest compile_log_id = @blobstore_client.create(@log_file) @logger.info("Uploaded #{@package_name} #{@package_version} " + "(sha1: #{compiled_sha1}, " + "blobstore_id: #{compiled_blobstore_id})") @logger = nil { "sha1" => compiled_sha1, "blobstore_id" => compiled_blobstore_id, "compile_log_id" => compile_log_id } end |