Class: Bosh::Cli::BlobManager
Overview
In order to avoid storing large objects in git repo, release might save them in the blobstore instead. BlobManager encapsulates most of the blob operations.
Constant Summary collapse
- DEFAULT_INDEX_NAME =
"blobs.yml"
Instance Attribute Summary collapse
-
#new_blobs ⇒ Object
readonly
Returns the value of attribute new_blobs.
-
#updated_blobs ⇒ Object
readonly
Returns the value of attribute updated_blobs.
Instance Method Summary collapse
-
#add_blob(local_path, blob_path) ⇒ void
Registers a file as BOSH blob.
-
#blobs_to_upload ⇒ Array
Returns a list of blobs that need to be uploaded.
-
#dirty? ⇒ Boolean
Returns whether blobs directory is dirty.
-
#download_blob(path) ⇒ Object
Downloads blob from a blobstore.
-
#initialize(release) ⇒ BlobManager
constructor
A new instance of BlobManager.
-
#print_status ⇒ void
Prints out blobs status.
-
#process_blobs_directory ⇒ void
Processes all files in blobs directory and only leaves non-symlinks.
-
#process_index ⇒ void
Processes blobs index, fetches any missing or mismatched blobs, establishes symlinks in blobs directory to any files present in index.
-
#remove_symlinks ⇒ void
Removes all symlinks from blobs directory.
-
#sync ⇒ void
Synchronizes the contents of blobs directory with blobs index.
-
#upload_blob(path) ⇒ Object
Uploads blob to a blobstore, updates blobs index.
Constructor Details
#initialize(release) ⇒ BlobManager
Returns a new instance of BlobManager.
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
# File 'lib/cli/blob_manager.rb', line 13 def initialize(release) @release = release @index_file = File.join(@release.dir, "config", DEFAULT_INDEX_NAME) legacy_index_file = File.join(@release.dir, "blob_index.yml") if File.exists?(legacy_index_file) if File.exists?(@index_file) err("Found both new and legacy blob index, please fix it") end FileUtils.mv(legacy_index_file, @index_file) end if File.file?(@index_file) @index = load_yaml_file(@index_file) else @index = {} end @src_dir = File.join(@release.dir, "src") unless File.directory?(@src_dir) err("`src' directory is missing") end @storage_dir = File.join(@release.dir, ".blobs") unless File.directory?(@storage_dir) FileUtils.mkdir(@storage_dir) end @blobs_dir = File.join(@release.dir, "blobs") unless File.directory?(@blobs_dir) FileUtils.mkdir(@blobs_dir) end @blobstore = @release.blobstore if @blobstore.nil? err("Blobstore is not configured") end @new_blobs = [] @updated_blobs = [] end |
Instance Attribute Details
#new_blobs ⇒ Object (readonly)
Returns the value of attribute new_blobs.
10 11 12 |
# File 'lib/cli/blob_manager.rb', line 10 def new_blobs @new_blobs end |
#updated_blobs ⇒ Object (readonly)
Returns the value of attribute updated_blobs.
10 11 12 |
# File 'lib/cli/blob_manager.rb', line 10 def updated_blobs @updated_blobs end |
Instance Method Details
#add_blob(local_path, blob_path) ⇒ void
This method returns an undefined value.
Registers a file as BOSH blob
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 |
# File 'lib/cli/blob_manager.rb', line 104 def add_blob(local_path, blob_path) unless File.exists?(local_path) err("File `#{local_path}' not found") end if File.directory?(local_path) err("`#{local_path}' is a directory") end if blob_path[0..0] == "/" err("Blob path should be a relative path") end if blob_path[0..5] == "blobs/" err("Blob path should not start with `blobs/'") end blob_dst = File.join(@blobs_dir, blob_path) if File.directory?(blob_dst) err("`#{blob_dst}' is a directory, please pick a different path") end update = false if File.exists?(blob_dst) if file_checksum(blob_dst) == file_checksum(local_path) err("Already tracking the same version of `#{blob_path}'") end update = true FileUtils.rm(blob_dst) end FileUtils.mkdir_p(File.dirname(blob_dst)) FileUtils.cp(local_path, blob_dst, :preserve => true) FileUtils.chmod(0644, blob_dst) if update say("Updated #{blob_path.make_yellow}") else say("Added #{blob_path.make_yellow}") end say("When you are done testing the new blob, please run\n" + "`#{"bosh upload blobs".make_green}' and commit changes.") end |
#blobs_to_upload ⇒ Array
Returns a list of blobs that need to be uploaded
58 59 60 |
# File 'lib/cli/blob_manager.rb', line 58 def blobs_to_upload @new_blobs + @updated_blobs end |
#dirty? ⇒ Boolean
Returns whether blobs directory is dirty
64 65 66 |
# File 'lib/cli/blob_manager.rb', line 64 def dirty? @new_blobs.size > 0 || @updated_blobs.size > 0 end |
#download_blob(path) ⇒ Object
Downloads blob from a blobstore
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 |
# File 'lib/cli/blob_manager.rb', line 269 def download_blob(path) unless @index.has_key?(path) err("Unknown blob path `#{path}'") end blob = @index[path] size = blob["size"].to_i tmp_file = File.open(File.join(Dir.mktmpdir, "bosh-blob"), "w") download_label = "downloading" if size > 0 download_label += " " + pretty_size(size) end = Thread.new do loop do break unless size > 0 if File.exists?(tmp_file.path) pct = 100 * File.size(tmp_file.path).to_f / size progress(path, "#{download_label} (#{pct.to_i}%)...") end sleep(0.2) end end progress(path, "#{download_label}...") @blobstore.get(blob["object_id"], tmp_file) tmp_file.close .kill progress(path, "downloaded\n".make_green) if file_checksum(tmp_file.path) != blob["sha"] err("Checksum mismatch for downloaded blob `#{path}'") end tmp_file.path end |
#print_status ⇒ void
This method returns an undefined value.
Prints out blobs status
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
# File 'lib/cli/blob_manager.rb', line 70 def print_status total_file_size = @index.inject(0) do |total, (_, entry)| total += entry["size"].to_i total end say("Total: #{@index.size}, #{pretty_size(total_file_size)}") process_blobs_directory unless dirty? say("No blobs to upload".make_green) return end nl say("You have some blobs that need to be uploaded:") @new_blobs.each do |blob| size = File.size(File.join(@blobs_dir, blob)) say("%s\t%s\t%s" % ["new".make_green, blob, pretty_size(size)]) end @updated_blobs.each do |blob| size = File.size(File.join(@blobs_dir, blob)) say("%s\t%s\t%s" % ["new version".make_yellow, blob, pretty_size(size)]) end nl say("When ready please run `#{"bosh upload blobs".make_green}'") end |
#process_blobs_directory ⇒ void
This method returns an undefined value.
Processes all files in blobs directory and only leaves non-symlinks. Marks blobs as dirty if there are any non-symlink files.
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 |
# File 'lib/cli/blob_manager.rb', line 161 def process_blobs_directory @updated_blobs = [] @new_blobs = [] Dir[File.join(@blobs_dir, "**", "*")].each do |file| next if File.directory?(file) || File.symlink?(file) # We don't care about symlinks because they represent blobs # that are already tracked. # Regular files are more interesting: it's either a new version # of an existing blob or a completely new blob. path = strip_blobs_dir(file) if File.exists?(File.join(@src_dir, path)) err("File `#{path}' is in both `blobs' and `src' directory.\n" + "Please fix release repo before proceeding") end if @index.has_key?(path) if file_checksum(file) == @index[path]["sha"] # Already have exactly the same file in the index, # no need to keep it around. Also handles the migration # scenario for people with old blobs checked out. local_path = File.join(@storage_dir, @index[path]["sha"]) if File.exists?(local_path) FileUtils.rm_rf(file) else FileUtils.mv(file, local_path) end install_blob(local_path, path, @index[path]["sha"]) else @updated_blobs << path end else @new_blobs << path end end end |
#process_index ⇒ void
This method returns an undefined value.
Processes blobs index, fetches any missing or mismatched blobs, establishes symlinks in blobs directory to any files present in index.
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 |
# File 'lib/cli/blob_manager.rb', line 210 def process_index @index.each_pair do |path, entry| if File.exists?(File.join(@src_dir, path)) err("File `#{path}' is in both blob index and src directory.\n" + "Please fix release repo before proceeding") end local_path = File.join(@storage_dir, entry["sha"]) need_download = true if File.exists?(local_path) checksum = file_checksum(local_path) if checksum == entry["sha"] need_download = false else progress(path, "checksum mismatch, re-downloading...\n".make_red) end end if need_download local_path = download_blob(path) end install_blob(local_path, path, entry["sha"]) end end |
#remove_symlinks ⇒ void
This method returns an undefined value.
Removes all symlinks from blobs directory
201 202 203 204 205 |
# File 'lib/cli/blob_manager.rb', line 201 def remove_symlinks Dir[File.join(@blobs_dir, "**", "*")].each do |file| FileUtils.rm_rf(file) if File.symlink?(file) end end |
#sync ⇒ void
This method returns an undefined value.
Synchronizes the contents of blobs directory with blobs index.
151 152 153 154 155 156 |
# File 'lib/cli/blob_manager.rb', line 151 def sync say("Syncing blobs...") remove_symlinks process_blobs_directory process_index end |
#upload_blob(path) ⇒ Object
Uploads blob to a blobstore, updates blobs index.
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 |
# File 'lib/cli/blob_manager.rb', line 239 def upload_blob(path) blob_path = File.join(@blobs_dir, path) unless File.exists?(blob_path) err("Cannot upload blob, local file `#{blob_path}' doesn't exist") end if File.symlink?(blob_path) err("`#{blob_path}' is a symlink") end checksum = file_checksum(blob_path) progress(path, "uploading...") object_id = @blobstore.create(File.open(blob_path, "r")) progress(path, "uploaded\n".make_green) @index[path] = { "object_id" => object_id, "sha" => checksum, "size" => File.size(blob_path) } update_index install_blob(blob_path, path, checksum) object_id end |