Class: Chef::CookbookSynchronizer
- Inherits:
-
Object
- Object
- Chef::CookbookSynchronizer
- Defined in:
- lib/chef/cookbook/synchronizer.rb
Overview
Synchronizes the locally cached copies of cookbooks with the files on the server.
Defined Under Namespace
Classes: CookbookFile
Instance Attribute Summary collapse
-
#remove_obsoleted_files ⇒ Object
Returns the value of attribute remove_obsoleted_files.
Instance Method Summary collapse
- #cache ⇒ Object
- #cached_copy_up_to_date?(local_path, expected_checksum) ⇒ Boolean
-
#clear_obsoleted_cookbooks ⇒ Object
Iterates over cached cookbooks' files, removing files belonging to cookbooks that don't appear in +cookbook_hash+.
- #cookbook_count ⇒ Object
- #cookbook_names ⇒ Object
- #cookbook_segment(cookbook_name, segment) ⇒ Object
- #cookbooks ⇒ Object
-
#download_file(url, destination) ⇒ Object
Unconditionally download the file from the given URL.
- #ensure_cookbook_paths ⇒ Object
- #files ⇒ Object
- #files_by_cookbook ⇒ Object
- #files_remaining_by_cookbook ⇒ Object
- #have_cookbook?(cookbook_name) ⇒ Boolean
-
#initialize(cookbooks_by_name, events) ⇒ CookbookSynchronizer
constructor
A new instance of CookbookSynchronizer.
-
#mark_cached_file_valid(cache_filename) ⇒ Object
Marks the given file as valid (non-stale).
- #mark_file_synced(file) ⇒ Object
-
#remove_deleted_files ⇒ Object
remove deleted files in cookbooks that are being used on the node.
-
#remove_old_cookbooks ⇒ Object
remove cookbooks that are not referenced in the expanded run_list at all (if we have an override run_list we may not want to do this).
-
#save_full_file_path(file, full_path) ⇒ Object
Saves the full_path to the file of the cookbook to be updated in the manifest later.
- #server_api ⇒ Object
-
#sync_cookbooks ⇒ Object
Synchronizes all the cookbooks from the chef-server.
-
#sync_file(file) ⇒ Object
Sync an individual file if needed.
- #update_cookbook_filenames ⇒ Object
Constructor Details
#initialize(cookbooks_by_name, events) ⇒ CookbookSynchronizer
Returns a new instance of CookbookSynchronizer.
87 88 89 90 91 92 93 94 |
# File 'lib/chef/cookbook/synchronizer.rb', line 87 def initialize(cookbooks_by_name, events) @cookbooks_by_name, @events = cookbooks_by_name, events @cookbook_full_file_paths = {} @remove_obsoleted_files = true @lazy_files = {} end |
Instance Attribute Details
#remove_obsoleted_files ⇒ Object
Returns the value of attribute remove_obsoleted_files.
85 86 87 |
# File 'lib/chef/cookbook/synchronizer.rb', line 85 def remove_obsoleted_files @remove_obsoleted_files end |
Instance Method Details
#cache ⇒ Object
96 97 98 |
# File 'lib/chef/cookbook/synchronizer.rb', line 96 def cache Chef::FileCache end |
#cached_copy_up_to_date?(local_path, expected_checksum) ⇒ Boolean
316 317 318 319 320 321 322 323 324 325 |
# File 'lib/chef/cookbook/synchronizer.rb', line 316 def cached_copy_up_to_date?(local_path, expected_checksum) return true if Chef::Config[:skip_cookbook_sync] if cache.key?(local_path) current_checksum = CookbookVersion.checksum_cookbook_file(cache.load(local_path, false)) expected_checksum == current_checksum else false end end |
#clear_obsoleted_cookbooks ⇒ Object
Iterates over cached cookbooks' files, removing files belonging to cookbooks that don't appear in +cookbook_hash+
258 259 260 261 262 263 264 265 266 267 268 269 270 271 |
# File 'lib/chef/cookbook/synchronizer.rb', line 258 def clear_obsoleted_cookbooks @events.cookbook_clean_start if remove_obsoleted_files remove_old_cookbooks else Chef::Log.info("Skipping removal of obsoleted cookbooks from the cache") CookbookCacheCleaner.instance.skip_removal = true end remove_deleted_files @events.cookbook_clean_complete end |
#cookbook_count ⇒ Object
108 109 110 |
# File 'lib/chef/cookbook/synchronizer.rb', line 108 def cookbook_count @cookbooks_by_name.size end |
#cookbook_names ⇒ Object
100 101 102 |
# File 'lib/chef/cookbook/synchronizer.rb', line 100 def cookbook_names @cookbooks_by_name.keys end |
#cookbook_segment(cookbook_name, segment) ⇒ Object
116 117 118 |
# File 'lib/chef/cookbook/synchronizer.rb', line 116 def cookbook_segment(cookbook_name, segment) @cookbooks_by_name[cookbook_name].files_for(segment) end |
#cookbooks ⇒ Object
104 105 106 |
# File 'lib/chef/cookbook/synchronizer.rb', line 104 def cookbooks @cookbooks_by_name.values end |
#download_file(url, destination) ⇒ Object
Unconditionally download the file from the given URL. File will be downloaded to the path +destination+ which is relative to the Chef file cache root.
330 331 332 333 334 335 |
# File 'lib/chef/cookbook/synchronizer.rb', line 330 def download_file(url, destination) raw_file = server_api.streaming_request(url) Chef::Log.info("Storing updated #{destination} in the cache.") cache.move_to(raw_file.path, destination) end |
#ensure_cookbook_paths ⇒ Object
283 284 285 286 287 288 |
# File 'lib/chef/cookbook/synchronizer.rb', line 283 def ensure_cookbook_paths cookbooks.each do |cookbook| cb_dir = File.join(Chef::Config[:file_cache_path], "cookbooks", cookbook.name) cookbook.root_paths = Array(cb_dir) end end |
#files ⇒ Object
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 |
# File 'lib/chef/cookbook/synchronizer.rb', line 120 def files lazy = unless Chef::Config[:no_lazy_load] %w{ files templates } else [] end @files ||= cookbooks.inject([]) do |memo, cookbook| cookbook.each_file do |manifest_record| part = manifest_record[:name].split("/")[0] if lazy.include?(part) manifest_record[:lazy] = true @lazy_files[cookbook] ||= [] @lazy_files[cookbook] << manifest_record else memo << CookbookFile.new(cookbook, manifest_record) end end memo end end |
#files_by_cookbook ⇒ Object
142 143 144 |
# File 'lib/chef/cookbook/synchronizer.rb', line 142 def files_by_cookbook files.group_by(&:cookbook) end |
#files_remaining_by_cookbook ⇒ Object
146 147 148 149 150 151 |
# File 'lib/chef/cookbook/synchronizer.rb', line 146 def files_remaining_by_cookbook @files_remaining_by_cookbook ||= files_by_cookbook.inject({}) do |memo, (cookbook, files)| memo[cookbook] = files.size memo end end |
#have_cookbook?(cookbook_name) ⇒ Boolean
112 113 114 |
# File 'lib/chef/cookbook/synchronizer.rb', line 112 def have_cookbook?(cookbook_name) @cookbooks_by_name.key?(cookbook_name) end |
#mark_cached_file_valid(cache_filename) ⇒ Object
Marks the given file as valid (non-stale).
338 339 340 |
# File 'lib/chef/cookbook/synchronizer.rb', line 338 def mark_cached_file_valid(cache_filename) CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename) end |
#mark_file_synced(file) ⇒ Object
153 154 155 156 157 158 159 |
# File 'lib/chef/cookbook/synchronizer.rb', line 153 def mark_file_synced(file) files_remaining_by_cookbook[file.cookbook] -= 1 if files_remaining_by_cookbook[file.cookbook] == 0 @events.synchronized_cookbook(file.cookbook.name, file.cookbook) end end |
#remove_deleted_files ⇒ Object
remove deleted files in cookbooks that are being used on the node
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 |
# File 'lib/chef/cookbook/synchronizer.rb', line 222 def remove_deleted_files cache_file_hash = {} @cookbooks_by_name.each_key do |k| cache_file_hash[k] = {} end # First populate files from cache cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file| md = cache_file.match(%r{^cookbooks/([^/]+)/([^/]+)/(.*)}) next unless md (cookbook_name, segment, file) = md[1..3] if have_cookbook?(cookbook_name) cache_file_hash[cookbook_name][segment] ||= {} cache_file_hash[cookbook_name][segment]["#{segment}/#{file}"] = cache_file end end # Determine which files don't match manifest @cookbooks_by_name.each_key do |cookbook_name| cache_file_hash[cookbook_name].each_key do |segment| manifest_segment = cookbook_segment(cookbook_name, segment) manifest_record_paths = manifest_segment.map { |manifest_record| manifest_record["path"] }.to_set to_be_removed = cache_file_hash[cookbook_name][segment].keys.to_set - manifest_record_paths to_be_removed.each do |path| cache_file = cache_file_hash[cookbook_name][segment][path] Chef::Log.info("Removing #{cache_file} from the cache; its is no longer in the cookbook manifest.") cache.delete(cache_file) @events.removed_cookbook_file(cache_file) end end end end |
#remove_old_cookbooks ⇒ Object
remove cookbooks that are not referenced in the expanded run_list at all (if we have an override run_list we may not want to do this)
210 211 212 213 214 215 216 217 218 219 |
# File 'lib/chef/cookbook/synchronizer.rb', line 210 def remove_old_cookbooks cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file| cache_file =~ %r{^cookbooks/([^/]+)/} unless have_cookbook?($1) Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.") cache.delete(cache_file) @events.removed_cookbook_file(cache_file) end end end |
#save_full_file_path(file, full_path) ⇒ Object
Saves the full_path to the file of the cookbook to be updated in the manifest later
203 204 205 206 |
# File 'lib/chef/cookbook/synchronizer.rb', line 203 def save_full_file_path(file, full_path) @cookbook_full_file_paths[file.cookbook] ||= [] @cookbook_full_file_paths[file.cookbook] << full_path end |
#server_api ⇒ Object
342 343 344 |
# File 'lib/chef/cookbook/synchronizer.rb', line 342 def server_api Thread.current[:server_api] ||= Chef::ServerAPI.new(Chef::Config[:chef_server_url], keepalives: true) end |
#sync_cookbooks ⇒ Object
Synchronizes all the cookbooks from the chef-server. ) === Returns true:: Always returns true
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 |
# File 'lib/chef/cookbook/synchronizer.rb', line 165 def sync_cookbooks Chef::Log.info("Loading cookbooks [#{cookbooks.map { |ckbk| ckbk.name + "@" + ckbk.version }.join(", ")}]") Chef::Log.trace("Cookbooks detail: #{cookbooks.inspect}") clear_obsoleted_cookbooks queue = Chef::Util::ThreadedJobQueue.new Chef::Log.warn("skipping cookbook synchronization! DO NOT LEAVE THIS ENABLED IN PRODUCTION!!!") if Chef::Config[:skip_cookbook_sync] files.each do |file| queue << lambda do |lock| full_file_path = sync_file(file) lock.synchronize do # Save the full_path of the downloaded file to be restored in the manifest later save_full_file_path(file, full_file_path) mark_file_synced(file) end end end @events.cookbook_sync_start(cookbook_count) queue.process(Chef::Config[:cookbook_sync_threads]) # Ensure that cookbooks know where they're rooted at, for manifest purposes. ensure_cookbook_paths # Update the full file paths in the manifest update_cookbook_filenames rescue Exception => e @events.cookbook_sync_failed(cookbooks, e) raise else @events.cookbook_sync_complete true end |
#sync_file(file) ⇒ Object
Sync an individual file if needed. If there is an up to date copy locally, nothing is done. Updates +file+'s manifest with the full path to the cached file.
=== Arguments
file
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 |
# File 'lib/chef/cookbook/synchronizer.rb', line 298 def sync_file(file) cache_filename = File.join("cookbooks", file.cookbook.name, file.manifest_record["path"]) mark_cached_file_valid(cache_filename) # If the checksums are different between on-disk (current) and on-server # (remote, per manifest), do the update. This will also execute if there # is no current checksum. if !cached_copy_up_to_date?(cache_filename, file.manifest_record["checksum"]) download_file(file.manifest_record["url"], cache_filename) @events.updated_cookbook_file(file.cookbook.name, cache_filename) else Chef::Log.trace("Not storing #{cache_filename}, as the cache is up to date.") end # Load the file in the cache and return the full file path to the loaded file cache.load(cache_filename, false) end |
#update_cookbook_filenames ⇒ Object
273 274 275 276 277 278 279 280 281 |
# File 'lib/chef/cookbook/synchronizer.rb', line 273 def update_cookbook_filenames @cookbook_full_file_paths.each do |cookbook, full_paths| cookbook.all_files = full_paths end @lazy_files.each do |cookbook, lazy_files| cookbook.cookbook_manifest.add_files_to_manifest(lazy_files) end end |