Class: Chef::CookbookUploader

Inherits:
Object
  • Object
show all
Defined in:
lib/chef/cookbook_uploader.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(cookbooks, opts = {}) ⇒ CookbookUploader

Creates a new CookbookUploader.

Arguments:

  • cookbooks:

    A Chef::CookbookVersion or array of them describing the cookbook(s) to be uploaded

  • path:

    A String or Array of Strings representing the base paths to the cookbook repositories.

  • opts:

    (optional) An options Hash

Options:

  • :force indicates that the uploader should set the force option when

    uploading the cookbook. This allows frozen CookbookVersion
documents on the server to be overwritten (otherwise a 409 is
returned by the server)

  • :rest A Chef::ServerAPI object that you have configured the way you like it.

    If you don't provide this, one will be created using the values
in Chef::Config.

  • :concurrency An integer that decided how many threads will be used to

    perform concurrent uploads
[View source]

39
40
41
42
43
44
45
 
# File 'lib/chef/cookbook_uploader.rb', line 39

def initialize(cookbooks, opts = {})
  @opts = opts
  @cookbooks = Array(cookbooks)
  @rest = opts[:rest] || Chef::ServerAPI.new(Chef::Config[:chef_server_url], version_class: Chef::CookbookManifestVersions)
  @concurrency = opts[:concurrency] || 10
  @policy_mode = opts[:policy_mode] || false
end

Instance Attribute Details

#concurrency ⇒ Object (readonly)

Returns the value of attribute concurrency.


20
21
22
 
# File 'lib/chef/cookbook_uploader.rb', line 20

def concurrency
  @concurrency
end

#cookbooks ⇒ Object (readonly)

Returns the value of attribute cookbooks.


16
17
18
 
# File 'lib/chef/cookbook_uploader.rb', line 16

def cookbooks
  @cookbooks
end

#opts ⇒ Object (readonly)

Returns the value of attribute opts.


18
19
20
 
# File 'lib/chef/cookbook_uploader.rb', line 18

def opts
  @opts
end

#path ⇒ Object (readonly)

Returns the value of attribute path.


17
18
19
 
# File 'lib/chef/cookbook_uploader.rb', line 17

def path
  @path
end

#rest ⇒ Object (readonly)

Returns the value of attribute rest.


19
20
21
 
# File 'lib/chef/cookbook_uploader.rb', line 19

def rest
  @rest
end

Instance Method Details

#policy_mode? ⇒ Boolean

Returns:

  • (Boolean)
[View source]

152
153
154
 
# File 'lib/chef/cookbook_uploader.rb', line 152

def policy_mode?
  @policy_mode
end

#upload_cookbooks ⇒ Object

[View source]

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/chef/cookbook_uploader.rb', line 47

def upload_cookbooks
  # Syntax Check
  validate_cookbooks unless opts[:skip_syntax_check]
  # generate checksums of cookbook files and create a sandbox
  checksum_files = {}
  cookbooks.each do |cb|
    Chef::Log.info("Saving #{cb.name}")
    checksum_files.merge!(cb.checksums)
  end

  checksums = checksum_files.inject({}) { |memo, elt| memo[elt.first] = nil; memo }
  new_sandbox = rest.post("sandboxes", { checksums: checksums })

  Chef::Log.info("Uploading files")

  queue = Chef::Util::ThreadedJobQueue.new

  checksums_to_upload = Set.new

  # upload the new checksums and commit the sandbox
  new_sandbox["checksums"].each do |checksum, info|
    if info["needs_upload"] == true
      checksums_to_upload << checksum
      Chef::Log.info("Uploading #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info["url"]}")
      queue << uploader_function_for(checksum_files[checksum], checksum, info["url"], checksums_to_upload)
    else
      Chef::Log.trace("#{checksum_files[checksum]} has not changed")
    end
  end

  queue.process(@concurrency)

  sandbox_url = new_sandbox["uri"]
  Chef::Log.trace("Committing sandbox")
  # Retry if S3 is claims a checksum doesn't exist (the eventual
  # in eventual consistency)
  retries = 0
  begin
    rest.put(sandbox_url, { is_completed: true })
  rescue Net::HTTPClientException => e
    if e.message =~ /^400/ && (retries += 1) <= 5
      sleep 2
      retry
    else
      raise
    end
  end

  # files are uploaded, so save the manifest
  cookbooks.each do |cb|

    manifest = Chef::CookbookManifest.new(cb, policy_mode: policy_mode?)

    save_url = opts[:force] ? manifest.force_save_url : manifest.save_url
    begin
      rest.put(save_url, manifest)
    rescue Net::HTTPClientException => e
      case e.response.code
      when "409"
        raise Chef::Exceptions::CookbookFrozen, "Version #{cb.version} of cookbook #{cb.name} is frozen. Use --force to override."
      else
        raise
      end
    end
  end

  Chef::Log.info("Upload complete!")
end

#uploader_function_for(file, checksum, url, checksums_to_upload) ⇒ Object

[View source]

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/chef/cookbook_uploader.rb', line 116

def uploader_function_for(file, checksum, url, checksums_to_upload)
  lambda do
    # Checksum is the hexadecimal representation of the md5,
    # but we need the base64 encoding for the content-md5
    # header
    checksum64 = Base64.encode64([checksum].pack("H*")).strip
    file_contents = File.binread(file)

    # Custom headers. 'content-type' disables JSON serialization of the request body.
    headers = { "content-type" => "application/x-binary", "content-md5" => checksum64, "accept" => "application/json" }

    begin
      rest.put(url, file_contents, headers)
      checksums_to_upload.delete(checksum)
    rescue Net::HTTPClientException, Net::HTTPFatalError, Errno::ECONNREFUSED, Timeout::Error, Errno::ETIMEDOUT, SocketError => e
      error_message = "Failed to upload #{file} (#{checksum}) to #{url} : #{e.message}"
      error_message << "\n#{e.response.body}" if e.respond_to?(:response)
      Chef::Knife.ui.error(error_message)
      raise
    end
  end
end

#validate_cookbooks ⇒ Object

[View source]

139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/chef/cookbook_uploader.rb', line 139

def validate_cookbooks
  cookbooks.each do |cb|
    next if cb.nil?

    syntax_checker = Chef::Cookbook::SyntaxCheck.new(cb.root_dir)
    Chef::Log.info("Validating ruby files")
    exit(1) unless syntax_checker.validate_ruby_files
    Chef::Log.info("Validating templates")
    exit(1) unless syntax_checker.validate_templates
    Chef::Log.info("Syntax OK")
  end
end