Class: Onceover::VendoredModules

Inherits:
Object
  • Object
show all
Defined in:
lib/onceover/vendored_modules.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ VendoredModules

Returns a new instance of VendoredModules.



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/onceover/vendored_modules.rb', line 27

def initialize(opts = {})
  @repo = opts[:repo] || Onceover::Controlrepo.new
  @cachedir = opts[:cachedir] || File.join(@repo.tempdir, 'vendored_modules')
  @puppet_version = Gem::Version.new(Puppet.version)
  @puppet_major_version = Gem::Version.new(@puppet_version.segments[0])
  @force_update = opts[:force_update] || false

  @missing_vendored = []

  # This only applies to puppet >= 6 so bail early
  raise 'Auto resolving vendored modules only applies to puppet versions >= 6' unless @puppet_major_version >= Gem::Version.new('6')

  # Create cachedir
  unless File.directory?(@cachedir)
    logger.debug "Creating #{@cachedir}"
    FileUtils.mkdir_p(@cachedir)
  end

  # Location of user provided caches:
  #   control-repo/spec/vendored_modules/<component>-puppet_agent-<agent version>.json
  @manual_vendored_dir = File.join(@repo.spec_dir, 'vendored_modules')

  # Get the entire file tree of the puppetlabs/puppet-agent repository
  # https://docs.github.com/en/rest/git/trees?apiVersion=2022-11-28#get-a-tree
  puppet_agent_tree = query_or_cache(
    "https://api.github.com/repos/puppetlabs/puppet-agent/git/trees/#{@puppet_version}",
    { recursive: true },
    component_cache('repo_tree'),
  )
  # Get only the module-puppetlabs-<something>_core.json component files
  vendored_components =  puppet_agent_tree['tree'].select { |file| %r{configs/components/module-puppetlabs-\w+\.json}.match(file['path']) }
  # Get the contents of each component file
  # https://docs.github.com/en/rest/git/blobs?apiVersion=2022-11-28#get-a-blob
  @vendored_references = vendored_components.map do |component|
    mod_slug = component['path'].match(/.*(puppetlabs-\w+).json$/)[1]
    mod_name = mod_slug.match(/puppetlabs-(\w+)/)[1]
    query_or_cache(
      component['url'],
      nil,
      component_cache(mod_name),
    )
  end
end

Instance Attribute Details

#missing_vendored ⇒ Object (readonly)

Returns the value of attribute missing_vendored.



25
26
27
 
# File 'lib/onceover/vendored_modules.rb', line 25

def missing_vendored
  @missing_vendored
end

#vendored_references ⇒ Object (readonly)

Returns the value of attribute vendored_references.



25
26
27
 
# File 'lib/onceover/vendored_modules.rb', line 25

def vendored_references
  @vendored_references
end

Instance Method Details

#component_cache(component) ⇒ Object 



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/onceover/vendored_modules.rb', line 71

def component_cache(component)
  # Ideally we want a cache for the version of the puppet agent used in tests
  desired_name = "#{component}-puppet_agent-#{@puppet_version}.json"
  # By default look for any caches created during previous runs
  cache_file = File.join(@cachedir, desired_name)

  # If the user provides their own cache
  if !@force_update && File.directory?(@manual_vendored_dir)
            # Check for any '<component>-puppet_agent-<puppet version>.json' files
            dg = Dir.glob(File.join(@manual_vendored_dir, "#{component}-puppet_agent*"))
            # Check if there are multiple versions of the component cache
            if dg.size > 1
              # If there is the same version supplied as whats being tested against use that
              if dg.any? { |s| s[desired_name] }
                cache_file = File.join(@manual_vendored_dir, desired_name)
              # If there are any with the same major version, use the latest supplied
              elsif dg.any? { |s| s["#{component}-puppet_agent-#{@puppet_major_version}"] }
                maj_match = dg.select { |f| /#{component}-puppet_agent-#{@puppet_major_version}.\d+\.\d+\.json/.match(f) }
                maj_match.each do |f|
                  next unless (version_from_file(cache_file) == version_from_file(desired_name)) || (version_from_file(f) >= version_from_file(cache_file))

                  # if the current cache version matches the desired version, use the first matching major version in user cache
                  # if there are multiple major version matches in user cache, use the latest
                  cache_file = f
                end
              # Otherwise just use the latest supplied
              else
                dg.each { |f| cache_file = f if version_from_file(f) >= version_from_file(cache_file) }
              end
            # If there is only one use that
            elsif dg.size == 1
              cache_file = dg[0]
            end
  end

  # Warn the user if cached version does not match whats being used to test
  cache_version = version_from_file(cache_file)
  logger.warn "Cache for #{component} is for puppet_agent #{cache_version}, while you are testing against puppet_agent #{@puppet_version}. Consider updating your cache to ensure consistent behavior in your tests" if cache_version != @puppet_version

  cache_file
end

#github_get(url, params) ⇒ Object

Given a github url and optional query parameters, return the parsed json body



155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/onceover/vendored_modules.rb', line 155

def github_get(url, params)
  uri = URI.parse(url)
  uri.query = URI.encode_www_form(params) if params
  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = (uri.scheme == 'https')
  request = Net::HTTP::Get.new(uri.request_uri)
  request['Accept'] = 'application/vnd.github.raw+json'
  request['X-GitHub-Api-Version'] = '2022-11-28'
  response = http.request(request)

  case response
  when Net::HTTPOK # 200
    MultiJson.load(response.body)
  else
    # Expose the ratelimit response headers
    # https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#checking-the-status-of-your-rate-limit
    ratelimit_headers = response.to_hash.select { |k, _v| k =~ /x-ratelimit.*/ }
    raise "#{response.code} #{response.message} #{ratelimit_headers}"
  end
end

#puppetfile_missing_vendored(puppetfile) ⇒ Object

Currently expects to be passed a R10K::Puppetfile object. ex: R10K::ModuleLoader::Puppetfile.new(basedir: ‘.’)



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/onceover/vendored_modules.rb', line 120

def puppetfile_missing_vendored(puppetfile)
  puppetfile.load
  @vendored_references.each do |mod|
    # Extract name and slug from url
    mod_slug = mod['url'].match(/.*(puppetlabs-\w+)\.git/)[1]
    mod_name = mod_slug.match(/^puppetlabs-(\w+)$/)[1]
    # Array of modules whos names match
    existing = puppetfile.modules.select { |e_mod| e_mod.name == mod_name }
    if existing.empty?
      # Change url to https instead of ssh to allow anonymous git clones
      # so that users do not need to have an ssh keypair associated with a Github account
      url = mod['url'].gsub('[email protected]:', 'https://github.com/')
      @missing_vendored << { mod_slug => { git: url, ref: mod['ref'] } }
      logger.debug "#{mod_name} found to be missing in Puppetfile"
    else
      logger.debug "#{mod_name} found in Puppetfile. Using the specified version"
    end
  end
end

#query_or_cache(url, params, filepath) ⇒ Object

Return json from a query whom caches, or from the cache to avoid spamming github



141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/onceover/vendored_modules.rb', line 141

def query_or_cache(url, params, filepath)
  if (File.exist? filepath) && (@force_update == false)
    logger.debug "Using cache: #{filepath}"
    json = read_json_dump(filepath)
  else
    logger.debug "Making GET request to: #{url}"
    json = github_get(url, params)
    logger.debug "Caching response to: #{filepath}"
    write_json_dump(filepath, json)
  end
  json
end

#read_json_dump(filepath) ⇒ Object

Returns parsed json of file



177
178
179
 
# File 'lib/onceover/vendored_modules.rb', line 177

def read_json_dump(filepath)
  MultiJson.load(File.read(filepath))
end

#version_from_file(cache_file) ⇒ Object 



113
114
115
116
 
# File 'lib/onceover/vendored_modules.rb', line 113

def version_from_file(cache_file)
  version_regex = /.*-puppet_agent-(\d+\.\d+\.\d+)\.json/
  Gem::Version.new(version_regex.match(cache_file)[1])
end

#write_json_dump(filepath, json_data) ⇒ Object

Writes json to a file



182
183
184
 
# File 'lib/onceover/vendored_modules.rb', line 182

def write_json_dump(filepath, json_data)
  File.write(filepath, MultiJson.dump(json_data))
end