Module: PuppetlabsSpecHelper::Tasks::FixtureHelpers

Defined in:
lib/puppetlabs_spec_helper/tasks/fixtures.rb

Overview

Helpers for working with fixtures.

Instance Method Summary collapse

Instance Method Details

Returns - returns a hash with the module name and the source directory.

Returns:

  • (Hash) —

    • returns a hash with the module name and the source directory



65
66
67
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 65

def auto_symlink
  { module_name => "\#{source_dir}" }
end

#clone_repo(scm, remote, target, _subdir = nil, ref = nil, branch = nil, flags = nil) ⇒ Object 



170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 170

def clone_repo(scm, remote, target, _subdir = nil, ref = nil, branch = nil, flags = nil)
  args = []
  case scm
  when 'hg'
    args.push('clone')
    args.push('-b', branch) if branch
    args.push(flags) if flags
    args.push(remote, target)
  when 'git'
    args.push('clone')
    args.push('--depth 1') unless ref
    args.push('-b', branch) if branch
    args.push(flags) if flags
    args.push(remote, target)
  else
    raise "Unfortunately #{scm} is not supported yet"
  end
  result = system("#{scm} #{args.flatten.join ' '}")
  raise "Failed to clone #{scm} repository #{remote} into #{target}" unless File.exist?(target)

  result
end

#current_thread_count(items) ⇒ Integer

returns the current thread count that is currently active a status of false or nil means the thread completed so when anything else we count that as a active thread

Returns:

  • (Integer) —

    • current thread count



277
278
279
280
281
282
283
284
285
286
287
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 277

def current_thread_count(items)
  active_threads = items.select do |_item, opts|
    if opts[:thread]
      opts[:thread].status
    else
      false
    end
  end
  logger.debug "Current thread count #{active_threads.count}"
  active_threads.count
end

#download_items(items) ⇒ Object

Parameters:

  • items (Hash) —

    • a hash of either repositories or forge modules

  • - (Block) —

    the method you wish to use to download the item



298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 298

def download_items(items)
  items.each do |remote, opts|
    # get the current active threads that are alive
    count = current_thread_count(items)
    if count < max_thread_limit
      logger.debug "New Thread started for #{remote}"
      # start up a new thread and store it in the opts hash
      opts[:thread] = Thread.new do
        yield(remote, opts)
      end
    else
      # the last thread started should be the longest wait
      # Rubocop seems to push towards using select here.. however the implementation today relies on the result being
      # an array. Select returns a hash which makes it unsuitable so we need to use find_all.last.
      item, item_opts = items.find_all { |_i, o| o.key?(:thread) }.last # rubocop:disable Performance/Detect
      logger.debug "Waiting on #{item}"
      item_opts[:thread].join # wait for the thread to finish
      # now that we waited lets try again
      redo
    end
  end
  # wait for all the threads to finish
  items.each_value { |opts| opts[:thread].join }
end

#download_module(remote, opts) ⇒ Boolean

Returns - returns true if the module was downloaded successfully, false otherwise.

Parameters:

  • - (String) —

    the remote url or namespace/name of the module to download

  • - (Hash) —

    list of options such as version

Returns:

  • (Boolean) —

    • returns true if the module was downloaded successfully, false otherwise



371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 371

def download_module(remote, opts)
  ref = ''
  flags = ''
  if opts.instance_of?(String)
    target = opts
  elsif opts.instance_of?(Hash)
    target = opts['target']
    ref = " --version #{opts['ref']}" unless opts['ref'].nil?
    flags = " #{opts['flags']}" if opts['flags']
  end

  forge_token = ENV.fetch('FORGE_API_KEY', nil)
  flags += " --forge_authorization \"Bearer #{forge_token}\"" if forge_token

  return false if File.directory?(target) && (ref.empty? || opts['ref'] == module_version(target))

  # The PMT cannot handle multi threaded runs due to cache directory collisons
  # so we randomize the directory instead.
  # Does working_dir even need to be passed?
  Dir.mktmpdir do |working_dir|
    command = "puppet module install#{ref}#{flags} --ignore-dependencies " \
              '--force ' \
              "--module_working_dir \"#{working_dir}\" " \
              "--target-dir \"#{module_target_dir}\" \"#{remote}\""

    raise "Failed to install module #{remote} to #{module_target_dir}" unless system(command)
  end
  $CHILD_STATUS.success?
end

#download_repository(remote, opts) ⇒ Boolean

Returns - returns true if the module was downloaded successfully, false otherwise.

Parameters:

  • - (String) —

    the remote url or namespace/name of the module to download

  • - (Hash) —

    list of options such as version, branch, ref

Returns:

  • (Boolean) —

    • returns true if the module was downloaded successfully, false otherwise



346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 346

def download_repository(remote, opts)
  scm = 'git'
  target = opts['target']
  subdir = opts['subdir']
  ref = opts['ref']
  scm = opts['scm'] if opts['scm']
  branch = opts['branch'] if opts['branch']
  flags = opts['flags']
  if valid_repo?(scm, target, remote)
    update_repo(scm, target)
  else
    clone_repo(scm, remote, target, subdir, ref, branch, flags)
  end
  revision(scm, target, ref) if ref
  remove_subdirectory(target, subdir) if subdir
end

#fixtures(category) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 74

def fixtures(category)
  fixtures_yaml = if ENV['FIXTURES_YML']
                    ENV['FIXTURES_YML']
                  elsif File.exist?('.fixtures.yml')
                    '.fixtures.yml'
                  elsif File.exist?('.fixtures.yaml')
                    '.fixtures.yaml'
                  else
                    false
                  end

  begin
    fixtures = if fixtures_yaml
                 YAML.load_file(fixtures_yaml) || { 'fixtures' => {} }
               else
                 { 'fixtures' => {} }
               end
  rescue Errno::ENOENT
    raise("Fixtures file not found: '#{fixtures_yaml}'")
  rescue Psych::SyntaxError => e
    raise("Found malformed YAML in '#{fixtures_yaml}' on line #{e.line} column #{e.column}: #{e.problem}")
  end

  unless fixtures.include?('fixtures')
    # File is non-empty, but does not specify fixtures
    raise("No 'fixtures' entries found in '#{fixtures_yaml}'; required")
  end

  fixture_defaults = if fixtures.include? 'defaults'
                       fixtures['defaults']
                     else
                       {}
                     end

  fixtures = fixtures['fixtures']
  fixtures['symlinks'] = (fixtures['symlinks'].nil? || fixtures['symlinks'].empty?) ? auto_symlink : auto_symlink.merge!(fixtures['symlinks'])

  result = {}
  if fixtures.include?(category) && !fixtures[category].nil?
    defaults = { 'target' => 'spec/fixtures/modules' }

    # load defaults from the `.fixtures.yml` `defaults` section
    # for the requested category and merge them into my defaults
    defaults = defaults.merge(fixture_defaults[category]) if fixture_defaults.include? category

    fixtures[category].each do |fixture, opts|
      # convert a simple string fixture to a hash, by
      # using the string fixture as the `repo` option of the hash.
      opts = { 'repo' => opts } if opts.instance_of?(String)
      # there should be a warning or something if it's not a hash...
      next unless opts.instance_of?(Hash)

      # merge our options into the defaults to get the
      # final option list
      opts = defaults.merge(opts)

      next unless include_repo?(opts['puppet_version'])

      real_target = eval("\"#{opts['target']}\"", binding, __FILE__, __LINE__) # evaluating target reference in this context (see auto_symlink)
      real_source = eval("\"#{opts['repo']}\"", binding, __FILE__, __LINE__) # evaluating repo reference in this context (see auto_symlink)

      result[real_source] = validate_fixture_hash!(
        'target' => File.join(real_target, fixture),
        'ref' => opts['ref'] || opts['tag'],
        'branch' => opts['branch'],
        'scm' => opts['scm'],
        'flags' => opts['flags'],
        'subdir' => opts['subdir'],
      )
    end
  end
  result
end

#forge_modules ⇒ Hash

“ref”=>nil, “branch”=>nil, “scm”=>nil, “flags”=>“–module_repository=myforge.example.com/”, “subdir”=>nil}

Returns:

  • (Hash) —

    • returns a hash of all the fixture forge modules



55
56
57
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 55

def forge_modules
  @forge_modules ||= fixtures('forge_modules') || {}
end

#git_remote_url(target) ⇒ Object 



236
237
238
239
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 236

def git_remote_url(target)
  output, status = Open3.capture2e('git', '--git-dir', File.join(target, '.git'), 'ls-remote', '--get-url', 'origin')
  status.success? ? output.strip : nil
end

#include_repo?(version_range) ⇒ Boolean

Returns:

  • (Boolean) 


158
159
160
161
162
163
164
165
166
167
168
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 158

def include_repo?(version_range)
  if version_range && defined?(SemanticPuppet)
    puppet_spec = Gem::Specification.find_by_name('puppet')
    puppet_version = SemanticPuppet::Version.parse(puppet_spec.version.to_s)

    constraint = SemanticPuppet::VersionRange.parse(version_range)
    constraint.include?(puppet_version)
  else
    true
  end
end

#logger ⇒ Object

creates a logger so we can log events with certain levels



252
253
254
255
256
257
258
259
260
261
262
263
264
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 252

def logger
  unless @logger
    require 'logger'
    level = if ENV['ENABLE_LOGGER']
              Logger::DEBUG
            else
              Logger::INFO
            end
    @logger = Logger.new($stderr)
    @logger.level = level
  end
  @logger
end

#max_thread_limit ⇒ Integer

Returns - returns the max_thread_count.

Returns:

  • (Integer) —

    • returns the max_thread_count



292
293
294
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 292

def max_thread_limit
  @max_thread_limit ||= (ENV['MAX_FIXTURE_THREAD_COUNT'] || 10).to_i
end

#module_name ⇒ String

Returns - the name of current module.

Returns:

  • (String) —

    • the name of current module



17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 17

def module_name
  raise ArgumentError unless File.file?('metadata.json') && File.readable?('metadata.json')

  metadata = JSON.parse(File.read('metadata.json'))
  metadata_name = metadata.fetch('name', nil) || ''

  raise ArgumentError if metadata_name.empty?

  metadata_name.split('-').last
rescue JSON::ParserError, ArgumentError
  File.basename(Dir.pwd).split('-').last
end

#module_target_dir ⇒ String

Returns - the spec/fixtures/modules directory in the module root folder.

Returns:

  • (String) —

    • the spec/fixtures/modules directory in the module root folder



364
365
366
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 364

def module_target_dir
  @module_target_dir ||= File.expand_path('spec/fixtures/modules')
end

#module_version(path) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 30

def module_version(path)
  metadata_path = File.join(path, 'metadata.json')
  raise ArgumentError unless File.file?(metadata_path) && File.readable?(metadata_path)

  metadata = JSON.parse(File.read(metadata_path))
  metadata.fetch('version', nil) || '0.0.1'
rescue JSON::ParserError, ArgumentError
  logger.warn "Failed to find module version at path #{path}"
  '0.0.1'
end

#module_working_directory ⇒ Object 



266
267
268
269
270
271
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 266

def module_working_directory
  # The problem with the relative path is that PMT doesn't expand the path properly and so passing in a relative path here
  # becomes something like C:\somewhere\backslashes/spec/fixtures/work-dir on Windows, and then PMT barfs itself.
  # This has been reported as https://tickets.puppetlabs.com/browse/PUP-4884
  File.expand_path(ENV['MODULE_WORKING_DIR'] || 'spec/fixtures/work-dir')
end

#remove_subdirectory(target, subdir) ⇒ Object 



241
242
243
244
245
246
247
248
249
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 241

def remove_subdirectory(target, subdir)
  return if subdir.nil?

  Dir.mktmpdir do |tmpdir|
    FileUtils.mv(Dir.glob("#{target}/#{subdir}/{.[^.]*,*}"), tmpdir)
    FileUtils.rm_rf("#{target}/#{subdir}")
    FileUtils.mv(Dir.glob("#{tmpdir}/{.[^.]*,*}"), target.to_s)
  end
end

#repositories ⇒ Hash

href="https://gitlab.com/puppetlabs/puppet-stdlib.git">gitlab.com/puppetlabs/puppet-stdlib.git”, “ref”=>nil, “branch”=>“main”, “scm”=>nil, }

Returns:

  • (Hash) —

    • returns a hash of all the fixture repositories



46
47
48
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 46

def repositories
  @repositories ||= fixtures('repositories') || {}
end

#revision(scm, target, ref) ⇒ Object 



211
212
213
214
215
216
217
218
219
220
221
222
223
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 211

def revision(scm, target, ref)
  args = []
  case scm
  when 'hg'
    args.push('update', '--clean', '-r', ref)
  when 'git'
    args.push('reset', '--hard', ref)
  else
    raise "Unfortunately #{scm} is not supported yet"
  end
  result = system("#{scm} #{args.flatten.join ' '}", chdir: target)
  raise "Invalid ref #{ref} for #{target}" unless result
end

#setup_symlink(target, link) ⇒ Object

works on windows and linux

Parameters:

  • target (String) —

    • the target directory

  • link (String) —

    • the name of the link you wish to create



326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 326

def setup_symlink(target, link)
  link = link['target']
  return if File.symlink?(link)

  logger.info("Creating symlink from #{link} to #{target}")
  if windows?
    target = File.join(File.dirname(link), target) unless Pathname.new(target).absolute?
    if Dir.respond_to?(:create_junction)
      Dir.create_junction(link, target)
    else
      system("call mklink /J \"#{link.tr('/', '\\')}\" \"#{target.tr('/', '\\')}\"")
    end
  else
    FileUtils.ln_sf(target, link)
  end
end

#shallow_git_repo? ⇒ Boolean

Returns:

  • (Boolean) 


207
208
209
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 207

def shallow_git_repo?
  File.file?(File.join('.git', 'shallow'))
end

#source_dir ⇒ Object

This is a helper for the self-symlink entry of fixtures.yml



12
13
14
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 12

def source_dir
  Dir.pwd
end

#symlinks ⇒ Hash

Returns - a hash of symlinks specified in the fixtures file.

Returns:

  • (Hash) —

    • a hash of symlinks specified in the fixtures file



60
61
62
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 60

def symlinks
  @symlinks ||= fixtures('symlinks') || {}
end

#update_repo(scm, target) ⇒ Object 



193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 193

def update_repo(scm, target)
  args = case scm
         when 'hg'
           ['pull']
         when 'git'
           ['fetch'].tap do |git_args|
             git_args << '--unshallow' if shallow_git_repo?
           end
         else
           raise "Unfortunately #{scm} is not supported yet"
         end
  system("#{scm} #{args.flatten.join(' ')}", chdir: target)
end

#valid_repo?(scm, target, remote) ⇒ Boolean

Returns:

  • (Boolean) 


225
226
227
228
229
230
231
232
233
234
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 225

def valid_repo?(scm, target, remote)
  return false unless File.directory?(target)
  return true if scm == 'hg'

  return true if git_remote_url(target) == remote

  warn "Git remote for #{target} has changed, recloning repository"
  FileUtils.rm_rf(target)
  false
end

#validate_fixture_hash!(hash) ⇒ Object

Raises:

  • (ArgumentError) 


148
149
150
151
152
153
154
155
156
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 148

def validate_fixture_hash!(hash)
  # Can only validate git based scm
  return hash unless hash['scm'] == 'git'

  # Forward slashes in the ref aren't allowed. And is probably a branch name.
  raise ArgumentError, "The ref for #{hash['target']} is invalid (Contains a forward slash). If this is a branch name, please use the 'branch' setting instead." if hash['ref']&.include?('/')

  hash
end

#windows? ⇒ Boolean

Returns - true if the os is a windows system.

Returns:

  • (Boolean) —

    • true if the os is a windows system



70
71
72
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 70

def windows?
  !!File::ALT_SEPARATOR
end