Module: PuppetlabsSpecHelper::Tasks::FixtureHelpers

Defined in:
lib/puppetlabs_spec_helper/tasks/fixtures.rb

Instance Method Summary collapse

Instance Method Details

Returns - returns a hash with the module name and the source directory.

Returns:

  • (Hash) —

    • returns a hash with the module name and the source directory



65
66
67
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 65

def auto_symlink
  { module_name => "\#{source_dir}" }
end

#clone_repo(scm, remote, target, _subdir = nil, ref = nil, branch = nil, flags = nil) ⇒ Object 



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 177

def clone_repo(scm, remote, target, _subdir = nil, ref = nil, branch = nil, flags = nil)
  args = []
  case scm
  when 'hg'
    args.push('clone')
    args.push('-b', branch) if branch
    args.push(flags) if flags
    args.push(remote, target)
  when 'git'
    args.push('clone')
    args.push('--depth 1') unless ref
    args.push('-b', branch) if branch
    args.push(flags) if flags
    args.push(remote, target)
  else
    raise "Unfortunately #{scm} is not supported yet"
  end
  result = system("#{scm} #{args.flatten.join ' '}")
  unless File.exist?(target)
    raise "Failed to clone #{scm} repository #{remote} into #{target}"
  end

  result
end

#current_thread_count(items) ⇒ Integer

returns the current thread count that is currently active a status of false or nil means the thread completed so when anything else we count that as a active thread

Returns:

  • (Integer) —

    • current thread count



286
287
288
289
290
291
292
293
294
295
296
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 286

def current_thread_count(items)
  active_threads = items.find_all do |_item, opts|
    if opts[:thread]
      opts[:thread].status
    else
      false
    end
  end
  logger.debug "Current thread count #{active_threads.count}"
  active_threads.count
end

#download_items(items) ⇒ Object

Parameters:

  • items (Hash) —

    • a hash of either repositories or forge modules

  • - (Block) —

    the method you wish to use to download the item



307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 307

def download_items(items)
  items.each do |remote, opts|
    # get the current active threads that are alive
    count = current_thread_count(items)
    if count < max_thread_limit
      logger.debug "New Thread started for #{remote}"
      # start up a new thread and store it in the opts hash
      opts[:thread] = Thread.new do
        yield(remote, opts)
      end
    else
      # the last thread started should be the longest wait
      item, item_opts = items.find_all { |_i, o| o.key?(:thread) }.last
      logger.debug "Waiting on #{item}"
      item_opts[:thread].join # wait for the thread to finish
      # now that we waited lets try again
      redo
    end
  end
  # wait for all the threads to finish
  items.each { |_remote, opts| opts[:thread].join }
end

#download_module(remote, opts) ⇒ Boolean

Returns - returns true if the module was downloaded successfully, false otherwise.

Parameters:

  • - (String) —

    the remote url or namespace/name of the module to download

  • - (Hash) —

    list of options such as version

Returns:

  • (Boolean) —

    • returns true if the module was downloaded successfully, false otherwise



378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 378

def download_module(remote, opts)
  ref = ''
  flags = ''
  if opts.instance_of?(String)
    target = opts
  elsif opts.instance_of?(Hash)
    target = opts['target']
    ref = " --version #{opts['ref']}" unless opts['ref'].nil?
    flags = " #{opts['flags']}" if opts['flags']
  end

  return false if File.directory?(target) && (ref.empty? || opts['ref'] == module_version(target))

  # The PMT cannot handle multi threaded runs due to cache directory collisons
  # so we randomize the directory instead.
  # Does working_dir even need to be passed?
  Dir.mktmpdir do |working_dir|
    command = "puppet module install#{ref}#{flags} --ignore-dependencies" \
    ' --force' \
    " --module_working_dir \"#{working_dir}\"" \
    " --target-dir \"#{module_target_dir}\" \"#{remote}\""

    unless system(command)
      raise "Failed to install module #{remote} to #{module_target_dir}"
    end
  end
  $CHILD_STATUS.success?
end

#download_repository(remote, opts) ⇒ Boolean

Returns - returns true if the module was downloaded successfully, false otherwise.

Parameters:

  • - (String) —

    the remote url or namespace/name of the module to download

  • - (Hash) —

    list of options such as version, branch, ref

Returns:

  • (Boolean) —

    • returns true if the module was downloaded successfully, false otherwise



353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 353

def download_repository(remote, opts)
  scm = 'git'
  target = opts['target']
  subdir = opts['subdir']
  ref = opts['ref']
  scm = opts['scm'] if opts['scm']
  branch = opts['branch'] if opts['branch']
  flags = opts['flags']
  if valid_repo?(scm, target, remote)
    update_repo(scm, target)
  else
    clone_repo(scm, remote, target, subdir, ref, branch, flags)
  end
  revision(scm, target, ref) if ref
  remove_subdirectory(target, subdir) if subdir
end

#fixtures(category) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 74

def fixtures(category)
  fixtures_yaml = if ENV['FIXTURES_YML']
                    ENV['FIXTURES_YML']
                  elsif File.exist?('.fixtures.yml')
                    '.fixtures.yml'
                  elsif File.exist?('.fixtures.yaml')
                    '.fixtures.yaml'
                  else
                    false
                  end

  begin
    fixtures = if fixtures_yaml
                 YAML.load_file(fixtures_yaml) || { 'fixtures' => {} }
               else
                 { 'fixtures' => {} }
               end
  rescue Errno::ENOENT
    raise("Fixtures file not found: '#{fixtures_yaml}'")
  rescue Psych::SyntaxError => e
    raise("Found malformed YAML in '#{fixtures_yaml}' on line #{e.line} column #{e.column}: #{e.problem}")
  end

  unless fixtures.include?('fixtures')
    # File is non-empty, but does not specify fixtures
    raise("No 'fixtures' entries found in '#{fixtures_yaml}'; required")
  end

  fixture_defaults = if fixtures.include? 'defaults'
                       fixtures['defaults']
                     else
                       {}
                     end

  fixtures = fixtures['fixtures']

  if fixtures['symlinks'].nil?
    fixtures['symlinks'] = auto_symlink
  end

  result = {}
  if fixtures.include?(category) && !fixtures[category].nil?
    defaults = { 'target' => 'spec/fixtures/modules' }

    # load defaults from the `.fixtures.yml` `defaults` section
    # for the requested category and merge them into my defaults
    if fixture_defaults.include? category
      defaults = defaults.merge(fixture_defaults[category])
    end

    fixtures[category].each do |fixture, opts|
      # convert a simple string fixture to a hash, by
      # using the string fixture as the `repo` option of the hash.
      if opts.instance_of?(String)
        opts = { 'repo' => opts }
      end
      # there should be a warning or something if it's not a hash...
      next unless opts.instance_of?(Hash)

      # merge our options into the defaults to get the
      # final option list
      opts = defaults.merge(opts)

      next unless include_repo?(opts['puppet_version'])

      real_target = eval("\"#{opts['target']}\"", binding, __FILE__, __LINE__) # evaluating target reference in this context (see auto_symlink)
      real_source = eval("\"#{opts['repo']}\"", binding, __FILE__, __LINE__) # evaluating repo reference in this context (see auto_symlink)

      result[real_source] = validate_fixture_hash!(
        'target' => File.join(real_target, fixture),
        'ref' => opts['ref'] || opts['tag'],
        'branch' => opts['branch'],
        'scm' => opts['scm'],
        'flags' => opts['flags'],
        'subdir' => opts['subdir'],
      )
    end
  end
  result
end

#forge_modules ⇒ Hash

“ref”=>nil, “branch”=>nil, “scm”=>nil, “flags”=>“–module_repository=myforge.example.com/”, “subdir”=>nil}

Returns:

  • (Hash) —

    • returns a hash of all the fixture forge modules



55
56
57
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 55

def forge_modules
  @forge_modules ||= fixtures('forge_modules') || {}
end

#git_remote_url(target) ⇒ Object 



245
246
247
248
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 245

def git_remote_url(target)
  output, status = Open3.capture2e('git', '--git-dir', File.join(target, '.git'), 'ls-remote', '--get-url', 'origin')
  status.success? ? output.strip : nil
end

#include_repo?(version_range) ⇒ Boolean

Returns:

  • (Boolean) 


165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 165

def include_repo?(version_range)
  if version_range && defined?(SemanticPuppet)
    puppet_spec = Gem::Specification.find_by_name('puppet')
    puppet_version = SemanticPuppet::Version.parse(puppet_spec.version.to_s)

    constraint = SemanticPuppet::VersionRange.parse(version_range)
    constraint.include?(puppet_version)
  else
    true
  end
end

#logger ⇒ Object

creates a logger so we can log events with certain levels



261
262
263
264
265
266
267
268
269
270
271
272
273
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 261

def logger
  unless @logger
    require 'logger'
    level = if ENV['ENABLE_LOGGER']
              Logger::DEBUG
            else
              Logger::INFO
            end
    @logger = Logger.new($stderr)
    @logger.level = level
  end
  @logger
end

#max_thread_limit ⇒ Integer

Returns - returns the max_thread_count.

Returns:

  • (Integer) —

    • returns the max_thread_count



301
302
303
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 301

def max_thread_limit
  @max_thread_limit ||= (ENV['MAX_FIXTURE_THREAD_COUNT'] || 10).to_i
end

#module_name ⇒ String

Returns - the name of current module.

Returns:

  • (String) —

    • the name of current module



17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 17

def module_name
  raise ArgumentError unless File.file?('metadata.json') && File.readable?('metadata.json')

  metadata = JSON.parse(File.read('metadata.json'))
  metadata_name = metadata.fetch('name', nil) || ''

  raise ArgumentError if metadata_name.empty?

  metadata_name.split('-').last
rescue JSON::ParserError, ArgumentError
  File.basename(Dir.pwd).split('-').last
end

#module_target_dir ⇒ String

Returns - the spec/fixtures/modules directory in the module root folder.

Returns:

  • (String) —

    • the spec/fixtures/modules directory in the module root folder



371
372
373
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 371

def module_target_dir
  @module_target_dir ||= File.expand_path('spec/fixtures/modules')
end

#module_version(path) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 30

def module_version(path)
  metadata_path = File.join(path, 'metadata.json')
  raise ArgumentError unless File.file?(metadata_path) && File.readable?(metadata_path)

  metadata = JSON.parse(File.read(metadata_path))
  metadata.fetch('version', nil) || '0.0.1'
rescue JSON::ParserError, ArgumentError
  logger.warn "Failed to find module version at path #{path}"
  '0.0.1'
end

#module_working_directory ⇒ Object 



275
276
277
278
279
280
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 275

def module_working_directory
  # The problem with the relative path is that PMT doesn't expand the path properly and so passing in a relative path here
  # becomes something like C:\somewhere\backslashes/spec/fixtures/work-dir on Windows, and then PMT barfs itself.
  # This has been reported as https://tickets.puppetlabs.com/browse/PUP-4884
  File.expand_path(ENV['MODULE_WORKING_DIR'] || 'spec/fixtures/work-dir')
end

#remove_subdirectory(target, subdir) ⇒ Object 



250
251
252
253
254
255
256
257
258
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 250

def remove_subdirectory(target, subdir)
  unless subdir.nil?
    Dir.mktmpdir do |tmpdir|
      FileUtils.mv(Dir.glob("#{target}/#{subdir}/{.[^\.]*,*}"), tmpdir)
      FileUtils.rm_rf("#{target}/#{subdir}")
      FileUtils.mv(Dir.glob("#{tmpdir}/{.[^\.]*,*}"), target.to_s)
    end
  end
end

#repositories ⇒ Hash

href="https://gitlab.com/puppetlabs/puppet-stdlib.git">gitlab.com/puppetlabs/puppet-stdlib.git”, “ref”=>nil, “branch”=>“master”, “scm”=>nil, }

Returns:

  • (Hash) —

    • returns a hash of all the fixture repositories



46
47
48
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 46

def repositories
  @repositories ||= fixtures('repositories') || {}
end

#revision(scm, target, ref) ⇒ Object 



220
221
222
223
224
225
226
227
228
229
230
231
232
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 220

def revision(scm, target, ref)
  args = []
  case scm
  when 'hg'
    args.push('update', '--clean', '-r', ref)
  when 'git'
    args.push('reset', '--hard', ref)
  else
    raise "Unfortunately #{scm} is not supported yet"
  end
  result = system("#{scm} #{args.flatten.join ' '}", chdir: target)
  raise "Invalid ref #{ref} for #{target}" unless result
end

#setup_symlink(target, link) ⇒ Object

works on windows and linux

Parameters:

  • target (String) —

    • the target directory

  • link (String) —

    • the name of the link you wish to create



333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 333

def setup_symlink(target, link)
  link = link['target']
  return if File.symlink?(link)

  logger.info("Creating symlink from #{link} to #{target}")
  if windows?
    target = File.join(File.dirname(link), target) unless Pathname.new(target).absolute?
    if Dir.respond_to?(:create_junction)
      Dir.create_junction(link, target)
    else
      system("call mklink /J \"#{link.tr('/', '\\')}\" \"#{target.tr('/', '\\')}\"")
    end
  else
    FileUtils.ln_sf(target, link)
  end
end

#shallow_git_repo? ⇒ Boolean

Returns:

  • (Boolean) 


216
217
218
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 216

def shallow_git_repo?
  File.file?(File.join('.git', 'shallow'))
end

#source_dir ⇒ Object

This is a helper for the self-symlink entry of fixtures.yml



12
13
14
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 12

def source_dir
  Dir.pwd
end

#symlinks ⇒ Hash

Returns - a hash of symlinks specified in the fixtures file.

Returns:

  • (Hash) —

    • a hash of symlinks specified in the fixtures file



60
61
62
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 60

def symlinks
  @symlinks ||= fixtures('symlinks') || {}
end

#update_repo(scm, target) ⇒ Object 



202
203
204
205
206
207
208
209
210
211
212
213
214
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 202

def update_repo(scm, target)
  args = case scm
         when 'hg'
           ['pull']
         when 'git'
           ['fetch'].tap do |git_args|
             git_args << '--unshallow' if shallow_git_repo?
           end
         else
           raise "Unfortunately #{scm} is not supported yet"
         end
  system("#{scm} #{args.flatten.join(' ')}", chdir: target)
end

#valid_repo?(scm, target, remote) ⇒ Boolean

Returns:

  • (Boolean) 


234
235
236
237
238
239
240
241
242
243
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 234

def valid_repo?(scm, target, remote)
  return false unless File.directory?(target)
  return true if scm == 'hg'

  return true if git_remote_url(target) == remote

  warn "Git remote for #{target} has changed, recloning repository"
  FileUtils.rm_rf(target)
  false
end

#validate_fixture_hash!(hash) ⇒ Object

Raises:

  • (ArgumentError) 


155
156
157
158
159
160
161
162
163
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 155

def validate_fixture_hash!(hash)
  # Can only validate git based scm
  return hash unless hash['scm'] == 'git'

  # Forward slashes in the ref aren't allowed. And is probably a branch name.
  raise ArgumentError, "The ref for #{hash['target']} is invalid (Contains a forward slash). If this is a branch name, please use the 'branch' setting instead." if hash['ref'] =~ %r{/}

  hash
end

#windows? ⇒ Boolean

Returns - true if the os is a windows system.

Returns:

  • (Boolean) —

    • true if the os is a windows system



70
71
72
 
# File 'lib/puppetlabs_spec_helper/tasks/fixtures.rb', line 70

def windows?
  !!File::ALT_SEPARATOR
end