1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
|
# frozen_string_literal: true
require 'yaml'
require 'open3'
require 'json'
module PuppetlabsSpecHelper
module Tasks
# Helpers for working with fixtures.
module FixtureHelpers
# This is a helper for the self-symlink entry of fixtures.yml
def source_dir
Dir.pwd
end
# @return [String] - the name of current module
def module_name
raise ArgumentError unless File.file?('metadata.json') && File.readable?('metadata.json')
metadata = JSON.parse(File.read('metadata.json'))
metadata_name = metadata.fetch('name', nil) || ''
raise ArgumentError if metadata_name.empty?
metadata_name.split('-').last
rescue JSON::ParserError, ArgumentError
File.basename(Dir.pwd).split('-').last
end
def module_version(path)
metadata_path = File.join(path, 'metadata.json')
raise ArgumentError unless File.file?(metadata_path) && File.readable?(metadata_path)
metadata = JSON.parse(File.read(metadata_path))
metadata.fetch('version', nil) || '0.0.1'
rescue JSON::ParserError, ArgumentError
logger.warn "Failed to find module version at path #{path}"
'0.0.1'
end
# @return [Hash] - returns a hash of all the fixture repositories
# @example
# {"puppetlabs-stdlib"=>{"target"=>"https://gitlab.com/puppetlabs/puppet-stdlib.git",
# "ref"=>nil, "branch"=>"main", "scm"=>nil,
# }}
def repositories
@repositories ||= fixtures('repositories') || {}
end
# @return [Hash] - returns a hash of all the fixture forge modules
# @example
# {"puppetlabs-stdlib"=>{"target"=>"spec/fixtures/modules/stdlib",
# "ref"=>nil, "branch"=>nil, "scm"=>nil,
# "flags"=>"--module_repository=https://myforge.example.com/", "subdir"=>nil}}
def forge_modules
@forge_modules ||= fixtures('forge_modules') || {}
end
# @return [Hash] - a hash of symlinks specified in the fixtures file
def symlinks
@symlinks ||= fixtures('symlinks') || {}
end
# @return [Hash] - returns a hash with the module name and the source directory
def auto_symlink
{ module_name => "\#{source_dir}" }
end
# @return [Boolean] - true if the os is a windows system
def windows?
!!File::ALT_SEPARATOR
end
def fixtures(category)
fixtures_yaml = if ENV['FIXTURES_YML']
ENV['FIXTURES_YML']
elsif File.exist?('.fixtures.yml')
'.fixtures.yml'
elsif File.exist?('.fixtures.yaml')
'.fixtures.yaml'
else
false
end
begin
fixtures = if fixtures_yaml
YAML.load_file(fixtures_yaml) || { 'fixtures' => {} }
else
{ 'fixtures' => {} }
end
rescue Errno::ENOENT
raise("Fixtures file not found: '#{fixtures_yaml}'")
rescue Psych::SyntaxError => e
raise("Found malformed YAML in '#{fixtures_yaml}' on line #{e.line} column #{e.column}: #{e.problem}")
end
unless fixtures.include?('fixtures')
# File is non-empty, but does not specify fixtures
raise("No 'fixtures' entries found in '#{fixtures_yaml}'; required")
end
fixture_defaults = if fixtures.include? 'defaults'
fixtures['defaults']
else
{}
end
fixtures = fixtures['fixtures']
fixtures['symlinks'] = (fixtures['symlinks'].nil? || fixtures['symlinks'].empty?) ? auto_symlink : auto_symlink.merge!(fixtures['symlinks'])
result = {}
if fixtures.include?(category) && !fixtures[category].nil?
defaults = { 'target' => 'spec/fixtures/modules' }
# load defaults from the `.fixtures.yml` `defaults` section
# for the requested category and merge them into my defaults
defaults = defaults.merge(fixture_defaults[category]) if fixture_defaults.include? category
fixtures[category].each do |fixture, opts|
# convert a simple string fixture to a hash, by
# using the string fixture as the `repo` option of the hash.
opts = { 'repo' => opts } if opts.instance_of?(String)
# there should be a warning or something if it's not a hash...
next unless opts.instance_of?(Hash)
# merge our options into the defaults to get the
# final option list
opts = defaults.merge(opts)
next unless include_repo?(opts['puppet_version'])
real_target = eval("\"#{opts['target']}\"", binding, __FILE__, __LINE__) # evaluating target reference in this context (see auto_symlink)
real_source = eval("\"#{opts['repo']}\"", binding, __FILE__, __LINE__) # evaluating repo reference in this context (see auto_symlink)
result[real_source] = validate_fixture_hash!(
'target' => File.join(real_target, fixture),
'ref' => opts['ref'] || opts['tag'],
'branch' => opts['branch'],
'scm' => opts['scm'],
'flags' => opts['flags'],
'subdir' => opts['subdir'],
)
end
end
result
end
def validate_fixture_hash!(hash)
# Can only validate git based scm
return hash unless hash['scm'] == 'git'
# Forward slashes in the ref aren't allowed. And is probably a branch name.
raise ArgumentError, "The ref for #{hash['target']} is invalid (Contains a forward slash). If this is a branch name, please use the 'branch' setting instead." if hash['ref']&.include?('/')
hash
end
def include_repo?(version_range)
if version_range && defined?(SemanticPuppet)
puppet_spec = Gem::Specification.find_by_name('puppet')
puppet_version = SemanticPuppet::Version.parse(puppet_spec.version.to_s)
constraint = SemanticPuppet::VersionRange.parse(version_range)
constraint.include?(puppet_version)
else
true
end
end
def clone_repo(scm, remote, target, _subdir = nil, ref = nil, branch = nil, flags = nil)
args = []
case scm
when 'hg'
args.push('clone')
args.push('-b', branch) if branch
args.push(flags) if flags
args.push(remote, target)
when 'git'
args.push('clone')
args.push('--depth 1') unless ref
args.push('-b', branch) if branch
args.push(flags) if flags
args.push(remote, target)
else
raise "Unfortunately #{scm} is not supported yet"
end
result = system("#{scm} #{args.flatten.join ' '}")
raise "Failed to clone #{scm} repository #{remote} into #{target}" unless File.exist?(target)
result
end
def update_repo(scm, target)
args = case scm
when 'hg'
['pull']
when 'git'
['fetch'].tap do |git_args|
git_args << '--unshallow' if shallow_git_repo?
end
else
raise "Unfortunately #{scm} is not supported yet"
end
system("#{scm} #{args.flatten.join(' ')}", chdir: target)
end
def shallow_git_repo?
File.file?(File.join('.git', 'shallow'))
end
def revision(scm, target, ref)
args = []
case scm
when 'hg'
args.push('update', '--clean', '-r', ref)
when 'git'
args.push('reset', '--hard', ref)
else
raise "Unfortunately #{scm} is not supported yet"
end
result = system("#{scm} #{args.flatten.join ' '}", chdir: target)
raise "Invalid ref #{ref} for #{target}" unless result
end
def valid_repo?(scm, target, remote)
return false unless File.directory?(target)
return true if scm == 'hg'
return true if git_remote_url(target) == remote
warn "Git remote for #{target} has changed, recloning repository"
FileUtils.rm_rf(target)
false
end
def git_remote_url(target)
output, status = Open3.capture2e('git', '--git-dir', File.join(target, '.git'), 'ls-remote', '--get-url', 'origin')
status.success? ? output.strip : nil
end
def remove_subdirectory(target, subdir)
return if subdir.nil?
Dir.mktmpdir do |tmpdir|
FileUtils.mv(Dir.glob("#{target}/#{subdir}/{.[^.]*,*}"), tmpdir)
FileUtils.rm_rf("#{target}/#{subdir}")
FileUtils.mv(Dir.glob("#{tmpdir}/{.[^.]*,*}"), target.to_s)
end
end
# creates a logger so we can log events with certain levels
def logger
unless @logger
require 'logger'
level = if ENV['ENABLE_LOGGER']
Logger::DEBUG
else
Logger::INFO
end
@logger = Logger.new($stderr)
@logger.level = level
end
@logger
end
def module_working_directory
# The problem with the relative path is that PMT doesn't expand the path properly and so passing in a relative path here
# becomes something like C:\somewhere\backslashes/spec/fixtures/work-dir on Windows, and then PMT barfs itself.
# This has been reported as https://tickets.puppetlabs.com/browse/PUP-4884
File.expand_path(ENV['MODULE_WORKING_DIR'] || 'spec/fixtures/work-dir')
end
# returns the current thread count that is currently active
# a status of false or nil means the thread completed
# so when anything else we count that as a active thread
# @return [Integer] - current thread count
def current_thread_count(items)
active_threads = items.select do |_item, opts|
if opts[:thread]
opts[:thread].status
else
false
end
end
logger.debug "Current thread count #{active_threads.count}"
active_threads.count
end
# @summary Set a limit on the amount threads used, defaults to 10
# MAX_FIXTURE_THREAD_COUNT can be used to set this limit
# @return [Integer] - returns the max_thread_count
def max_thread_limit
@max_thread_limit ||= (ENV['MAX_FIXTURE_THREAD_COUNT'] || 10).to_i
end
# @param items [Hash] - a hash of either repositories or forge modules
# @param [Block] - the method you wish to use to download the item
def download_items(items)
items.each do |remote, opts|
# get the current active threads that are alive
count = current_thread_count(items)
if count < max_thread_limit
logger.debug "New Thread started for #{remote}"
# start up a new thread and store it in the opts hash
opts[:thread] = Thread.new do
yield(remote, opts)
end
else
# the last thread started should be the longest wait
# Rubocop seems to push towards using select here.. however the implementation today relies on the result being
# an array. Select returns a hash which makes it unsuitable so we need to use find_all.last.
item, item_opts = items.find_all { |_i, o| o.key?(:thread) }.last # rubocop:disable Performance/Detect
logger.debug "Waiting on #{item}"
item_opts[:thread].join # wait for the thread to finish
# now that we waited lets try again
redo
end
end
# wait for all the threads to finish
items.each_value { |opts| opts[:thread].join }
end
# @param target [String] - the target directory
# @param link [String] - the name of the link you wish to create
# works on windows and linux
def setup_symlink(target, link)
link = link['target']
return if File.symlink?(link)
logger.info("Creating symlink from #{link} to #{target}")
if windows?
target = File.join(File.dirname(link), target) unless Pathname.new(target).absolute?
if Dir.respond_to?(:create_junction)
Dir.create_junction(link, target)
else
system("call mklink /J \"#{link.tr('/', '\\')}\" \"#{target.tr('/', '\\')}\"")
end
else
FileUtils.ln_sf(target, link)
end
end
# @return [Boolean] - returns true if the module was downloaded successfully, false otherwise
# @param [String] - the remote url or namespace/name of the module to download
# @param [Hash] - list of options such as version, branch, ref
def download_repository(remote, opts)
scm = 'git'
target = opts['target']
subdir = opts['subdir']
ref = opts['ref']
scm = opts['scm'] if opts['scm']
branch = opts['branch'] if opts['branch']
flags = opts['flags']
if valid_repo?(scm, target, remote)
update_repo(scm, target)
else
clone_repo(scm, remote, target, subdir, ref, branch, flags)
end
revision(scm, target, ref) if ref
remove_subdirectory(target, subdir) if subdir
end
# @return [String] - the spec/fixtures/modules directory in the module root folder
def module_target_dir
@module_target_dir ||= File.expand_path('spec/fixtures/modules')
end
# @return [Boolean] - returns true if the module was downloaded successfully, false otherwise
# @param [String] - the remote url or namespace/name of the module to download
# @param [Hash] - list of options such as version
def download_module(remote, opts)
ref = ''
flags = ''
if opts.instance_of?(String)
target = opts
elsif opts.instance_of?(Hash)
target = opts['target']
ref = " --version #{opts['ref']}" unless opts['ref'].nil?
flags = " #{opts['flags']}" if opts['flags']
end
forge_token = ENV.fetch('FORGE_API_KEY', nil)
flags += " --forge_authorization \"Bearer #{forge_token}\"" if forge_token
return false if File.directory?(target) && (ref.empty? || opts['ref'] == module_version(target))
# The PMT cannot handle multi threaded runs due to cache directory collisons
# so we randomize the directory instead.
# Does working_dir even need to be passed?
Dir.mktmpdir do |working_dir|
command = "puppet module install#{ref}#{flags} --ignore-dependencies " \
'--force ' \
"--module_working_dir \"#{working_dir}\" " \
"--target-dir \"#{module_target_dir}\" \"#{remote}\""
raise "Failed to install module #{remote} to #{module_target_dir}" unless system(command)
end
$CHILD_STATUS.success?
end
end
end
end
include PuppetlabsSpecHelper::Tasks::FixtureHelpers # DSL include
desc 'Create the fixtures directory'
task :spec_prep do
# Ruby only sets File::ALT_SEPARATOR on Windows and Rubys standard library
# uses this to check for Windows
if windows?
begin
require 'win32/dir'
rescue LoadError
warn 'win32-dir gem not installed, falling back to executing mklink directly'
end
end
# git has a race condition creating that directory, that would lead to aborted clone operations
FileUtils.mkdir_p('spec/fixtures/modules')
symlinks.each { |target, link| setup_symlink(target, link) }
download_items(repositories) { |remote, opts| download_repository(remote, opts) }
download_items(forge_modules) { |remote, opts| download_module(remote, opts) }
FileUtils.mkdir_p('spec/fixtures/manifests')
FileUtils.touch('spec/fixtures/manifests/site.pp')
end
desc 'Clean up the fixtures directory'
task :spec_clean do
repositories.each_value do |opts|
target = opts['target']
FileUtils.rm_rf(target)
end
forge_modules.each_value do |opts|
target = opts['target']
FileUtils.rm_rf(target)
end
FileUtils.rm_rf(module_working_directory)
Rake::Task[:spec_clean_symlinks].invoke
FileUtils.rm_f('spec/fixtures/manifests/site.pp') if File.empty?('spec/fixtures/manifests/site.pp')
end
desc 'Clean up any fixture symlinks'
task :spec_clean_symlinks do
fixtures('symlinks').each_value do |opts|
target = opts['target']
FileUtils.rm_f(target)
end
end
|