1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
|
require 'open3'
require 'open-uri'
require 'json'
require 'pp'
PROJECT_ROOT = File.dirname(__FILE__)
ACCEPTANCE_ROOT = ENV['ACCEPTANCE_ROOT'] ||
File.join(PROJECT_ROOT, 'acceptance')
BEAKER_OPTIONS_FILE = File.join(ACCEPTANCE_ROOT, 'config', 'beaker', 'options.rb')
PUPPET_SRC = File.join(PROJECT_ROOT, 'ruby', 'puppet')
PUPPET_LIB = File.join(PROJECT_ROOT, 'ruby', 'puppet', 'lib')
FACTER_LIB = File.join(PROJECT_ROOT, 'ruby', 'facter', 'lib')
PUPPET_SERVER_RUBY_SRC = File.join(PROJECT_ROOT, 'src', 'ruby', 'puppetserver-lib')
PUPPET_SERVER_RUBY_SPEC = File.join(PROJECT_ROOT, 'spec')
PUPPET_SUBMODULE_PATH = File.join('ruby','puppet')
FACTER_SUBMODULE_PATH = File.join('ruby','facter')
# Branch of puppetserver for which to update submodule pins
PUPPETSERVER_BRANCH = ENV['PUPPETSERVER_BRANCH'] || 'main'
# Branch of puppet-agent to track for passing puppet SHA
PUPPET_AGENT_BRANCH = ENV['PUPPET_AGENT_BRANCH'] || 'main'
# Branch of puppet-agent to track for Facter's passing SHA.
# This needs to be separate because for 6.x, we want to use facter#main
# instead of Facter 3.
FACTER_BRANCH = ENV['FACTER_BRANCH'] || 'main'
TEST_GEMS_DIR = File.join(PROJECT_ROOT, 'vendor', 'test_gems')
TEST_BUNDLE_DIR = File.join(PROJECT_ROOT, 'vendor', 'test_bundle')
GEM_SOURCE = ENV['GEM_SOURCE'] || "https://artifactory.delivery.puppetlabs.net/artifactory/api/gems/rubygems/"
# Jenkins uses `$LEIN` configure the lein version to use
LEIN_PATH = ENV['LEIN'] || 'lein'
def assemble_default_beaker_config
if ENV["BEAKER_CONFIG"]
return ENV["BEAKER_CONFIG"]
end
platform = ENV['PLATFORM']
layout = ENV['LAYOUT']
if platform and layout
beaker_config = "#{ACCEPTANCE_ROOT}/config/beaker/jenkins/"
beaker_config += "#{platform}-#{layout}.cfg"
else
abort "Must specify an appropriate value for BEAKER_CONFIG. See acceptance/README.md"
end
return beaker_config
end
def setup_smoke_hosts_config
sh "bundle exec beaker-hostgenerator centos7-64m-64a > acceptance/scripts/hosts.cfg"
end
def basic_smoke_test(package_version)
beaker = "PACKAGE_BUILD_VERSION=#{package_version}"
beaker += " bundle exec beaker --debug --root-keys --repo-proxy"
beaker += " --preserve-hosts always"
beaker += " --type aio"
beaker += " --helper acceptance/lib/helper.rb"
beaker += " --options-file #{BEAKER_OPTIONS_FILE}"
beaker += " --load-path acceptance/lib"
beaker += " --config acceptance/scripts/hosts.cfg"
beaker += " --keyfile ~/.ssh/id_rsa-acceptance"
beaker += " --pre-suite acceptance/suites/pre_suite/foss"
beaker += " --post-suite acceptance/suites/post_suite"
beaker += " --tests acceptance/suites/tests/00_smoke"
sh beaker
end
# TODO: this could be DRY'd up with the method above, but it seemed like it
# might make it a little harder to read and didn't seem worth the effort yet
def re_run_basic_smoke_test
beaker = "bundle exec beaker --debug --root-keys --repo-proxy"
beaker += " --preserve-hosts always"
beaker += " --type aio"
beaker += " --helper acceptance/lib/helper.rb"
beaker += " --options-file #{BEAKER_OPTIONS_FILE}"
beaker += " --load-path acceptance/lib"
beaker += " --config acceptance/scripts/hosts.cfg"
beaker += " --keyfile ~/.ssh/id_rsa-acceptance"
beaker += " --tests acceptance/suites/tests/00_smoke"
sh beaker
end
def jenkins_passing_json_parsed(branch)
passing_url = "http://builds.delivery.puppetlabs.net/passing-agent-SHAs/api/v1/json/report-#{branch}"
uri = URI.parse(passing_url)
begin
# DO NOT use uri-open if accepting user input for the uri
# we've done some simple correction here,
# but not enough to cleanse malicious user input
jenkins_result = uri.open(redirect: false)
rescue OpenURI::HTTPError => e
abort "ERROR: Could not get last passing run data for #{branch} of puppet-agent: '#{e.message}'"
end
begin
jenkins_result_parsed = JSON.parse(jenkins_result.read)
rescue JSON::ParserError => e
abort "ERROR: Could not get valid json for last passing run of #{branch}: '#{e.message}'"
end
end
def lookup_passing_puppetagent_sha(my_jenkins_passing_json)
begin
my_jenkins_passing_json['suite-commit']
rescue => e
abort "ERROR: Could not get last passing suite-commit value for #{PUPPET_AGENT_BRANCH}\n\n #{e}"
end
end
def lookup_passing_puppet_sha(my_jenkins_passing_json)
begin
my_jenkins_passing_json['puppet']
rescue => e
abort "ERROR: Could not get puppet's last passing SHA for #{PUPPET_AGENT_BRANCH}\n\n #{e}"
end
end
def lookup_passing_facter_sha(my_jenkins_passing_json)
begin
my_jenkins_passing_json['facter']
rescue => e
abort "ERROR: Could not get facter's last passing SHA for #{FACTER_BRANCH}\n\n #{e}"
end
end
def update_submodule(submodule_path, submodule_sha, submodule_name)
# ensure we fetch here, or the describe done later could be wrong
git_checkout_command = "cd #{submodule_path} && git fetch origin && " \
"git checkout #{submodule_sha}"
puts("checking out known passing #{submodule_name} version in submodule: `#{git_checkout_command}`")
system(git_checkout_command)
end
def replace_puppet_pins(passing_puppetagent_sha)
# read beaker options hash from its file
puts("replacing puppet-agent SHA in #{BEAKER_OPTIONS_FILE} " \
"with #{passing_puppetagent_sha}")
beaker_options_from_file = eval(File.read(BEAKER_OPTIONS_FILE))
# add puppet-agent version value
beaker_options_from_file[:puppet_build_version] = passing_puppetagent_sha
File.write(BEAKER_OPTIONS_FILE, beaker_options_from_file.pretty_inspect)
end
namespace :dev_deps_update do
desc 'update puppet submodule commit'
task :update_puppet_version do
my_jenkins_passing_json = jenkins_passing_json_parsed(PUPPET_AGENT_BRANCH)
puppet_sha = lookup_passing_puppet_sha(my_jenkins_passing_json)
update_submodule(PUPPET_SUBMODULE_PATH, puppet_sha, 'puppet')
# replace puppet-agent sha pin in beaker options file
replace_puppet_pins(lookup_passing_puppetagent_sha(my_jenkins_passing_json))
end
desc 'update facter submodule commit'
task :update_facter_version do
my_jenkins_passing_json = jenkins_passing_json_parsed(FACTER_BRANCH)
facter_sha = lookup_passing_facter_sha(my_jenkins_passing_json)
update_submodule(FACTER_SUBMODULE_PATH, facter_sha, 'facter')
end
desc 'commit and push; CAUTION: WILL commit and push, upstream, local changes to the puppet submodule and acceptance options'
task :commit_push do
git_commit_command = "git checkout #{PUPPETSERVER_BRANCH} && git add #{PUPPET_SUBMODULE_PATH} " \
"&& git add #{FACTER_SUBMODULE_PATH} && git add #{BEAKER_OPTIONS_FILE} " \
"&& git commit -m '(maint) update submodule versions and agent pin'"
git_push_command = "git checkout #{PUPPETSERVER_BRANCH} && git push origin HEAD:#{PUPPETSERVER_BRANCH}"
puts "committing submodules and agent pin via: `#{git_commit_command}`"
system(git_commit_command)
puts "pushing submodules and agent pin via: `#{git_push_command}`"
system(git_push_command)
end
desc 'update puppet versions and commit and push; CAUTION: WILL commit and push, upstream, local changes to the puppet submodule and acceptance options'
task :update_dev_deps_w_push => [:update_puppet_version, :update_facter_version, :commit_push]
end
namespace :spec do
task :init do
if ! Dir.exist? TEST_GEMS_DIR
## Install bundler
## Line 1 launches the JRuby that we depend on via leiningen
## Line 2 programmatically runs 'gem install bundler' via the gem command that comes with JRuby
gem_install_bundler = <<-CMD
GEM_HOME='#{TEST_GEMS_DIR}' GEM_PATH='#{TEST_GEMS_DIR}' \
#{LEIN_PATH} gem install -i '#{TEST_GEMS_DIR}' bundler --version 2.5.10 --no-document --source '#{GEM_SOURCE}'
CMD
sh gem_install_bundler
path = ENV['PATH']
## Install gems via bundler
## Line 1 makes sure that our local bundler script is on the path first
## Line 2 tells bundler to use puppet's Gemfile
## Line 3 tells JRuby where to look for gems
## Line 4 launches the JRuby that we depend on via leiningen
## Line 5 runs our bundle install script
bundle_install = <<-CMD
PATH='#{TEST_GEMS_DIR}/bin:#{path}' \
BUNDLE_GEMFILE='#{PUPPET_SRC}/Gemfile' \
GEM_HOME='#{TEST_GEMS_DIR}' GEM_PATH='#{TEST_GEMS_DIR}' \
#{LEIN_PATH} run -m org.jruby.Main \
-S bundle install --without extra development packaging --path='#{TEST_BUNDLE_DIR}' --retry=3
CMD
sh bundle_install
end
end
end
desc "Run rspec tests"
task :spec, [:rspec_opts] => ["spec:init"] do |t, args|
rspec_opts = args[:rspec_opts] || './spec'
## Run RSpec via our JRuby dependency
## Line 1 tells bundler to use puppet's Gemfile
## Line 2 tells JRuby where to look for gems
## Line 3 launches the JRuby that we depend on via leiningen
## Line 4 adds all our Ruby source to the JRuby LOAD_PATH
## Line 5 runs our rspec wrapper script
## <sarcasm-font>dang ole real easy man</sarcasm-font>
run_rspec_with_jruby = <<-CMD
BUNDLE_GEMFILE='#{PUPPET_SRC}/Gemfile' \
GEM_HOME='#{TEST_GEMS_DIR}' GEM_PATH='#{TEST_GEMS_DIR}' \
#{LEIN_PATH} run -m org.jruby.Main \
-I'#{PUPPET_SERVER_RUBY_SPEC}' -I'#{PUPPET_LIB}' -I'#{FACTER_LIB}' -I'#{PUPPET_SERVER_RUBY_SRC}' \
./spec/run_specs.rb #{rspec_opts}
CMD
sh run_rspec_with_jruby
end
namespace :test do
namespace :acceptance do
desc "Run beaker based acceptance tests"
task :beaker do |t, args|
# variables that take a limited set of acceptable strings
type = ENV["BEAKER_TYPE"] || "pe"
# variables that take pathnames
beakeropts = ENV["BEAKER_OPTS"] || ""
presuite = ENV["BEAKER_PRESUITE"] || "#{ACCEPTANCE_ROOT}/suites/pre_suite/#{type}"
postsuite = ENV["BEAKER_POSTSUITE"] || ""
helper = ENV["BEAKER_HELPER"] || "#{ACCEPTANCE_ROOT}/lib/helper.rb"
testsuite = ENV["BEAKER_TESTSUITE"] || "#{ACCEPTANCE_ROOT}/suites/tests"
loadpath = ENV["BEAKER_LOADPATH"] || ""
options = ENV["BEAKER_OPTIONSFILE"] || "#{ACCEPTANCE_ROOT}/config/beaker/options.rb"
# variables requiring some assembly
config = assemble_default_beaker_config
beaker = "beaker "
beaker += " -c #{config}"
beaker += " --helper #{helper}"
beaker += " --type #{type}"
beaker += " --options-file #{options}" if options != ''
beaker += " --load-path #{loadpath}" if loadpath != ''
beaker += " --pre-suite #{presuite}" if presuite != ''
beaker += " --post-suite #{postsuite}" if postsuite != ''
beaker += " --tests " + testsuite if testsuite != ''
beaker += " " + beakeropts
sh beaker
end
desc "Do an ezbake build, and then a beaker smoke test off of that build, preserving the vmpooler host"
task :bakeNbeak do
package_version = nil
Open3.popen3("#{LEIN_PATH} with-profile ezbake ezbake build 2>&1") do |stdin, stdout, stderr, thread|
# sleep 5
# puts "STDOUT IS: #{stdout}"
success = true
stdout.each do |line|
if match = line.match(%r|^Your packages will be available at http://builds.delivery.puppetlabs.net/puppetserver/(.*)$|)
package_version = match[1]
elsif line =~ /^Packaging FAILURE\s*$/
success = false
end
puts line
end
exit_code = thread.value
if success == true
puts "PACKAGE VERSION IS #{package_version}"
else
puts "\n\nPACKAGING FAILED! exit code is '#{exit_code}'. STDERR IS:"
puts stderr.read
exit 1
end
end
begin
setup_smoke_hosts_config()
basic_smoke_test(package_version)
rescue => e
puts "\n\nJOB FAILED; PACKAGE VERSION WAS: #{package_version}\n\n"
raise e
end
end
desc "Do a basic smoke test, using the package version specified by PACKAGE_BUILD_VERSION, preserving the vmpooler host"
task :smoke do
package_version = ENV["PACKAGE_BUILD_VERSION"]
unless package_version
STDERR.puts("'smoke' task requires PACKAGE_BUILD_VERSION environment variable")
exit 1
end
setup_smoke_hosts_config()
basic_smoke_test(package_version)
end
desc "Re-run the basic smoke test on the host preserved from a previous run of the 'smoke' task"
task :resmoke do
re_run_basic_smoke_test()
end
end
end
namespace :package do
task :bootstrap do
puts 'Bootstrap is no longer needed, using packaging-as-a-gem'
end
task :implode do
puts 'Implode is no longer needed, using packaging-as-a-gem'
end
end
begin
require 'packaging'
Pkg::Util::RakeUtils.load_packaging_tasks
rescue LoadError => e
puts "Error loading packaging rake tasks: #{e}"
end
|