File: cts-fencing.in

package info (click to toggle)
pacemaker 3.0.1-1
links: PTS, VCS
area: main
in suites: forky, sid
size: 68,576 kB
sloc: xml: 160,564; ansic: 143,744; python: 5,670; sh: 2,969; makefile: 2,426
file content (915 lines) | stat: -rw-r--r-- 57,885 bytes
#!@PYTHON@
"""Regression tests for Pacemaker's fencer."""

# pylint doesn't like the module name "cts-fencing" which is an invalid complaint for this file
# but probably something we want to continue warning about elsewhere
# pylint: disable=invalid-name
# pacemaker imports need to come after we modify sys.path, which pylint will complain about.
# pylint: disable=wrong-import-position

__copyright__ = "Copyright 2012-2025 the Pacemaker project contributors"
__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

import argparse
import os
import sys
import subprocess
import tempfile

from pacemaker.buildoptions import BuildOptions
from pacemaker.exitstatus import ExitStatus
from pacemaker._cts.corosync import Corosync, localname
from pacemaker._cts.process import killall, exit_if_proc_running
from pacemaker._cts.test import Test, Tests

TEST_DIR = sys.path[0]


def update_path():
    """Set the PATH environment variable appropriately for the tests."""
    new_path = os.environ['PATH']
    if os.path.exists(f"{TEST_DIR}/cts-fencing.in"):
        # pylint: disable=protected-access
        print(f"Running tests from the source tree: {BuildOptions._BUILD_DIR} ({TEST_DIR})")
        # For pacemaker-fenced and cts-fence-helper
        new_path = f"{BuildOptions._BUILD_DIR}/daemons/fenced:{new_path}"
        new_path = f"{BuildOptions._BUILD_DIR}/tools:{new_path}"  # For stonith_admin
        new_path = f"{BuildOptions._BUILD_DIR}/cts/support:{new_path}"  # For cts-support

    else:
        print(f"Running tests from the install tree: {BuildOptions.DAEMON_DIR} (not {TEST_DIR})")
        # For pacemaker-fenced, cts-fence-helper, and cts-support
        new_path = f"{BuildOptions.DAEMON_DIR}:{new_path}"

    print(f'Using PATH="{new_path}"')
    os.environ['PATH'] = new_path


class FenceTest(Test):
    """Executor for a single test."""

    def __init__(self, name, description, **kwargs):
        """
        Create a new FenceTest instance.

        Arguments:
        name        -- A unique name for this test.  This can be used on the
                       command line to specify that only a specific test should
                       be executed.
        description -- A meaningful description for the test.
        """
        Test.__init__(self, name, description, **kwargs)

        self._daemon_location = "pacemaker-fenced"

    def _kill_daemons(self):
        killall(["pacemakerd", "pacemaker-fenced"])

    def _start_daemons(self):
        cmd = ["pacemaker-fenced", "--stand-alone", "--logfile", self.logpath]
        if self.verbose:
            cmd += ["-V"]
            s = " ".join(cmd)
            print(f"Starting {s}")

        # pylint: disable=consider-using-with
        self._daemon_process = subprocess.Popen(cmd)


class FenceTests(Tests):
    """Collection of all fencing regression tests."""

    def __init__(self, **kwargs):
        """Create a new FenceTests instance."""
        Tests.__init__(self, **kwargs)

        self._corosync = Corosync(self.verbose, self.logdir, "cts-fencing")

    def new_test(self, name, description):
        """Create a named test."""
        test = FenceTest(name, description, verbose=self.verbose,
                         timeout=self.timeout, force_wait=self.force_wait,
                         logdir=self.logdir)
        self._tests.append(test)
        return test

    def build_api_sanity_tests(self):
        """Register tests to verify basic API usage."""
        verbose_arg = ""
        if self.verbose:
            verbose_arg = "-V"

        test = self.new_test("low_level_api_test", "Sanity-test client API")
        test.add_cmd("cts-fence-helper", args=f"-t {verbose_arg}", validate=False)

        test = self.new_test("low_level_api_mainloop_test",
                             "Sanity-test client API using mainloop")
        test.add_cmd("cts-fence-helper", args=f"-m {verbose_arg}", validate=False)

    def build_custom_timeout_tests(self):
        """Register tests to verify custom timeout usage."""
        # custom timeout without topology
        test = self.new_test("custom_timeout_1",
                             "Verify per device timeouts work as expected without using topology")
        test.add_cmd('stonith_admin',
                     args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd('stonith_admin',
                     args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_list=node3 -o pcmk_off_timeout=1')
        test.add_cmd('stonith_admin',
                     args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3 -o pcmk_off_timeout=4')
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5")
        # timeout is 5+1+4 = 10
        test.add_log_pattern("Total timeout set to 12s")

        # custom timeout _WITH_ topology
        test = self.new_test("custom_timeout_2",
                             "Verify per device timeouts work as expected _WITH_ topology")
        test.add_cmd('stonith_admin',
                     args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd('stonith_admin',
                     args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_list=node3 -o pcmk_off_timeout=1000ms')
        test.add_cmd('stonith_admin',
                     args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3 -o pcmk_off_timeout=4000s')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v false2")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5")
        # timeout is 5+1+4000 = 4006
        test.add_log_pattern("Total timeout set to 4807s")

    def build_fence_merge_tests(self):
        """Register tests to verify when fence operations should be merged."""
        # Simple test that overlapping fencing operations get merged
        test = self.new_test("custom_merge_single",
                             "Verify overlapping identical fencing operations are merged, no fencing levels used")
        test.add_cmd("stonith_admin", args="--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin", args="--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10")
        # one merger will happen
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        # the pattern below signifies that both the original and duplicate operation completed
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")

        # Test that multiple mergers occur
        test = self.new_test("custom_merge_multiple",
                             "Verify multiple overlapping identical fencing operations are merged")
        test.add_cmd("stonith_admin", args="--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin",
                     args="--output-as=xml -R true1  -a fence_dummy -o \"mode=pass\" -o delay=2 -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin", args="--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10")
        # 4 mergers should occur
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        # the pattern below signifies that both the original and duplicate operation completed
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")

        # Test that multiple mergers occur with topologies used
        test = self.new_test("custom_merge_with_topology",
                             "Verify multiple overlapping identical fencing operations are merged with fencing levels")
        test.add_cmd("stonith_admin", args="--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin", args="--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10")
        # 4 mergers should occur
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client")
        # the pattern below signifies that both the original and duplicate operation completed
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")
        test.add_log_pattern("Operation 'off' targeting node3 by ")

    def build_fence_no_merge_tests(self):
        """Register tests to verify when fence operations should not be merged."""
        test = self.new_test("custom_no_merge",
                             "Verify differing fencing operations are not merged")
        test.add_cmd("stonith_admin", args="--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node3 node2")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_list=node3 node2")
        test.add_cmd("stonith_admin", args="--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node3 node2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node2 -t 10", no_wait=True)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 10")
        test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client",
                             negative=True)

    def build_standalone_tests(self):
        """Register a grab bag of tests."""
        # test what happens when all devices timeout
        test = self.new_test("fence_multi_device_failure",
                             "Verify that all devices timeout, a fencing failure is returned")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false2  -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false3 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 2", expected_exitcode=ExitStatus.TIMEOUT)
        test.add_log_pattern("Total timeout set to 7s")
        test.add_log_pattern("targeting node3 using false1 returned ")
        test.add_log_pattern("targeting node3 using false2 returned ")
        test.add_log_pattern("targeting node3 using false3 returned ")

        # test what happens when multiple devices can fence a node, but the first device fails
        test = self.new_test("fence_device_failure_rollover",
                             "Verify that when one fence device fails for a node, the others are tried")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5")
        test.add_log_pattern("Total timeout set to 18s")

        # test what happens when we try to use a missing fence-agent
        test = self.new_test("fence_missing_agent",
                             "Verify proper error-handling when using a non-existent fence-agent")
        test.add_cmd("stonith_admin",
                     args="--output-as=xml -R true1 -a fence_missing -o mode=pass -o pcmk_host_list=node3")
        test.add_cmd("stonith_admin",
                     args="--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_list=node2")

        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5", expected_exitcode=ExitStatus.NOSUCH)
        test.add_cmd("stonith_admin", args="--output-as=xml -F node2 -t 5")

        # simple topology test for one device
        test = self.new_test("topology_simple",
                             "Verify all fencing devices at a level are used")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5")

        test.add_log_pattern("Total timeout set to 6s")
        test.add_log_pattern("targeting node3 using true returned 0")

        # add topology, delete topology, verify fencing still works
        test = self.new_test("topology_add_remove",
                             "Verify fencing occurrs after all topology levels are removed")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true")
        test.add_cmd("stonith_admin", args="--output-as=xml -d node3 -i 1")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5")

        test.add_log_pattern("Total timeout set to 6s")
        test.add_log_pattern("targeting node3 using true returned 0")

        # test what happens when the first fencing level has multiple devices
        test = self.new_test("topology_device_fails",
                             "Verify if one device in a level fails, the other is tried")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false  -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 20")
        test.add_log_pattern("Total timeout set to 48s")
        test.add_log_pattern("targeting node3 using false returned 1")
        test.add_log_pattern("targeting node3 using true returned 0")

        # test what happens when the first fencing level fails
        test = self.new_test("topology_multi_level_fails",
                             "Verify if one level fails, the next leve is tried")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true2  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true3  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true4  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')

        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v false2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true3")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true4")

        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 3")

        test.add_log_pattern("Total timeout set to 21s")
        test.add_log_pattern("targeting node3 using false1 returned 1")
        test.add_log_pattern("targeting node3 using false2 returned 1")
        test.add_log_pattern("targeting node3 using true3 returned 0")
        test.add_log_pattern("targeting node3 using true4 returned 0")

        # test what happens when the first fencing level had devices that no one has registered
        test = self.new_test("topology_missing_devices",
                             "Verify topology can continue with missing devices")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true2  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true3  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true4  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')

        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v false2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true3")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true4")

        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5")

        # Test what happens if multiple fencing levels are defined, and then the first one is removed
        test = self.new_test("topology_level_removal",
                             "Verify level removal works")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true2  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true3  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true4  -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3"')

        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true1")

        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v false2")

        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true3")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 3 -v true4")

        # Now remove level 2, verify none of the devices in level two are hit
        test.add_cmd("stonith_admin", args="--output-as=xml -d node3 -i 2")

        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 20")

        test.add_log_pattern("Total timeout set to 96s")
        test.add_log_pattern("targeting node3 using false1 returned 1")
        test.add_log_pattern("targeting node3 using false2 returned ",
                             negative=True)
        test.add_log_pattern("targeting node3 using true3 returned 0")
        test.add_log_pattern("targeting node3 using true4 returned 0")

        # Test targeting a topology level by node name pattern
        test = self.new_test("topology_level_pattern",
                             "Verify targeting topology by node name pattern works")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -r '@node.*' -i 1 -v true")
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5")
        test.add_log_pattern("targeting node3 using true returned 0")

        # test allowing commas and semicolons as delimiters in pcmk_host_list
        test = self.new_test("host_list_delimiters",
                             "Verify commas and semicolons can be used as pcmk_host_list delimiters")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1,node2,node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=pcmk1;pcmk2;pcmk3"')
        test.add_cmd("stonith_admin", args="stonith_admin --output-as=xml -F node2 -t 5")
        test.add_cmd("stonith_admin", args="stonith_admin --output-as=xml -F pcmk3 -t 5")
        test.add_log_pattern("targeting node2 using true1 returned 0")
        test.add_log_pattern("targeting pcmk3 using true2 returned 0")

        # test the stonith builds the correct list of devices that can fence a node
        test = self.new_test("list_devices",
                             "Verify list of devices that can fence a node is correct")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -l node1 -V",
                     stdout_match="true2", stdout_no_match="true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -l node1 -V",
                     stdout_match="true3", stdout_no_match="true1")

        # simple test of device monitor
        test = self.new_test("monitor", "Verify device is reachable")
        test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node3"')
        test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node3"')

        test.add_cmd("stonith_admin", args="--output-as=xml -Q true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -Q false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -Q true2", expected_exitcode=ExitStatus.NOSUCH)

        # Verify monitor occurs for duration of timeout period on failure
        test = self.new_test("monitor_timeout",
                             "Verify monitor uses duration of timeout period given")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1 -a fence_dummy -o mode=fail -o monitor_mode=fail -o pcmk_host_list=node3')
        test.add_cmd("stonith_admin", args="--output-as=xml -Q true1 -t 5", expected_exitcode=ExitStatus.ERROR)
        test.add_log_pattern("Attempt 2 to execute")

        # Verify monitor occurs for duration of timeout period on failure, but stops at max retries
        test = self.new_test("monitor_timeout_max_retries",
                             "Verify monitor retries until max retry value or timeout is hit")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1  -a fence_dummy -o mode=fail -o monitor_mode=fail -o pcmk_host_list=node3')
        test.add_cmd("stonith_admin", args="--output-as=xml -Q true1 -t 15", expected_exitcode=ExitStatus.ERROR)
        test.add_log_pattern("Attempted to execute agent fence_dummy (list) the maximum number of times")

        # simple register test
        test = self.new_test("register",
                             "Verify devices can be registered and un-registered")
        test.add_cmd("stonith_admin", args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_list=node3')
        test.add_cmd("stonith_admin", args="--output-as=xml -Q true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -D true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -Q true1", expected_exitcode=ExitStatus.NOSUCH)

        # simple reboot test
        test = self.new_test("reboot", "Verify devices can be rebooted")
        test.add_cmd("stonith_admin", args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_list=node3')
        test.add_cmd("stonith_admin", args="--output-as=xml -B node3 -t 5")
        test.add_cmd("stonith_admin", args="--output-as=xml -D true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -Q true1", expected_exitcode=ExitStatus.NOSUCH)

        # test fencing history
        test = self.new_test("fence_history",
                             "Verify last fencing operation is returned")
        test.add_cmd("stonith_admin", args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_list=node3')
        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 -t 5 -V")
        test.add_cmd("stonith_admin", args="--output-as=xml -H node3",
                     stdout_match='action="off" target="node3" .* status="success"')

        # simple test of dynamic list query
        test = self.new_test("dynamic_list_query",
                             "Verify dynamic list of fencing devices can be retrieved")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true2 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true3 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")

        test.add_cmd("stonith_admin", args="--output-as=xml -l fake_port_1",
                     stdout_match='count="3"')

        # fence using dynamic list query
        test = self.new_test("fence_dynamic_list_query",
                             "Verify dynamic list of fencing devices can be retrieved")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true1 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true2 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")
        test.add_cmd("stonith_admin", args="--output-as=xml -R true3 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1")

        test.add_cmd("stonith_admin", args="--output-as=xml -F fake_port_1 -t 5 -V")

        # simple test of  query using status action
        test = self.new_test("status_query",
                             "Verify dynamic list of fencing devices can be retrieved")
        test.add_cmd("stonith_admin", args='--output-as=xml -R true1  -a fence_dummy -o mode=pass -o pcmk_host_check=status')
        test.add_cmd("stonith_admin", args='--output-as=xml -R true2  -a fence_dummy -o mode=pass -o pcmk_host_check=status')
        test.add_cmd("stonith_admin", args='--output-as=xml -R true3  -a fence_dummy -o mode=pass -o pcmk_host_check=status')

        test.add_cmd("stonith_admin", args="--output-as=xml -l fake_port_1",
                     stdout_match='count="3"')

        # test what happens when no reboot action is advertised
        test = self.new_test("no_reboot_support",
                             "Verify reboot action defaults to off when no reboot action is advertised by agent")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1 -a fence_dummy_no_reboot -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -B node1 -t 5 -V")
        test.add_log_pattern("does not support reboot")
        test.add_log_pattern("using true1 returned 0")

        # make sure reboot is used when reboot action is advertised
        test = self.new_test("with_reboot_support",
                             "Verify reboot action can be used when metadata advertises it")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')
        test.add_cmd("stonith_admin", args="--output-as=xml -B node1 -t 5 -V")
        test.add_log_pattern("does not advertise support for 'reboot', performing 'off'",
                             negative=True)
        test.add_log_pattern("using true1 returned 0")

        # make sure all fencing delays are applied correctly and taken into account by fencing timeouts with topology
        test = self.new_test("topology_delays",
                             "Verify all fencing delays are applied correctly and taken into account by fencing timeouts with topology")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3" -o pcmk_delay_base=1')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list=node1 node2 node3" -o pcmk_delay_base=1')
        # Resulting "random" delay will always be 1 since (rand() % (delay_max - delay_base)) is always 0 here
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3" -o pcmk_delay_base=1 -o pcmk_delay_max=2')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o "pcmk_host_list=node1 node2 node3"')

        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node3 -i 2 -v true3")

        test.add_cmd("stonith_admin", args="--output-as=xml -F node3 --delay 1")

        # Total fencing timeout takes all fencing delays into account
        test.add_log_pattern("Total timeout set to 582s")

        # Fencing timeout for the first device takes the requested fencing delay
        # and pcmk_delay_base into account
        test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using true1 .*146s.*",
                             regex=True)
        # Requested fencing delay is applied only for the first device in the
        # first level, with the static delay from pcmk_delay_base added
        test.add_log_pattern("Delaying 'off' action targeting node3 using true1 for 2s | timeout=120s requested_delay=1s base=1s max=1s")

        # Fencing timeout no longer takes the requested fencing delay into account for further devices
        test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using false1 .*145s.*",
                             regex=True)
        # Requested fencing delay is no longer applied for further devices
        test.add_log_pattern("Delaying 'off' action targeting node3 using false1 for 1s | timeout=120s requested_delay=0s base=1s max=1s")

        # Fencing timeout takes pcmk_delay_max into account
        test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using true2 .*146s.*",
                             regex=True)
        test.add_log_pattern("Delaying 'off' action targeting node3 using true2 for 1s | timeout=120s requested_delay=0s base=1s max=2s")

        test.add_log_pattern("Delaying 'off' action targeting node3 using true3",
                             negative=True)

    def build_unfence_tests(self):
        """Register tests that verify unfencing."""
        our_uname = localname()

        # verify unfencing using automatic unfencing
        test = self.new_test("unfence_required_1",
                             "Verify require unfencing on all devices when automatic=true in agent's metadata")
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname}"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname}"')
        test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3")
        # both devices should be executed
        test.add_log_pattern("using true1 returned 0")
        test.add_log_pattern("using true2 returned 0")

        # verify unfencing using automatic unfencing fails if any of the required agents fail
        test = self.new_test("unfence_required_2",
                             "Verify require unfencing on all devices when automatic=true in agent's metadata")
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname}"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=fail -o "pcmk_host_list={our_uname}"')
        test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 6", expected_exitcode=ExitStatus.ERROR)

        # verify unfencing using automatic devices with topology
        test = self.new_test("unfence_required_3",
                             "Verify require unfencing on all devices even when at different topology levels")
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 1 -v true1")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v true2")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3")
        test.add_log_pattern("using true1 returned 0")
        test.add_log_pattern("using true2 returned 0")

        # verify unfencing using automatic devices with topology
        test = self.new_test("unfence_required_4",
                             "Verify all required devices are executed even with topology levels fail")
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true1 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true2 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true3 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R true4 -a fence_dummy_auto_unfence -o mode=pass -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R false1 -a fence_dummy -o mode=fail -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R false2 -a fence_dummy -o mode=fail -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R false3 -a fence_dummy -o mode=fail -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd('stonith_admin',
                     args=f'--output-as=xml -R false4 -a fence_dummy -o mode=fail -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 1 -v true1")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 1 -v false1")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v false2")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v true2")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v false3")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v true3")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 3 -v false4")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 4 -v true4")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3")
        test.add_log_pattern("using true1 returned 0")
        test.add_log_pattern("using true2 returned 0")
        test.add_log_pattern("using true3 returned 0")
        test.add_log_pattern("using true4 returned 0")

    def build_unfence_on_target_tests(self):
        """Register tests that verify unfencing that runs on the target."""
        our_uname = localname()

        # verify unfencing using on_target device
        test = self.new_test("unfence_on_target_1",
                             "Verify unfencing with on_target = true")
        test.add_cmd("stonith_admin",
                     args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname}"')
        test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3")
        test.add_log_pattern("(on) to be executed on target")

        # verify failure of unfencing using on_target device
        test = self.new_test("unfence_on_target_2",
                             "Verify failure unfencing with on_target = true")
        test.add_cmd("stonith_admin",
                     args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node_fake_1234"')
        test.add_cmd("stonith_admin", args="--output-as=xml -U node_fake_1234 -t 3", expected_exitcode=ExitStatus.NOSUCH)
        test.add_log_pattern("(on) to be executed on target")

        # verify unfencing using on_target device with topology
        test = self.new_test("unfence_on_target_3",
                             "Verify unfencing with on_target = true using topology")

        test.add_cmd("stonith_admin",
                     args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node3"')
        test.add_cmd("stonith_admin",
                     args=f'--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node3"')

        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 1 -v true1")
        test.add_cmd("stonith_admin", args=f"--output-as=xml -r {our_uname} -i 2 -v true2")

        test.add_cmd("stonith_admin", args=f"--output-as=xml -U {our_uname} -t 3")
        test.add_log_pattern("(on) to be executed on target")

        # verify unfencing using on_target device with topology fails when target node doesn't exist
        test = self.new_test("unfence_on_target_4",
                             "Verify unfencing failure with on_target = true using topology")
        test.add_cmd("stonith_admin",
                     args=f'--output-as=xml -R true1 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node_fake"')
        test.add_cmd("stonith_admin",
                     args=f'--output-as=xml -R true2 -a fence_dummy -o mode=pass -o "pcmk_host_list={our_uname} node_fake"')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v true1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 2 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -U node_fake -t 3", expected_exitcode=ExitStatus.NOSUCH)
        test.add_log_pattern("(on) to be executed on target")

    def build_remap_tests(self):
        """Register tests that verify remapping of reboots to off-on."""
        test = self.new_test("remap_simple",
                             "Verify sequential topology reboot is remapped to all-off-then-all-on")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake '
                          '-o pcmk_off_timeout=1 -o pcmk_reboot_timeout=10')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake '
                          '-o pcmk_off_timeout=2 -o pcmk_reboot_timeout=20')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v true1 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5")
        test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
        # timeout should be sum of off timeouts (1+2=3), not reboot timeouts (10+20=30)
        test.add_log_pattern("Total timeout set to 3s for peer's fencing targeting node_fake")
        test.add_log_pattern("perform 'off' action targeting node_fake using true1")
        test.add_log_pattern("perform 'off' action targeting node_fake using true2")
        test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'")
        # fence_dummy sets "on" as an on_target action
        test.add_log_pattern("Ignoring true1 'on' failure (no capable peers) targeting node_fake")
        test.add_log_pattern("Ignoring true2 'on' failure (no capable peers) targeting node_fake")
        test.add_log_pattern("Undoing remap of reboot targeting node_fake")

        test = self.new_test("remap_simple_off",
                             "Verify sequential topology reboot skips 'on' if "
                             "pcmk_reboot_action=off or agent doesn't support "
                             "'on'")
        test.add_cmd("stonith_admin",
                     args="--output-as=xml -R true1 -a fence_dummy -o mode=pass "
                          "-o pcmk_host_list=node_fake -o pcmk_off_timeout=1 "
                          "-o pcmk_reboot_timeout=10 -o pcmk_reboot_action=off")
        test.add_cmd("stonith_admin",
                     args="--output-as=xml -R true2 -a fence_dummy_no_on "
                          "-o mode=pass -o pcmk_host_list=node_fake "
                          "-o pcmk_off_timeout=2 -o pcmk_reboot_timeout=20")
        test.add_cmd("stonith_admin",
                     args="--output-as=xml -r node_fake -i 1 -v true1 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5")
        test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
        # timeout should be sum of off timeouts (1+2=3), not reboot timeouts (10+20=30)
        test.add_log_pattern("Total timeout set to 3s for peer's fencing targeting node_fake")
        test.add_log_pattern("perform 'off' action targeting node_fake using true1")
        test.add_log_pattern("perform 'off' action targeting node_fake using true2")
        test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'")
        # "on" should be skipped
        test.add_log_pattern("Not turning node_fake back on using "
                             "true1 because the device is configured "
                             "to stay off")
        test.add_log_pattern("Not turning node_fake back on using true2"
                             " because the agent doesn't support 'on'")
        test.add_log_pattern("Undoing remap of reboot targeting node_fake")

        test = self.new_test("remap_automatic",
                             "Verify remapped topology reboot skips automatic 'on'")
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true1 -a fence_dummy_auto_unfence '
                          '-o mode=pass -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin",
                     args='--output-as=xml -R true2 -a fence_dummy_auto_unfence '
                          '-o "mode=pass" -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v true1 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5")
        test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
        test.add_log_pattern("perform 'off' action targeting node_fake using true1")
        test.add_log_pattern("perform 'off' action targeting node_fake using true2")
        test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'")
        test.add_log_pattern("Undoing remap of reboot targeting node_fake")
        test.add_log_pattern("perform 'on' action targeting node_fake using",
                             negative=True)
        test.add_log_pattern("'on' failure",
                             negative=True)

        test = self.new_test("remap_complex_1",
                             "Verify remapped topology reboot in second level works if non-remapped first level fails")
        test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 2 -v true1 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5")
        test.add_log_pattern("perform 'reboot' action targeting node_fake using false1")
        test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
        test.add_log_pattern("perform 'off' action targeting node_fake using true1")
        test.add_log_pattern("perform 'off' action targeting node_fake using true2")
        test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'")
        test.add_log_pattern("Ignoring true1 'on' failure (no capable peers) targeting node_fake")
        test.add_log_pattern("Ignoring true2 'on' failure (no capable peers) targeting node_fake")
        test.add_log_pattern("Undoing remap of reboot targeting node_fake")

        test = self.new_test("remap_complex_2",
                             "Verify remapped topology reboot failure in second level proceeds to third level")
        test.add_cmd("stonith_admin", args='--output-as=xml -R false1 -a fence_dummy -o mode=fail -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args='--output-as=xml -R false2 -a fence_dummy -o mode=fail -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args='--output-as=xml -R true1 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args='--output-as=xml -R true2 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args='--output-as=xml -R true3 -a fence_dummy -o mode=pass -o pcmk_host_list=node_fake')
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 1 -v false1")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 2 -v true1 -v false2 -v true3")
        test.add_cmd("stonith_admin", args="--output-as=xml -r node_fake -i 3 -v true2")
        test.add_cmd("stonith_admin", args="--output-as=xml -B node_fake -t 5")
        test.add_log_pattern("perform 'reboot' action targeting node_fake using false1")
        test.add_log_pattern("Remapping multiple-device reboot targeting node_fake")
        test.add_log_pattern("perform 'off' action targeting node_fake using true1")
        test.add_log_pattern("perform 'off' action targeting node_fake using false2")
        test.add_log_pattern("Attempted to execute agent fence_dummy (off) the maximum number of times")
        test.add_log_pattern("Undoing remap of reboot targeting node_fake")
        test.add_log_pattern("perform 'reboot' action targeting node_fake using true2")
        test.add_log_pattern("node_fake with true3",
                             negative=True)

    def build_query_tests(self):
        """Run stonith_admin --metadata for the fence_dummy agent and check command output."""
        test = self.new_test("get_metadata",
                             "Run stonith_admin --metadata for the fence_dummy agent")
        test.add_cmd("stonith_admin", args="--output-as=xml -a fence_dummy --metadata",
                     stdout_match='<shortdesc lang')

    def build_metadata_tests(self):
        """Run fence-agents coming with pacemaker with -o metadata and check for valid xml."""
        test = self.new_test("check_metadata_dummy",
                             "Run fence_dummy -o metadata and check for valid xml")
        test.add_cmd("fence_dummy", args="-o metadata", check_rng=False, check_stderr=False)
        # fence_dummy prints on stderr to check that tools just listen on stdout

        test = self.new_test("check_metadata_watchdog",
                             "Run fence_watchdog -o metadata and check for valid xml")
        test.add_cmd("fence_watchdog", args="-o metadata", check_rng=False)

    def build_validate_tests(self):
        """Run stonith_admin --validate for the fence_dummy agent and check command output."""
        test = self.new_test("validate_dummy",
                             "Run stonith_admin --validate-all and check output")
        test.add_cmd("stonith_admin", args="-a fence_dummy --validate --output-as=xml",
                     expected_exitcode=ExitStatus.ERROR)
        test.add_cmd("stonith_admin", args='-a fence_dummy --validate -o delay=5 --output-as=xml', check_rng=False)
        test.add_cmd("stonith_admin", args='-a fence_dummy --validate -o delay=15 --output-as=xml',
                     expected_exitcode=ExitStatus.ERROR)

    def setup_environment(self):
        """Prepare the host before executing any tests."""
        self._corosync.start(kill_first=True)
        subprocess.call(["cts-support", "install"])

    def cleanup_environment(self):
        """Clean up the host after executing desired tests."""
        self._corosync.stop()
        subprocess.call(["cts-support", "uninstall"])


def build_options():
    """Handle command line arguments."""
    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
                                     description="Run pacemaker-fenced regression tests",
                                     epilog="Example: Run only the test 'start_stop'\n"
                                            f"\t {sys.argv[0]} --run-only start_stop\n\n"
                                            "Example: Run only the tests with the string 'systemd' present in them\n"
                                            f"\t {sys.argv[0]} --run-only-pattern systemd")
    parser.add_argument("-l", "--list-tests", action="store_true",
                        help="Print out all registered tests")
    parser.add_argument("-p", "--run-only-pattern", metavar='PATTERN',
                        help="Run only tests matching the given pattern")
    parser.add_argument("-r", "--run-only", metavar='TEST',
                        help="Run a specific test")
    parser.add_argument("-t", "--timeout", type=float, default=2,
                        help="Up to how many seconds each test case waits for the daemon to "
                             "be initialized.  Defaults to 2.  The value 0 means no limit.")
    parser.add_argument("-w", "--force-wait", action="store_true",
                        help="Each test case waits the default/specified --timeout for the "
                             "daemon without tracking the log")
    parser.add_argument("-V", "--verbose", action="store_true",
                        help="Verbose output")

    args = parser.parse_args()
    return args


def main():
    """Run fencing regression tests as specified by arguments."""
    update_path()

    # Ensure all command output is in portable locale for comparison
    os.environ['LC_ALL'] = "C"

    opts = build_options()

    exit_if_proc_running("pacemaker-fenced")
    exit_if_proc_running("corosync")

    # Create a temporary directory for log files (the directory and its
    # contents will automatically be erased when done)
    with tempfile.TemporaryDirectory(prefix="cts-fencing-") as logdir:
        tests = FenceTests(verbose=opts.verbose, timeout=opts.timeout,
                           force_wait=opts.force_wait, logdir=logdir)

        tests.build_standalone_tests()
        tests.build_custom_timeout_tests()
        tests.build_api_sanity_tests()
        tests.build_fence_merge_tests()
        tests.build_fence_no_merge_tests()
        tests.build_unfence_tests()
        tests.build_unfence_on_target_tests()
        tests.build_remap_tests()
        tests.build_query_tests()
        tests.build_metadata_tests()
        tests.build_validate_tests()

        if opts.list_tests:
            tests.print_list()
            sys.exit(ExitStatus.OK)

        print("Starting ...")

        try:
            tests.setup_environment()
        except TimeoutError:
            print("corosync did not start in time, exiting")
            sys.exit(ExitStatus.TIMEOUT)

        if opts.run_only_pattern:
            tests.run_tests_matching(opts.run_only_pattern)
            tests.print_results()
        elif opts.run_only:
            tests.run_single(opts.run_only)
            tests.print_results()
        else:
            tests.run_tests()
            tests.print_results()

        tests.cleanup_environment()

    tests.exit()


if __name__ == "__main__":
    main()