File: testing.html

package info (click to toggle)
petsc 3.23.1%2Bdfsg1-1exp1
links: PTS, VCS
area: main
in suites: experimental
size: 515,576 kB
sloc: ansic: 751,607; cpp: 51,542; python: 38,598; f90: 17,352; javascript: 3,493; makefile: 3,157; sh: 1,502; xml: 619; objc: 445; java: 13; csh: 1
file content (1677 lines) | stat: -rw-r--r-- 116,192 bytes

<!DOCTYPE html>


<html lang="en" data-content_root="../" >

  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />

    <title>PETSc Testing System &#8212; PETSc 3.23.1 documentation</title>
  
  
  
  <script data-cfasync="false">
    document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
    document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
  </script>
  
  <!-- Loaded before other Sphinx assets -->
  <link href="../_static/styles/theme.css?digest=bd9e20870c6007c4c509" rel="stylesheet" />
<link href="../_static/styles/bootstrap.css?digest=bd9e20870c6007c4c509" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=bd9e20870c6007c4c509" rel="stylesheet" />

  
  <link href="../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=bd9e20870c6007c4c509" rel="stylesheet" />
  <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" />

    <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
    <link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
    <link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=87e54e7c" />
    <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.css" />
    <link rel="stylesheet" type="text/css" href="../_static/katex-math.css?v=91adb8b6" />
    <link rel="stylesheet" type="text/css" href="../_static/css/custom.css?v=dbe1606d" />
  
  <!-- Pre-loaded scripts that we'll load fully later -->
  <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=bd9e20870c6007c4c509" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=bd9e20870c6007c4c509" />
  <script src="../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=bd9e20870c6007c4c509"></script>

    <script src="../_static/documentation_options.js?v=34da53a5"></script>
    <script src="../_static/doctools.js?v=9a2dae69"></script>
    <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
    <script src="../_static/clipboard.min.js?v=a7894cd8"></script>
    <script src="../_static/copybutton.js?v=a56c686a"></script>
    <script src="../_static/design-tabs.js?v=f930bc37"></script>
    <script src="../_static/katex.min.js?v=be8ff15f"></script>
    <script src="../_static/auto-render.min.js?v=ad136472"></script>
    <script src="../_static/katex_autorenderer.js?v=bebc588a"></script>
    <script>DOCUMENTATION_OPTIONS.pagename = 'developers/testing';</script>
    <link rel="icon" href="../_static/petsc_favicon.png"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="Developing PETSc Documentation" href="documentation.html" />
    <link rel="prev" title="BuildSystem" href="buildsystem.html" />
  <meta name="viewport" content="width=device-width, initial-scale=1"/>
  <meta name="docsearch:language" content="en"/>
    <meta name="docbuild:last-update" content="2025-04-30T13:10:40-0500 (v3.23.1)"/>
  </head>
  
  
  <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">

  
  
  <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a>
  
  <div id="pst-scroll-pixel-helper"></div>

  
  <button type="button" class="btn rounded-pill" id="pst-back-to-top">
    <i class="fa-solid fa-arrow-up"></i>
    Back to top
  </button>

  
  <input type="checkbox"
          class="sidebar-toggle"
          name="__primary"
          id="__primary"/>
  <label class="overlay overlay-primary" for="__primary"></label>
  
  <input type="checkbox"
          class="sidebar-toggle"
          name="__secondary"
          id="__secondary"/>
  <label class="overlay overlay-secondary" for="__secondary"></label>
  
  <div class="search-button__wrapper">
    <div class="search-button__overlay"></div>
    <div class="search-button__search-container">
<form class="bd-search d-flex align-items-center"
      action="../search.html"
      method="get">
  <i class="fa-solid fa-magnifying-glass"></i>
  <input type="search"
         class="form-control"
         name="q"
         id="search-input"
         placeholder="Search the docs ..."
         aria-label="Search the docs ..."
         autocomplete="off"
         autocorrect="off"
         autocapitalize="off"
         spellcheck="false"/>
  <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form></div>
  </div>

  <header>
  
    <div class="bd-header navbar navbar-expand-lg bd-navbar">
<div class="bd-header__inner bd-page-width">
  <label class="sidebar-toggle primary-toggle" for="__primary">
    <span class="fa-solid fa-bars"></span>
  </label>
  
  
  <div class="col-lg-3 navbar-header-items__start">
    
      <div class="navbar-item">

  

<a class="navbar-brand logo" href="../index.html">
  
  
  
  
  
    
    
      
    
    
    <img src="../_static/PETSc-TAO_RGB.svg" class="logo__image only-light" alt="PETSc 3.23.1 documentation - Home"/>
    <script>document.write(`<img src="../_static/PETSc-TAO_RGB_white.svg" class="logo__image only-dark" alt="PETSc 3.23.1 documentation - Home"/>`);</script>
  
  
</a></div>
    
  </div>
  
  <div class="col-lg-9 navbar-header-items">
    
    <div class="me-auto navbar-header-items__center">
      
        <div class="navbar-item">
<nav class="navbar-nav">
  <ul class="bd-navbar-elements navbar-nav">
    
                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../overview/index.html">
                        Overview
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../install/index.html">
                        Install
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../tutorials/index.html">
                        Tutorials
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../manual/index.html">
                        User-Guide
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../manualpages/index.html">
                        C/Fortran API
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../petsc4py/index.html">
                        petsc4py API
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../faq/index.html">
                        FAQ
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../community/index.html">
                        Community
                      </a>
                    </li>
                

                    <li class="nav-item current active">
                      <a class="nav-link nav-internal" href="index.html">
                        Developers
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../miscellaneous/index.html">
                        Misc.
                      </a>
                    </li>
                
  </ul>
</nav></div>
      
    </div>
    
    
    <div class="navbar-header-items__end">
      
        <div class="navbar-item navbar-persistent--container">
          

 <script>
 document.write(`
   <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
    <i class="fa-solid fa-magnifying-glass"></i>
    <span class="search-button__default-text">Search</span>
    <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
   </button>
 `);
 </script>
        </div>
      
      
        <div class="navbar-item">

<script>
document.write(`
  <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
    <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
    <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
    <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
  </button>
`);
</script></div>
      
        <div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
    aria-label="Icon Links">
        <li class="nav-item">
          
          
          
          
          
          
          
          
          <a href="https://gitlab.com/petsc/petsc" title="GitLab" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fab fa-gitlab fa-lg" aria-hidden="true"></i></span>
            <span class="sr-only">GitLab</span></a>
        </li>
</ul></div>
      
    </div>
    
  </div>
  
  
    <div class="navbar-persistent--mobile">

 <script>
 document.write(`
   <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
    <i class="fa-solid fa-magnifying-glass"></i>
    <span class="search-button__default-text">Search</span>
    <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
   </button>
 `);
 </script>
    </div>
  

  
    <label class="sidebar-toggle secondary-toggle" for="__secondary" tabindex="0">
      <span class="fa-solid fa-outdent"></span>
    </label>
  
</div>

    </div>
  
  </header>

  <div class="bd-container">
    <div class="bd-container__inner bd-page-width">
      
      
      
      <div class="bd-sidebar-primary bd-sidebar">
        

  
  <div class="sidebar-header-items sidebar-primary__section">
    
    
      <div class="sidebar-header-items__center">
        
          <div class="navbar-item">
<nav class="navbar-nav">
  <ul class="bd-navbar-elements navbar-nav">
    
                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../overview/index.html">
                        Overview
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../install/index.html">
                        Install
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../tutorials/index.html">
                        Tutorials
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../manual/index.html">
                        User-Guide
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../manualpages/index.html">
                        C/Fortran API
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../petsc4py/index.html">
                        petsc4py API
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../faq/index.html">
                        FAQ
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../community/index.html">
                        Community
                      </a>
                    </li>
                

                    <li class="nav-item current active">
                      <a class="nav-link nav-internal" href="index.html">
                        Developers
                      </a>
                    </li>
                

                    <li class="nav-item">
                      <a class="nav-link nav-internal" href="../miscellaneous/index.html">
                        Misc.
                      </a>
                    </li>
                
  </ul>
</nav></div>
        
      </div>
    
    
    
      <div class="sidebar-header-items__end">
        
          <div class="navbar-item">

<script>
document.write(`
  <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
    <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
    <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
    <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
  </button>
`);
</script></div>
        
          <div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
    aria-label="Icon Links">
        <li class="nav-item">
          
          
          
          
          
          
          
          
          <a href="https://gitlab.com/petsc/petsc" title="GitLab" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fab fa-gitlab fa-lg" aria-hidden="true"></i></span>
            <span class="sr-only">GitLab</span></a>
        </li>
</ul></div>
        
      </div>
    
  </div>
  
    <div class="sidebar-primary-items__start sidebar-primary__section">
        <div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links"
     aria-label="Section Navigation">
  <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
  <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="communication.html">PETSc Developers Communication Channels</a></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="contributing/index.html">Contributing to PETSc</a><input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-1"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="contributing/developingmr.html">Developing a Merge Request</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributing/submittingmr.html">Submitting a Merge Request</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributing/pipelines.html">GitLab CI Pipelines</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="mrmanagement.html">Merge request management</a></li>
<li class="toctree-l1"><a class="reference internal" href="development.html">PETSc Development Environment</a></li>
<li class="toctree-l1"><a class="reference internal" href="style.html">PETSc Style and Usage Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="buildsystem.html">BuildSystem</a></li>
<li class="toctree-l1 current active"><a class="current reference internal" href="#">PETSc Testing System</a></li>
<li class="toctree-l1"><a class="reference internal" href="documentation.html">Developing PETSc Documentation</a></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="design.html">The Design of PETSc</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-2"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="kernel.html">The PETSc Kernel</a></li>
<li class="toctree-l2"><a class="reference internal" href="objects.html">Basic Object Design and Implementation</a></li>
<li class="toctree-l2"><a class="reference internal" href="callbacks.html">How the Solvers Handle User Provided Callbacks</a></li>
<li class="toctree-l2"><a class="reference internal" href="matrices.html">The Various Matrix Classes</a></li>
<li class="toctree-l2"><a class="reference internal" href="articles.html">Articles about PETSc Design</a></li>
</ul>
</li>
</ul>
</div>
</nav></div>
    </div>
  
  
  <div class="sidebar-primary-items__end sidebar-primary__section">
  </div>
  
  <div id="rtd-footer-container"></div>


      </div>
      
      <main id="main-content" class="bd-main">
        
        
          <div class="bd-content">
            <div class="bd-article-container">
              
              <div class="bd-header-article">
<div class="header-article-items header-article__inner">
  
    <div class="header-article-items__start">
      
        <div class="header-article-item">



<nav aria-label="Breadcrumb">
  <ul class="bd-breadcrumbs">
    
    <li class="breadcrumb-item breadcrumb-home">
      <a href="../index.html" class="nav-link" aria-label="Home">
        <i class="fa-solid fa-home"></i>
      </a>
    </li>
    
    <li class="breadcrumb-item"><a href="index.html" class="nav-link">Developers</a></li>
    
    <li class="breadcrumb-item active" aria-current="page">PETSc Testing System</li>
  </ul>
</nav>
</div>
      
    </div>
  
  
</div>
</div>
              
              
              
                
<div id="searchbox"></div>
                <article class="bd-article">
                  
  <section class="tex2jax_ignore mathjax_ignore" id="petsc-testing-system">
<span id="test-harness"></span><h1>PETSc Testing System<a class="headerlink" href="#petsc-testing-system" title="Link to this heading">#</a></h1>
<p>The PETSc test system consists of</p>
<ul class="simple">
<li><p>Formatted comments at the bottom of the tutorials and test source files that describes the tests to be run.</p></li>
<li><p>The <em>test generator</em> (<code class="docutils notranslate"><span class="pre">config/gmakegentest.py</span></code>) that parses the tutorial and test source files and generates the makefiles and shell scripts. This is run
automatically by the make system and rarely is run directly.</p></li>
<li><p>The <em>PETSc test harness</em> that consists of makefile and shell scripts that runs the executables with several logging and reporting features.</p></li>
</ul>
<p>Details on using the harness may be found in the <a class="reference internal" href="../manual/tests.html#sec-runningtests"><span class="std std-ref">user’s manual</span></a>. The testing system is used by <a class="reference internal" href="contributing/pipelines.html#pipelines"><span class="std std-ref">GitLab CI Pipelines</span></a>.</p>
<section id="petsc-test-description-language">
<h2>PETSc Test Description Language<a class="headerlink" href="#petsc-test-description-language" title="Link to this heading">#</a></h2>
<p>PETSc tests and tutorials contain at the bottom of the their source files a simple language to
describe tests and subtests required to run executables associated with
compilation of that file. The general skeleton of the file is</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">help</span><span class="p">[]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">&quot;A simple MOAB example</span><span class="se">\n</span><span class="s">&quot;</span><span class="p">;</span>

<span class="p">...</span>
<span class="o">&lt;</span><span class="n">source</span><span class="w"> </span><span class="n">code</span><span class="o">&gt;</span>
<span class="p">...</span>

<span class="cm">/*TEST</span>
<span class="cm">   build:</span>
<span class="cm">     requires: moab</span>
<span class="cm">   testset:</span>
<span class="cm">     suffix: 1</span>
<span class="cm">     requires: !complex</span>
<span class="cm">   testset:</span>
<span class="cm">     suffix: 2</span>
<span class="cm">     args: -debug -fields v1,v2,v3</span>
<span class="cm">     test:</span>
<span class="cm">     test:</span>
<span class="cm">       args: -foo bar</span>
<span class="cm">TEST*/</span>
</pre></div>
</div>
<p>For our language, a <em>test</em> is associated with the following</p>
<ul class="simple">
<li><p>A single shell script</p></li>
<li><p>A single makefile</p></li>
<li><p>An output file that represents the <em>expected results</em>. It is also possible – though unusual – to have multiple output files for a single test</p></li>
<li><p>Two or more command tests, usually:</p>
<ul>
<li><p>one or more <code class="docutils notranslate"><span class="pre">mpiexec</span></code> tests that run the executable</p></li>
<li><p>one or more <code class="docutils notranslate"><span class="pre">diff</span></code> tests to compare output with the expected result</p></li>
</ul>
</li>
</ul>
<p>Our language also supports a <em>testset</em> that specifies either a new test
entirely or multiple executable/diff tests within a single test. At the
core, the executable/diff test combination will look something like
this:</p>
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>mpiexec<span class="w"> </span>-n<span class="w"> </span><span class="m">1</span><span class="w"> </span>../ex1<span class="w"> </span><span class="m">1</span>&gt;<span class="w"> </span>ex1.tmp<span class="w"> </span><span class="m">2</span>&gt;<span class="w"> </span>ex1.err
diff<span class="w"> </span>ex1.tmp<span class="w"> </span>output/ex1.out<span class="w"> </span><span class="m">1</span>&gt;<span class="w"> </span>diff-ex1.tmp<span class="w"> </span><span class="m">2</span>&gt;<span class="w"> </span>diff-ex1.err
</pre></div>
</div>
<p>In practice, we want to do various logging and counting by the test
harness; as are explained further below. The input language supports
simple yet flexible test control.</p>
<section id="datafiles-needed-for-some-tests">
<span id="test-harness-data"></span><h3>Datafiles needed for some tests<a class="headerlink" href="#datafiles-needed-for-some-tests" title="Link to this heading">#</a></h3>
<p>Some tests require matrices or meshes that are too large for the primary PETSc Git repository.
The repository <a class="reference external" href="https://gitlab.com/petsc/datafiles">datafiles</a> contains all the test files needed for the test suite.
To run these tests one must first clone the datafiles repository and then set the environmental variable <code class="docutils notranslate"><span class="pre">DATAFILESPATH</span></code>.
For these tests <code class="docutils notranslate"><span class="pre">requires:</span> <span class="pre">datafilespath</span></code> should be specified.</p>
</section>
<section id="runtime-language-options">
<h3>Runtime Language Options<a class="headerlink" href="#runtime-language-options" title="Link to this heading">#</a></h3>
<p>At the end of each test file, a marked comment block is
inserted to describe the test(s) to be run. The elements of the test are
done with a set of supported key words that sets up the test.</p>
<p>The goals of the language are to be</p>
<ul class="simple">
<li><p>as minimal as possible with the simplest test requiring only one keyword,</p></li>
<li><p>independent of the filename such that a file can be renamed without rewriting the tests, and</p></li>
<li><p>intuitive.</p></li>
</ul>
<p>In order to enable the second goal, the <em>basestring</em> of the filename is
defined as the filename without the extension; for example, if the
filename is <code class="docutils notranslate"><span class="pre">ex1.c</span></code>, then <code class="docutils notranslate"><span class="pre">basestring=ex1</span></code>.</p>
<p>With this background, these keywords are as follows.</p>
<ul>
<li><p><strong>testset</strong> or <strong>test</strong>: (<em>Required</em>)</p>
<ul class="simple">
<li><p>At the top level either a single test or a test set must be
specified. All other keywords are sub-entries of this keyword.</p></li>
</ul>
</li>
<li><p><strong>suffix</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">suffix=&quot;&quot;</span></code>)</p>
<ul class="simple">
<li><p>The test name is given by <code class="docutils notranslate"><span class="pre">testname</span> <span class="pre">=</span> <span class="pre">basestring</span></code> if the suffix
is set to an empty string, and by
<code class="docutils notranslate"><span class="pre">testname</span> <span class="pre">=</span> <span class="pre">basestring</span> <span class="pre">+</span> <span class="pre">&quot;_&quot;</span> <span class="pre">+</span> <span class="pre">suffix</span></code> otherwise.</p></li>
<li><p>This can be specified only for top level test nodes.</p></li>
</ul>
</li>
<li><p><strong>output_file</strong>: (<em>Optional</em>; <em>Default:</em>
<code class="docutils notranslate"><span class="pre">output_file</span> <span class="pre">=</span> <span class="pre">&quot;output/&quot;</span> <span class="pre">+</span> <span class="pre">testname</span> <span class="pre">+</span> <span class="pre">&quot;.out&quot;</span></code>)</p>
<ul class="simple">
<li><p>The output of the test is to be compared with an <em>expected result</em>
whose name is given by <code class="docutils notranslate"><span class="pre">output_file</span></code>.</p></li>
<li><p>This file is described relative to the source directory of the
source file and should be in the output subdirectory (for example,
<code class="docutils notranslate"><span class="pre">output/ex1.out</span></code>)</p></li>
</ul>
</li>
<li><p><strong>nsize</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">nsize=1</span></code>)</p>
<ul class="simple">
<li><p>This integer is passed to mpiexec; i.e., <code class="docutils notranslate"><span class="pre">mpiexec</span> <span class="pre">-n</span> <span class="pre">nsize</span></code></p></li>
</ul>
</li>
<li><p><strong>args</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul class="simple">
<li><p>These arguments are passed to the executable.</p></li>
</ul>
</li>
<li><p><strong>diff_args</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul class="simple">
<li><p>These arguments are passed to the <code class="docutils notranslate"><span class="pre">lib/petsc/bin/petscdiff</span></code> script that
is used in the diff part of the test. For example, <code class="docutils notranslate"><span class="pre">-j</span></code> enables testing
the floating point numbers.</p></li>
</ul>
</li>
<li><p><strong>TODO</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">False</span></code>)</p>
<ul class="simple">
<li><p>Setting this Boolean to True will tell the test to appear in the
test harness but report only TODO per the TAP standard. Optionally
provide a string indicating why it is todo.</p></li>
<li><p>A runscript will be generated and can easily be modified by hand
to run.</p></li>
</ul>
</li>
<li><p><strong>filter</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul class="simple">
<li><p>Sometimes only a subset of the output is meant to be tested
against the expected result. If this keyword is used, it filters
the executable output to
compare with <code class="docutils notranslate"><span class="pre">output_file</span></code>.</p></li>
<li><p>The value of this is the command to be run, for example,
<code class="docutils notranslate"><span class="pre">grep</span> <span class="pre">foo</span></code> or <code class="docutils notranslate"><span class="pre">sort</span> <span class="pre">-nr</span></code>.</p></li>
<li><p><strong>NOTE: this method of testing error output is NOT recommended. See section on</strong>
<a class="reference internal" href="#sec-testing-error-testing"><span class="std std-ref">testing errors</span></a> <strong>instead.</strong> If the filter begins
with <code class="docutils notranslate"><span class="pre">Error:</span></code>, then the test is assumed to be testing the <code class="docutils notranslate"><span class="pre">stderr</span></code> output, and the
error code and output are set up to be tested.</p></li>
</ul>
</li>
<li><p><strong>filter_output</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul class="simple">
<li><p>Sometimes filtering the output file is useful for standardizing
tests. For example, in order to handle the issues related to
parallel output, both the output from the test example and the
output file need to be sorted (since sort does not produce the
same output on all machines). This works the same as filter to
implement this feature</p></li>
</ul>
</li>
<li><p><strong>localrunfiles</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul class="simple">
<li><p>Some tests
require runtime files that are maintained in the source tree.
Files in this (space-delimited) list will be copied over to the
testing directory so they will be found by the executable. If you
list a directory instead of files, it will copy the entire
directory (this is limited currently to a single directory)</p></li>
<li><p>The copying is done by the test generator and not by creating
makefile dependencies.</p></li>
</ul>
</li>
<li><p><strong>temporaries</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul class="simple">
<li><p>Some tests produce temporary files that are read by the filter
to compare to expected results.
Files in this (space-delimited) list will cleared before
the test is run to ensure that stale temporary files are not read.</p></li>
</ul>
</li>
<li><p><strong>requires</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul class="simple">
<li><p>This is a space-delimited list of run requirements (not build
requirements; see Build requirements below).</p></li>
<li><p>In general, the language supports <code class="docutils notranslate"><span class="pre">and</span></code> and <code class="docutils notranslate"><span class="pre">not</span></code> constructs
using <code class="docutils notranslate"><span class="pre">!</span> <span class="pre">=&gt;</span> <span class="pre">not</span></code> and <code class="docutils notranslate"><span class="pre">,</span> <span class="pre">=&gt;</span> <span class="pre">and</span></code>.</p></li>
<li><p>MPIUNI should work for all -n 1 examples so this need not be in
the requirements list.</p></li>
<li><p>Some tests require matrices or meshes contained in the
directory given by the environmental variable <code class="docutils notranslate"><span class="pre">DATAFILESPATH</span></code>.
For these tests <code class="docutils notranslate"><span class="pre">requires:</span> <span class="pre">datafilespath</span></code> is
specified. See <a class="reference internal" href="#test-harness-data"><span class="std std-ref">test harness data</span></a></p></li>
<li><p>Packages are indicated with lower-case specification, for example,
<code class="docutils notranslate"><span class="pre">requires:</span> <span class="pre">superlu_dist</span></code>.</p></li>
<li><p>Any defined variable in petscconf.h can be specified with the
<code class="docutils notranslate"><span class="pre">defined(...)</span></code> syntax, for example, <code class="docutils notranslate"><span class="pre">defined(PETSC_USE_INFO)</span></code>.</p></li>
<li><p>Any definition of the form <code class="docutils notranslate"><span class="pre">PETSC_HAVE_FOO</span></code> can just use
<code class="docutils notranslate"><span class="pre">requires:</span> <span class="pre">foo</span></code> similar to how third-party packages are handled.</p></li>
</ul>
</li>
<li><p><strong>timeoutfactor</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;1&quot;</span></code>)</p>
<ul class="simple">
<li><p>This parameter allows you to extend the default timeout for an
individual test such that the new timeout time is
<code class="docutils notranslate"><span class="pre">timeout=(default</span> <span class="pre">timeout)</span> <span class="pre">x</span> <span class="pre">(timeoutfactor)</span></code>.</p></li>
<li><p>Tests are limited to a set time that is found at the top of
<code class="docutils notranslate"><span class="pre">&quot;config/petsc_harness.sh&quot;</span></code> and can be overwritten by passing in
the <code class="docutils notranslate"><span class="pre">TIMEOUT</span></code> argument to <code class="docutils notranslate"><span class="pre">gmakefile</span></code></p></li>
</ul>
</li>
<li><p><strong>env</strong>: (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">env=&quot;&quot;</span></code>)</p>
<ul>
<li><p>Allows you to set environment variables for the test. Values are copied verbatim to
the runscript and defined and exported prior to all other variables.</p></li>
<li><p>Variables defined within <code class="docutils notranslate"><span class="pre">env:</span></code> blocks are expanded and processed by the shell that
runs the runscript. No prior preprocessing (other than splitting the lines into
separate declarations) is done. This means that any escaping of special characters
must be done in the text of the <code class="docutils notranslate"><span class="pre">TEST</span></code> block.</p></li>
<li><p>Defining the <code class="docutils notranslate"><span class="pre">env:</span></code> keyword more than once is allowed. Subsequent declarations are
then appended to prior list of declarations . Multiple environment variables may also
be defined in the same <code class="docutils notranslate"><span class="pre">env:</span></code> block, i.e. given a test <code class="docutils notranslate"><span class="pre">ex1.c</span></code> with the following
spec:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">test</span><span class="p">:</span>
<span class="w">  </span><span class="nt">env</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">FOO=1 BAR=1</span>

<span class="c1"># equivalently</span>
<span class="nt">test</span><span class="p">:</span>
<span class="w">  </span><span class="nt">env</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">FOO=1</span>
<span class="w">  </span><span class="nt">env</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">BAR=1</span>
</pre></div>
</div>
<p>results in</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><span class="nv">FOO</span><span class="o">=</span><span class="m">1</span><span class="p">;</span><span class="w"> </span><span class="nb">export</span><span class="w"> </span><span class="nv">BAR</span><span class="o">=</span><span class="m">1</span><span class="p">;</span><span class="w"> </span>./ex1
</pre></div>
</div>
</li>
<li><p>Variables defined in an <code class="docutils notranslate"><span class="pre">env:</span></code> block are evaluated by the runscript in the order in
which they are defined in the <code class="docutils notranslate"><span class="pre">TEST</span></code> block. Thus it is possible for later variables
to refer to previously defined ones:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">test</span><span class="p">:</span>
<span class="w">  </span><span class="nt">env</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">FOO=&#39;hello&#39; BAR=${FOO}</span>
</pre></div>
</div>
<p>results in</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><span class="nv">FOO</span><span class="o">=</span><span class="s1">&#39;hello&#39;</span><span class="p">;</span><span class="w"> </span><span class="nb">export</span><span class="w"> </span><span class="nv">BAR</span><span class="o">=</span><span class="si">${</span><span class="nv">FOO</span><span class="si">}</span><span class="p">;</span><span class="w"> </span>./ex1
<span class="gp"># </span>expanded<span class="w"> </span>by<span class="w"> </span>shell<span class="w"> </span>to
<span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><span class="nv">FOO</span><span class="o">=</span><span class="s1">&#39;hello&#39;</span><span class="p">;</span><span class="w"> </span><span class="nb">export</span><span class="w"> </span><span class="nv">BAR</span><span class="o">=</span><span class="s1">&#39;hello&#39;</span><span class="p">;</span><span class="w"> </span>./ex1
</pre></div>
</div>
<p>Note this also implies that</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">test</span><span class="p">:</span>
<span class="w">  </span><span class="nt">env</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">FOO=1 FOO=0</span>
</pre></div>
</div>
<p>results in</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><span class="nv">FOO</span><span class="o">=</span><span class="m">1</span><span class="p">;</span><span class="w"> </span><span class="nb">export</span><span class="w"> </span><span class="nv">FOO</span><span class="o">=</span><span class="m">0</span><span class="p">;</span><span class="w"> </span>./ex1
</pre></div>
</div>
</li>
</ul>
</li>
</ul>
</section>
<section id="additional-specifications">
<h3>Additional Specifications<a class="headerlink" href="#additional-specifications" title="Link to this heading">#</a></h3>
<p>In addition to the above keywords, other language features are
supported.</p>
<ul>
<li><p><strong>for loops</strong>: Specifying <code class="docutils notranslate"><span class="pre">{{list</span> <span class="pre">of</span> <span class="pre">values}}</span></code> will generate a loop over
an enclosed space-delimited list of values.
It is supported within <code class="docutils notranslate"><span class="pre">nsize</span></code> and <code class="docutils notranslate"><span class="pre">args</span></code>. For example,</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="nl">nsize</span><span class="p">:</span><span class="w"> </span><span class="p">{{</span><span class="mi">1</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="mi">4</span><span class="p">}}</span>
<span class="nl">args</span><span class="p">:</span><span class="w"> </span><span class="o">-</span><span class="n">matload_block_size</span><span class="w"> </span><span class="p">{{</span><span class="mi">2</span><span class="w"> </span><span class="mi">3</span><span class="p">}</span><span class="n">shared</span><span class="w"> </span><span class="n">output</span><span class="p">}</span>
</pre></div>
</div>
<p>Here the output for each <code class="docutils notranslate"><span class="pre">-matload_block_size</span></code> value is assumed to be
the same so that only one output file is needed.</p>
<p>If the loop causes different output for each loop iteration, then <code class="docutils notranslate"><span class="pre">separate</span> <span class="pre">output</span></code> needs to be used:</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="nl">args</span><span class="p">:</span><span class="w"> </span><span class="o">-</span><span class="n">matload_block_size</span><span class="w"> </span><span class="p">{{</span><span class="mi">2</span><span class="w"> </span><span class="mi">3</span><span class="p">}</span><span class="n">separate</span><span class="w"> </span><span class="n">output</span><span class="p">}</span>
</pre></div>
</div>
<p>In this case, each loop value generates a separate script,
and uses a separate output file for comparison.</p>
<p>Note that <code class="docutils notranslate"><span class="pre">{{...}}</span></code> is equivalent to <code class="docutils notranslate"><span class="pre">{{...}shared</span> <span class="pre">output}</span></code>.</p>
</li>
</ul>
</section>
<section id="testing-errors-and-exceptional-code">
<span id="sec-testing-error-testing"></span><h3>Testing Errors And Exceptional Code<a class="headerlink" href="#testing-errors-and-exceptional-code" title="Link to this heading">#</a></h3>
<p>It is possible (and encouraged!) to test error conditions within the test harness. Since
error messages produced by <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/SETERRQ.html">SETERRQ</a>()</span></code> and friends are not portable between systems,
additional arguments must be passed to tests to modify error handling, specifically:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-petsc_ci_portable_error_output -error_output_stdout</span>
</pre></div>
</div>
<p>These arguments have the following effect:</p>
<ul>
<li><p><code class="docutils notranslate"><span class="pre">-petsc_ci_portable_error_output</span></code>: Strips system or configuration-specific information
from error messages. Specifically this:</p>
<ul class="simple">
<li><p>Removes all path components except the file name from the traceback</p></li>
<li><p>Removes line and column numbers from the traceback</p></li>
<li><p>Removes PETSc version information</p></li>
<li><p>Removes <code class="docutils notranslate"><span class="pre">configure</span></code> options used</p></li>
<li><p>Removes system name</p></li>
<li><p>Removes hostname</p></li>
<li><p>Removes date</p></li>
</ul>
<p>With this option error messages will be identical across systems, runs, and PETSc
configurations (barring of course configurations in which the error is not raised).</p>
<p>Furthermore, this option also changes the default behavior of the error handler to
<strong>gracefully</strong> exit where possible. For single-ranked runs this means returning with
exit-code <code class="docutils notranslate"><span class="pre">0</span></code> and calling <code class="docutils notranslate"><span class="pre"><a href="http://www.mpich.org/static/docs/latest/www3/MPI_Finalize.html#MPI_Finalize">MPI_Finalize</a>()</span></code> instead of <code class="docutils notranslate"><span class="pre"><a href="http://www.mpich.org/static/docs/latest/www3/MPI_Abort.html#MPI_Abort">MPI_Abort</a>()</span></code>. Multi-rank
tests will call <code class="docutils notranslate"><span class="pre"><a href="http://www.mpich.org/static/docs/latest/www3/MPI_Abort.html#MPI_Abort">MPI_Abort</a>()</span></code> on errors raised on <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PETSC_COMM_SELF.html">PETSC_COMM_SELF</a></span></code>, but will call
<code class="docutils notranslate"><span class="pre"><a href="http://www.mpich.org/static/docs/latest/www3/MPI_Finalize.html#MPI_Finalize">MPI_Finalize</a>()</span></code> otherwise.</p>
</li>
<li><p><code class="docutils notranslate"><span class="pre">-error_output_stdout</span></code>: Forces <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/SETERRQ.html">SETERRQ</a>()</span></code> and friends to dump error messages to
<code class="docutils notranslate"><span class="pre">stdout</span></code> instead of <code class="docutils notranslate"><span class="pre">stderr</span></code>. While using <code class="docutils notranslate"><span class="pre">stderr</span></code> (alongside the <code class="docutils notranslate"><span class="pre">Error:</span></code>
sub-directive under <code class="docutils notranslate"><span class="pre">filter:</span></code>) also works it appears to be unstable under heavy
load, especially in CI.</p></li>
</ul>
<p>Using both options in tandem allows one to use the normal <code class="docutils notranslate"><span class="pre">output:</span></code> mechanism to compare
expected and actual error outputs.</p>
<p>When writing ASCII output that may be not portable, so one wants <code class="docutils notranslate"><span class="pre">-petsc_ci_portable_error_output</span></code> to
cause the output to be skipped, enclose the output with code such as</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="o">!</span><span class="n">PetscCIEnabledPortableErrorOutput</span><span class="p">)</span>
</pre></div>
</div>
<p>to prevent it from being output when the CI test harness is running.</p>
</section>
<section id="test-block-examples">
<h3>Test Block Examples<a class="headerlink" href="#test-block-examples" title="Link to this heading">#</a></h3>
<p>The following is the simplest test block:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">/*TEST</span>
<span class="w">  </span><span class="l l-Scalar l-Scalar-Plain">test</span><span class="p p-Indicator">:</span>
<span class="l l-Scalar l-Scalar-Plain">TEST*/</span>
</pre></div>
</div>
<p>If this block is in <code class="docutils notranslate"><span class="pre">src/a/b/examples/tutorials/ex1.c</span></code>, then it will
create <code class="docutils notranslate"><span class="pre">a_b_tutorials-ex1</span></code> test that requires only one
process, with no arguments, and diff the resultant output with
<code class="docutils notranslate"><span class="pre">src/a/b/examples/tutorials/output/ex1.out</span></code>.</p>
<p>For Fortran, the equivalent is</p>
<div class="highlight-fortran notranslate"><div class="highlight"><pre><span></span><span class="c">!/*TEST</span>
<span class="c">!  test:</span>
<span class="c">!TEST*/</span>
</pre></div>
</div>
<p>A more complete example, showing just the lines between <code class="docutils notranslate"><span class="pre">/*TEST</span></code> and <code class="docutils notranslate"><span class="pre">TEST*/</span></code>:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">test</span><span class="p">:</span>
<span class="nt">test</span><span class="p">:</span>
<span class="w">  </span><span class="nt">suffix</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1</span>
<span class="w">  </span><span class="nt">nsize</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">2</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-t 2 -pc_type jacobi -ksp_monitor_short -ksp_type gmres</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-ksp_gmres_cgs_refinement_type refine_always -s2_ksp_type bcgs</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-s2_pc_type jacobi -s2_ksp_monitor_short</span>
<span class="w">  </span><span class="nt">requires</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">x</span>
</pre></div>
</div>
<p>This creates two tests. Assuming that this is
<code class="docutils notranslate"><span class="pre">src/a/b/examples/tutorials/ex1.c</span></code>, the tests would be
<code class="docutils notranslate"><span class="pre">a_b_tutorials-ex1</span></code> and <code class="docutils notranslate"><span class="pre">a_b_tutorials-ex1_1</span></code>.</p>
<p>Following is an example of how to test a permutation of arguments
against the same output file:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">testset</span><span class="p">:</span>
<span class="w">  </span><span class="nt">suffix</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">19</span>
<span class="w">  </span><span class="nt">requires</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">datafilespath</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-f0 ${DATAFILESPATH}/matrices/poisson1</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-ksp_type cg -pc_type icc -pc_factor_levels 2</span>
<span class="w">  </span><span class="nt">test</span><span class="p">:</span>
<span class="w">  </span><span class="nt">test</span><span class="p">:</span>
<span class="w">    </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-mat_type seqsbaij</span>
</pre></div>
</div>
<p>Assuming that this is <code class="docutils notranslate"><span class="pre">ex10.c</span></code>, there would be two mpiexec/diff
invocations in <code class="docutils notranslate"><span class="pre">runex10_19.sh</span></code>.</p>
<p>Here is a similar example, but the permutation of arguments creates
different output:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">testset</span><span class="p">:</span>
<span class="w">  </span><span class="nt">requires</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">datafilespath</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-f0 ${DATAFILESPATH}/matrices/medium</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-ksp_type bicg</span>
<span class="w">  </span><span class="nt">test</span><span class="p">:</span>
<span class="w">    </span><span class="nt">suffix</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">4</span>
<span class="w">    </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-pc_type lu</span>
<span class="w">  </span><span class="nt">test</span><span class="p">:</span>
<span class="w">    </span><span class="nt">suffix</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5</span>
</pre></div>
</div>
<p>Assuming that this is <code class="docutils notranslate"><span class="pre">ex10.c</span></code>, two shell scripts will be created:
<code class="docutils notranslate"><span class="pre">runex10_4.sh</span></code> and <code class="docutils notranslate"><span class="pre">runex10_5.sh</span></code>.</p>
<p>An example using a for loop is:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">testset</span><span class="p">:</span>
<span class="w">  </span><span class="nt">suffix</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-f ${DATAFILESPATH}/matrices/small -mat_type aij</span>
<span class="w">  </span><span class="nt">requires</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">datafilespath</span>
<span class="nt">testset</span><span class="p">:</span>
<span class="w">  </span><span class="nt">suffix</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">2</span>
<span class="w">  </span><span class="nt">output_file</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">output/ex138_1.out</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-f ${DATAFILESPATH}/matrices/small</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-mat_type baij -matload_block_size {{2 3}shared output}</span>
<span class="w">  </span><span class="nt">requires</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">datafilespath</span>
</pre></div>
</div>
<p>In this example, <code class="docutils notranslate"><span class="pre">ex138_2</span></code> will invoke <code class="docutils notranslate"><span class="pre">runex138_2.sh</span></code> twice with
two different arguments, but both are diffed with the same file.</p>
<p>Following is an example showing the hierarchical nature of the test
specification.</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">testset</span><span class="p">:</span>
<span class="w">  </span><span class="l l-Scalar l-Scalar-Plain">suffix:2</span>
<span class="w">  </span><span class="l l-Scalar l-Scalar-Plain">output_file</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">output/ex138_1.out</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-f ${DATAFILESPATH}/matrices/small -mat_type baij</span>
<span class="w">  </span><span class="nt">test</span><span class="p">:</span>
<span class="w">    </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-matload_block_size 2</span>
<span class="w">  </span><span class="nt">test</span><span class="p">:</span>
<span class="w">    </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-matload_block_size 3</span>
</pre></div>
</div>
<p>This is functionally equivalent to the for loop shown above.</p>
<p>Here is a more complex example using for loops:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">testset</span><span class="p">:</span>
<span class="w">  </span><span class="nt">suffix</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">19</span>
<span class="w">  </span><span class="nt">requires</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">datafilespath</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-f0 ${DATAFILESPATH}/matrices/poisson1</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-ksp_type cg -pc_type icc</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-pc_factor_levels {{0 2 4}separate output}</span>
<span class="w">  </span><span class="nt">test</span><span class="p">:</span>
<span class="w">  </span><span class="nt">test</span><span class="p">:</span>
<span class="w">    </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-mat_type seqsbaij</span>
</pre></div>
</div>
<p>If this is in <code class="docutils notranslate"><span class="pre">ex10.c</span></code>, then the shell scripts generated would be</p>
<ul class="simple">
<li><p><code class="docutils notranslate"><span class="pre">runex10_19_pc_factor_levels-0.sh</span></code></p></li>
<li><p><code class="docutils notranslate"><span class="pre">runex10_19_pc_factor_levels-2.sh</span></code></p></li>
<li><p><code class="docutils notranslate"><span class="pre">runex10_19_pc_factor_levels-4.sh</span></code></p></li>
</ul>
<p>Each shell script would invoke twice.</p>
</section>
<section id="build-language-options">
<h3>Build Language Options<a class="headerlink" href="#build-language-options" title="Link to this heading">#</a></h3>
<p>You can specify issues related to the compilation of the source file
with the <code class="docutils notranslate"><span class="pre">build:</span></code> block. The language is as follows.</p>
<ul class="simple">
<li><p><strong>requires:</strong> (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul>
<li><p>Same as the runtime requirements (for example, can include
<code class="docutils notranslate"><span class="pre">requires:</span> <span class="pre">fftw</span></code>) but also requirements related to types:</p>
<ol class="arabic simple">
<li><p>Precision types: <code class="docutils notranslate"><span class="pre">single</span></code>, <code class="docutils notranslate"><span class="pre">double</span></code>, <code class="docutils notranslate"><span class="pre">quad</span></code>, <code class="docutils notranslate"><span class="pre">int32</span></code></p></li>
<li><p>Scalar types: <code class="docutils notranslate"><span class="pre">complex</span></code> (and <code class="docutils notranslate"><span class="pre">!complex</span></code>)</p></li>
</ol>
</li>
<li><p>In addition, <code class="docutils notranslate"><span class="pre">TODO</span></code> is available to allow you to skip the build
of this file but still maintain it in the source tree.</p></li>
</ul>
</li>
<li><p><strong>depends:</strong> (<em>Optional</em>; <em>Default:</em> <code class="docutils notranslate"><span class="pre">&quot;&quot;</span></code>)</p>
<ul>
<li><p>List any dependencies required to compile the file</p></li>
</ul>
</li>
</ul>
<p>A typical example for compiling for only real numbers is</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cm">/*TEST</span>
<span class="cm">  build:</span>
<span class="cm">    requires: !complex</span>
<span class="cm">  test:</span>
<span class="cm">TEST*/</span>
</pre></div>
</div>
</section>
</section>
<section id="running-the-tests">
<h2>Running the tests<a class="headerlink" href="#running-the-tests" title="Link to this heading">#</a></h2>
<p>The make rules for running tests are contained in <code class="docutils notranslate"><span class="pre">gmakefile.test</span></code> in the PETSc root directory. They can usually be accessed by
simply using commands such as</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span><span class="nb">test</span>
</pre></div>
</div>
<p>or, for a list of test options,</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>help-test
</pre></div>
</div>
<section id="determining-the-failed-jobs-of-a-given-run">
<h3>Determining the failed jobs of a given run<a class="headerlink" href="#determining-the-failed-jobs-of-a-given-run" title="Link to this heading">#</a></h3>
<p>The running of the test harness will show which tests fail, but you may not have
logged the output or run without showing the full error. The best way of
examining the errors is with this command:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span><span class="nv">$EDITOR</span><span class="w"> </span><span class="nv">$PETSC_DIR</span>/<span class="nv">$PETSC_ARCH</span>/tests/test*err.log
</pre></div>
</div>
<p>This method can also be used for the PETSc continuous integration (CI) pipeline jobs. For failed jobs you can download the
log files from the <code class="docutils notranslate"><span class="pre">artifacts</span> <span class="pre">download</span></code> tab on the right side:</p>
<figure class="align-default" id="id1">
<img alt="Test Artifacts at Gitlab" src="../_images/test-artifacts.png" />
<figcaption>
<p><span class="caption-number">Fig. 46 </span><span class="caption-text">Test artifacts can be downloaded from GitLab.</span><a class="headerlink" href="#id1" title="Link to this image">#</a></p>
</figcaption>
</figure>
<p>To see the list of all tests that failed from the last run, you can also run this command:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span>test-fail<span class="o">=</span><span class="m">1</span>
</pre></div>
</div>
<p>To print it out in a column format:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span>test-fail<span class="o">=</span><span class="m">1</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>tr<span class="w"> </span><span class="s1">&#39; &#39;</span><span class="w"> </span><span class="s1">&#39;\n&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>sort
</pre></div>
</div>
<p>Once you know which tests failed, the question is how to debug them.</p>
</section>
<section id="introduction-to-debugging-workflows">
<h3>Introduction to debugging workflows<a class="headerlink" href="#introduction-to-debugging-workflows" title="Link to this heading">#</a></h3>
<p>Here, two different workflows on developing with the test harness are presented,
and then the language for adding a new test is described. Before describing the
workflow, we first discuss the output of the test harness and how it maps onto
makefile targets and shell scripts.</p>
<p>Consider this line from running the PETSc test system:</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="n">TEST</span><span class="w"> </span><span class="n">arch</span><span class="o">-</span><span class="n">ci</span><span class="o">-</span><span class="n">linux</span><span class="o">-</span><span class="n">uni</span><span class="o">-</span><span class="n">pkgs</span><span class="o">/</span><span class="n">tests</span><span class="o">/</span><span class="n">counts</span><span class="o">/</span><span class="n">vec_is_sf_tests</span><span class="o">-</span><span class="n">ex1_basic_1</span><span class="p">.</span><span class="n">counts</span>
</pre></div>
</div>
<p>The string <code class="docutils notranslate"><span class="pre">vec_is_sf_tests-ex1_basic_1</span></code> gives the following information:</p>
<ul class="simple">
<li><p>The file generating the tests is found in <code class="docutils notranslate"><span class="pre">$PETSC_DIR/src/vec/is/sf/tests/ex1.c</span></code></p></li>
<li><p>The makefile target for the <em>test</em> is <code class="docutils notranslate"><span class="pre">vec_is_sf_tests-ex1_basic_1</span></code></p></li>
<li><p>The makefile target for the <em>executable</em> is <code class="docutils notranslate"><span class="pre">$PETSC_ARCH/tests/vec/is/sf/tests/ex1</span></code></p></li>
<li><p>The shell script running the test is located at: <code class="docutils notranslate"><span class="pre">$PETSC_DIR/$PETSC_ARCH/tests/vec/is/sf/tests/runex1_basic_1.sh</span></code></p></li>
</ul>
<p>Let’s say that you want to debug a single test as part of development. There
are two basic methods of doing this: 1) use shell script directly in test
directory, or 2) use the gmakefile.test from the top level directory. We present both
workflows.</p>
</section>
<section id="debugging-a-test-using-shell-the-generated-scripts">
<h3>Debugging a test using shell the generated scripts<a class="headerlink" href="#debugging-a-test-using-shell-the-generated-scripts" title="Link to this heading">#</a></h3>
<p>First, look at the working directory and the options for the
scripts:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span><span class="nb">cd</span><span class="w"> </span><span class="nv">$PETSC_ARCH</span>/tests/vec/is/sf/tests
<span class="gp">$ </span>./runex1_basic_1.sh<span class="w"> </span>-h
<span class="go">Usage: ./runex1_basic_1.sh [options]</span>

<span class="go">OPTIONS</span>
<span class="go">  -a &lt;args&gt; ......... Override default arguments</span>
<span class="go">  -c ................ Cleanup (remove generated files)</span>
<span class="go">  -C ................ Compile</span>
<span class="go">  -d ................ Launch in debugger</span>
<span class="go">  -e &lt;args&gt; ......... Add extra arguments to default</span>
<span class="go">  -f ................ force attempt to run test that would otherwise be skipped</span>
<span class="go">  -h ................ help: print this message</span>
<span class="go">  -n &lt;integer&gt; ...... Override the number of processors to use</span>
<span class="go">  -j ................ Pass -j to petscdiff (just use diff)</span>
<span class="go">  -J &lt;arg&gt; .......... Pass -J to petscdiff (just use diff with arg)</span>
<span class="go">  -m ................ Update results using petscdiff</span>
<span class="go">  -M ................ Update alt files using petscdiff</span>
<span class="go">  -o &lt;arg&gt; .......... Output format: &#39;interactive&#39;, &#39;err_only&#39;</span>
<span class="go">  -p ................ Print command: Print first command and exit</span>
<span class="go">  -t ................ Override the default timeout (default=60 sec)</span>
<span class="go">  -U ................ run cUda-memcheck</span>
<span class="go">  -V ................ run Valgrind</span>
<span class="go">  -v ................ Verbose: Print commands</span>
</pre></div>
</div>
<p>We will be using the <code class="docutils notranslate"><span class="pre">-C</span></code>, <code class="docutils notranslate"><span class="pre">-V</span></code>, and <code class="docutils notranslate"><span class="pre">-p</span></code> flags.</p>
<p>A basic workflow is something similar to:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>&lt;edit&gt;
<span class="gp">$ </span>runex1_basic_1.sh<span class="w"> </span>-C
<span class="gp">$ </span>&lt;edit&gt;
<span class="gp">$ </span>...
<span class="gp">$ </span>runex1_basic_1.sh<span class="w"> </span>-m<span class="w"> </span><span class="c1"># If need to update results</span>
<span class="gp">$ </span>...
<span class="gp">$ </span>runex1_basic_1.sh<span class="w"> </span>-V<span class="w"> </span><span class="c1"># Make sure valgrind clean</span>
<span class="gp">$ </span><span class="nb">cd</span><span class="w"> </span><span class="nv">$PETSC_DIR</span>
<span class="gp">$ </span>git<span class="w"> </span>commit<span class="w"> </span>-a
</pre></div>
</div>
<p>For loops it sometimes can become onerous to run the whole test.
In this case, you can use the <code class="docutils notranslate"><span class="pre">-p</span></code> flag to print just the first
command. It will print a command suitable for running from
<code class="docutils notranslate"><span class="pre">$PETSC_DIR</span></code>, but it is easy to modify for execution in the test
directory:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>runex1_basic_1.sh<span class="w"> </span>-p
</pre></div>
</div>
</section>
<section id="debugging-a-petsc-test-using-the-gmakefile-test">
<h3>Debugging a PETSc test using the gmakefile.test<a class="headerlink" href="#debugging-a-petsc-test-using-the-gmakefile-test" title="Link to this heading">#</a></h3>
<p>First recall how to find help for the options:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>help-test
<span class="go">Test usage:</span>
<span class="go">   /usr/bin/gmake --no-print-directory test &lt;options&gt;</span>

<span class="go">Options:</span>
<span class="go">  NO_RM=1           Do not remove the executables after running</span>
<span class="go">  REPLACE=1         Replace the output in PETSC_DIR source tree (-m to test scripts)</span>
<span class="go">  OUTPUT=1          Show only the errors on stdout</span>
<span class="go">  ALT=1             Replace &#39;alt&#39; output in PETSC_DIR source tree (-M to test scripts)</span>
<span class="go">  DIFF_NUMBERS=1    Diff the numbers in the output (-j to test scripts and petscdiff)</span>
<span class="go">  CUDAMEMCHECK=1    Execute the tests using cuda-memcheck (-U to test scripts)</span>
<span class="go">                    Use PETSC_CUDAMEMCHECK_COMMAND to change the executable to run and</span>
<span class="go">                    PETSC_CUDAMEMCHECK_ARGS to change the arguments (note: both</span>
<span class="go">                    cuda-memcheck and compute-sanitizer are supported)</span>
<span class="go">  VALGRIND=1        Execute the tests using valgrind (-V to test scripts)</span>
<span class="go">  DEBUG=1           Launch tests in the debugger (-d to the scripts)</span>
<span class="go">  NP=&lt;num proc&gt;     Set a number of processors to pass to scripts.</span>
<span class="go">  FORCE=1           Force SKIP or TODO tests to run</span>
<span class="go">  PRINTONLY=1       Print the command, but do not run.  For loops print first command</span>
<span class="go">  TIMEOUT=&lt;time&gt;    Test timeout limit in seconds (default in config/petsc_harness.sh)</span>
<span class="go">  TESTDIR=&#39;tests&#39;   Subdirectory where tests are run ($PETSC_DIR/$PETSC_ARCH</span>
<span class="go">                    or /</span>
<span class="go">                    or /share/petsc/examples/)</span>
<span class="go">  TESTBASE=&#39;tests&#39;   Subdirectory where tests are run ($PETSC_DIR/$PETSC_ARCH)</span>
<span class="go">  OPTIONS=&#39;&lt;args&gt;&#39;  Override options to scripts (-a to test scripts)</span>
<span class="go">  EXTRA_OPTIONS=&#39;&lt;args&gt;&#39;  Add options to scripts (-e to test scripts)</span>

<span class="go">Special options for macOS:</span>
<span class="go">  MACOS_FIREWALL=1  Add each built test to the macOS firewall list to prevent popups. Configure --with-macos-firewall-rules to make this default</span>

<span class="go">Tests can be generated by searching with multiple methods</span>
<span class="go">  For general searching (using config/query_tests.py):</span>
<span class="go">    /usr/bin/gmake --no-print-directory test search=&#39;sys*ex2*&#39;</span>
<span class="go">   or the shortcut using s</span>
<span class="go">    /usr/bin/gmake --no-print-directory test s=&#39;sys*ex2*&#39;</span>
<span class="go">  You can also use the full path to a file directory</span>
<span class="go">    /usr/bin/gmake --no-print-directory test s=&#39;src/sys/tests/&#39;</span>
<span class="go">   or a file</span>
<span class="go">    /usr/bin/gmake --no-print-directory test s=&#39;src/sys/tests/ex1.c&#39;</span>

<span class="go">  To search for fields from the original test definitions:</span>
<span class="go">    /usr/bin/gmake --no-print-directory test query=&#39;requires&#39; queryval=&#39;*MPI_PROCESS_SHARED_MEMORY*&#39;</span>
<span class="go">   or the shortcut using q and qv</span>
<span class="go">    /usr/bin/gmake --no-print-directory test q=&#39;requires&#39; qv=&#39;*MPI_PROCESS_SHARED_MEMORY*&#39;</span>
<span class="go">  To filter results from other searches, use searchin</span>
<span class="go">    /usr/bin/gmake --no-print-directory test s=&#39;src/sys/tests/&#39; searchin=&#39;*options*&#39;</span>

<span class="go">  To re-run the last tests which failed:</span>
<span class="go">    /usr/bin/gmake --no-print-directory test test-fail=&#39;1&#39;</span>

<span class="go">  To see which targets match a given pattern (useful for doing a specific target):</span>
<span class="go">    /usr/bin/gmake --no-print-directory print-test search=sys*</span>

<span class="go">  To build an executable, give full path to location:</span>
<span class="go">    /usr/bin/gmake --no-print-directory ${PETSC_ARCH}/tests/sys/tests/ex1</span>
<span class="go">  or make the test with NO_RM=1</span>
</pre></div>
</div>
<p>To compile the test and run it:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="nv">search</span><span class="o">=</span>vec_is_sf_tests-ex1_basic_1
</pre></div>
</div>
<p>This can consist of your basic workflow. However,
for the normal compile and edit, running the entire harness with search can be
cumbersome. So first get the command:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>vec_is_sf_tests-ex1_basic_1<span class="w"> </span><span class="nv">PRINTONLY</span><span class="o">=</span><span class="m">1</span>
<span class="go">&lt;copy command&gt;</span>
<span class="go">&lt;edit&gt;</span>
<span class="gp">$ </span>make<span class="w"> </span><span class="nv">$PETSC_ARCH</span>/tests/vec/is/sf/tests/ex1
<span class="gp">$ </span>/scratch/kruger/contrib/petsc-mpich-cxx/bin/mpiexec<span class="w"> </span>-n<span class="w"> </span><span class="m">1</span><span class="w"> </span>arch-mpich-cxx-py3/tests/vec/is/sf/tests/ex1
<span class="go">...</span>
<span class="gp">$ </span><span class="nb">cd</span><span class="w"> </span><span class="nv">$PETSC_DIR</span>
<span class="gp">$ </span>git<span class="w"> </span>commit<span class="w"> </span>-a
</pre></div>
</div>
</section>
<section id="advanced-searching">
<h3>Advanced searching<a class="headerlink" href="#advanced-searching" title="Link to this heading">#</a></h3>
<p>For forming a search, it is recommended to always use <code class="docutils notranslate"><span class="pre">print-test</span></code> instead of
<code class="docutils notranslate"><span class="pre">test</span></code> to make sure it is returning the values that you want.</p>
<p>The three basic and recommended arguments are:</p>
<ul>
<li><p><code class="docutils notranslate"><span class="pre">search</span></code> (or <code class="docutils notranslate"><span class="pre">s</span></code>)</p>
<ul>
<li><p>Searches based on name of test target (see above)</p></li>
<li><p>Use the familiar glob syntax (like the Unix <code class="docutils notranslate"><span class="pre">ls</span></code> command). Example:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">search</span><span class="o">=</span><span class="s1">&#39;vec_is*ex1*basic*1&#39;</span>
</pre></div>
</div>
<p>Equivalently:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">s</span><span class="o">=</span><span class="s1">&#39;vec_is*ex1*basic*1&#39;</span>
</pre></div>
</div>
</li>
<li><p>It also takes full paths. Examples:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">s</span><span class="o">=</span><span class="s1">&#39;src/vec/is/tests/ex1.c&#39;</span>
</pre></div>
</div>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">s</span><span class="o">=</span><span class="s1">&#39;src/dm/impls/plex/tests/&#39;</span>
</pre></div>
</div>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">s</span><span class="o">=</span><span class="s1">&#39;src/dm/impls/plex/tests/ex1.c&#39;</span>
</pre></div>
</div>
</li>
</ul>
</li>
<li><p><code class="docutils notranslate"><span class="pre">query</span></code> and <code class="docutils notranslate"><span class="pre">queryval</span></code> (or <code class="docutils notranslate"><span class="pre">q</span></code> and <code class="docutils notranslate"><span class="pre">qv</span></code>)</p>
<ul>
<li><p><code class="docutils notranslate"><span class="pre">query</span></code> corresponds to test harness keyword, <code class="docutils notranslate"><span class="pre">queryval</span></code> to the value. Example:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;suffix&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;basic_1&#39;</span>
</pre></div>
</div>
</li>
<li><p>Invokes <code class="docutils notranslate"><span class="pre">config/query_tests.py</span></code> to query the tests (see
<code class="docutils notranslate"><span class="pre">config/query_tests.py</span> <span class="pre">--help</span></code> for more information).</p></li>
<li><p>See below for how to use as it has many features</p></li>
</ul>
</li>
<li><p><code class="docutils notranslate"><span class="pre">searchin</span></code> (or <code class="docutils notranslate"><span class="pre">i</span></code>)</p>
<ul>
<li><p>Filters results of above searches. Example:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">s</span><span class="o">=</span><span class="s1">&#39;src/dm/impls/plex/tests/ex1.c&#39;</span><span class="w"> </span><span class="nv">i</span><span class="o">=</span><span class="s1">&#39;*refine_overlap_2d*&#39;</span>
</pre></div>
</div>
</li>
</ul>
</li>
</ul>
<p>Searching using GNU make’s native regexp functionality is kept for people who like it, but most developers will likely prefer the above methods:</p>
<ul>
<li><p><code class="docutils notranslate"><span class="pre">gmakesearch</span></code></p>
<ul>
<li><p>Use GNU make’s own filter capability.</p></li>
<li><p>Fast, but requires knowing GNU make regex syntax which uses <code class="docutils notranslate"><span class="pre">%</span></code> instead of <code class="docutils notranslate"><span class="pre">*</span></code></p></li>
<li><p>Also very limited (cannot use two <code class="docutils notranslate"><span class="pre">%</span></code>’s for example)</p></li>
<li><p>Example:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="nv">gmakesearch</span><span class="o">=</span><span class="s1">&#39;vec_is%ex1_basic_1&#39;</span>
</pre></div>
</div>
</li>
</ul>
</li>
<li><p><code class="docutils notranslate"><span class="pre">gmakesearchin</span></code></p>
<ul>
<li><p>Use GNU make’s own filter capability to search in previous results. Example:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="nv">gmakesearch</span><span class="o">=</span><span class="s1">&#39;vec_is%1&#39;</span><span class="w"> </span><span class="nv">gmakesearchin</span><span class="o">=</span><span class="s1">&#39;basic&#39;</span>
</pre></div>
</div>
</li>
</ul>
</li>
</ul>
</section>
<section id="query-based-searching">
<h3>Query-based searching<a class="headerlink" href="#query-based-searching" title="Link to this heading">#</a></h3>
<p>Note the use of glob style matching is also accepted in the value field:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;suffix&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;basic_1&#39;</span>
</pre></div>
</div>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;cuda&#39;</span>
</pre></div>
</div>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;defined(PETSC_HAVE_MPI_GPU_AWARE)&#39;</span>
</pre></div>
</div>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;*GPU_AWARE*&#39;</span>
</pre></div>
</div>
<p>Using the <code class="docutils notranslate"><span class="pre">name</span></code> field is equivalent to the search above:</p>
<ul>
<li><p>Example:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;name&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;vec_is*ex1*basic*1&#39;</span>
</pre></div>
</div>
</li>
<li><p>This can be combined with union/intersect queries as discussed below</p></li>
</ul>
<p>Arguments are tricky to search for. Consider</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>args: -ksp_monitor_short -pc_type ml -ksp_max_it 3
</pre></div>
</div>
<p>Search terms are</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>ksp_monitor, pc_type ml, ksp_max_it
</pre></div>
</div>
<p>Certain items are ignored:</p>
<ul class="simple">
<li><p>Numbers (see <code class="docutils notranslate"><span class="pre">ksp_max_it</span></code> above), but floats are ignored as well.</p></li>
<li><p>Loops: <code class="docutils notranslate"><span class="pre">args:</span> <span class="pre">-pc_fieldsplit_diag_use_amat</span> <span class="pre">{{0</span> <span class="pre">1}}</span></code> gives <code class="docutils notranslate"><span class="pre">pc_fieldsplit_diag_use_amat</span></code> as the search term</p></li>
<li><p>Input files: <code class="docutils notranslate"><span class="pre">-f</span> <span class="pre">*</span></code></p></li>
</ul>
<p>Examples of argument searching:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;args&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;ksp_monitor&#39;</span>
</pre></div>
</div>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;args&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;*monitor*&#39;</span>
</pre></div>
</div>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;args&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;pc_type ml&#39;</span>
</pre></div>
</div>
<p>Multiple simultaneous queries can be performed with union (<code class="docutils notranslate"><span class="pre">,</span></code>), and intersection
(<code class="docutils notranslate"><span class="pre">|</span></code>) operators in the <code class="docutils notranslate"><span class="pre">query</span></code> field. One may also use their alternate spellings
(<code class="docutils notranslate"><span class="pre">%AND%</span></code> and <code class="docutils notranslate"><span class="pre">%OR%</span></code> respectively). The alternate spellings are useful in cases where
one cannot avoid (possibly multiple) shell expansions that might otherwise interpret the
<code class="docutils notranslate"><span class="pre">|</span></code> operator as a shell pipe. Examples:</p>
<ul>
<li><p>All examples using <code class="docutils notranslate"><span class="pre">cuda</span></code> and all examples using <code class="docutils notranslate"><span class="pre">hip</span></code>:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires,requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;cuda,hip&#39;</span>
<span class="gp"># </span>equivalently
<span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires%AND%requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;cuda%AND%hip&#39;</span>
</pre></div>
</div>
</li>
<li><p>Examples that require both triangle and ctetgen (intersection of tests)</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires|requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;ctetgen,triangle&#39;</span>
<span class="gp"># </span>equivalently
<span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires%OR%requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;ctetgen%AND%triangle&#39;</span>
</pre></div>
</div>
</li>
<li><p>Tests that require either <code class="docutils notranslate"><span class="pre">ctetgen</span></code> or <code class="docutils notranslate"><span class="pre">triangle</span></code></p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires,requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;ctetgen,triangle&#39;</span>
<span class="gp"># </span>equivalently
<span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires%AND%requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;ctetgen%AND%triangle&#39;</span>
</pre></div>
</div>
</li>
<li><p>Find <code class="docutils notranslate"><span class="pre">cuda</span></code> examples in the <code class="docutils notranslate"><span class="pre">dm</span></code> package.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires|name&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;cuda,dm*&#39;</span>
<span class="gp"># </span>equivalently
<span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires%OR%name&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;cuda%AND%dm*&#39;</span>
</pre></div>
</div>
</li>
</ul>
<p>Here is a way of getting a feel for how the union and intersect operators work:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;ctetgen&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>tr<span class="w"> </span><span class="s1">&#39; &#39;</span><span class="w"> </span><span class="s1">&#39;\n&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>wc<span class="w"> </span>-l
<span class="go">170</span>
<span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;triangle&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>tr<span class="w"> </span><span class="s1">&#39; &#39;</span><span class="w"> </span><span class="s1">&#39;\n&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>wc<span class="w"> </span>-l
<span class="go">330</span>
<span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires,requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;ctetgen,triangle&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>tr<span class="w"> </span><span class="s1">&#39; &#39;</span><span class="w"> </span><span class="s1">&#39;\n&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>wc<span class="w"> </span>-l
<span class="go">478</span>
<span class="gp">$ </span>make<span class="w"> </span>print-test<span class="w"> </span><span class="nv">query</span><span class="o">=</span><span class="s1">&#39;requires|requires&#39;</span><span class="w"> </span><span class="nv">queryval</span><span class="o">=</span><span class="s1">&#39;ctetgen,triangle&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>tr<span class="w"> </span><span class="s1">&#39; &#39;</span><span class="w"> </span><span class="s1">&#39;\n&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>wc<span class="w"> </span>-l
<span class="go">22</span>
</pre></div>
</div>
<p>The total number of tests for running only ctetgen or triangle is 500. They have 22 tests in common, and 478 that
run independently of each other.</p>
<p>The union and intersection have fixed grouping. So this string argument</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>query=&#39;requires,requires|args&#39; queryval=&#39;cuda,hip,*log*&#39;
# equivalently
query=&#39;requires%AND%requires%OR%args&#39; queryval=&#39;cuda%AND%hip%AND%*log*&#39;
</pre></div>
</div>
<p>will can be read as</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>requires:cuda &amp;&amp; (requires:hip || args:*log*)
</pre></div>
</div>
<p>which is probably not what is intended.</p>
<p><code class="docutils notranslate"><span class="pre">query/queryval</span></code> also support negation (<code class="docutils notranslate"><span class="pre">!</span></code>, alternate <code class="docutils notranslate"><span class="pre">%NEG%</span></code>), but is limited.
The negation only applies to tests that have a related field in it. So for example, the
arguments of</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">query=requires queryval=&#39;!cuda&#39;</span>
<span class="gp"># </span>equivalently
<span class="go">query=requires queryval=&#39;%NEG%cuda&#39;</span>
</pre></div>
</div>
<p>will only match if they explicitly have:</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="nl">requires</span><span class="p">:</span><span class="w"> </span><span class="o">!</span><span class="n">cuda</span>
</pre></div>
</div>
<p>It does not match all cases that do not require cuda.</p>
</section>
<section id="debugging-for-loops">
<h3>Debugging for loops<a class="headerlink" href="#debugging-for-loops" title="Link to this heading">#</a></h3>
<p>One of the more difficult issues is how to debug for loops when a subset of the
arguments are the ones that cause a code crash. The default naming scheme is
not always helpful for figuring out the argument combination.</p>
<p>For example:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="nv">s</span><span class="o">=</span><span class="s1">&#39;src/ksp/ksp/tests/ex9.c&#39;</span><span class="w"> </span><span class="nv">i</span><span class="o">=</span><span class="s1">&#39;*1&#39;</span>
<span class="go">Using MAKEFLAGS: i=*1 s=src/ksp/ksp/tests/ex9.c</span>
<span class="go">        TEST arch-osx-pkgs-opt-new/tests/counts/ksp_ksp_tests-ex9_1.counts</span>
<span class="go"> ok ksp_ksp_tests-ex9_1+pc_fieldsplit_diag_use_amat-0_pc_fieldsplit_diag_use_amat-0_pc_fieldsplit_type-additive</span>
<span class="go"> not ok diff-ksp_ksp_tests-ex9_1+pc_fieldsplit_diag_use_amat-0_pc_fieldsplit_diag_use_amat-0_pc_fieldsplit_type-additive</span>
<span class="go"> ok ksp_ksp_tests-ex9_1+pc_fieldsplit_diag_use_amat-0_pc_fieldsplit_diag_use_amat-0_pc_fieldsplit_type-multiplicative</span>
<span class="go"> ...</span>
</pre></div>
</div>
<p>In this case, the trick is to use the verbose option, <code class="docutils notranslate"><span class="pre">V=1</span></code> (or for the shell script workflows, <code class="docutils notranslate"><span class="pre">-v</span></code>) to have it show the commands:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="nv">s</span><span class="o">=</span><span class="s1">&#39;src/ksp/ksp/tests/ex9.c&#39;</span><span class="w"> </span><span class="nv">i</span><span class="o">=</span><span class="s1">&#39;*1&#39;</span><span class="w"> </span><span class="nv">V</span><span class="o">=</span><span class="m">1</span>
<span class="go">Using MAKEFLAGS: V=1 i=*1 s=src/ksp/ksp/tests/ex9.c</span>
<span class="go">arch-osx-pkgs-opt-new/tests/ksp/ksp/tests/runex9_1.sh  -v</span>
<span class="go"> ok ksp_ksp_tests-ex9_1+pc_fieldsplit_diag_use_amat-0_pc_fieldsplit_diag_use_amat-0_pc_fieldsplit_type-additive # mpiexec  -n 1 ../ex9 -ksp_converged_reason -ksp_error_if_not_converged  -pc_fieldsplit_diag_use_amat 0 -pc_fieldsplit_diag_use_amat 0 -pc_fieldsplit_type additive &gt; ex9_1.tmp 2&gt; runex9_1.err</span>
<span class="go">...</span>
</pre></div>
</div>
<p>This can still be hard to read and pick out what you want. So use the fact that you want <code class="docutils notranslate"><span class="pre">not</span> <span class="pre">ok</span></code>
combined with the fact that <code class="docutils notranslate"><span class="pre">#</span></code> is the delimiter:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span><span class="nb">test</span><span class="w"> </span><span class="nv">s</span><span class="o">=</span><span class="s1">&#39;src/ksp/ksp/tests/ex9.c&#39;</span><span class="w"> </span><span class="nv">i</span><span class="o">=</span><span class="s1">&#39;*1&#39;</span><span class="w"> </span><span class="nv">v</span><span class="o">=</span><span class="m">1</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span><span class="s1">&#39;not ok&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>cut<span class="w"> </span>-d#<span class="w"> </span>-f2
<span class="go">mpiexec  -n 1 ../ex9 -ksp_converged_reason -ksp_error_if_not_converged  -pc_fieldsplit_diag_use_amat 0 -pc_fieldsplit_diag_use_amat 0 -pc_fieldsplit_type multiplicative &gt; ex9_1.tmp 2&gt; runex9_1.err</span>
</pre></div>
</div>
</section>
</section>
<section id="petsc-test-harness">
<h2>PETSC Test Harness<a class="headerlink" href="#petsc-test-harness" title="Link to this heading">#</a></h2>
<p>The goals of the PETSc test harness are threefold.</p>
<ol class="arabic simple">
<li><p>Provide standard output used by other testing tools</p></li>
<li><p>Be as lightweight as possible and easily fit within the PETSc build chain</p></li>
<li><p>Provide information on all tests, even those that are not built or run because they do not meet the configuration requirements</p></li>
</ol>
<p>Before understanding the test harness, you should first understand the
desired requirements for reporting and logging.</p>
<section id="testing-the-parsing">
<h3>Testing the Parsing<a class="headerlink" href="#testing-the-parsing" title="Link to this heading">#</a></h3>
<p>After inserting the language into the file, you can test the parsing by
executing</p>
<p>A dictionary will be pretty-printed. From this dictionary printout, any
problems in the parsing are is usually obvious. This python file is used
by</p>
<p>in generating the test harness.</p>
</section>
</section>
<section id="test-output-standards-tap">
<h2>Test Output Standards: TAP<a class="headerlink" href="#test-output-standards-tap" title="Link to this heading">#</a></h2>
<p>The PETSc test system is designed to be compliant with the <a class="reference external" href="https://testanything.org/tap-specification.html">Test Anything Protocol (TAP)</a>.</p>
<p>This is a simple standard designed to allow testing tools to work
together easily. There are libraries to enable the output to be used
easily, including sharness, which is used by the Git team. However, the
simplicity of the PETSc tests and TAP specification means that we use
our own simple harness given by a single shell script that each file
sources: <code class="docutils notranslate"><span class="pre">$PETSC_DIR/config/petsc_harness.sh</span></code>.</p>
<p>As an example, consider this test input:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">test</span><span class="p">:</span>
<span class="w">  </span><span class="nt">suffix</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">2</span>
<span class="w">  </span><span class="nt">output_file</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">output/ex138.out</span>
<span class="w">  </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-f ${DATAFILESPATH}/matrices/small -mat_type {{aij baij sbaij}} -matload_block_size {{2 3}}</span>
<span class="w">  </span><span class="nt">requires</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">datafilespath</span>
</pre></div>
</div>
<p>A sample output from this would be:</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="n">ok</span><span class="w"> </span><span class="mi">1</span><span class="w"> </span><span class="n">In</span><span class="w"> </span><span class="n">mat</span><span class="p">...</span><span class="n">tests</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;./ex138 -f ${DATAFILESPATH}/matrices/small -mat_type aij -matload_block_size 2&quot;</span>
<span class="n">ok</span><span class="w"> </span><span class="mi">2</span><span class="w"> </span><span class="n">In</span><span class="w"> </span><span class="n">mat</span><span class="p">...</span><span class="n">tests</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;Diff of ./ex138 -f ${DATAFILESPATH}/matrices/small -mat_type aij -matload_block_size 2&quot;</span>
<span class="n">ok</span><span class="w"> </span><span class="mi">3</span><span class="w"> </span><span class="n">In</span><span class="w"> </span><span class="n">mat</span><span class="p">...</span><span class="n">tests</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;./ex138 -f ${DATAFILESPATH}/matrices/small -mat_type aij -matload_block_size 3&quot;</span>
<span class="n">ok</span><span class="w"> </span><span class="mi">4</span><span class="w"> </span><span class="n">In</span><span class="w"> </span><span class="n">mat</span><span class="p">...</span><span class="n">tests</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;Diff of ./ex138 -f ${DATAFILESPATH}/matrices/small -mat_type aij -matload_block_size 3&quot;</span>
<span class="n">ok</span><span class="w"> </span><span class="mi">5</span><span class="w"> </span><span class="n">In</span><span class="w"> </span><span class="n">mat</span><span class="p">...</span><span class="n">tests</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;./ex138 -f ${DATAFILESPATH}/matrices/small -mat_type baij -matload_block_size 2&quot;</span>
<span class="n">ok</span><span class="w"> </span><span class="mi">6</span><span class="w"> </span><span class="n">In</span><span class="w"> </span><span class="n">mat</span><span class="p">...</span><span class="n">tests</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;Diff of ./ex138 -f ${DATAFILESPATH}/matrices/small -mat_type baij -matload_block_size 2&quot;</span>
<span class="p">...</span>

<span class="n">ok</span><span class="w"> </span><span class="mi">11</span><span class="w"> </span><span class="n">In</span><span class="w"> </span><span class="n">mat</span><span class="p">...</span><span class="n">tests</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;./ex138 -f ${DATAFILESPATH}/matrices/small -mat_type saij -matload_block_size 2&quot;</span>
<span class="n">ok</span><span class="w"> </span><span class="mi">12</span><span class="w"> </span><span class="n">In</span><span class="w"> </span><span class="n">mat</span><span class="p">...</span><span class="n">tests</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;Diff of ./ex138 -f ${DATAFILESPATH}/matrices/small -mat_type aij -matload_block_size 2&quot;</span>
</pre></div>
</div>
</section>
<section id="test-harness-implementation">
<h2>Test Harness Implementation<a class="headerlink" href="#test-harness-implementation" title="Link to this heading">#</a></h2>
<p>Most of the requirements for being TAP-compliant lie in the shell
scripts, so we focus on that description.</p>
<p>A sample shell script is given the following.</p>
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span><span class="ch">#!/bin/sh</span>
.<span class="w"> </span>petsc_harness.sh

petsc_testrun<span class="w"> </span>./ex1<span class="w"> </span>ex1.tmp<span class="w"> </span>ex1.err
petsc_testrun<span class="w"> </span><span class="s1">&#39;diff ex1.tmp output/ex1.out&#39;</span><span class="w"> </span>diff-ex1.tmp<span class="w"> </span>diff-ex1.err

petsc_testend
</pre></div>
</div>
<p><code class="docutils notranslate"><span class="pre">petsc_harness.sh</span></code> is a small shell script that provides the logging and reporting
functions <code class="docutils notranslate"><span class="pre">petsc_testrun</span></code> and <code class="docutils notranslate"><span class="pre">petsc_testend</span></code>.</p>
<p>A small sample of the output from the test harness is as follows.</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>ok 1 ./ex1
ok 2 diff ex1.tmp output/ex1.out
not ok 4 ./ex2
#   ex2: Error: cannot read file
not ok 5 diff ex2.tmp output/ex2.out
ok 7 ./ex3 -f /matrices/small -mat_type aij -matload_block_size 2
ok 8 diff ex3.tmp output/ex3.out
ok 9 ./ex3 -f /matrices/small -mat_type aij -matload_block_size 3
ok 10 diff ex3.tmp output/ex3.out
ok 11 ./ex3 -f /matrices/small -mat_type baij -matload_block_size 2
ok 12 diff ex3.tmp output/ex3.out
ok 13 ./ex3 -f /matrices/small -mat_type baij -matload_block_size 3
ok 14 diff ex3.tmp output/ex3.out
ok 15 ./ex3 -f /matrices/small -mat_type sbaij -matload_block_size 2
ok 16 diff ex3.tmp output/ex3.out
ok 17 ./ex3 -f /matrices/small -mat_type sbaij -matload_block_size 3
ok 18 diff ex3.tmp output/ex3.out
# FAILED   4 5
# failed 2/16 tests; 87.500% ok
</pre></div>
</div>
<p>For developers, modifying the lines that get written to the file can be
done by modifying <code class="docutils notranslate"><span class="pre">$PETSC_DIR/config/example_template.py</span></code>.</p>
<p>To modify the test harness, you can modify <code class="docutils notranslate"><span class="pre">$PETSC_DIR/config/petsc_harness.sh</span></code>.</p>
<section id="additional-tips">
<h3>Additional Tips<a class="headerlink" href="#additional-tips" title="Link to this heading">#</a></h3>
<p>To rerun just the reporting use</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>config/report_tests.py
</pre></div>
</div>
<p>To see the full options use</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>config/report_tests.py<span class="w"> </span>-h
</pre></div>
</div>
<p>To see the full timing information for the five most expensive tests use</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>config/report_tests.py<span class="w"> </span>-t<span class="w"> </span><span class="m">5</span>
</pre></div>
</div>
</section>
</section>
</section>


                </article>
              
              
              
              
              
                <footer class="prev-next-footer">
                  
<div class="prev-next-area">
    <a class="left-prev"
       href="buildsystem.html"
       title="previous page">
      <i class="fa-solid fa-angle-left"></i>
      <div class="prev-next-info">
        <p class="prev-next-subtitle">previous</p>
        <p class="prev-next-title">BuildSystem</p>
      </div>
    </a>
    <a class="right-next"
       href="documentation.html"
       title="next page">
      <div class="prev-next-info">
        <p class="prev-next-subtitle">next</p>
        <p class="prev-next-title">Developing PETSc Documentation</p>
      </div>
      <i class="fa-solid fa-angle-right"></i>
    </a>
</div>
                </footer>
              
            </div>
            
            
              
                <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">


  <div class="sidebar-secondary-item">
<div
    id="pst-page-navigation-heading-2"
    class="page-toc tocsection onthispage">
    <i class="fa-solid fa-list"></i> On this page
  </div>
  <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
    <ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#petsc-test-description-language">PETSc Test Description Language</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafiles-needed-for-some-tests">Datafiles needed for some tests</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#runtime-language-options">Runtime Language Options</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#additional-specifications">Additional Specifications</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#testing-errors-and-exceptional-code">Testing Errors And Exceptional Code</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#test-block-examples">Test Block Examples</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#build-language-options">Build Language Options</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#running-the-tests">Running the tests</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#determining-the-failed-jobs-of-a-given-run">Determining the failed jobs of a given run</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#introduction-to-debugging-workflows">Introduction to debugging workflows</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#debugging-a-test-using-shell-the-generated-scripts">Debugging a test using shell the generated scripts</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#debugging-a-petsc-test-using-the-gmakefile-test">Debugging a PETSc test using the gmakefile.test</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#advanced-searching">Advanced searching</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#query-based-searching">Query-based searching</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#debugging-for-loops">Debugging for loops</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#petsc-test-harness">PETSC Test Harness</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#testing-the-parsing">Testing the Parsing</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#test-output-standards-tap">Test Output Standards: TAP</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#test-harness-implementation">Test Harness Implementation</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#additional-tips">Additional Tips</a></li>
</ul>
</li>
</ul>
  </nav></div>

  <div class="sidebar-secondary-item">

  
  <div class="tocsection editthispage">
    <a href="https://gitlab.com/petsc/petsc/-/edit/release/doc/developers/testing.md">
      <i class="fa-solid fa-pencil"></i>
      
      
        
          Edit on GitLab
        
      
    </a>
  </div>
</div>

  <div class="sidebar-secondary-item">

  <div class="tocsection sourcelink">
    <a href="../_sources/developers/testing.md.txt">
      <i class="fa-solid fa-file-lines"></i> Show Source
    </a>
  </div>
</div>

</div></div>
              
            
          </div>
          <footer class="bd-footer-content">
            
          </footer>
        
      </main>
    </div>
  </div>
  
  <!-- Scripts loaded after <body> so the DOM is not blocked -->
  <script src="../_static/scripts/bootstrap.js?digest=bd9e20870c6007c4c509"></script>
<script src="../_static/scripts/pydata-sphinx-theme.js?digest=bd9e20870c6007c4c509"></script>

  <footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
  
    <div class="footer-items__start">
      
        <div class="footer-item">

  <p class="copyright">
    
      © Copyright 1991-2025, UChicago Argonne, LLC and the PETSc Development Team.
      <br/>
    
  </p>
</div>
      
        <div class="footer-item">

  <p class="sphinx-version">
    Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 7.3.7.
    <br/>
  </p>
</div>
      
    </div>
  
  
  
    <div class="footer-items__end">
      
        <div class="footer-item">
<p class="theme-version">
  Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.1.
</p></div>
      
        <div class="footer-item"><p class="last-updated">
  Last updated on 2025-04-30T13:10:40-0500 (v3.23.1).
  <br/>
</p></div>
      
    </div>
  
</div>

  </footer>
  </body>
</html>