1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
|
# -*- perl -*-
#
# tests for xref-helpmsgs-manpages
#
use v5.20;
use strict;
use warnings;
use Clone qw(clone);
use File::Basename;
use File::Path qw(make_path);
use File::Temp qw(tempdir);
use FindBin;
use Test::More;
use Test::Differences;
plan tests => 10;
require_ok "$FindBin::Bin/xref-helpmsgs-manpages";
my $workdir = tempdir(basename($0) . ".XXXXXXXX", TMPDIR => 1, CLEANUP => 1);
# Copy man pages to tmpdir, so we can fiddle with them
my $doc_path = do {
no warnings 'once';
"$LibPod::CI::XrefHelpmsgsManpages::Markdown_Path";
};
make_path "$workdir/$doc_path"
or die "Internal error: could not make_path $workdir/$doc_path";
system('rsync', '-a', "$doc_path/." => "$workdir/$doc_path/.") == 0
or die "Internal error: could not rsync $doc_path to $workdir";
chdir $workdir
or die "Internal error: could not cd $workdir: $!";
my @warnings_seen;
$SIG{__WARN__} = sub {
my $msg = shift;
chomp $msg;
$msg =~ s/^xref-\S+?:\s+//; # strip "$ME: "
$msg =~ s!(^|\s+)$doc_path/!$1!g; # strip "doc/source/markdown"
$msg =~ s!:\d+:!:NNN:!; # file line numbers can change
push @warnings_seen, $msg;
};
# When we get errors (hopefully only when adding new functionality
# to this test!), this format is MUCH easier to read and copy-paste.
unified_diff;
# Helper function for running xref tests.
sub test_xref {
my $name = shift;
my $h = shift;
my $m = shift;
my $expect_by_help = shift;
my $expect_by_man = shift;
@warnings_seen = ();
LibPod::CI::XrefHelpmsgsManpages::xref_by_help($h, $m);
eq_or_diff_text \@warnings_seen, $expect_by_help, "$name: xref_by_help()";
@warnings_seen = ();
LibPod::CI::XrefHelpmsgsManpages::xref_by_man($h, $m);
eq_or_diff_text \@warnings_seen, $expect_by_man, "$name: xref_by_man()";
}
###############################################################################
# BEGIN Baseline tests
#
# Confirm that everything passes in the current tree
my $help = LibPod::CI::XrefHelpmsgsManpages::podman_help();
eq_or_diff_text \@warnings_seen, [], "podman_help() runs cleanly, no warnings";
@warnings_seen = ();
my $man = LibPod::CI::XrefHelpmsgsManpages::podman_man('podman');
eq_or_diff_text \@warnings_seen, [], "podman_man() runs cleanly, no warnings";
# If this doesn't pass, we've got big problems.
test_xref("baseline", $help, $man, [], []);
#use Data::Dump; dd $man; exit 0;
# END Baseline tests
##########################################################################
# BEGIN fault injection tests on xref_by_man()
#
# These are really simple: only two different warnings.
my $hclone = clone($help);
my $mclone = clone($man);
delete $hclone->{network}{ls}{"--format"};
delete $hclone->{save};
$mclone->{"command-in-man"} = {};
$mclone->{"system"}{"subcommand-in-man"} = {};
# --format field documented in man page but not in autocomplete
delete $hclone->{events}{"--format"}{".HealthStatus"};
test_xref("xref_by_man injection", $hclone, $mclone,
[],
[
"'podman ': 'command-in-man' in podman.1.md, but not in --help",
"'podman events --format': '.HealthStatus' in podman-events.1.md, but not in command completion",
"'podman network ls': --format options documented in man page, but not available via autocomplete",
"'podman ': 'save' in podman.1.md, but not in --help",
"'podman system': 'subcommand-in-man' in podman-system.1.md, but not in --help",
],
);
# END fault injection tests on xref_by_man()
###############################################################################
# BEGIN fault injection tests on xref_by_help()
#
# These are much more complicated.
$hclone = clone($help);
$mclone = clone($man);
# --format is not documented in man page
delete $mclone->{"auto-update"}{"--format"};
# --format is documented, but without a table
$mclone->{container}{list}{"--format"} = 1;
# --format is documented, with a table, but entries are wrong
$mclone->{events}{"--format"}{".Attributes"} = 0;
$mclone->{events}{"--format"}{".Image"} = '...';
$mclone->{events}{"--format"}{".Status"} = 1;
$hclone->{events}{"--format"}{".Status"} = '...';
$mclone->{pod}{ps}{"--format"}{".Label"} = 3;
$mclone->{ps}{"--format"}{".Label"} = 0;
# --format is documented, with a table, but one entry missing
delete $mclone->{events}{"--format"}{".Type"};
# -l option is not documented
delete $mclone->{pod}{inspect}{"-l"};
# Command and subcommand in podman --help, but not in man pages
$hclone->{"new-command-in-help"} = {};
$hclone->{"secret"}{"subcommand-in-help"} = {};
# Can happen if podman-partlydocumented exists in --help, and is listed
# in podman.1.md, but does not have its own actual man page.
$hclone->{partlydocumented} = { "something" => 1 };
$mclone->{partlydocumented} = undef;
test_xref("xref_by_help() injection", $hclone, $mclone,
[
"'podman auto-update --help' lists '--format', which is not in podman-auto-update.1.md",
"'podman container list': --format options are available through autocomplete, but are not documented in podman-ps.1.md",
"'podman events --format {{.Attributes' is a nested structure. Please add '...' to man page.",
"'podman events --format {{.Image' is a simple value, not a nested structure. Please remove '...' from man page.",
"'podman events --format {{.Status' is a nested structure, but the man page documents it as a function?!?",
"'podman events --format <TAB>' lists '.Type', which is not in podman-events.1.md",
"'podman --help' lists 'new-command-in-help', which is not in podman.1.md",
"'podman partlydocumented' is not documented in man pages!",
"'podman pod inspect --help' lists '-l', which is not in podman-pod-inspect.1.md",
"'podman pod ps --format {{.Label' is a function that calls for 1 args; the man page lists 3. Please fix the man page.",
"'podman ps --format {{.Label' is a function that calls for 1 args. Please investigate what those are, then add them to the man page. E.g., '.Label *bool*' or '.Label *path* *bool*'",
"'podman secret --help' lists 'subcommand-in-help', which is not in podman-secret.1.md",
],
[],
);
# END fault injection tests on xref_by_help()
###############################################################################
# BEGIN fault injection tests on podman_man()
#
# This function has a ton of sanity checks. To test them we need to
# perform minor surgery on lots of .md files: reordering lines,
# adding inconsistencies.
#
# Ordered list of the warnings we expect to see
my @expect_warnings;
# Helper function: given a filename and a function, reads filename
# line by line, invoking filter on each line and writing out the
# results.
sub sed {
my $path = shift; # in: filename (something.md)
my $action = shift; # in: filter function
# The rest of our arguments are the warnings introduced into this man page
push @expect_warnings, @_;
open my $fh_in, '<', "$doc_path/$path"
or die "Cannot read $doc_path/$path: $!";
my $tmpfile = "$doc_path/$path.tmp.$$";
open my $fh_out, '>', $tmpfile
or die "Cannot create $doc_path/$tmpfile: $!";
while (my $line = <$fh_in>) {
# This is what does all the magic
print { $fh_out } $action->($line);
}
close $fh_in;
close $fh_out
or die "Error writing $doc_path/$tmpfile: $!";
rename "$tmpfile" => "$doc_path/$path"
or die "Could not rename $doc_path/$tmpfile: $!";
}
# Start filtering.
# podman-attach is a deliberate choice here: it also serves as the man page
# for podman-container-attach. Prior to 2023-12-20 we would read the file
# twice, issuing two warnings, which is anti-helpful. Here we confirm that
# the dup-removing code works.
sed('podman-attach.1.md', sub {
my $line = shift;
$line =~ s/^(%\s+podman)-(attach\s+1)/$1 $2/;
$line;
},
"podman-attach.1.md:NNN: wrong title line '% podman attach 1'; should be '% podman-attach 1'",
);
# Tests for broken command-line options
# IMPORTANT NOTE: podman-exec precedes podman-container (below),
# because podman-exec.1.md is actually read while podman-container.1.md
# is still processing; so these messages are printed earlier:
# podman-container.1.md -> list of subcommands -> exec -> read -exec.1.md
# Sorry for the confusion.
sed('podman-exec.1.md', sub {
my $line = shift;
if ($line =~ /^#### \*\*--env\*\*/) {
$line = $line . "\ndup dup dup\n\n" . $line;
}
elsif ($line =~ /^#### \*\*--privileged/) {
$line = "#### \*\*--put-me-back-in-order\*\*\n\nbogus option\n\n" . $line;
}
elsif ($line =~ /^#### \*\*--tty\*\*/) {
chomp $line;
$line .= " xyz\n";
}
elsif ($line =~ /^#### \*\*--workdir\*\*/) {
$line = <<"END_FOO";
#### **--workdir**=*dir*, **-w**
blah blah bogus description
#### **--yucca**=*cactus*|*starch*|*both*
blah blah
#### **--zydeco**=*true* | *false*
END_FOO
}
return $line;
},
"podman-exec.1.md:NNN: flag '--env' is a dup",
"podman-exec.1.md:NNN: --privileged should precede --put-me-back-in-order",
"podman-exec.1.md:NNN: could not parse ' xyz' in option description",
"podman-exec.1.md:NNN: please rewrite as ', **-w**=*dir*'",
"podman-exec.1.md:NNN: values must be space-separated: '=*cactus*|*starch*|*both*'",
"podman-exec.1.md:NNN: Do not enumerate true/false for boolean-only options",
);
# Tests for subcommands in a table
sed('podman-container.1.md', sub {
my $line = shift;
# "podman container diff": force an out-of-order error
state $diff;
if ($line =~ /^\|\s+diff\s+\|/) {
$diff = $line;
return '';
}
if ($diff) {
$line .= $diff;
$diff = undef;
}
# "podman init": force a duplicate-command error
if ($line =~ /^\|\s+init\s+\|/) {
$line .= $line;
}
# "podman container port": force a wrong-man-page error
if ($line =~ /^\|\s+port\s+\|/) {
$line =~ s/-port\.1\.md/-top.1.md/;
}
return $line;
},
"podman-container.1.md:NNN: 'exec' and 'diff' are out of order",
"podman-container.1.md:NNN: duplicate subcommand 'init'",
# FIXME: this is not technically correct; it could be the other way around.
"podman-container.1.md:NNN: 'podman-port' should be 'podman-top' in '[podman-port(1)](podman-top.1.md)'",
);
# Tests for --format specifiers in a table
sed('podman-image-inspect.1.md', sub {
my $line = shift;
state $digest;
if ($line =~ /^\|\s+\.Digest\s+\|/) {
$digest = $line;
return '';
}
if ($digest) {
$line .= $digest;
$digest = undef;
}
if ($line =~ /^\|\s+\.ID\s+\|/) {
$line = $line . $line;
}
$line =~ s/^\|\s+\.Parent\s+\|/| .Parent BAD-ARG |/;
$line =~ s/^\|\s+\.Size\s+\|/| .Size *arg1* arg2 |/;
return $line;
},
"podman-image-inspect.1.md:NNN: format specifier '.Digest' should precede '.GraphDriver'",
"podman-image-inspect.1.md:NNN: format specifier '.ID' is a dup",
"podman-image-inspect.1.md:NNN: unknown args 'BAD-ARG' for '.Parent'. Valid args are '...' for nested structs or, for functions, one or more asterisk-wrapped argument names.",
"podman-image-inspect.1.md:NNN: unknown args '*arg1* arg2' for '.Size'. Valid args are '...' for nested structs or, for functions, one or more asterisk-wrapped argument names.",
);
# Tests for SEE ALSO section
sed('podman-version.1.md', sub {
my $line = shift;
if ($line =~ /^## SEE ALSO/) {
$line .= "**foo**,**bar**"
. ", **baz**baz**"
. ", missingstars"
. ", **[podman-info(1)](podman-cp.1.md)**"
. ", **[podman-foo(1)](podman-wait.1.md)**"
. ", **[podman-x](podman-bar.1.md)**"
. ", **podman-logs(1)**"
. ", **podman-image-rm(1)**"
. ", **sdfsdf**"
. "\n";
}
return $line;
},
"podman-version.1.md:NNN: please add space after comma: '**foo**,**bar**'",
"podman-version.1.md:NNN: invalid token 'baz**baz'",
"podman-version.1.md:NNN: 'missingstars' should be bracketed by '**'",
"podman-version.1.md:NNN: inconsistent link podman-info(1) -> podman-cp.1.md, expected podman-info.1.md",
"podman-version.1.md:NNN: invalid link podman-foo(1) -> podman-wait.1.md",
"podman-version.1.md:NNN: could not parse 'podman-x' as 'manpage(N)'",
"podman-version.1.md:NNN: 'podman-logs(1)' should be '[podman-logs(1)](podman-logs.1.md)'",
"podman-version.1.md:NNN: 'podman-image-rm(1)' refers to a command alias; please use the canonical command name instead",
"podman-version.1.md:NNN: invalid token 'sdfsdf'"
);
# Tests for --filter specifiers
sed('podman-volume-prune.1.md', sub {
my $line = shift;
if ($line =~ /^\|\s+driver\s+\|/) {
$line = "| name! | sdfsdf |\n" . $line;
}
if ($line =~ /^\|\s+opt\s+\|/) {
$line .= $line;
}
return $line;
},
"podman-volume-prune.1.md:NNN: filter 'name!' only allowed immediately after its positive",
"podman-volume-prune.1.md:NNN: filter specifier 'opt' is a dup",
);
# DONE with fault injection. Reread man pages and verify warnings.
@warnings_seen = ();
{
no warnings 'once';
%LibPod::CI::XrefHelpmsgsManpages::Man_Seen = ();
}
$man = LibPod::CI::XrefHelpmsgsManpages::podman_man('podman');
eq_or_diff_text \@warnings_seen, \@expect_warnings, "podman_man() with faults";
# END fault injection tests on podman_man()
###############################################################################
# BEGIN fault injection tests on podman_help()
#
# Nope, this is not likely to happen. In order to do this we'd need to:
#
# * instrument podman and cobra to emit fake output; or
# * write a podman wrapper that selectively munges output; or
# * write a dummy podman that generates the right form of (broken) output.
#
# podman_help() has few sanity checks, and those are unlikely, so doing this
# is way more effort than it's worth.
#
# END fault injection tests on podman_help()
###############################################################################
1;
|