diff options
Diffstat (limited to 'regress')
-rwxr-xr-x | regress/regress.pl | 404 | ||||
-rw-r--r-- | regress/regress.pl.1 | 64 |
2 files changed, 236 insertions, 232 deletions
diff --git a/regress/regress.pl b/regress/regress.pl index 6c37039b..8c62a147 100755 --- a/regress/regress.pl +++ b/regress/regress.pl @@ -36,12 +36,17 @@ sub usage ($) { exit 1; } +# Modifier arguments provided on the command line, +# inspected by the main program and by the utility functions. +my %targets; + # Run a command and send STDOUT and STDERR to a file. # 1st argument: path to the output file # 2nd argument: command name # The remaining arguments are passed to the command. sub sysout ($@) { my $outfile = shift; + print "@_\n" if $targets{verbose}; local *OUT_FH; open OUT_FH, '>', $outfile or die "$outfile: $!"; my $pid = open3 undef, ">&OUT_FH", undef, @_; @@ -53,6 +58,7 @@ sub sysout ($@) { # Simlar, but filter the output as needed for the lint test. sub syslint ($@) { my $outfile = shift; + print "@_\n" if $targets{verbose}; open my $outfd, '>', $outfile or die "$outfile: $!"; my $infd; my $pid = open3 undef, $infd, undef, @_; @@ -69,6 +75,7 @@ sub syslint ($@) { # Simlar, but filter the output as needed for the html test. sub syshtml ($@) { my $outfile = shift; + print "@_\n" if $targets{verbose}; open my $outfd, '>', $outfile or die "$outfile: $!"; my $infd; my $pid = open3 undef, $infd, undef, @_; @@ -97,7 +104,7 @@ sub syshtml ($@) { } my @failures; -sub fail ($$$) { +sub fail ($$) { warn "FAILED: @_\n"; push @failures, [@_]; } @@ -105,14 +112,8 @@ sub fail ($$$) { # --- process command line arguments ----------------------------------- -my ($subdir, $onlytest) = split ':', (shift // '.'); -my $displaylevel = 2; -my %targets; +my $onlytest = shift // ''; for (@ARGV) { - if (/^[0123]$/) { - $displaylevel = int; - next; - } /^(all|ascii|utf8|man|html|markdown|lint|clean|verbose)$/ or usage "$_: invalid modifier"; $targets{$_} = 1; @@ -123,14 +124,12 @@ $targets{all} = 1 $targets{lint} || $targets{clean}; $targets{ascii} = $targets{utf8} = $targets{man} = $targets{html} = $targets{markdown} = $targets{lint} = 1 if $targets{all}; -$displaylevel = 3 if $targets{verbose}; # --- parse Makefiles -------------------------------------------------- -my %vars = (MOPTS => ''); -sub parse_makefile ($) { - my $filename = shift; +sub parse_makefile ($%) { + my ($filename, $vars) = @_; open my $fh, '<', $filename or die "$filename: $!"; while (<$fh>) { chomp; @@ -143,242 +142,293 @@ sub parse_makefile ($) { my $var = $1; my $opt = $2; my $val = $3; - $val =~ s/\$\{(\w+)\}/$vars{$1}/; - $val = "$vars{$var} $val" if $opt eq '+'; - $vars{$var} = $val - unless $opt eq '?' && defined $vars{$var}; + $val =~ s/\$\{(\w+)\}/$vars->{$1}/; + $val = "$vars->{$var} $val" if $opt eq '+'; + $vars->{$var} = $val + unless $opt eq '?' && defined $vars->{$var}; } close $fh; } -if ($subdir eq '.') { - $vars{SUBDIR} = 'roff char mdoc man tbl eqn'; -} else { - parse_makefile "$subdir/Makefile"; - parse_makefile "$subdir/../Makefile.inc" - if -e "$subdir/../Makefile.inc"; -} - -my @mandoc = '../mandoc'; -my @subdir_names; -my (@regress_testnames, @utf8_testnames, @lint_testnames); -my (@html_testnames, @markdown_testnames); +my (@regress_tests, @utf8_tests, @lint_tests, @html_tests); my (%skip_ascii, %skip_man, %skip_markdown); - -push @mandoc, split ' ', $vars{MOPTS} if $vars{MOPTS}; -delete $vars{MOPTS}; -delete $vars{SKIP_GROFF}; -delete $vars{SKIP_GROFF_ASCII}; -delete $vars{TBL}; -delete $vars{EQN}; -if (defined $vars{SUBDIR}) { - @subdir_names = split ' ', $vars{SUBDIR}; - delete $vars{SUBDIR}; -} -if (defined $vars{REGRESS_TARGETS}) { - @regress_testnames = split ' ', $vars{REGRESS_TARGETS}; - delete $vars{REGRESS_TARGETS}; -} -if (defined $vars{UTF8_TARGETS}) { - @utf8_testnames = split ' ', $vars{UTF8_TARGETS}; - delete $vars{UTF8_TARGETS}; -} -if (defined $vars{HTML_TARGETS}) { - @html_testnames = split ' ', $vars{HTML_TARGETS}; - delete $vars{HTML_TARGETS}; -} -if (defined $vars{MARKDOWN_TARGETS}) { - @markdown_testnames = split ' ', $vars{MARKDOWN_TARGETS}; - delete $vars{MARKDOWN_TARGETS}; -} -if (defined $vars{LINT_TARGETS}) { - @lint_testnames = split ' ', $vars{LINT_TARGETS}; - delete $vars{LINT_TARGETS}; -} -if (defined $vars{SKIP_ASCII}) { - for (split ' ', $vars{SKIP_ASCII}) { - $skip_ascii{$_} = 1; - $skip_man{$_} = 1; +foreach my $module (qw(roff char mdoc man tbl eqn)) { + my %modvars; + parse_makefile "$module/Makefile", \%modvars; + foreach my $subdir (split ' ', $modvars{SUBDIR}) { + my %subvars = (MOPTS => ''); + parse_makefile "$module/$subdir/Makefile", \%subvars; + parse_makefile "$module/Makefile.inc", \%subvars; + delete $subvars{SKIP_GROFF}; + delete $subvars{SKIP_GROFF_ASCII}; + delete $subvars{TBL}; + delete $subvars{EQN}; + my @mandoc = ('../mandoc', split ' ', $subvars{MOPTS}); + delete $subvars{MOPTS}; + my @regress_testnames; + if (defined $subvars{REGRESS_TARGETS}) { + push @regress_testnames, + split ' ', $subvars{REGRESS_TARGETS}; + push @regress_tests, { + NAME => "$module/$subdir/$_", + MANDOC => \@mandoc, + } foreach @regress_testnames; + delete $subvars{REGRESS_TARGETS}; + } + if (defined $subvars{UTF8_TARGETS}) { + push @utf8_tests, { + NAME => "$module/$subdir/$_", + MANDOC => \@mandoc, + } foreach split ' ', $subvars{UTF8_TARGETS}; + delete $subvars{UTF8_TARGETS}; + } + if (defined $subvars{HTML_TARGETS}) { + push @html_tests, { + NAME => "$module/$subdir/$_", + MANDOC => \@mandoc, + } foreach split ' ', $subvars{HTML_TARGETS}; + delete $subvars{HTML_TARGETS}; + } + if (defined $subvars{LINT_TARGETS}) { + push @lint_tests, { + NAME => "$module/$subdir/$_", + MANDOC => \@mandoc, + } foreach split ' ', $subvars{LINT_TARGETS}; + delete $subvars{LINT_TARGETS}; + } + if (defined $subvars{SKIP_ASCII}) { + for (split ' ', $subvars{SKIP_ASCII}) { + $skip_ascii{"$module/$subdir/$_"} = 1; + $skip_man{"$module/$subdir/$_"} = 1; + } + delete $subvars{SKIP_ASCII}; + } + if (defined $subvars{SKIP_TMAN}) { + $skip_man{"$module/$subdir/$_"} = 1 + for split ' ', $subvars{SKIP_TMAN}; + delete $subvars{SKIP_TMAN}; + } + if (defined $subvars{SKIP_MARKDOWN}) { + $skip_markdown{"$module/$subdir/$_"} = 1 + for split ' ', $subvars{SKIP_MARKDOWN}; + delete $subvars{SKIP_MARKDOWN}; + } + if (keys %subvars) { + my @vars = keys %subvars; + die "unknown var(s) @vars in dir $module/$subdir"; + } + map { + $skip_ascii{"$module/$subdir/$_"} = 1; + } @regress_testnames if $skip_ascii{"$module/$subdir/ALL"}; + map { + $skip_man{"$module/$subdir/$_"} = 1; + } @regress_testnames if $skip_man{"$module/$subdir/ALL"}; + map { + $skip_markdown{"$module/$subdir/$_"} = 1; + } @regress_testnames if $skip_markdown{"$module/$subdir/ALL"}; + } + delete $modvars{SUBDIR}; + if (keys %modvars) { + my @vars = keys %modvars; + die "unknown var(s) @vars in module $module"; } - delete $vars{SKIP_ASCII}; -} -if (defined $vars{SKIP_TMAN}) { - $skip_man{$_} = 1 for split ' ', $vars{SKIP_TMAN}; - delete $vars{SKIP_TMAN}; -} -if (defined $vars{SKIP_MARKDOWN}) { - $skip_markdown{$_} = 1 for split ' ', $vars{SKIP_MARKDOWN}; - delete $vars{SKIP_MARKDOWN}; -} -if (keys %vars) { - my @vars = keys %vars; - die "unknown var(s) @vars"; } -map { $skip_ascii{$_} = 1; } @regress_testnames if $skip_ascii{ALL}; -map { $skip_man{$_} = 1; } @regress_testnames if $skip_man{ALL}; -map { $skip_markdown{$_} = 1; } @regress_testnames if $skip_markdown{ALL}; # --- run targets ------------------------------------------------------ my $count_total = 0; -for my $dirname (@subdir_names) { - $count_total++; - print "\n" if $targets{verbose}; - system './regress.pl', "$subdir/$dirname", keys %targets, - ($displaylevel ? $displaylevel - 1 : 0), - and fail $subdir, $dirname, 'subdir'; -} - my $count_ascii = 0; my $count_man = 0; -for my $testname (@regress_testnames) { - next if $onlytest && $testname ne $onlytest; - my $i = "$subdir/$testname.in"; - my $o = "$subdir/$testname.mandoc_ascii"; - my $w = "$subdir/$testname.out_ascii"; - if ($targets{ascii} && !$skip_ascii{$testname}) { +my $count_rm = 0; +if ($targets{ascii} || $targets{man}) { + print "Running ascii and man tests "; + print "...\n" if $targets{verbose}; +} +for my $test (@regress_tests) { + my $i = "$test->{NAME}.in"; + my $o = "$test->{NAME}.mandoc_ascii"; + my $w = "$test->{NAME}.out_ascii"; + if ($targets{ascii} && !$skip_ascii{$test->{NAME}} && + $test->{NAME} =~ /^$onlytest/) { $count_ascii++; $count_total++; - print "@mandoc -T ascii $i\n" if $targets{verbose}; - sysout $o, @mandoc, qw(-I os=OpenBSD -T ascii), $i - and fail $subdir, $testname, 'ascii:mandoc'; + sysout $o, @{$test->{MANDOC}}, qw(-I os=OpenBSD -T ascii), $i + and fail $test->{NAME}, 'ascii:mandoc'; system @diff, $w, $o - and fail $subdir, $testname, 'ascii:diff'; + and fail $test->{NAME}, 'ascii:diff'; + print "." unless $targets{verbose}; } - my $m = "$subdir/$testname.in_man"; - my $mo = "$subdir/$testname.mandoc_man"; - if ($targets{man} && !$skip_man{$testname}) { + my $m = "$test->{NAME}.in_man"; + my $mo = "$test->{NAME}.mandoc_man"; + if ($targets{man} && !$skip_man{$test->{NAME}} && + $test->{NAME} =~ /^$onlytest/) { $count_man++; $count_total++; - print "@mandoc -T man $i\n" if $targets{verbose}; - sysout $m, @mandoc, qw(-I os=OpenBSD -T man), $i - and fail $subdir, $testname, 'man:man'; - print "@mandoc -man -T ascii $m\n" if $targets{verbose}; - sysout $mo, @mandoc, qw(-man -I os=OpenBSD -T ascii -O mdoc), $m - and fail $subdir, $testname, 'man:mandoc'; + sysout $m, @{$test->{MANDOC}}, qw(-I os=OpenBSD -T man), $i + and fail $test->{NAME}, 'man:man'; + sysout $mo, @{$test->{MANDOC}}, + qw(-man -I os=OpenBSD -T ascii -O mdoc), $m + and fail $test->{NAME}, 'man:mandoc'; system @diff, $w, $mo - and fail $subdir, $testname, 'man:diff'; + and fail $test->{NAME}, 'man:diff'; + print "." unless $targets{verbose}; } if ($targets{clean}) { - print "rm $o\n" - if $targets{verbose} && !$skip_ascii{$testname}; - unlink $o; - print "rm $m $mo\n" - if $targets{verbose} && !$skip_man{$testname}; - unlink $m, $mo; + print "rm $o $m $mo\n" if $targets{verbose}; + $count_rm += unlink $o, $m, $mo; } } +if ($targets{ascii} || $targets{man}) { + print "Number of ascii and man tests:" if $targets{verbose}; + print " $count_ascii + $count_man tests run.\n"; +} my $count_utf8 = 0; -for my $testname (@utf8_testnames) { - next if $onlytest && $testname ne $onlytest; - my $i = "$subdir/$testname.in"; - my $o = "$subdir/$testname.mandoc_utf8"; - my $w = "$subdir/$testname.out_utf8"; - if ($targets{utf8}) { +if ($targets{utf8}) { + print "Running utf8 tests "; + print "...\n" if $targets{verbose}; +} +for my $test (@utf8_tests) { + my $i = "$test->{NAME}.in"; + my $o = "$test->{NAME}.mandoc_utf8"; + my $w = "$test->{NAME}.out_utf8"; + if ($targets{utf8} && $test->{NAME} =~ /^$onlytest/o) { $count_utf8++; $count_total++; - print "@mandoc -T utf8 $i\n" if $targets{verbose}; - sysout $o, @mandoc, qw(-I os=OpenBSD -T utf8), $i - and fail $subdir, $testname, 'utf8:mandoc'; + sysout $o, @{$test->{MANDOC}}, qw(-I os=OpenBSD -T utf8), $i + and fail $test->{NAME}, 'utf8:mandoc'; system @diff, $w, $o - and fail $subdir, $testname, 'utf8:diff'; + and fail $test->{NAME}, 'utf8:diff'; + print "." unless $targets{verbose}; } if ($targets{clean}) { print "rm $o\n" if $targets{verbose}; - unlink $o; + $count_rm += unlink $o; } } +if ($targets{utf8}) { + print "Number of utf8 tests:" if $targets{verbose}; + print " $count_utf8 tests run.\n"; +} my $count_html = 0; -for my $testname (@html_testnames) { - next if $onlytest && $testname ne $onlytest; - my $i = "$subdir/$testname.in"; - my $o = "$subdir/$testname.mandoc_html"; - my $w = "$subdir/$testname.out_html"; - if ($targets{html}) { +if ($targets{html}) { + print "Running html tests "; + print "...\n" if $targets{verbose}; +} +for my $test (@html_tests) { + my $i = "$test->{NAME}.in"; + my $o = "$test->{NAME}.mandoc_html"; + my $w = "$test->{NAME}.out_html"; + if ($targets{html} && $test->{NAME} =~ /^$onlytest/) { $count_html++; $count_total++; - print "@mandoc -T html $i\n" if $targets{verbose}; - syshtml $o, @mandoc, qw(-T html), $i - and fail $subdir, $testname, 'html:mandoc'; + syshtml $o, @{$test->{MANDOC}}, qw(-T html), $i + and fail $test->{NAME}, 'html:mandoc'; system @diff, $w, $o - and fail $subdir, $testname, 'html:diff'; + and fail $test->{NAME}, 'html:diff'; + print "." unless $targets{verbose}; } if ($targets{clean}) { print "rm $o\n" if $targets{verbose}; - unlink $o; + $count_rm += unlink $o; } } +if ($targets{html}) { + print "Number of html tests:" if $targets{verbose}; + print " $count_html tests run.\n"; +} my $count_markdown = 0; -for my $testname (@regress_testnames) { - next if $onlytest && $testname ne $onlytest; - my $i = "$subdir/$testname.in"; - my $o = "$subdir/$testname.mandoc_markdown"; - my $w = "$subdir/$testname.out_markdown"; - if ($targets{markdown} && !$skip_markdown{$testname}) { +if ($targets{markdown}) { + print "Running markdown tests "; + print "...\n" if $targets{verbose}; +} +for my $test (@regress_tests) { + my $i = "$test->{NAME}.in"; + my $o = "$test->{NAME}.mandoc_markdown"; + my $w = "$test->{NAME}.out_markdown"; + if ($targets{markdown} && !$skip_markdown{$test->{NAME}} && + $test->{NAME} =~ /^$onlytest/) { $count_markdown++; $count_total++; - print "@mandoc -T markdown $i\n" if $targets{verbose}; - sysout $o, @mandoc, qw(-I os=OpenBSD -T markdown), $i - and fail $subdir, $testname, 'markdown:mandoc'; + sysout $o, @{$test->{MANDOC}}, + qw(-I os=OpenBSD -T markdown), $i + and fail $test->{NAME}, 'markdown:mandoc'; system @diff, $w, $o - and fail $subdir, $testname, 'markdown:diff'; + and fail $test->{NAME}, 'markdown:diff'; + print "." unless $targets{verbose}; } if ($targets{clean}) { print "rm $o\n" if $targets{verbose}; - unlink $o; + $count_rm += unlink $o; } } +if ($targets{markdown}) { + print "Number of markdown tests:" if $targets{verbose}; + print " $count_markdown tests run.\n"; +} my $count_lint = 0; -for my $testname (@lint_testnames) { - next if $onlytest && $testname ne $onlytest; - my $i = "$subdir/$testname.in"; - my $o = "$subdir/$testname.mandoc_lint"; - my $w = "$subdir/$testname.out_lint"; - if ($targets{lint}) { +if ($targets{lint}) { + print "Running lint tests "; + print "...\n" if $targets{verbose}; +} +for my $test (@lint_tests) { + my $i = "$test->{NAME}.in"; + my $o = "$test->{NAME}.mandoc_lint"; + my $w = "$test->{NAME}.out_lint"; + if ($targets{lint} && $test->{NAME} =~ /^$onlytest/) { $count_lint++; $count_total++; - print "@mandoc -T lint -W all $i\n" if $targets{verbose}; - syslint $o, @mandoc, qw(-I os=OpenBSD -T lint -W all), $i - and fail $subdir, $testname, 'lint:mandoc'; + syslint $o, @{$test->{MANDOC}}, + qw(-I os=OpenBSD -T lint -W all), $i + and fail $test->{NAME}, 'lint:mandoc'; system @diff, $w, $o - and fail $subdir, $testname, 'lint:diff'; + and fail $test->{NAME}, 'lint:diff'; + print "." unless $targets{verbose}; } if ($targets{clean}) { print "rm $o\n" if $targets{verbose}; - unlink $o; + $count_rm += unlink $o; } } - -exit 0 unless $displaylevel or @failures; - -print "\n" if $targets{verbose}; -if ($onlytest) { - print "test $subdir:$onlytest finished"; -} else { - print "testsuite $subdir finished"; +if ($targets{lint}) { + print "Number of lint tests:" if $targets{verbose}; + print " $count_lint tests run.\n"; } -print ' ', (scalar @subdir_names), ' subdirectories' if @subdir_names; -print " $count_ascii ascii" if $count_ascii; -print " $count_man man" if $count_man; -print " $count_utf8 utf8" if $count_utf8; -print " $count_html html" if $count_html; -print " $count_markdown markdown" if $count_markdown; -print " $count_lint lint" if $count_lint; + +# --- final report ----------------------------------------------------- if (@failures) { - print " (FAIL)\n\nSOME TESTS FAILED:\n\n"; + print "\nNUMBER OF FAILED TESTS: ", scalar @failures, + " (of $count_total tests run.)\n"; print "@$_\n" for @failures; print "\n"; exit 1; -} elsif ($count_total == 1) { - print " (OK)\n"; +} +print "\n" if $targets{verbose}; +if ($count_total == 1) { + print "Test succeeded.\n"; } elsif ($count_total) { - print " (all $count_total tests OK)\n"; + print "All $count_total tests OK:"; + print " $count_ascii ascii" if $count_ascii; + print " $count_man man" if $count_man; + print " $count_utf8 utf8" if $count_utf8; + print " $count_html html" if $count_html; + print " $count_markdown markdown" if $count_markdown; + print " $count_lint lint" if $count_lint; + print "\n"; } else { - print " (no tests run)\n"; + print "No tests were run.\n"; } +if ($targets{clean}) { + if ($count_rm) { + print "Deleted $count_rm test output files.\n"; + print "The tree is now clean.\n"; + } else { + print "No test output files were found.\n"; + print "The tree was already clean.\n"; + } +} exit 0; diff --git a/regress/regress.pl.1 b/regress/regress.pl.1 index a61c31ed..33c89a1a 100644 --- a/regress/regress.pl.1 +++ b/regress/regress.pl.1 @@ -23,7 +23,7 @@ .Sh SYNOPSIS .Nm ./regress.pl .Oo -.Ar directory Ns Op Pf : Ar test +.Ar directory Ns Op Pf / Ar test .Op Ar modifier ... .Oc .Sh DESCRIPTION @@ -32,29 +32,18 @@ The steering script allows running the .Xr mandoc 1 regression suite on arbitrary operating systems, -even though the suite was designed for OpenBSD only. +even though the suite was originally designed for OpenBSD only. .Pp When run without an argument, .Nm runs the complete regression suite. .Pp -When run with one argument, that argument can be: -.Bl -enum -.It -A single dot to run the complete suite. -.It -One of the top level directories, for example -.Pa mdoc , -to run the test suite for a complete language or feature group. -.It -A subdirectory, for example -.Pa man/IP , -to run the tests for a specific macro or an individual feature. -.It -A subdirectory with a test name appended with a colon, for example -.Pa char/unicode : Ns Pa named , -to run the tests for one particular input file. -.El +The first argument is a Perl regular expression to match test names, +automatically anchored at the beginning of the names. +Test names are names of test input files without the file name extension +.Pa .in , +for example +.Pa char/unicode/named . .Pp Any additional arguments modify the way the tests are run. The default is @@ -96,35 +85,6 @@ output mode. .It Cm verbose Display approximate indications of what is being done. .El -.Pp -The amount of summary lines shown can be modified by giving an -argument consisting of a single digit: -.Bl -tag -width verbose -.It Cm 3 -Show all summary lines for all directories entered. -Even without -.Cm verbose , -this generates more than hundred lines of output when running the -complete regression suite. -.It Cm 2 -This is the default. -It shows the summary lines for the -.Ar directory -given on the command line and its immediate children. -Except for -.Pa mdoc , -the output usually fits on one screen. -.It Cm 1 -Only show a single summary line for the whole run. -.It Cm 0 -Do not show any summary lines. -No output means success. -Success or failure can also be seen from the exit status. -.El -.Pp -All failed tests are always reported, even when the -.Cm 0 -modifier is given. .Sh EXIT STATUS .Ex -std .Sh EXAMPLES @@ -140,15 +100,9 @@ Complete check, but keep the tree clean: .Pp .Dl ./regress.pl \&. all clean .Pp -Test all of -.Pa mdoc , -but don't print the usual 65 lines of output: -.Pp -.Dl ./regress.pl mdoc 1 -.Pp Investigate a specific failure: .Pp -.Dl ./regress.pl mdoc/Bd:broken man verbose +.Dl ./regress.pl mdoc/Bd/broken man verbose .Sh HISTORY The .Nm |