[PATCH] testsuite: improvements to our shell TAP library and APIs

[Stefano Lattarini]

These improvements impact only the Automake's own testsuite.

* tests/tap-functions.sh (plan_): Allow new special arguments
"unknown" a.k.a. "later" and "lazy" a.k.a. "now".  Perform
sanity check the given arguments.  Adjust incorrect comments.
(lazy_plan_): Removed, superseded by the new version of `plan_'.
(not): New function, run a command and inverts its exit status.
(ok_, not_ok_, skip_): These are just shorthands for common
`result_' usages.
(xfail_, xpass_, reset_test_count_): Removed, they are not used,
nor does a legitimate use for them seem in sight.
(not): New function, to run a command inverting its exit status.
(command_not_ok_): Removed, superseded by `command_ok_' + `not'.
(command_ok_): Extended to accept TAP directives.
* tests/defs (trap): Don't use `late_plan_' anymore, use
`plan_ "now"' instead.
* tests/ac-output-old.tap: Update to take advantage of the new
APIs.
* tests/self-check-env-sanitize.tap: Likewise.
* tests/suffix-chain.tap: Likewise.
* tests/suffix10.tap: Likewise.
* tests/suffix11.tap: Likewise.
* tests/suffix3.tap: Likewise.
* tests/suffix8.tap: Likewise.
* tests/tap-bad-prog.tap: Likewise.
* tests/self-check-exit.tap: Likewise, plus a small improvement
in the name of a variable.
---
 ChangeLog                         |   30 +++++++
 tests/ac-output-old.tap           |    2 +-
 tests/defs                        |    2 +-
 tests/self-check-env-sanitize.tap |    2 +-
 tests/self-check-exit.tap         |   15 ++--
 tests/suffix-chain.tap            |    6 +-
 tests/suffix10.tap                |    4 +-
 tests/suffix11.tap                |    6 +-
 tests/suffix3.tap                 |   10 +-
 tests/suffix8.tap                 |    6 +-
 tests/tap-bad-prog.tap            |   10 +--
 tests/tap-functions.sh            |  152 +++++++++++++------------------------
 12 files changed, 111 insertions(+), 134 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 394af5f..c34bd51 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,35 @@
 2011-08-09  Stefano Lattarini  <address@hidden>
 
+       testsuite: improvements to our shell TAP library and APIs
+       These improvements impact only the Automake's own testsuite.
+       * tests/tap-functions.sh (plan_): Allow new special arguments
+       "unknown" a.k.a. "later" and "lazy" a.k.a. "now".  Perform
+       sanity check the given arguments.  Adjust incorrect comments.
+       (lazy_plan_): Removed, superseded by the new version of `plan_'.
+       (not): New function, run a command and inverts its exit status.
+       (ok_, not_ok_, skip_): These are just shorthands for common
+       `result_' usages.
+       (xfail_, xpass_, reset_test_count_): Removed, they are not used,
+       nor does a legitimate use for them seem in sight.
+       (not): New function, to run a command inverting its exit status.
+       (command_not_ok_): Removed, superseded by `command_ok_' + `not'.
+       (command_ok_): Extended to accept TAP directives.
+       * tests/defs (trap): Don't use `late_plan_' anymore, use
+       `plan_ "now"' instead.
+       * tests/ac-output-old.tap: Update to take advantage of the new
+       APIs.
+       * tests/self-check-env-sanitize.tap: Likewise.
+       * tests/suffix-chain.tap: Likewise.
+       * tests/suffix10.tap: Likewise.
+       * tests/suffix11.tap: Likewise.
+       * tests/suffix3.tap: Likewise.
+       * tests/suffix8.tap: Likewise.
+       * tests/tap-bad-prog.tap: Likewise.
+       * tests/self-check-exit.tap: Likewise, plus a small improvement
+       in the name of a variable.
+
+2011-08-09  Stefano Lattarini  <address@hidden>
+
        testsuite: convert some `suffix*' tests to TAP
        This will allow us to explicitly declare some expected failures
        involving Solaris make as such, instead of partly papering over
diff --git a/tests/ac-output-old.tap b/tests/ac-output-old.tap
index ee2d150..5eeb672 100755
--- a/tests/ac-output-old.tap
+++ b/tests/ac-output-old.tap
@@ -48,7 +48,7 @@ acoutbs_check ()
   command_ok_ "can ./configure in $dir" ./configure
   command_ok_ "zot created in $dir" test -f zot
   ls -a > lst || bailout_ "cannot get file listing in $dir"
-  command_not_ok_ "'\\' not leaked in filenames in $dir" grep '\\' lst
+  command_ok_ "'\\' not leaked in filenames in $dir" not grep '\\' lst
   cd ..
 }
 
diff --git a/tests/defs b/tests/defs
index affb1c1..047e22f 100644
--- a/tests/defs
+++ b/tests/defs
@@ -869,7 +869,7 @@ if test "$sh_errexit_works" = yes; then
     fi
     if test $using_tap = yes; then
       if test $have_tap_plan_ != yes; then
-        late_plan_
+        plan_ "now"
       fi
       test $exit_status -eq 0 \
         && test $tap_xfail_count_ -eq 0 \
diff --git a/tests/self-check-env-sanitize.tap 
b/tests/self-check-env-sanitize.tap
index 16e3a99..4556e47 100755
--- a/tests/self-check-env-sanitize.tap
+++ b/tests/self-check-env-sanitize.tap
@@ -49,7 +49,7 @@ do_grep ()
 }
 
 for var in $vars; do
-  command_not_ok_ "$var [err status]" do_run $var
+  command_ok_ "$var [err status]" not do_run $var
   command_ok_ "$var [err message]" do_grep $var
 done
 
diff --git a/tests/self-check-exit.tap b/tests/self-check-exit.tap
index ff0278a..fc3fdca 100755
--- a/tests/self-check-exit.tap
+++ b/tests/self-check-exit.tap
@@ -29,9 +29,9 @@ AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC
 init='. ./defs; stderr_fileno_=2; unset am_explicit_skips'
 
 for st in 1 2 3 4 5 77 99 126 127 128 129 130 255; do
-  for ex in "Exit $st" "sh -c 'exit $st'"; do
-    rc=0; $SHELL -c  "$init; $ex; :" || rc=$?
-    command_ok_ "$ex" test $rc -eq $st
+  for exit_cmd in "Exit $st" "sh -c 'exit $st'"; do
+    rc=0; $SHELL -c  "$init; $exit_cmd; :" || rc=$?
+    command_ok_ "$exit_cmd" test $rc -eq $st
   done
 done
 
@@ -74,12 +74,9 @@ if $SHELL -c 'set -e; trap "exit \$?" 0; 
non-existent-program'; then
 else
   maybe_todo="" reason=""
 fi
-if $SHELL -c  "$init; non-existent-program; :"; then
-  r='not ok'
-else
-  r='ok'
-fi
-result_ "$r" -D "$maybe_todo" -r "$reason" "command not found"
+rc=0; $SHELL -c  "$init; non-existent-program; :" || rc=$?
+command_ok_ "command not found" -D "$maybe_todo" -r "$reason" \
+            -- test $rc -gt 0
 
 : Non-executable command.
 test -f Makefile && test ! -x Makefile || \
diff --git a/tests/suffix-chain.tap b/tests/suffix-chain.tap
index 0ea685a..974c3f7 100755
--- a/tests/suffix-chain.tap
+++ b/tests/suffix-chain.tap
@@ -48,9 +48,9 @@ command_ok_ "configure" ./configure
 directive=''; make_can_chain_suffix_rules || directive=TODO
 
 for target in all distcheck; do
-  if $MAKE $target; then r='ok'; else r='not ok'; fi
-  result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" \
-               -- "make $target"
+  command_ok_ "make $target" \
+              -D "$directive" -r "suffix rules not chained" \
+              $MAKE $target
 done
 
 command_ok_ "clean" $MAKE clean
diff --git a/tests/suffix10.tap b/tests/suffix10.tap
index 95c4533..d71450d 100755
--- a/tests/suffix10.tap
+++ b/tests/suffix10.tap
@@ -62,7 +62,7 @@ command_ok_ "configure"  ./configure
 command_ok_ "make test"  $MAKE test
 
 directive=''; make_can_chain_suffix_rules || directive=TODO
-if $MAKE all; then r='ok'; else r='not ok'; fi
-result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" "make all"
+command_ok_ "make all" -D "$directive" -r "suffix rules not chained" \
+            $MAKE all
 
 :
diff --git a/tests/suffix11.tap b/tests/suffix11.tap
index 30fa874..e740f6c 100755
--- a/tests/suffix11.tap
+++ b/tests/suffix11.tap
@@ -74,9 +74,9 @@ command_ok_ "make test-real" $MAKE test-real
 directive=''; make_can_chain_suffix_rules || directive=TODO
 
 for target in '' distcheck; do
-  if $MAKE $target; then r='ok'; else r='not ok'; fi
-  result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" \
-               -- "make $target"
+  command_ok_ "make $target" \
+              -D "$directive" -r "suffix rules not chained" \
+              $MAKE $target
 done
 
 :
diff --git a/tests/suffix3.tap b/tests/suffix3.tap
index a4908d4..f1596ce 100755
--- a/tests/suffix3.tap
+++ b/tests/suffix3.tap
@@ -45,8 +45,8 @@ command_ok_ "automake" $AUTOMAKE
 # The foo.cc intermediate step is implicit, it's a mistake if
 # Automake requires this file somewhere.  Also, Automake should
 # not require the file `foo.c' anywhere.
-command_not_ok_ "intermediate files not mentioned" \
-  $FGREP foo.c Makefile.in
+command_ok_ "intermediate files not mentioned" \
+  not $FGREP foo.c Makefile.in
 # However Automake must figure that foo.zoo is eventually
 # transformed into foo.o, and use this latter file (to link foo).
 command_ok_ "final object file figured out" \
@@ -68,9 +68,9 @@ END
 directive=''; make_can_chain_suffix_rules || directive=TODO
 
 for target in all distcheck; do
-  if $MAKE $target; then r='ok'; else r='not ok'; fi
-  result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" \
-               -- "make $target"
+  command_ok_ "make $target"  \
+              -D "$directive" -r "suffix rules not chained" \
+              $MAKE $target
 done
 
 # FIXME: should we check that intermediate file `foo.cc' has
diff --git a/tests/suffix8.tap b/tests/suffix8.tap
index 50d0967..7b07496 100755
--- a/tests/suffix8.tap
+++ b/tests/suffix8.tap
@@ -83,9 +83,9 @@ command_ok_ "make test1" $MAKE test1
 directive=''; make_can_chain_suffix_rules || directive=TODO
 
 for target in test2 all distcheck; do
-  if $MAKE $target; then r='ok'; else r='not ok'; fi
-  result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" \
-               -- "make $target"
+  command_ok_ "make $target"  \
+              -D "$directive" -r "suffix rules not chained" \
+              $MAKE $target
 done
 
 :
diff --git a/tests/tap-bad-prog.tap b/tests/tap-bad-prog.tap
index cf475a1..8ba153f 100755
--- a/tests/tap-bad-prog.tap
+++ b/tests/tap-bad-prog.tap
@@ -77,13 +77,7 @@ fi
 
 # Check that no spurious test results is reported.  This is lower-priority
 # (and in fact the check currently fails.
-if count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3
-then
-  r='ok'
-else
-  r='not ok'
-fi
-result_ "$r" --directive TODO --reason 'still get "missing plan"' \
-            'no spurious results'
+command_ok_ 'no spurious results' -D TODO -r 'still get "missing plan"' \
+  count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3
 
 :
diff --git a/tests/tap-functions.sh b/tests/tap-functions.sh
index 5cd6c1a..603b53c 100644
--- a/tests/tap-functions.sh
+++ b/tests/tap-functions.sh
@@ -42,31 +42,41 @@ else
   incr_ () { eval "$1=\`expr \${$1} + 1\`"; }
 fi
 
-# plan_ NUMBER-OF-PLANNED-TESTS
-# -----------------------------
-# Print a TAP plan for the given number of tests.  This must be called
-# before reporting any test result; in fact, it must be called before
-# emitting anything on standard output.
-plan_ ()
+# not COMMAND [ARGS...]
+# ---------------------
+# Run the given command and invert its exit status.
+not ()
 {
-  echo "1..$1"
-  have_tap_plan_=yes
+  if "$@"; then return 1; else return 0; fi
 }
 
-# late_plan_
-# ----------
-# Print a TAP plan that accounts for the number of tests seen so far.
-# This must be called after all the tests result have been reported;
-# in fact, after this has been called, nothing more can be print on
-# standard output.
-late_plan_ ()
+# plan_ [unknown|later|lazy|now|NUMBER-OF-PLANNED-TESTS]
+# ------------------------------------------------------
+# Print a TAP plan for the given number of tests.  This must be called
+# before reporting any test result.  If called with the special argument
+# "unknown" or "later", it will do nothing, expecting the calling script
+# to declare the plan later.  If called with the special argument "lazy"
+# or "now", it will print a TAP plan that accounts for the number of tests
+# seen so far.
+plan_ ()
 {
-  echo "1..$tap_count_"
-  have_tap_plan_=yes
+  if test $# -eq 0; then
+    bailout_ "plan_: missing argument"
+  elif test $# -ge 2; then
+    bailout_ "plan_: too many arguments"
+  elif test x"$1" = x"unknown" || test x"$1" = x"later"; then
+    : No-op.
+  elif test x"$1" = x"lazy" || test x"$1" = x"now"; then
+    echo "1..$tap_count_" # Number of test results seen so far.
+    have_tap_plan_=yes
+  elif test $1 -ge 0; then
+    echo "1..$1"
+    have_tap_plan_=yes
+  else
+    bailout_ "plan_: invalid argument '$1'"
+  fi
 }
-
-# Initialize it to avoid interferences from the environment.
-have_tap_plan_=no
+have_tap_plan_=no # Avoid interferences from the environment.
 
 # diag_ [EXPLANATION]
 # ------------------
@@ -139,30 +149,10 @@ result_ ()
   set -x # Restore shell xtraces.
 }
 
-# ok_ [DESCRIPTION...]
-# --------------------
-# Report a successful test.
-ok_ ()
-{
-  result_ 'ok' -- ${1+"$@"}
-}
-
-# not_ok_ [DESCRIPTION...]
-# ------------------------
-# Report a failed test.
-not_ok_ ()
-{
-  result_ 'not ok' -- ${1+"$@"}
-}
-
-# skip_ [-r REASON] [--] [DESCRIPTION...]
-# ---------------------------------------
-# Report a skipped test.  If the `-r' option is present, its argument is
-# give as the reason of the skip.
-skip_ ()
-{
-  result_ 'ok' -D SKIP ${1+"$@"}
-}
+#  Shorthands for common usages of `result_'.
+ok_ () { result_ 'ok' ${1+"$@"}; }
+not_ok_ () { result_ 'not ok' ${1+"$@"}; }
+skip_ () { result_ 'ok' -D SKIP ${1+"$@"}; }
 
 # skip_row_ COUNT [-r REASON] [--] [DESCRIPTION...]
 # -------------------------------------------------
@@ -175,24 +165,6 @@ skip_row_ ()
   for i_ in `seq $skip_count_`; do skip_ ${1+"$@"}; done
 }
 
-# xfail_ [-r REASON] [DESCRIPTION...]
-# ----------------------------------
-# Report a test that failed expectedly.  If the `-r' option is present, its
-# argument is give as the reason why the failure is expected.
-xfail_ ()
-{
-  result_ 'not ok' -D TODO ${1+"$@"}
-}
-
-# xpass_ [-r REASON] [DESCRIPTION...]
-# -----------------------------------
-# Report a test that failed unexpectedly.  If the `-r' option is present, its
-# argument is give as the reason why the failure is expected.
-xpass_ ()
-{
-  result_ 'ok' -D TODO ${1+"$@"}
-}
-
 # skip_all_ [REASON ...]
 # ----------------------
 # Skip all the tests in a test script.  Must be used before calling `plan_'
@@ -233,44 +205,28 @@ framework_failure_ ()
   bailout_ "set-up failure"${1+": $*"}
 }
 
-# command_ok_ TEST-DESCRIPTION [--] CMD [ARGS...]
-# -----------------------------------------------
-# Report a passed test if the given command returns with success,
-# a failed test otherwise.
+# command_ok_ TEST-DESCRIPTION [OPTIONS..] [--] CMD [ARGS...]
+# -----------------------------------------------------------
+# Helper subroutine for when a TAP result must be determined by the
+# outcome of a command.
 command_ok_ ()
 {
-  tap_desc_=$1; shift
-  test x"$1" != x"--" || shift
-  if "$@"; then
-    ok_ "$tap_desc_"
-  else
-    not_ok_ "$tap_desc_"
-  fi
-}
-
-# command_not_ok_ TEST-DESCRIPTION [--] CMD [ARGS...]
-# ---------------------------------------------------
-# Report a failed test if the given command returns with success,
-# a failed test otherwise.
-command_not_ok_ ()
-{
-  tap_desc_=$1; shift
-  test x"$1" != x"--" || shift
-  if "$@"; then
-    not_ok_ "$tap_desc_"
-  else
-    ok_ "$tap_desc_"
-  fi
-}
-
-# reset_test_count_ COUNT
-# -----------------------
-# Reset the count of the TAP test results seen so far to COUNT.
-# This function is for use in corner cases only (e.g., when `ok_' and
-# `not_ok_' must be used inside a subshell).  Be careful when using it!
-reset_test_count_ ()
-{
-  tap_count_=$1
+  tap_directive_= tap_reason_=
+  test $# -gt 0 || bailout_ "command_ok_: missing argument"
+  tap_description_=$1; shift
+  while test $# -gt 0; do
+    case $1 in
+      -D|--directive) tap_directive_=$2; shift;;
+      -r|--reason) tap_reason_=$2; shift;;
+      --) shift; break;;
+      -*) bailout_ "command_ok_: invalid option '$1'";;
+       *) break;;
+    esac
+    shift
+  done
+  tap_result_="ok"; "$@" || tap_result_="not ok"
+  result_ "$tap_result_" -D "$tap_directive_" -r "$tap_reason_" \
+          -- "$tap_description_"
 }
 
 :
-- 
1.7.2.3