[FYI] {testsuite-work} testsuite: use $SHELL to run tests which are shell scripts

[Stefano Lattarini]

Reference:
 <http://lists.gnu.org/archive/html/automake-patches/2011-01/msg00089.html>

On Sunday 09 January 2011, Stefano Lattarini wrote:
> On Sunday 09 January 2011, Ralf Wildenhues wrote:
> > * Stefano Lattarini wrote on Mon, Jan 03, 2011 at 04:04:26PM CET:
> > > References:
> > >  
> > > <http://lists.gnu.org/archive/html/automake-patches/2010-12/msg00010.html>
> > >  
> > > <http://lists.gnu.org/archive/html/automake-patches/2010-09/msg00004.html>
> > 
> > > I have rebased the patch on latest master again.  I'll wait until *sunday*
> > > evening before pushing to master.
> > > 
> > > That said, even if the patch is pretty simple, it's not completely 
> > > obvious,
> > > and a review before I push it would be very welcome.
> > 
> > I have some gripes with this patch.  I'm sorry, but can I ask you to
> > wait another couple of days?
> >
> Yes, of course. JTBS, I'll wait at least until next Sunday evening for
> feedback.
> 
> Regards,
>   Stefano
> 
I've finally rebased the patch, thrown in a couple of improvements, and pushed
to the `testsuite-work' branch.  Attached is what I've pushed.

Regards,
  Stefano

From 1321be7068464238d1c626abad0f52cb1cd6cba2 Mon Sep 17 00:00:00 2001
Message-Id: <address@hidden>
From: Stefano Lattarini <address@hidden>
Date: Fri, 3 Dec 2010 15:24:14 +0100
Subject: [PATCH] testsuite: use $SHELL to run tests which are shell scripts

* tests/Makefile.am (TEST_LOG_COMPILER): Define so that the
configure-time $SHELL is used to run the tests.
* tests/defs: Add code to re-execute by default the running
test script with configure-time $SHELL.  Updated comments.
* configure.ac: Check that `set -e' is working for $SHELL,
not for /bin/sh.
* tests/defs-static.in: Update comments.
* tests/README (Supported shells): Updated.
(Getting details from failures): Don't tell that tests are
run by /bin/sh by default.
---
 ChangeLog            |   14 +++++++++++++
 configure            |    6 ++--
 configure.ac         |    4 +-
 tests/Makefile.am    |    7 ++++++
 tests/Makefile.in    |    7 ++++++
 tests/README         |   24 +++++++++++++++++----
 tests/defs           |   54 +++++++++++++++++++++++++++++++++++++++++++++++--
 tests/defs-static.in |   10 ++++++--
 8 files changed, 110 insertions(+), 16 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 24183b7..d3219af 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2011-05-27  Stefano Lattarini  <address@hidden>
+
+       testsuite: use $SHELL to run tests which are shell scripts
+       * tests/Makefile.am (TEST_LOG_COMPILER): Define so that the
+       configure-time $SHELL is used to run the tests.
+       * tests/defs: Add code to re-execute by default the running
+       test script with configure-time $SHELL.  Updated comments.
+       * configure.ac: Check that `set -e' is working for $SHELL,
+       not for /bin/sh.
+       * tests/defs-static.in: Update comments.
+       * tests/README (Supported shells): Updated.
+       (Getting details from failures): Don't tell that tests are
+       run by /bin/sh by default.
+
 2011-05-26  Stefano Lattarini  <address@hidden>
 
        tests: split 'subst2.test' to improve modularity and coverage
diff --git a/configure b/configure
index 084d8d2..723491c 100755
--- a/configure
+++ b/configure
@@ -2796,12 +2796,12 @@ $as_echo "$ac_cv_path_FGREP" >&6; }
 
 
 
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether /bin/sh has working 
'set -e' with exit trap" >&5
-$as_echo_n "checking whether /bin/sh has working 'set -e' with exit trap... " 
>&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $SHELL has working 
'set -e' with exit trap" >&5
+$as_echo_n "checking whether $SHELL has working 'set -e' with exit trap... " 
>&6; }
 if ${am_cv_sh_errexit_works+:} false; then :
   $as_echo_n "(cached) " >&6
 else
-  if /bin/sh -ec "trap 'exit \$?' 0; (exit 77); exit 77"; test $? = 77
+  if $SHELL -ec "trap 'exit \$?' 0; (exit 77); exit 77"; test $? = 77
 then
   am_cv_sh_errexit_works=yes
 else
diff --git a/configure.ac b/configure.ac
index 857177b..b6a5586 100644
--- a/configure.ac
+++ b/configure.ac
@@ -179,9 +179,9 @@ AC_SUBST([MODIFICATION_DELAY])
 AC_PROG_EGREP
 AC_PROG_FGREP
 
-AC_CACHE_CHECK([whether /bin/sh has working 'set -e' with exit trap],
+AC_CACHE_CHECK([whether $SHELL has working 'set -e' with exit trap],
 [am_cv_sh_errexit_works],
-[if /bin/sh -ec "trap 'exit \$?' 0; (exit 77); exit 77"; test $? = 77
+[if $SHELL -ec "trap 'exit \$?' 0; (exit 77); exit 77"; test $? = 77
 then
   am_cv_sh_errexit_works=yes
 else
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 09adc65..d8f8599 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -20,6 +20,8 @@ MAINTAINERCLEANFILES =
 EXTRA_DIST = ChangeLog-old
 
 TEST_EXTENSIONS = .test
+# Run the tests with the shell detected at configure time.
+TEST_LOG_COMPILER = $(SHELL)
 
 XFAIL_TESTS = \
 all.test \
@@ -109,12 +111,17 @@ $(depmod_tests):
 # test scripts, but not from the environment.
 # We want warning messages and explanations for skipped tests to go to
 # the console if possible, so set up `stderr_fileno_' properly.
+# The `AM_TESTS_REEXEC=no' setting tells the tests not to needlessly
+# re-execute themselves with the shell detected at configure time, since
+# we are already running them under it explicitly in our setup (see e.g.
+# the definition of TEST_LOG_COMPILER above).
 AM_TESTS_ENVIRONMENT = \
   test x"$$me" = x || unset me; \
   test x"$$required" = x || unset required; \
   test x"$$parallel_tests" = x || unset parallel_tests; \
   test x"$$original_AUTOMAKE" = x || unset original_AUTOMAKE; \
   test x"$$original_ACLOCAL" = x || unset original_ACLOCAL; \
+  AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC; \
   exec 9>&2; stderr_fileno_=9; export stderr_fileno_;
 
 TESTS = \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index ad8b2c0..b64da1f 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -299,6 +299,8 @@ MAINTAINERCLEANFILES = $(parallel_tests)
 EXTRA_DIST = ChangeLog-old gen-parallel-tests instspc-tests.sh \
        depmod-tests.sh $(handwritten_tests) $(generated_tests)
 TEST_EXTENSIONS = .test .instspc .depmod
+# Run the tests with the shell detected at configure time.
+TEST_LOG_COMPILER = $(SHELL)
 XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test \
        override-conditional-2.test pr8365-remake-timing.test \
        yacc-dist-nobuild-subdir.test txinfo5.test \
@@ -377,12 +379,17 @@ DEPMOD_LOG_COMPILER = $(SHELL) $(srcdir)/depmod-tests.sh
 # test scripts, but not from the environment.
 # We want warning messages and explanations for skipped tests to go to
 # the console if possible, so set up `stderr_fileno_' properly.
+# The `AM_TESTS_REEXEC=no' setting tells the tests not to needlessly
+# re-execute themselves with the shell detected at configure time, since
+# we are already running them under it explicitly in our setup (see e.g.
+# the definition of TEST_LOG_COMPILER above).
 AM_TESTS_ENVIRONMENT = \
   test x"$$me" = x || unset me; \
   test x"$$required" = x || unset required; \
   test x"$$parallel_tests" = x || unset parallel_tests; \
   test x"$$original_AUTOMAKE" = x || unset original_AUTOMAKE; \
   test x"$$original_ACLOCAL" = x || unset original_ACLOCAL; \
+  AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC; \
   exec 9>&2; stderr_fileno_=9; export stderr_fileno_;
 
 TESTS = \
diff --git a/tests/README b/tests/README
index 14a128d..2db4e22 100644
--- a/tests/README
+++ b/tests/README
@@ -61,10 +61,10 @@ Interpretation
 Getting details from failures
 -----------------------------
 
-  Each test is a shell script, and by default is run by /bin/sh.
-  In a non-VPATH build you can run them directly, they will be verbose.
-  By default, verbose output of a test foo.test is retained in the log
-  file foo.log.  A summary log is created in the file test-suite.log.
+  Each test is a shell script.  In a non-VPATH build you can run the
+  tests directly, they will be verbose.  By default, verbose output of
+  a test foo.test is retained in the log file foo.log.  A summary log
+  is created in the file test-suite.log.
 
   You can limit the set of files using the TESTS variable, and enable
   detailed test output at the end of the test run with the VERBOSE
@@ -76,6 +76,20 @@ Getting details from failures
 Supported shells
 ----------------
 
+  By default, the tests are run by the $SHELL detected at configure
+  time.  They also take care to re-execute themselves with that shell,
+  unless told not to.  So, to run the tests with a different shell, say
+  `/path/to/another/sh', the user must use:
+
+    AM_TESTS_REEXEC=no /path/to/another/sh ./foo.test
+
+  to run a test directly, and:
+
+    make check TEST_LOG_COMPILER=/path/to/sh            (GNU make)
+    TEST_LOG_COMPILER=/path/to/sh make -e check         (non-GNU make)
+
+  to run the test(s) through the makefile test driver.
+
   The test scripts are written with portability in mind, so that they
   should run with any decent Bourne-compatible shell.
 
@@ -87,7 +101,7 @@ Supported shells
   version of Zsh is used.  Thus, if you want to run a test script, say
   foo.test, with Zsh 4.2, you *can't* simply do `zsh foo.test', but
   you *must* resort to:
-    zsh -o no_function_argzero foo.test
+    AM_TESTS_REEXEC=no zsh -o no_function_argzero foo.test
 
   Note that this problem does not occur if Zsh is executed through
   a symlink with a basename of `sh', since in that case Zsh starts
diff --git a/tests/defs b/tests/defs
index e3c922c..5494625 100644
--- a/tests/defs
+++ b/tests/defs
@@ -17,6 +17,12 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+
+# IMPORTANT NOTE: This file should execute correctly with any system's
+# /bin/sh shell, and not only with configure-time detected $CONFIG_SHELL,
+# until differently and explicitly specified.
+
+
 ## -------------------------------------------------------- ##
 ##  Source static setup and definitions for the testsuite.  ##
 ## -------------------------------------------------------- ##
@@ -42,9 +48,9 @@ if test -z "$me"; then
     || { echo "$argv0: failed to define \$me" >&2; exit 99; }
 fi
 
-## ---------------------------------------- ##
-##  Sanity checks and environment cleanup.  ##
-## ---------------------------------------- ##
+## ---------------------- ##
+##  Early sanity checks.  ##
+## ---------------------- ##
 
 # A single whitespace character.
 sp=' '
@@ -70,6 +76,48 @@ test -f "$testbuilddir/defs-static" || {
    exit 99
 }
 
+# Ensure we can find ourselves.
+test -f "$0" || {
+   echo "$me: unable to find myself: $0" >&2
+   exit 1
+}
+
+
+## ------------------------------------ ##
+##  Ensure we run with a proper shell.  ##
+## ------------------------------------ ##
+
+# Make sure we run with the shell detected at configure time (unless
+# the user forbids it).
+case ${AM_TESTS_REEXEC-yes} in
+  n|no|false|0)
+    ;;
+  *)
+    AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC
+    # Cannot simply do `opts=$-', since the content of $- is not
+    # portable among different shells.  So try to propagate only
+    # the portable and interesting options.
+    case $- in
+      *x*v*|*v*x) opts=-vx;;
+      *v*) opts=-v;;
+      *x*) opts=-x;;
+      *) opts=;;
+    esac
+    echo $me: exec $SHELL $opts "$0" "$*"
+    exec $SHELL $opts "$0" ${1+"$@"}
+    echo "$me: failed to re-execute with $SHELL" >&2
+    exit 99
+    ;;
+esac
+
+# NOTE: From this point on, we can assume this file is being executed
+# by the configure-time detected $CONFIG_SHELL.
+
+
+## ---------------------- ##
+##  Environment cleanup.  ##
+## ---------------------- ##
+
 # Unset some MAKE... variables that may cause $MAKE to act like a
 # recursively invoked sub-make.  Any $MAKE invocation in a test is
 # conceptually an independent invocation, not part of the main
diff --git a/tests/defs-static.in b/tests/defs-static.in
index df5072f..8f64dc6 100644
--- a/tests/defs-static.in
+++ b/tests/defs-static.in
@@ -19,8 +19,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Defines and minimal setup for Automake testing environment.
-# Multiple inclusions of this file should be idempotent!
-# This code needs to be 'set -e' clean.
+
+# IMPORTANT NOTES AND REQUIREMENTS
+#   - Multiple inclusions of this file should be idempotent.
+#   - This code has to be 'set -e' clean.
+#   - This file should execute correctly with any system's /bin/sh
+#     shell, not only with configure-time detected $CONFIG_SHELL.
 
 # Be more Bourne compatible.
 # (Snippet inspired to configure's initialization in Autoconf 2.64)
@@ -133,7 +137,7 @@ FGREP='@FGREP@'
 # this variable.
 TEX='@TEX@'
 
-# Whether /bin/sh has working 'set -e' with exit trap.
+# Whether $SHELL has working 'set -e' with exit trap.
 sh_errexit_works='@sh_errexit_works@'
 
 # The amount we should wait after modifying files depends on the platform.
-- 
1.7.2.3