[FYI] {parallel-tests-maint} check: document and test $(TEST_SUITE_LOG) overriding

[Stefano Lattarini]

* doc/automake.texi (Simple Tests using parallel-tests): Explain
how and why TEST_SUITE_LOG can be overridden at runtime.
* tests/parallel-tests-log-override-1.test: New test, check that
the newly documented idiom and similar usages are truly supported.
* tests/parallel-tests-log-override-2.test: Likewise.
* tests/parallel-tests-log-override-recheck.test: Likewise.
* tests/Makefile.am (TESTS): Update.
---
 ChangeLog                                      |   11 +++
 doc/automake.texi                              |   17 ++++
 tests/Makefile.am                              |    3 +
 tests/Makefile.in                              |    3 +
 tests/parallel-tests-log-override-1.test       |  112 ++++++++++++++++++++++++
 tests/parallel-tests-log-override-2.test       |   88 +++++++++++++++++++
 tests/parallel-tests-log-override-recheck.test |   92 +++++++++++++++++++
 7 files changed, 326 insertions(+), 0 deletions(-)
 create mode 100755 tests/parallel-tests-log-override-1.test
 create mode 100755 tests/parallel-tests-log-override-2.test
 create mode 100755 tests/parallel-tests-log-override-recheck.test

diff --git a/ChangeLog b/ChangeLog
index b9c71f3..06b9788 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2011-05-11  Stefano Lattarini  <address@hidden>
+
+       check: document and test $(TEST_SUITE_LOG) overriding
+       * doc/automake.texi (Simple Tests using parallel-tests): Explain
+       how and why TEST_SUITE_LOG can be overridden at runtime.
+       * tests/parallel-tests-log-override-1.test: New test, check that
+       the newly documented idiom and similar usages are truly supported.
+       * tests/parallel-tests-log-override-2.test: Likewise.
+       * tests/parallel-tests-log-override-recheck.test: Likewise.
+       * tests/Makefile.am (TESTS): Update.
+
 2011-05-07  Stefano Lattarini  <address@hidden>
 
        tests: fix spurious failure of txinfo21.test on FreeBSD
diff --git a/doc/automake.texi b/doc/automake.texi
index 3c4bddd..3d5dcc5 100644
--- a/doc/automake.texi
+++ b/doc/automake.texi
@@ -8772,6 +8772,20 @@ use a command like this to run only a subset of the 
tests:
 env TESTS="foo.test bar.test" make -e check
 @end example
 
+Note however that the command above will unconditionally overwrite the
address@hidden file, thus clobbering the recorded results
+of any previous testsuite run.  This might be undesirable for packages
+whose testsuite takes long time to execute.  Luckily, this problem can
+easily be avoided by overriding also @code{TEST_SUITE_LOG} at runtime;
+for example,
+
address@hidden
+env TEST_SUITE_LOG=partial.log TESTS="..." make -e check
address@hidden example
+
+will write the result of the partial testsuite runs to the
address@hidden, without touching @file{test-suite.log}.
+
 @item
 You can set the @code{TEST_LOGS} variable.  By default, this variable is
 computed at @command{make} run time from the value of @code{TESTS} as
@@ -8782,6 +8796,9 @@ set x subset*.log; shift
 env TEST_LOGS="foo.log $*" make -e check
 @end example
 
+The comments made above about @code{TEST_SUITE_LOG} overriding applies
+here too.
+
 @item
 @vindex RECHECK_LOGS
 @cindex lazy test execution
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 49d8942..33599ad 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -590,6 +590,9 @@ parallel-tests9.test \
 parallel-tests10.test \
 parallel-tests-unreadable-log.test \
 parallel-tests-subdir.test \
+parallel-tests-log-override-1.test \
+parallel-tests-log-override-2.test \
+parallel-tests-log-override-recheck.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 78d723c..44aac47 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -861,6 +861,9 @@ parallel-tests9.test \
 parallel-tests10.test \
 parallel-tests-unreadable-log.test \
 parallel-tests-subdir.test \
+parallel-tests-log-override-1.test \
+parallel-tests-log-override-2.test \
+parallel-tests-log-override-recheck.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/parallel-tests-log-override-1.test 
b/tests/parallel-tests-log-override-1.test
new file mode 100755
index 0000000..6e98380
--- /dev/null
+++ b/tests/parallel-tests-log-override-1.test
@@ -0,0 +1,112 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: runtime redefinition of $(TEST_SUITE_LOG).
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = pass.test skip.test xfail.test
+XFAIL_TESTS = xfail.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > pass.test <<'END'
+#! /bin/sh
+exit 0
+END
+
+cat > skip.test <<'END'
+#! /bin/sh
+echo "% test skipped %"
+exit 77
+END
+
+cat > xfail.test <<'END'
+#! /bin/sh
+echo "# expected failure #"
+exit 1
+END
+
+chmod a+x *.test
+
+test_log_edit ()
+{
+  sed -e "s|^  *$me 1\.0:.*$|  $me 1.0: ???|" \
+      -e "s|^=====*|=======================|" $*
+}
+
+test_log_expected ()
+{
+  test_log_edit orig > exp
+  test_log_edit $1   > got
+  diff exp got || Exit 1
+  rm -f exp got
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check
+ls -l
+cat test-suite.log
+cp test-suite.log orig
+
+$MAKE clean
+test -f test-suite.log && Exit 99 # Sanity check.
+
+# Check that we can override the testsuite log file at runtime.
+TEST_SUITE_LOG=zardoz.log $MAKE -e check
+ls -l
+test ! -f test-suite.log
+cat zardoz.log
+test_log_expected zardoz.log
+# Sanity check the distribution too (this also does minimal checks on
+# VPATH support).
+TEST_SUITE_LOG=zardoz.log $MAKE -e distcheck
+
+# Check that cleanup rules remove the correct file even when
+# user overrides are in place.
+cp orig test-suite.log
+TEST_SUITE_LOG=zardoz.log $MAKE -e clean
+ls -l
+test ! -f zardoz.log
+diff orig test-suite.log
+
+# Check that the default testsuite log doesn't get unduly modified.
+# Also check that the testsuite log file doesn't need to be named
+# accordingly to the `*.log' pattern.
+chmod a-w test-suite.log
+TEST_SUITE_LOG=TheLogFile $MAKE -e check
+ls -l
+diff orig test-suite.log
+test_log_expected TheLogFile
+TEST_SUITE_LOG=TheLogFile $MAKE -e clean
+ls -l
+test ! -f TheLogFile
+diff orig test-suite.log
+
+:
diff --git a/tests/parallel-tests-log-override-2.test 
b/tests/parallel-tests-log-override-2.test
new file mode 100755
index 0000000..649360c
--- /dev/null
+++ b/tests/parallel-tests-log-override-2.test
@@ -0,0 +1,88 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: runtime redefinition of:
+#  - $(TEST_SUITE_LOG) and $(TESTS)
+#  - $(TEST_SUITE_LOG) and $(TEST_LOGS)
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = pass.test pass2.test skip.test skip2.test fail.test
+END
+
+cat > pass.test <<'END'
+#! /bin/sh
+exit 0
+END
+
+cp pass.test pass2.test
+
+cat > skip.test <<'END'
+#! /bin/sh
+echo "% skipped test %"
+exit 77
+END
+
+cp skip.test skip2.test
+
+cat > fail.test <<'END'
+#! /bin/sh
+exit 1
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+for test_list_override in \
+  'TESTS=pass.test skip.test' \
+  'TEST_LOGS=pass.log skip.log'
+do
+  env TEST_SUITE_LOG=partial.log "$test_list_override" \
+    $MAKE -e check >stdout || { cat stdout; Exit 1; }
+  cat stdout
+  ls -l
+  cat pass.log
+  cat skip.log
+  cat partial.log
+  test ! -f test-suite.log
+  test ! -f pass2.log
+  test ! -f skip2.log
+  test ! -f fail.log
+  grep '^PASS: .*pass\.test' stdout
+  grep '^SKIP: .*skip\.test' stdout
+  $FGREP 'skip.test' partial.log
+  $FGREP '% skipped test %' partial.log
+  for t in pass2 skip2 fail; do
+    $FGREP "$t.test" stdout && Exit 1
+    $FGREP "$t.test" partial.log && Exit 1
+  done
+  rm -f *.log
+done
+
+:
diff --git a/tests/parallel-tests-log-override-recheck.test 
b/tests/parallel-tests-log-override-recheck.test
new file mode 100755
index 0000000..8a50d29
--- /dev/null
+++ b/tests/parallel-tests-log-override-recheck.test
@@ -0,0 +1,92 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: runtime redefinition of $(TEST_SUITE_LOG)
+# for the recheck target.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = foo.test bar.test baz.test
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 0
+END
+
+cat > bar.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 99
+END
+
+cat > baz.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit ${BAZ_EXIT_STATUS-1}
+END
+
+chmod a+x *.test
+
+unset BAZ_EXIT_STATUS || :
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+chmod a-rw test-suite.log
+TEST_SUITE_LOG=my.log $MAKE -e recheck >stdout \
+  && { cat stdout; Exit 1; }
+cat stdout
+ls -l
+grep '^2 of 2 .*failed' stdout
+for x in stdout my.log; do
+  $FGREP foo.test $x && Exit 1
+  $FGREP bar.test $x
+  $FGREP baz.test $x
+done
+
+chmod a-rw my.log
+BAZ_EXIT_STATUS=0 TEST_SUITE_LOG=my2.log $MAKE -e recheck >stdout \
+  && { cat stdout; Exit 1; }
+cat stdout
+ls -l
+grep '^1 of 2 .*failed' stdout
+$FGREP foo.test stdout && Exit 1
+$FGREP bar.test stdout
+$FGREP baz.test stdout
+$FGREP foo.test my2.log && Exit 1
+$FGREP bar.test my2.log
+$FGREP baz.test my2.log && Exit 1
+
+chmod u+r test-suite.log my.log
+$FGREP baz.test test-suite.log
+$FGREP baz.test my.log
+
+:
-- 
1.7.2.3