[FYI] {parallel-tests-maint} parallel-tests: stricter checks on DISABLE_HARD_ERRORS support

[Stefano Lattarini]

* tests/parallel-tests-harderror.test: New test, doing more
in-depth checks on DISABLE_HARD_ERRORS.
* tests/parallel-tests.test: Remove tests on DISABLE_HARD_ERRORS,
now redundant.
* tests/Makefile.am (TESTS): Update.
---
 ChangeLog                           |    9 +++
 tests/Makefile.am                   |    1 +
 tests/Makefile.in                   |    1 +
 tests/parallel-tests-harderror.test |   98 +++++++++++++++++++++++++++++++++++
 tests/parallel-tests.test           |   18 ++-----
 5 files changed, 114 insertions(+), 13 deletions(-)
 create mode 100755 tests/parallel-tests-harderror.test

diff --git a/ChangeLog b/ChangeLog
index 6eb09d7..6ec063d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+2011-05-22  Stefano Lattarini  <address@hidden>
+
+       parallel-tests: stricter checks on DISABLE_HARD_ERRORS support
+       * tests/parallel-tests-harderror.test: New test, doing more
+       in-depth checks on DISABLE_HARD_ERRORS.
+       * tests/parallel-tests.test: Remove tests on DISABLE_HARD_ERRORS,
+       now redundant.
+       * tests/Makefile.am (TESTS): Update.
+
 2011-05-20  Stefano Lattarini  <address@hidden>
 
        testsuite: avoid re-running few tests with 'parallel-tests' option
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 33599ad..86a49c0 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -588,6 +588,7 @@ parallel-tests7.test \
 parallel-tests8.test \
 parallel-tests9.test \
 parallel-tests10.test \
+parallel-tests-harderror.test \
 parallel-tests-unreadable-log.test \
 parallel-tests-subdir.test \
 parallel-tests-log-override-1.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 80f6b15..42272c7 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -856,6 +856,7 @@ parallel-tests7.test \
 parallel-tests8.test \
 parallel-tests9.test \
 parallel-tests10.test \
+parallel-tests-harderror.test \
 parallel-tests-unreadable-log.test \
 parallel-tests-subdir.test \
 parallel-tests-log-override-1.test \
diff --git a/tests/parallel-tests-harderror.test 
b/tests/parallel-tests-harderror.test
new file mode 100755
index 0000000..8680e29
--- /dev/null
+++ b/tests/parallel-tests-harderror.test
@@ -0,0 +1,98 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: DISABLE_HARD_ERRORS
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_CONFIG_FILES([sub/Makefile])
+AC_OUTPUT
+END
+
+mkdir sub
+
+cat > Makefile.am << 'END'
+SUBDIRS = . sub
+TESTS = foo.test
+XFAIL_TESTS = foo.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > sub/Makefile.am << 'END'
+TESTS = bar.test
+XFAIL_TESTS = $(TESTS)
+EXTRA_DIST = $(TESTS)
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+exit 99
+END
+chmod a+x foo.test
+cp foo.test sub/bar.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+# DISABLE_HARD_ERRORS is not defined as a make variable, so that it
+# should be possible to define either from the environment or from
+# command-line, also when recursive make invocations are involved,
+# and also without using the `-e' make flag; this should work also
+# for non-GNU make. Moreover, it shouldn't be necessary to define
+# DISABLE_HARD_ERRORS to "yes" to really disable hard errors: any
+# non-empty value should do.
+$MAKE check DISABLE_HARD_ERRORS=yes
+DISABLE_HARD_ERRORS=x $MAKE check
+
+# But an empty values for DISABLE_HARD_ERRORS means that hard errors
+# are not to be counted like normal failures.
+
+$MAKE check DISABLE_HARD_ERRORS='' && Exit 1
+cat test-suite.log
+grep '^FAIL: foo\.test .*exit.*99' test-suite.log
+
+cd sub
+# The `-e' is wanted here.
+DISABLE_HARD_ERRORS='' $MAKE -e check && Exit 1
+cat test-suite.log
+grep '^FAIL: bar\.test .*exit.*99' test-suite.log
+cd ..
+
+# Check the distributions.
+$MAKE DISABLE_HARD_ERRORS=y distcheck
+
+# Finally, DISABLE_HARD_ERRORS should work also when the developer
+# sets it directly in Makefile.am or Makefile.  And its effects
+# should remain local to that specific Makefile, obviously.
+
+echo 'DISABLE_HARD_ERRORS = yes' >> Makefile.am
+$AUTOMAKE Makefile
+./config.status Makefile
+VERBOSE=yes $MAKE check && Exit 1
+grep '^FAIL' test-suite.log && Exit 1
+grep '^FAIL: bar\.test .*exit.*99' sub/test-suite.log
+
+echo 'DISABLE_HARD_ERRORS = zardoz' >> sub/Makefile
+VERBOSE=yes $MAKE check
+
+:
diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test
index 399b23b..54f0cd5 100755
--- a/tests/parallel-tests.test
+++ b/tests/parallel-tests.test
@@ -19,7 +19,6 @@
 # - clean
 # - TEST_SUITE_LOG
 # - dependencies between tests
-# - DISABLE_HARD_ERRORS
 # - TESTS
 # - TEST_LOGS
 # - RECHECK_LOGS
@@ -68,12 +67,13 @@ $AUTOCONF
 $AUTOMAKE -a
 
 ./configure
-# No hard errors: all tests should be run, there should be one failure.
-env DISABLE_HARD_ERRORS=yes $MAKE -e check >stdout && { cat stdout; Exit 1; }
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
 cat stdout
-test `grep -c '^FAIL' stdout` -eq 1
+# There should be two errors: bar.test is a hard error.
+test `grep -c '^FAIL' stdout` -eq 2
 test -f mylog.log
-test `grep -c '^FAIL' mylog.log` -eq 1
+test `grep -c '^FAIL' mylog.log` -eq 2
 test -f baz.log
 test -f bar.log
 test -f foo.log
@@ -89,19 +89,11 @@ test ! -f foo.log
 test ! -f mylog.log
 test -f unrelated.log
 
-$MAKE clean
-$MAKE check >stdout && { cat stdout; Exit 1; }
-cat stdout
-# Now, there should be two errors: bar.test is a hard error.
-test `grep -c '^FAIL' stdout` -eq 2
-test `grep -c '^FAIL' mylog.log` -eq 2
-
 # Check dependencies: baz.test needs to run before bar.test,
 # but foo.test is not needed.
 # Note that this usage has a problem: the summary will only
 # take bar.log into account, because the $(TEST_SUITE_LOG) rule
 # does not "see" baz.log.  Hmm.
-$MAKE clean
 env TESTS='bar.test' $MAKE -e check && Exit 1
 test -f baz.log
 test -f bar.log
-- 
1.7.2.3