Re: [PATCH] {maint} tests: new subroutines for test skipping/failing

[Stefano Lattarini]

Hello automakers.

> check: new developer-reserved AM_TESTS_SETUP variable
>
> For reference, see the discussion at:
> <http://lists.gnu.org/archive/html/automake-patches/2011-01/msg00213.html>
>
> * lib/am/check.am [%?PARALLEL_TESTS%] (am__check_pre): Pass also
> $(AM_TESTS_SETUP).  Comments updated, and some typos fixed.
> * doc/automake.texi (Simple Tests using parallel-tests): Document
> AM_TESTS_SETUP.  Reorder some of the existing documentation a bit.
> * tests/parallel-tests-am_tests_setup.test: New test.
>
> From a suggestion by Ralf Wildenhues.
>
While writing the above patch, I noticed that the coverage of the
simple-tests and parallel-tests drivers is still lacking in some
respects.  The attached patch improves the situation somewhat.

OK for maint?

Regards,
  Stefano

From 1ff065e251caecf589f1d7b183a6b236accc5542 Mon Sep 17 00:00:00 2001
From: Stefano Lattarini <address@hidden>
Date: Mon, 24 Jan 2011 15:56:52 +0100
Subject: [PATCH] coverage: more tests on simple and parallel test drivers

* tests/parallel-tests-subdir.test: New test.
* tests/check-exported-srcdir.test: Likewise.
* tests/check-tests-in-builddir.test: Likewise.
* tests/check-tests_environment.test: Likewise.
* tests/Makefile.am (TESTS): Update.
---
 ChangeLog                          |    9 ++++
 tests/Makefile.am                  |    4 ++
 tests/Makefile.in                  |    7 +++
 tests/check-exported-srcdir.test   |   65 +++++++++++++++++++++++++++++
 tests/check-tests-in-builddir.test |   81 ++++++++++++++++++++++++++++++++++++
 tests/check-tests_environment.test |   47 +++++++++++++++++++++
 tests/parallel-tests-subdir.test   |   51 ++++++++++++++++++++++
 7 files changed, 264 insertions(+), 0 deletions(-)
 create mode 100755 tests/check-exported-srcdir.test
 create mode 100755 tests/check-tests-in-builddir.test
 create mode 100755 tests/check-tests_environment.test
 create mode 100755 tests/parallel-tests-subdir.test

diff --git a/ChangeLog b/ChangeLog
index c632b44..cf55538 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+2011-01-24  Stefano Lattarini  <address@hidden>
+
+       coverage: more tests on simple and parallel test drivers
+       * tests/parallel-tests-subdir.test: New test.
+       * tests/check-exported-srcdir.test: Likewise.
+       * tests/check-tests-in-builddir.test: Likewise.
+       * tests/check-tests_environment.test: Likewise.
+       * tests/Makefile.am (TESTS): Update.
+
 2011-01-23  Stefano Lattarini  <address@hidden>
 
        coverage: test semantics of "dummy" per-target flags
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 78bacf2..c9f8467 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -141,6 +141,9 @@ check8.test \
 check9.test \
 check10.test \
 check11.test \
+check-exported-srcdir.test \
+check-tests-in-builddir.test \
+check-tests_environment.test \
 checkall.test \
 clean.test \
 clean2.test \
@@ -568,6 +571,7 @@ parallel-tests8.test \
 parallel-tests9.test \
 parallel-tests10.test \
 parallel-tests-unreadable-log.test \
+parallel-tests-subdir.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 18cb337..1370461 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -280,6 +280,9 @@ gcj6.test \
 txinfo5.test
 
 parallel_tests = \
+check-exported-srcdir-p.test \
+check-tests-in-builddir-p.test \
+check-tests_environment-p.test \
 check-p.test \
 check10-p.test \
 check11-p.test \
@@ -408,6 +411,9 @@ check8.test \
 check9.test \
 check10.test \
 check11.test \
+check-exported-srcdir.test \
+check-tests-in-builddir.test \
+check-tests_environment.test \
 checkall.test \
 clean.test \
 clean2.test \
@@ -835,6 +841,7 @@ parallel-tests8.test \
 parallel-tests9.test \
 parallel-tests10.test \
 parallel-tests-unreadable-log.test \
+parallel-tests-subdir.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/check-exported-srcdir.test b/tests/check-exported-srcdir.test
new file mode 100755
index 0000000..9209fc8
--- /dev/null
+++ b/tests/check-exported-srcdir.test
@@ -0,0 +1,65 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check that the "Simple Tests" driver (either with or without the
+# parallel-tests option enabled) exports the `srcdir' value in the
+# environment of the tests.  This is documented in the manual.
+
+. ./defs || Exit 1
+
+set -e
+
+show_info ()
+{
+  if test x"$parallel_tests" = x"yes"; then
+     cat foo.log
+     cat test-suite.log
+  else
+     :
+  fi
+}
+
+mkdir SrcDir BuildDir
+
+mv [a-z]* SrcDir
+cd SrcDir
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test
+END
+
+cat > foo.test <<'END'
+#!/bin/sh
+echo "foo.test: srcdir='$srcdir'"
+test x"$srcdir" = x"../SrcDir"
+END
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+cd ../BuildDir
+../SrcDir/configure
+
+$MAKE check || { show_info; Exit 1; }
+show_info
+
+:
diff --git a/tests/check-tests-in-builddir.test 
b/tests/check-tests-in-builddir.test
new file mode 100755
index 0000000..b30999b
--- /dev/null
+++ b/tests/check-tests-in-builddir.test
@@ -0,0 +1,81 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check that the "Simple Tests" driver can find test in the srcdir as
+# well as in builddir, and that is prefers those in the builddir.
+
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > foo.test << 'END'
+#! /bin/sh
+exit ${FOO_EXIT_STATUS-0}
+END
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+mkdir build
+cd build
+
+../configure
+
+cat > bar.test << 'END'
+#! /bin/sh
+exit 0
+END
+chmod a+x bar.test
+
+$MAKE check >out 2>&1 || { cat out; Exit1; }
+cat out
+grep '\.\./foo' out && Exit 1
+grep '^PASS: foo.test *$' out
+grep '^PASS: bar.test *$' out
+
+rm -f test-suite.log foo.log bar.log
+
+FOO_EXIT_STATUS=1 $MAKE check >out 2>&1 && { cat out; Exit1; }
+cat out
+grep '\.\./foo' out && Exit 1
+grep '^FAIL: foo.test *$' out
+grep '^PASS: bar.test *$' out
+
+rm -f test-suite.log foo.log bar.log
+
+# Check that if the same test is present in srcdir and builddir,
+# the one in builddir is preferred.
+cp bar.test foo.test
+FOO_EXIT_STATUS=1 $MAKE check >out 2>&1 || { cat out; Exit1; }
+cat out
+grep '^PASS: foo.test *$' out
+grep '^PASS: bar.test *$' out
+
+# The tests in the builddir must be preferred also by "make dist".
+FOO_EXIT_STATUS=1 $MAKE distcheck
+
+:
diff --git a/tests/check-tests_environment.test 
b/tests/check-tests_environment.test
new file mode 100755
index 0000000..a07ab95
--- /dev/null
+++ b/tests/check-tests_environment.test
@@ -0,0 +1,47 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# "Simple Tests" testsuite driver: check TESTS_ENVIRONMENT support.
+
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > foo.test << 'END'
+#! /bin/sh
+test x"$FOO" = x"ok"
+END
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+FOO=bad TESTS_ENVIRONMENT='FOO=ok'  $MAKE check
+FOO=ok  TESTS_ENVIRONMENT='FOO=bad' $MAKE check && Exit 1
+
+:
diff --git a/tests/parallel-tests-subdir.test b/tests/parallel-tests-subdir.test
new file mode 100755
index 0000000..18695fe
--- /dev/null
+++ b/tests/parallel-tests-subdir.test
@@ -0,0 +1,51 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check that the parallel-tests driver creates parent directories for
+# the log files when needed.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = dir1/foo.test dir2/dir3/foo.test
+TEST_LOG_COMPILER = sh
+END
+
+mkdir dir1 dir2 dir2/dir3
+echo : > dir1/foo.test
+echo : > dir2/dir3/foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+mkdir build
+cd build
+../configure
+$MAKE check
+find . # For debugging.
+test -f test-suite.log
+test -f dir1/foo.log
+test -f dir2/dir3/foo.log
+
+:
-- 
1.7.2.3