[PATCH 1/6] Maint-check: check consistency of list of test scripts.

[Stefano Lattarini]

This patch add a new maintainer-specific check:
  `maintainer-check-list-of-tests'.

This check verifies that the tests listed in $(TESTS) correspond to the 
test scripts on the filesystem, counting both the "committed" (e.g. 
`aclibobj.test')  and "generated" (e.g. `check-p.test') test scripts.
Note that maintainer-check fails after applying this patch, as some 
test scripts are missing from $(TESTS).  This will be fixed by the next 
patch.

From 45f9809c052f3237b6f346e1f4c5dd82572b4be8 Mon Sep 17 00:00:00 2001
From: Stefano Lattarini <address@hidden>
Date: Fri, 25 Dec 2009 23:17:07 +0100
Subject: [PATCH 1/6] Maint-check: check consistency of list of test scripts.

* Makefile.am (maintainer-check-list-of-tests): New target.
(maintainer-check): Added dependency from new target
`maintainer-check-list-of-tests'.
* tests/Makefile.am (maintainer-check-list-of-tests): New target,
check for consistency between list of tests defined in the Makefile
and list of tests on the filesystem.
(MOSTLYCLEANFILES): Extended to get rid of temporary files created
by `maintainer-check-list-of-tests'.
* lib/Automake/tests/Makefile.am: Likewise.
---
 ChangeLog                      |   13 ++++++++++++
 Makefile.am                    |   19 +++++++++++++++++-
 Makefile.in                    |   17 +++++++++++++++-
 lib/Automake/tests/Makefile.am |   42 ++++++++++++++++++++++++++++++++++++++++
 lib/Automake/tests/Makefile.in |   36 ++++++++++++++++++++++++++++++++++
 tests/Makefile.am              |   42 ++++++++++++++++++++++++++++++++++++++++
 tests/Makefile.in              |   36 ++++++++++++++++++++++++++++++++++
 7 files changed, 203 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 399a33e..7777d57 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+2009-12-25  Stefano Lattarini  <address@hidden>
+
+       Maint-check: check consistency of list of test scripts.
+       * Makefile.am (maintainer-check-list-of-tests): New target.
+       (maintainer-check): Added dependency from new target
+       `maintainer-check-list-of-tests'.
+       * tests/Makefile.am (maintainer-check-list-of-tests): New target,
+       check for consistency between list of tests defined in the Makefile
+       and list of tests on the filesystem.
+       (MOSTLYCLEANFILES): Extended to get rid of temporary files created
+       by `maintainer-check-list-of-tests'.
+       * lib/Automake/tests/Makefile.am: Likewise.
+
 2009-12-21  Julien Danjou  <address@hidden>  (tiny change)
 
        Add support for newer python version
diff --git a/Makefile.am b/Makefile.am
index 0c3f438..08fdc6f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -197,7 +197,24 @@ sc_at_in_texi
 
 .PHONY: $(syntax_check_rules)
 $(syntax_check_rules): automake aclocal
-maintainer-check: $(syntax_check_rules)
+
+## Check that the list of tests given in the Makefile is equal to the
+## list of all test scripts in the Automake testsuite.
+.PHONY: maintainer-check-list-of-tests
+maintainer-check-list-of-tests:
+       @failcom='exit 1'; \
+       for f in x $$MAKEFLAGS; do \
+         case $$f in \
+           *=* | --[!k]*);; \
+           *k*) failcom='fail=yes';; \
+         esac; \
+       done; \
+       for subdir in $(TEST_SUBDIRS); do \
+         (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) $@) || eval $$failcom; \
+       done; \
+       test -z "$$fail"
+
+maintainer-check: $(syntax_check_rules) maintainer-check-list-of-tests
 
 ## This check avoids accidental configure substitutions in the source.
 ## There are exactly 6 lines that should be modified.  This works out
diff --git a/Makefile.in b/Makefile.in
index d038a5c..c8fb016 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -940,7 +940,22 @@ clean-local: clean-coverage
 
 .PHONY: $(syntax_check_rules)
 $(syntax_check_rules): automake aclocal
-maintainer-check: $(syntax_check_rules)
+
+.PHONY: maintainer-check-list-of-tests
+maintainer-check-list-of-tests:
+       @failcom='exit 1'; \
+       for f in x $$MAKEFLAGS; do \
+         case $$f in \
+           *=* | --[!k]*);; \
+           *k*) failcom='fail=yes';; \
+         esac; \
+       done; \
+       for subdir in $(TEST_SUBDIRS); do \
+         (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) $@) || eval $$failcom; \
+       done; \
+       test -z "$$fail"
+
+maintainer-check: $(syntax_check_rules) maintainer-check-list-of-tests
 
 sc_diff_automake_in_automake:
        @if test `diff $(srcdir)/automake.in automake | wc -l` -ne 22; then \
diff --git a/lib/Automake/tests/Makefile.am b/lib/Automake/tests/Makefile.am
index 722c75c..dbd4f00 100644
--- a/lib/Automake/tests/Makefile.am
+++ b/lib/Automake/tests/Makefile.am
@@ -42,3 +42,45 @@ Version2.pl \
 Version3.pl
 
 EXTRA_DIST = $(TESTS)
+
+## temporary files used in the `check-list-of-tests' target
+tmk = tests-in-makefile-list.tmp
+tfs = tests-on-filesystem-list.tmp
+tdf = diff-in-tests-lists.tmp
+
+MOSTLYCLEANFILES = $(tmk) $(tfs) $(tdf)
+
+# Check that the list of tests given in the Makefile is equal to the
+# list of all test scripts in the Automake testsuite.
+maintainer-check-list-of-tests:
+       @:; \
+        ## prefer unified diffs over plain diffs, for readability
+        if diff -u /dev/null /dev/null >/dev/null 2>&1; then \
+          diff='diff -u'; \
+        else \
+          diff='diff'; \
+        fi; \
+        ## list of tests in Makefile
+        for t in $(TESTS); do \
+          echo "$$t"; \
+        done | sort >$(tmk); \
+        ## list of tests on filesystem
+        glob="$(srcdir)/*.pl"; \
+        if test x"$(srcdir)" != x"$(builddir)"; then \
+          glob="$$glob $(builddir)/*.pl"; \
+        fi; \
+        for t in $$glob; do \
+          echo "$$t"; \
+        done | sed 's,^.*/,,' | sort >$(tfs); \
+        ## compare the two lists
+        if $$diff $(tmk) $(tfs) >$(tdf); then \
+           e=0; \
+        else \
+           echo '$@: list of tests in Makefile an on filesystem differ' >&2; \
+           echo "+ $$diff in-makefile on-filesystem" >&2; \
+           cat $(tdf) >&2; \
+           e=1; \
+        fi; \
+        rm -f $(tmk) $(tfs) $(tdf); \
+        exit $$e;
+.PHONY: maintainer-check-list-of-tests
diff --git a/lib/Automake/tests/Makefile.in b/lib/Automake/tests/Makefile.in
index 0bbced3..c3712b9 100644
--- a/lib/Automake/tests/Makefile.in
+++ b/lib/Automake/tests/Makefile.in
@@ -255,6 +255,10 @@ Version2.pl \
 Version3.pl
 
 EXTRA_DIST = $(TESTS)
+tmk = tests-in-makefile-list.tmp
+tfs = tests-on-filesystem-list.tmp
+tdf = diff-in-tests-lists.tmp
+MOSTLYCLEANFILES = $(tmk) $(tfs) $(tdf)
 all: all-am
 
 .SUFFIXES:
@@ -503,6 +507,7 @@ install-strip:
          `test -z '$(STRIP)' || \
            echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
 mostlyclean-generic:
+       -test -z "$(MOSTLYCLEANFILES)" || rm -f $(MOSTLYCLEANFILES)
        -test -z "$(TEST_LOGS)" || rm -f $(TEST_LOGS)
        -test -z "$(TEST_LOGS_TMP)" || rm -f $(TEST_LOGS_TMP)
        -test -z "$(TEST_SUITE_HTML)" || rm -f $(TEST_SUITE_HTML)
@@ -598,6 +603,37 @@ uninstall-am:
        pdf-am ps ps-am recheck recheck-html uninstall uninstall-am
 
 
+# Check that the list of tests given in the Makefile is equal to the
+# list of all test scripts in the Automake testsuite.
+maintainer-check-list-of-tests:
+       @:; \
+        if diff -u /dev/null /dev/null >/dev/null 2>&1; then \
+          diff='diff -u'; \
+        else \
+          diff='diff'; \
+        fi; \
+        for t in $(TESTS); do \
+          echo "$$t"; \
+        done | sort >$(tmk); \
+        glob="$(srcdir)/*.pl"; \
+        if test x"$(srcdir)" != x"$(builddir)"; then \
+          glob="$$glob $(builddir)/*.pl"; \
+        fi; \
+        for t in $$glob; do \
+          echo "$$t"; \
+        done | sed 's,^.*/,,' | sort >$(tfs); \
+        if $$diff $(tmk) $(tfs) >$(tdf); then \
+           e=0; \
+        else \
+           echo '$@: list of tests in Makefile an on filesystem differ' >&2; \
+           echo "+ $$diff in-makefile on-filesystem" >&2; \
+           cat $(tdf) >&2; \
+           e=1; \
+        fi; \
+        rm -f $(tmk) $(tfs) $(tdf); \
+        exit $$e;
+.PHONY: maintainer-check-list-of-tests
+
 # Tell versions [3.59,3.63) of GNU make to not export all variables.
 # Otherwise a system limit (for SysV at least) may be exceeded.
 .NOEXPORT:
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 8936ff1..fcf5f0e 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -752,6 +752,48 @@ $(parallel_tests)
 
 EXTRA_DIST = ChangeLog-old gen-parallel-tests $(TESTS)
 
+## temporary files used in the `check-list-of-tests' target
+tmk = tests-in-makefile-list.tmp
+tfs = tests-on-filesystem-list.tmp
+tdf = diff-in-tests-lists.tmp
+
+MOSTLYCLEANFILES = $(tmk) $(tfs) $(tdf)
+
+# Check that the list of tests given in the Makefile is equal to the
+# list of all test scripts in the Automake testsuite.
+maintainer-check-list-of-tests: $(parallel_tests)
+       @:; \
+        ## prefer unified diffs over plain diffs, for readability
+        if diff -u /dev/null /dev/null >/dev/null 2>&1; then \
+          diff='diff -u'; \
+        else \
+          diff='diff'; \
+        fi; \
+        ## list of tests in Makefile
+        for t in $(TESTS); do \
+          echo "$$t"; \
+        done | sort >$(tmk); \
+        ## list of tests on filesystem
+        glob="$(srcdir)/*.test"; \
+        if test x"$(srcdir)" != x"$(builddir)"; then \
+          glob="$$glob $(builddir)/*.test"; \
+        fi; \
+        for t in $$glob; do \
+          echo "$$t"; \
+        done | sed 's,^.*/,,' | sort >$(tfs); \
+        ## compare the two lists
+        if $$diff $(tmk) $(tfs) >$(tdf); then \
+           e=0; \
+        else \
+           echo '$@: list of tests in Makefile an on filesystem differ' >&2; \
+           echo "+ $$diff in-makefile on-filesystem" >&2; \
+           cat $(tdf) >&2; \
+           e=1; \
+        fi; \
+        rm -f $(tmk) $(tfs) $(tdf); \
+        exit $$e;
+.PHONY: maintainer-check-list-of-tests
+
 # Each test case depends on defs, aclocal, and automake.
 check_SCRIPTS = defs defs-p aclocal-$(APIVERSION) automake-$(APIVERSION)
 
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 248493b..12c39a3 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -992,6 +992,10 @@ yaccvpath.test \
 $(parallel_tests)
 
 EXTRA_DIST = ChangeLog-old gen-parallel-tests $(TESTS)
+tmk = tests-in-makefile-list.tmp
+tfs = tests-on-filesystem-list.tmp
+tdf = diff-in-tests-lists.tmp
+MOSTLYCLEANFILES = $(tmk) $(tfs) $(tdf)
 
 # Each test case depends on defs, aclocal, and automake.
 check_SCRIPTS = defs defs-p aclocal-$(APIVERSION) automake-$(APIVERSION)
@@ -1250,6 +1254,7 @@ install-strip:
          `test -z '$(STRIP)' || \
            echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
 mostlyclean-generic:
+       -test -z "$(MOSTLYCLEANFILES)" || rm -f $(MOSTLYCLEANFILES)
        -test -z "$(TEST_LOGS)" || rm -f $(TEST_LOGS)
        -test -z "$(TEST_LOGS_TMP)" || rm -f $(TEST_LOGS_TMP)
        -test -z "$(TEST_SUITE_HTML)" || rm -f $(TEST_SUITE_HTML)
@@ -1357,6 +1362,37 @@ $(parallel_tests): $(parallel_tests:-p.test=.test) 
Makefile.am
 defs-p: defs Makefile.am
        $(AM_V_GEN)sed 's,^AM_INIT_AUTOMAKE$$,&([parallel-tests]),' < defs >$@
 
+# Check that the list of tests given in the Makefile is equal to the
+# list of all test scripts in the Automake testsuite.
+maintainer-check-list-of-tests: $(parallel_tests)
+       @:; \
+        if diff -u /dev/null /dev/null >/dev/null 2>&1; then \
+          diff='diff -u'; \
+        else \
+          diff='diff'; \
+        fi; \
+        for t in $(TESTS); do \
+          echo "$$t"; \
+        done | sort >$(tmk); \
+        glob="$(srcdir)/*.test"; \
+        if test x"$(srcdir)" != x"$(builddir)"; then \
+          glob="$$glob $(builddir)/*.test"; \
+        fi; \
+        for t in $$glob; do \
+          echo "$$t"; \
+        done | sed 's,^.*/,,' | sort >$(tfs); \
+        if $$diff $(tmk) $(tfs) >$(tdf); then \
+           e=0; \
+        else \
+           echo '$@: list of tests in Makefile an on filesystem differ' >&2; \
+           echo "+ $$diff in-makefile on-filesystem" >&2; \
+           cat $(tdf) >&2; \
+           e=1; \
+        fi; \
+        rm -f $(tmk) $(tfs) $(tdf); \
+        exit $$e;
+.PHONY: maintainer-check-list-of-tests
+
 clean-local: clean-local-check
 .PHONY: clean-local-check
 clean-local-check:
-- 
1.6.5