[PATCH] {GSoC} tests defs: new auxiliary function 'count_test_results'

[Stefano Lattarini]

Hello automakers.

Posted below is a patch that I'll soon apply to the temporary branch
'GSoC/experimental/test-results-work'; it is aimed at reducing code
duplication a bit in the current testsuite, at the same time offering
a simpler and clearer idiom for checks about test counting.  Note that
the real usefulness of the patch will become manifest only after I've
posted the new testcases on TAP support (I hope to get to it by this
evening).

Regards,
  Stefano

-*-*-

From 00fab5bc78f3a89d5f42a677d51b27f3917e6c1c Mon Sep 17 00:00:00 2001
Message-Id: <address@hidden>
From: Stefano Lattarini <address@hidden>
Date: Mon, 4 Jul 2011 18:12:07 +0200
Subject: [PATCH] tests defs: new auxiliary function 'count_test_results'

* tests/defs (count_test_results): New function.
* tests/test-driver-custom-multitest.test: Use it.
* tests/test-driver-custom-multitest-recheck.test: Likewise.
* tests/test-driver-custom-multitest-recheck2.test: Likewise.
---
 ChangeLog                                        |    8 ++++
 tests/defs                                       |   32 +++++++++++++++++
 tests/test-driver-custom-multitest-recheck.test  |   31 +++++------------
 tests/test-driver-custom-multitest-recheck2.test |   31 +++++------------
 tests/test-driver-custom-multitest.test          |   41 +--------------------
 5 files changed, 60 insertions(+), 83 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 19b1536..3ab3742 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2011-07-04  Stefano Lattarini  <address@hidden>
+
+       tests defs: new auxiliary function 'count_test_results'
+       * tests/defs (count_test_results): New function.
+       * tests/test-driver-custom-multitest.test: Use it.
+       * tests/test-driver-custom-multitest-recheck.test: Likewise.
+       * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+
 2011-07-03  Stefano Lattarini  <address@hidden>
 
        parallel-tests: simplify testsuite summary
diff --git a/tests/defs b/tests/defs
index fb0b535..4ed34c2 100644
--- a/tests/defs
+++ b/tests/defs
@@ -203,6 +203,38 @@ using_gmake ()
   esac
 }
 
+# count_test_results total=N pass=N fail=N xpass=N xfail=N skip=N error=N
+# -----------------------------------------------------------------------
+# Check that a testsuite run driven by the parallel-tests harness has
+# had the specified numbers of test results (specified by kind).
+# This function assumes that the output of "make check" or "make recheck"
+# has been saved in the `stdout' file in the current directory.
+count_test_results ()
+{
+  # Use a subshell so that we won't pollute the script namespace.
+  (
+    # TODO: Do proper checks on the arguments?
+    total=ERR pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
+    eval "$@"
+    $EGREP -i '(x?pass|x?fail|skip|error)' stdout || : # For debugging.
+    rc=0
+    test `grep -c '^PASS:'  stdout` -eq $pass  || rc=1
+    test `grep -c '^XFAIL:' stdout` -eq $xfail || rc=1
+    test `grep -c '^SKIP:'  stdout` -eq $skip  || rc=1
+    test `grep -c '^FAIL:'  stdout` -eq $fail  || rc=1
+    test `grep -c '^XPASS:' stdout` -eq $xpass || rc=1
+    test `grep -c '^ERROR:' stdout` -eq $error || rc=1
+    grep "^# tests: *$total$" stdout || rc=1
+    grep "^# pass: *$pass$"   stdout || rc=1
+    grep "^# xfail: *$xfail$" stdout || rc=1
+    grep "^# skip: *$skip$"   stdout || rc=1
+    grep "^# fail: *$fail$"   stdout || rc=1
+    grep "^# xpass: *$xpass$" stdout || rc=1
+    grep "^# error: *$error$" stdout || rc=1
+    test $rc -eq 0
+  )
+}
+
 commented_sed_unindent_prog='
   /^$/b                    # Nothing to do for empty lines.
   x                        # Get x<indent> into pattern space.
diff --git a/tests/test-driver-custom-multitest-recheck.test 
b/tests/test-driver-custom-multitest-recheck.test
index c178fa0..137ff96 100755
--- a/tests/test-driver-custom-multitest-recheck.test
+++ b/tests/test-driver-custom-multitest-recheck.test
@@ -95,19 +95,6 @@ do_recheck ()
   cat stdout; ls -l
 }
 
-do_count ()
-{
-  pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
-  eval "$@"
-  $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP|ERROR)' stdout || : # For debugging.
-  test `grep -c '^PASS:' stdout` -eq $pass
-  test `grep -c '^FAIL:' stdout` -eq $fail
-  test `grep -c '^XPASS:' stdout` -eq $xpass
-  test `grep -c '^XFAIL:' stdout` -eq $xfail
-  test `grep -c '^SKIP:' stdout` -eq $skip
-  test `grep -c '^ERROR:' stdout` -eq $error
-}
-
 for vpath in : false; do
   if $vpath; then
     mkdir build
@@ -130,7 +117,7 @@ for vpath in : false; do
   test ! -r c.log
   test ! -r d.run
   test ! -r d.log
-  do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
 
   : Run the tests for the first time.
   $MAKE check >stdout && { cat stdout; Exit 1; }
@@ -141,7 +128,7 @@ for vpath in : false; do
   test -f b.run
   test -f c.run
   test -f d.run
-  do_count pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
+  count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
 
   : Let us make b.test pass.
   echo OK > b.ok
@@ -152,7 +139,7 @@ for vpath in : false; do
   test -f b.run
   test -f c.run
   test -f d.run
-  do_count pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
+  count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
 
   : Let us make the first part of c.test pass.
   echo OK > c.pass
@@ -161,7 +148,7 @@ for vpath in : false; do
   test ! -r b.run
   test -f c.run
   test -f d.run
-  do_count pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
+  count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
 
   : Let us make also the second part of c.test pass.
   echo KO > c.xfail
@@ -170,7 +157,7 @@ for vpath in : false; do
   test ! -r b.run
   test -f c.run
   test -f d.run
-  do_count pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
+  count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
 
   : Nothing changed, so only d.test should be run.
   for i in 1 2; do
@@ -179,7 +166,7 @@ for vpath in : false; do
     test ! -r b.run
     test ! -r c.run
     test -f d.run
-    do_count pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
+    count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
   done
 
   : Let us make d.test run more testcases, and experience _more_ failures.
@@ -200,7 +187,7 @@ END
   test ! -r b.run
   test ! -r c.run
   test -f d.run
-  do_count pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
+  count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
 
   : Let us finally make d.test pass.
   echo : > d.extra
@@ -209,7 +196,7 @@ END
   test ! -r b.run
   test ! -r c.run
   test -f d.run
-  do_count pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
+  count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
 
   : All tests have been successful or skipped, nothing should be re-run.
   do_recheck --pass
@@ -217,7 +204,7 @@ END
   test ! -r b.run
   test ! -r c.run
   test ! -r d.run
-  do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
 
   cd $srcdir
 
diff --git a/tests/test-driver-custom-multitest-recheck2.test 
b/tests/test-driver-custom-multitest-recheck2.test
index 36cb302..ed08675 100755
--- a/tests/test-driver-custom-multitest-recheck2.test
+++ b/tests/test-driver-custom-multitest-recheck2.test
@@ -72,19 +72,6 @@ $ACLOCAL
 $AUTOCONF
 $AUTOMAKE
 
-do_count ()
-{
-  pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
-  eval "$@"
-  $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP|ERROR)' stdout || : # For debugging.
-  test `grep -c '^PASS:' stdout` -eq $pass
-  test `grep -c '^FAIL:' stdout` -eq $fail
-  test `grep -c '^XPASS:' stdout` -eq $xpass
-  test `grep -c '^XFAIL:' stdout` -eq $xfail
-  test `grep -c '^SKIP:' stdout` -eq $skip
-  test `grep -c '^ERROR:' stdout` -eq $error
-}
-
 for vpath in : false; do
   if $vpath; then
     mkdir build
@@ -103,7 +90,7 @@ for vpath in : false; do
   test -f a.run
   test -f b.run
   test -f c.run
-  do_count pass=2 fail=1 xpass=1 xfail=0 skip=1 error=0
+  count_test_results total=5 pass=2 fail=1 xpass=1 xfail=0 skip=1 error=0
 
   rm -f *.run
 
@@ -111,7 +98,7 @@ for vpath in : false; do
   for var in TESTS TEST_LOGS; do
     env "$var=" $MAKE -e recheck >stdout || { cat stdout; Exit 1; }
     cat stdout
-    do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+    count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
     test ! -r a.run
     test ! -r b.run
     test ! -r c.run
@@ -122,7 +109,7 @@ for vpath in : false; do
   env TESTS=a.test $MAKE -e recheck >stdout \
     || { cat stdout; Exit 1; }
   cat stdout
-  do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
   test ! -r a.run
   test ! -r b.run
   test ! -r c.run
@@ -135,7 +122,7 @@ for vpath in : false; do
   test ! -r a.run
   test -f b.run
   test ! -r c.run
-  do_count pass=0 fail=0 xpass=0 xfail=1 skip=1 error=0
+  count_test_results total=2 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=0
 
   rm -f *.run
 
@@ -143,14 +130,14 @@ for vpath in : false; do
   TEST_LOGS=b.log $MAKE -e recheck >stdout \
     || { cat stdout; Exit 1; }
   cat stdout
-  do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
   test ! -r a.run
   test ! -r b.run
   test ! -r c.run
   TESTS='a.test b.test' $MAKE -e recheck >stdout \
     || { cat stdout; Exit 1; }
   cat stdout
-  do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
   test ! -r a.run
   test ! -r b.run
   test ! -r c.run
@@ -162,7 +149,7 @@ for vpath in : false; do
   env TEST_LOGS='a.log c.log' $MAKE -e recheck >stdout \
     && { cat stdout; Exit 1; }
   cat stdout
-  do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
+  count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
   test ! -r a.run
   test ! -r b.run
   test -f c.run
@@ -175,7 +162,7 @@ for vpath in : false; do
   env TESTS='c.test a.test' $MAKE -e recheck >stdout \
     || { cat stdout; Exit 1; }
   cat stdout
-  do_count pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+  count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
   test ! -r a.run
   test ! -r b.run
   test -f c.run
@@ -186,7 +173,7 @@ for vpath in : false; do
   : succesful.
   $MAKE recheck >stdout || { cat stdout; Exit 1; }
   cat stdout
-  do_count pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+  count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
   test ! -r a.run
   test ! -r b.run
   test ! -r c.run
diff --git a/tests/test-driver-custom-multitest.test 
b/tests/test-driver-custom-multitest.test
index 797572d..df0568c 100755
--- a/tests/test-driver-custom-multitest.test
+++ b/tests/test-driver-custom-multitest.test
@@ -43,14 +43,6 @@ TESTS = \
   pass-xpass-fail-xfail-skip-error.t
 END
 
-expected_tests=23
-expected_pass=10
-expected_fail=5
-expected_skip=4
-expected_xfail=2
-expected_xpass=1
-expected_error=1
-
 cat > pass.t << 'END'
 echo %% pass %%
 echo PASS: pass
@@ -143,15 +135,8 @@ for vpath in : false; do
   cat pass4-skip.log
   cat pass3-skip2-xfail.log
   cat pass-xpass-fail-xfail-skip-error.log
-  # For debugging.
-  $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP|ERROR)' stdout
 
-  test `grep -c '^PASS:' stdout` -eq $expected_pass
-  test `grep -c '^FAIL:' stdout` -eq $expected_fail
-  test `grep -c '^XPASS:' stdout` -eq $expected_xpass
-  test `grep -c '^XFAIL:' stdout` -eq $expected_xfail
-  test `grep -c '^SKIP:' stdout` -eq $expected_skip
-  test `grep -c '^ERROR:' stdout` -eq $expected_error
+  count_test_results total=23 pass=10 fail=5 skip=4 xfail=2 xpass=1 error=1
 
   tst=pass-xpass-fail-xfail-skip-error
   grep  "^PASS: $tst\.t, testcase 1" stdout
@@ -161,15 +146,6 @@ for vpath in : false; do
   grep  "^SKIP: $tst\.t, testcase 5" stdout
   grep "^ERROR: $tst\.t, testcase 6" stdout
 
-  # Check counts of testcases in testsuite summary printed on console.
-  grep "^# tests: *$expected_tests$" stdout
-  grep "^# pass: *$expected_pass$" stdout
-  grep "^# xpass: *$expected_xpass$" stdout
-  grep "^# fail: *$expected_fail$" stdout
-  grep "^# xfail: *$expected_xfail$" stdout
-  grep "^# skip: *$expected_skip$" stdout
-  grep "^# error: *$expected_error$" stdout
-
   # Check that the content of, and only of, the test logs with at least
   # one failing test case has been copied into `test-suite.log'.  Note
   # that test logs containing skipped or failed test cases are *not*
@@ -186,20 +162,7 @@ for vpath in : false; do
     || { cat stdout; cat test-suite.log; Exit 1; }
   cat test-suite.log
   cat stdout
-  # For debugging.
-  $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP|ERROR)' stdout
-  test `grep -c '^PASS:' stdout` -eq 4
-  test `grep -c '^SKIP:' stdout` -eq 2
-  test `grep -c '^XFAIL:' stdout` -eq 1
-  $EGREP '^(FAIL|XPASS|ERROR)' stdout && Exit 1
-  # Check counts of testcases in testsuite summary printed on console.
-  grep "^# tests: *7$" stdout
-  grep "^# pass: *4$" stdout
-  grep "^# xpass: *0$" stdout
-  grep "^# fail: *0$" stdout
-  grep "^# xfail: *1$" stdout
-  grep "^# skip: *2$" stdout
-  grep "^# error: *0$" stdout
+  count_test_results total=7 pass=4 fail=0 skip=2 xfail=1 xpass=0 error=0
 
   cd $srcdir
 
-- 
1.7.2.3