Re: Libtool stresstest: echo marker with loop variables

[Ralf Wildenhues]

Hi Peter,

* Peter Ekberg wrote on Fri, Sep 16, 2005 at 09:31:58AM CEST:
> 
> I can see that you are on top of all of my "complaints" on
> stresstest.at. I will continue with a locally patched version
> and will leave it up to you to improve it, the m4 magic and
> autotest stuff is beyond me, really...

Here's something for you to play with, which should incorporate
most of the outlined ideas.

stresstest should now complete all desired tests, and report
failing commands or skipped executions in the output files
`failures' and `skips' respectively.  It should exit with one
of these, if any test failed or was skipped.

You should be able to limit the number and type of tests that
should be run, like this: per default, all tests are run.
Per switch, you have a command line variable with which you can
override the set of switches to run.  Switch lists will be eval'ed,
so that you can add empty arguments like this: ''

Best explanation is an example:

  stresstest_undef_opts="''"    \
  stresstest_expsyms="''"       \
  stresstest_rpath="''"         \
  stresstest_relpath="''"       \
  stresstest_static="''"        \
  stresstest_selfsyms="''"      \
  make check TESTS= TESTSUITE_FLAGS="-v -d 22"

Cut'n'paste these lines to run a minimal set of tests.
Change the third line to contain

  stresstest_rpath="'' '-rpath /nonexistent'"

if you want to see both convenience archives and installed libraries.
Note that if you override a variable, you have to set an empty argument
in order for the tests to be run at all

If any link tests failed, you might be able to paste and run them from
the failures file (given that the prerequisite files are present).

Does this work for you?  Could you play around and test this a bit, to
see if there are any errors in the new code?  It might still be much
work to find the errors from the runs in the noise, but at least a
quicker rerun should be possible.  Maybe I should revert to capturing
or putting stdout/stderr in `stdout'/`stderr' files again..

I have not changed any of the actual stuff to test yet.  But the next
useful thing would be to copy $top_builddir/libtool to the test dir,
adjust $LIBTOOL, and set max_cmd_len so low that the
piecewise/reload/bla stuff all triggers, as done in pdemo.

Probably there are shells which do not like the assignments in this
code..

Cheers,
Ralf

        * tests/testsuite.at (PREPARE_TESTS): Fetch $ECHO as well from
        libtool script.
        * tests/link-order.at (Link order test): Adjusted.
        * tests/stresstest.at (LT_AT_EXPAND_NOFAIL_CHECK)
        (LT_AT_EXEC_NOFAIL_CHECK): New macros.
        (Libtool stress test): Use these to avoid stopping on failure.
        Capture failed and skipped tests in the files `failures' and
        `skips'.
        (stresstest_undef_opts, stresstest_expsyms, stresstest_rpath,
        stresstest_relpath, stresstest_static, stresstest_selfsyms):
        New variables, to override default set of test options.

Index: tests/testsuite.at
===================================================================
RCS file: /cvsroot/libtool/libtool/tests/testsuite.at,v
retrieving revision 1.19
diff -u -r1.19 testsuite.at
--- tests/testsuite.at  16 Sep 2005 08:33:24 -0000      1.19
+++ tests/testsuite.at  19 Sep 2005 17:03:41 -0000
@@ -25,7 +25,7 @@
 : ${AUTOCONF=autoconf}
 export LIBTOOLIZE LIBTOOL ACLOCAL AUTOCONF
 eval `$LIBTOOL --config | grep ^EGREP=`
-eval `$LIBTOOL --config | $EGREP '^(host|host_os|build)='`
+eval `$LIBTOOL --config | $EGREP '^(host|host_os|build|ECHO)='`
 m4_divert_pop([PREPARE_TESTS])dnl
 
 # LT_AT_LIBTOOLIZE([ARGS])
Index: tests/link-order.at
===================================================================
RCS file: /cvsroot/libtool/libtool/tests/link-order.at,v
retrieving revision 1.4
diff -u -r1.4 link-order.at
--- tests/link-order.at 13 Aug 2005 06:57:39 -0000      1.4
+++ tests/link-order.at 19 Sep 2005 17:03:41 -0000
@@ -19,7 +19,6 @@
 # link-order.test - make sure that library linking order matches
 
 AT_SETUP([Link order test.])
-eval `$LIBTOOL --config | grep ECHO=`
 LDFLAGS="$LDFLAGS -no-undefined"
 
 prefix_old=`pwd`/old
Index: tests/stresstest.at
===================================================================
RCS file: /cvsroot/libtool/libtool/tests/stresstest.at,v
retrieving revision 1.6
diff -u -r1.6 stresstest.at
--- tests/stresstest.at 13 Sep 2005 07:28:14 -0000      1.6
+++ tests/stresstest.at 19 Sep 2005 17:17:10 -0000
@@ -170,32 +170,68 @@
 AT_CHECK([$LIBTOOL --mode=compile $CC $CFLAGS -c main.c],[0],[ignore],[ignore])
 AT_CHECK([$LIBTOOL --mode=compile $CC $CFLAGS -c dlself.c -o 
sub3/dlself.lo],[0],[ignore],[ignore])
 
+
+# LT_AT_EXPAND_NOFAIL_CHECK(COMMANDS)
+# -----------------------------------
+# Do not use single quotes in the m4 literal COMMANDS, please.
+m4_define([LT_AT_EXPAND_NOFAIL_CHECK],
+[eval \$ECHO \'$1\'
+AT_CHECK([$1 || eval \$ECHO \'$1\' >>failures], [ignore], [ignore], [ignore])
+eval last_link=\'$1\'
+])
+
+# LT_AT_EXEC_NOFAIL_CHECK(EXECUTABLE)
+# -----------------------------------
+# EXECUTABLE always gives 0 in the good case.
+m4_define([LT_AT_EXEC_NOFAIL_CHECK],
+[AT_CHECK(
+  [if $1; then :; else
+     if test "X$host" != "X$build" && { test -x "$1" || test -x "$1"$EXEEXT; }
+     then out=skips; else out=failures; fi
+   eval \$ECHO \'\# failure of $1 after: $last_link\' >>$out
+   fi], [ignore], [ignore], [ignore])
+])
+
+
+# list of arguments we want to check
 case $host_os in
-  cygwin* | mingw* | pw32*) undef_opts=-no-undefined ;;
-  *)                        undef_opts='"" -no-undefined' ;;
+cygwin*|mingw*|pw32*)
+   : ${stresstest_undef_opts="-no-undefined"} ;;
+*) : ${stresstest_undef_opts="'' -no-undefined"} ;;
 esac
+: ${stresstest_expsyms="'' '-export-symbols-regex v.*' '-export-symbols 
asyms'"}
+: ${stresstest_rpath="'' '-rpath /nonexistent'"}
+: ${stresstest_relpath="'' ./ `pwd`/"}
+: ${stresstest_static="'' '-static'"}
+: ${stresstest_selfsyms="'' '-export-symbols dlselfsyms'"}
 
-for l1 in $undef_opts
+eval "set x $stresstest_undef_opts"; shift
+for l1
 do
-  for l2 in '' '-export-symbols-regex v.*' '-export-symbols asyms'
+  eval "set x $stresstest_expsyms"; shift
+  for l2
   do
-    for l3 in '' '-rpath /nonexistent'
+    eval "set x $stresstest_rpath"; shift
+    for l3
     do
       linkargs="$l1 $l2 $l3"
-      for rel in '' ./ `pwd`/
+      eval "set x $stresstest_relpath"; shift
+      for rel
       do
-       AT_CHECK([$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS -o 
"$rel"sub2/liba.la "$rel"sub/a.lo $linkargs],
-                [0],[ignore],[ignore])
-       for st in '' '-static'
+       LT_AT_EXPAND_NOFAIL_CHECK(
+           [$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS -o "$rel"sub2/liba.la 
"$rel"sub/a.lo $linkargs])
+       eval "set x $stresstest_static"; shift
+       for st
        do
-         AT_CHECK([$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS $st -o "$rel"main 
"$rel"main.lo "$rel"sub2/liba.la],
-                  [0],[ignore],[ignore])
-         LT_AT_EXEC_CHECK([./main],[0])
-         for l10 in '' '-export-symbols dlselfsyms'
+         LT_AT_EXPAND_NOFAIL_CHECK(
+             [$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS $st -o "$rel"main 
"$rel"main.lo "$rel"sub2/liba.la])
+         LT_AT_EXEC_NOFAIL_CHECK([./main])
+         eval "set x $stresstest_selfsyms"; shift
+         for l10
          do
-           AT_CHECK([$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS $st -o 
"$rel"sub3/dlself "$rel"sub3/dlself.lo "$rel"sub2/liba.la sub/b.lo -dlopen self 
$l10],
-                    [0],[ignore],[ignore])
-           LT_AT_EXEC_CHECK([./sub3/dlself],[0])
+           LT_AT_EXPAND_NOFAIL_CHECK(
+               [$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS $st -o 
"$rel"sub3/dlself "$rel"sub3/dlself.lo "$rel"sub2/liba.la sub/b.lo -dlopen self 
$l10])
+           LT_AT_EXEC_NOFAIL_CHECK([./sub3/dlself])
          done
        done
       done
@@ -203,4 +239,6 @@
   done
 done
 
+AT_CHECK([test -s failures], [1])
+AT_CHECK([test -s skips && (exit 77)], [1])
 AT_CLEANUP