[Automake-commit] [SCM] GNU Automake branch, master, updated. Release-1-10-212-g6a0dc32

[Ralf Wildenhues]

This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU Automake".

http://git.sv.gnu.org/gitweb/?p=automake.git;a=commitdiff;h=6a0dc321fa3d45e0deaf28ad82f3b5e3c8e8ea4a

The branch, master has been updated
       via  6a0dc321fa3d45e0deaf28ad82f3b5e3c8e8ea4a (commit)
      from  be6de492fc71c2249e6d080a3e5050a4e0ad4334 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
commit 6a0dc321fa3d45e0deaf28ad82f3b5e3c8e8ea4a
Author: Ralf Wildenhues <address@hidden>
Date:   Sun Oct 26 20:49:26 2008 +0100

    Parallel automake --add-missing: serialized file installs.
    
    * automake.in (QUEUE_CONF_FILE, QUEUE_LOCATION, QUEUE_STRING):
    New serialization keys.
    ($required_conf_file_queue): New file global.
    (queue_required_conf_file, require_queued_conf_file): New
    functions, to queue and dequeue requirements for aux dir files.
    (require_conf_file): Enqueue if needed.
    (get_number_of_threads): Can do threads with --add-missing now.
    (handle_makefiles_threaded): Let worker threads enqueue, let
    master attend to queued requirements at the right time.
    * tests/parallel-am.test: Explain the purpose of the include
    chain used here.
    * tests/parallel-am2.test: Also cope with --add-missing.
    * tests/parallel-am3.test: New test, test absence of races with
    concurrent same-file installs stemming from --add-missing.
    * tests/Makefile.am: Adjust.
    
    Signed-off-by: Ralf Wildenhues <address@hidden>

-----------------------------------------------------------------------

Summary of changes:
 ChangeLog                                      |   17 +++++
 automake.in                                    |   86 ++++++++++++++++++++++--
 tests/Makefile.am                              |    1 +
 tests/Makefile.in                              |    1 +
 tests/parallel-am.test                         |    2 +
 tests/parallel-am2.test                        |    5 +-
 tests/{parallel-am2.test => parallel-am3.test} |   56 ++++++++--------
 7 files changed, 130 insertions(+), 38 deletions(-)
 copy tests/{parallel-am2.test => parallel-am3.test} (50%)

diff --git a/ChangeLog b/ChangeLog
index 10b0b5f..f1f7ff6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,22 @@
 2008-10-26  Ralf Wildenhues  <address@hidden>
 
+       Parallel automake --add-missing: serialized file installs.
+       * automake.in (QUEUE_CONF_FILE, QUEUE_LOCATION, QUEUE_STRING):
+       New serialization keys.
+       ($required_conf_file_queue): New file global.
+       (queue_required_conf_file, require_queued_conf_file): New
+       functions, to queue and dequeue requirements for aux dir files.
+       (require_conf_file): Enqueue if needed.
+       (get_number_of_threads): Can do threads with --add-missing now.
+       (handle_makefiles_threaded): Let worker threads enqueue, let
+       master attend to queued requirements at the right time.
+       * tests/parallel-am.test: Explain the purpose of the include
+       chain used here.
+       * tests/parallel-am2.test: Also cope with --add-missing.
+       * tests/parallel-am3.test: New test, test absence of races with
+       concurrent same-file installs stemming from --add-missing.
+       * tests/Makefile.am: Adjust.
+
        Implement serialization for Locations.
        * lib/Automake/Location.pm (serialize, deserialize): New
        functions.  They allows to serialize a Location in an array, and
diff --git a/automake.in b/automake.in
index d7db627..0815773 100755
--- a/automake.in
+++ b/automake.in
@@ -280,6 +280,9 @@ use constant INTERNAL => new Automake::Location;
 # Serialization keys for message queues.
 use constant {
   QUEUE_MESSAGE   => "msg",
+  QUEUE_CONF_FILE => "conf file",
+  QUEUE_LOCATION  => "location",
+  QUEUE_STRING    => "string"
 };
 
 
@@ -7475,13 +7478,80 @@ sub require_libsource_with_macro ($$$@)
       }
 }
 
+# Queue to push require_conf_file requirements to.
+my $required_conf_file_queue;
+
+# &queue_required_conf_file ($QUEUE, $KEY, $DIR, $WHERE, $MYSTRICT, @FILES)
+# -------------------------------------------------------------------------
+sub queue_required_conf_file ($$$$@)
+{
+    my ($queue, $key, $dir, $where, $mystrict, @files) = @_;
+    my @serial_loc;
+    if (ref $where)
+      {
+        @serial_loc = (QUEUE_LOCATION, $where->serialize ());
+      }
+    else
+      {
+        @serial_loc = (QUEUE_STRING, $where);
+      }
+    $queue->enqueue ($key, $dir, @serial_loc, $mystrict, 0 + @files, @files);
+}
+
+# &require_queued_conf_file ($QUEUE)
+# ----------------------------------
+sub require_queued_conf_file ($)
+{
+    my ($queue) = @_;
+    my $where;
+    my $dir = $queue->dequeue ();
+    my $loc_key = $queue->dequeue ();
+    if ($loc_key eq QUEUE_LOCATION)
+      {
+       $where = Automake::Location::deserialize ($queue);
+      }
+    elsif ($loc_key eq QUEUE_STRING)
+      {
+       $where = $queue->dequeue ();
+      }
+    else
+      {
+       prog_error "unexpected key $loc_key";
+      }
+    my $mystrict = $queue->dequeue ();
+    my $nfiles = $queue->dequeue ();
+    my @files;
+    push @files, $queue->dequeue ()
+      foreach (1 .. $nfiles);
+
+    # Dequeuing happens outside of per-makefile context, so we have to
+    # set the variables used by require_file_internal and the functions
+    # it calls.  Gross!
+    $relative_dir = $dir;
+    require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+}
+
 # &require_conf_file ($WHERE, $MYSTRICT, @FILES)
 # ----------------------------------------------
-# Looks in configuration path, as specified by AC_CONFIG_AUX_DIR.
+# Looks in configuration path, as specified by AC_CONFIG_AUX_DIR;
+# worker threads may queue up the action to be serialized by the master.
+#
+# FIXME: this seriously relies on the semantics of require_file_internal
+# and maybe_push_required_file, in that we exploit the fact that only the
+# contents of the last handled output file may be impacted (which in turn
+# is dealt with by the master thread).
 sub require_conf_file ($$@)
 {
     my ($where, $mystrict, @files) = @_;
-    require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+    if (defined $required_conf_file_queue)
+      {
+       queue_required_conf_file ($required_conf_file_queue, QUEUE_CONF_FILE,
+                                 $relative_dir, $where, $mystrict, @files);
+      }
+    else
+      {
+       require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+      }
 }
 
 
@@ -8051,11 +8121,6 @@ sub get_number_of_threads
     {
       $nthreads = $max_threads;
     }
-
-  # We cannot deal with --add-missing (yet).
-  $nthreads = 0
-    if ($add_missing);
-
   return $nthreads;
 }
 
@@ -8066,6 +8131,7 @@ sub get_number_of_threads
 # worker threads push back everything that needs serialization:
 # * warning and (normal) error messages, for stable stderr output
 #   order and content (avoiding duplicates, for example),
+# * races when installing aux files (and respective messages),
 # * races when collecting aux files for distribution.
 #
 # The latter requires that the makefile that deals with the aux dir
@@ -8101,9 +8167,11 @@ sub handle_makefiles_threaded ($)
              verb "handling $file";
              my $queue = $msg_queues{$file};
              setup_channel_queue ($queue, QUEUE_MESSAGE);
+             $required_conf_file_queue = $queue;
              handle_makefile ($file);
              $queue->enqueue (undef);
              setup_channel_queue (undef, undef);
+             $required_conf_file_queue = undef;
            }
          return $exit_code;
        });
@@ -8125,6 +8193,10 @@ sub handle_makefiles_threaded ($)
            {
              pop_channel_queue ($queue);
            }
+         elsif ($key eq QUEUE_CONF_FILE)
+           {
+             require_queued_conf_file ($queue);
+           }
          else
            {
              prog_error "unexpected key $key";
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 7c6d4a2..2526acf 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -451,6 +451,7 @@ output-order.test \
 overrid.test \
 parallel-am.test \
 parallel-am2.test \
+parallel-am3.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 9d8b17e..05bf946 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -604,6 +604,7 @@ output-order.test \
 overrid.test \
 parallel-am.test \
 parallel-am2.test \
+parallel-am3.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/parallel-am.test b/tests/parallel-am.test
index d57e014..de86271 100755
--- a/tests/parallel-am.test
+++ b/tests/parallel-am.test
@@ -60,6 +60,8 @@ for i in $list; do
   mkdir sub$i
   echo > sub$i/Makefile.am
 done
+# Use an include chain to cause a nontrivial location object to be
+# serialized through a thread queue.
 echo 'include foo.am' >> sub7/Makefile.am
 echo 'include bar.am' > sub7/foo.am
 echo 'python_PYTHON = foo.py' > sub7/bar.am
diff --git a/tests/parallel-am2.test b/tests/parallel-am2.test
index b5a4ac7..b92b79e 100755
--- a/tests/parallel-am2.test
+++ b/tests/parallel-am2.test
@@ -62,15 +62,16 @@ $ACLOCAL
 
 # Generate expected output using non-threaded code.
 unset AUTOMAKE_JOBS
+rm -f install-sh missing depcomp
 AUTOMAKE_fails --add-missing
-AUTOMAKE_fails
 mv stderr expected
 
 AUTOMAKE_JOBS=5
 export AUTOMAKE_JOBS
 
 for i in 1 2 3 4 5 6 7 8; do
-  AUTOMAKE_fails
+  rm -f install-sh missing depcomp
+  AUTOMAKE_fails --add-missing
   diff expected stderr
 done
 
diff --git a/tests/parallel-am2.test b/tests/parallel-am3.test
similarity index 50%
copy from tests/parallel-am2.test
copy to tests/parallel-am3.test
index b5a4ac7..b3b0aac 100755
--- a/tests/parallel-am2.test
+++ b/tests/parallel-am3.test
@@ -17,19 +17,25 @@
 # Test parallel automake execution.
 
 # This tests:
-# 4) warning and normal error output should be identical, in that duplicate
-#    warnings should be omitted in the same way as without threads,
+# 3) normal automake output should be identical and ordered in the same way
+#    with --add-missing, even with concurrent file requirements, and the
+#    installation of aux files should be race-free,
 
 . ./defs || Exit 1
 
 set -e
 
-mkdir sub
+cat > configure.in << 'END'
+AC_INIT([parallel-am], [1.0])
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE
+AC_PROG_CC
+AM_PATH_LISPDIR
+AM_PATH_PYTHON
+AC_CONFIG_FILES([Makefile])
+END
 
 cat > Makefile.am << 'END'
-AUTOMAKE_OPTIONS = subdir-objects
-bin_PROGRAMS = main
-main_SOURCES = sub/main.c
 SUBDIRS =
 END
 
@@ -37,41 +43,33 @@ list='1 2 3'
 for i in $list; do
   echo "AC_CONFIG_FILES([sub$i/Makefile])" >> configure.in
   echo "SUBDIRS += sub$i" >> Makefile.am
-  mkdir sub$i sub$i/sub
-  cat > sub$i/Makefile.am << END
-AUTOMAKE_OPTIONS = subdir-objects
-bin_PROGRAMS = sub$i
-sub${i}_SOURCES = sub/main$i.c
+  mkdir sub$i
+  cat > sub$i/Makefile.am <<END
+python_PYTHON = foo$i.py
+lisp_LISP = foo$i.el
+bin_PROGRAMS = p$i
 END
 done
 
+rm -f install-sh missing depcomp
 mkdir build-aux
 
 $ACLOCAL
 
-# Independently of the number of worker threads, automake output
-# should be
-# - stable (multiple runs should produce the same output),
-# - properly uniquified,
-# - complete (output from worker threads should not be lost).
-#
-# The parts output by --add-missing are unstable not only wrt. order
-# but also wrt. content: any of the Makefile.am files may cause the
-# depcomp script to be installed (or several of them).
-# Thus we install the auxiliary files in a prior step.
-
-# Generate expected output using non-threaded code.
+# Generate expected output using the non-threaded code.
 unset AUTOMAKE_JOBS
-AUTOMAKE_fails --add-missing
-AUTOMAKE_fails
+AUTOMAKE_run 0 --add-missing
 mv stderr expected
+mv Makefile.in Makefile.in.exp
 
-AUTOMAKE_JOBS=5
+AUTOMAKE_JOBS=3
 export AUTOMAKE_JOBS
 
-for i in 1 2 3 4 5 6 7 8; do
-  AUTOMAKE_fails
-  diff expected stderr
+for run in 1 2 3 4 5 6 7; do
+  rm -f build-aux/* sub*/Makefile.in
+  AUTOMAKE_run 0 --add-missing
+  diff stderr expected
+  diff Makefile.in Makefile.in.exp
 done
 
 :


hooks/post-receive
--
GNU Automake