Re: parallel compression in 'make dist'

[Ralf Wildenhues]

* Peter Breitenlohner wrote on Mon, Oct 12, 2009 at 10:32:07AM CEST:
> this looks very nice. Here my timing results for 'make -j dist' in the TeX
> Live tree:

Thanks for testing.

> >... Notably, -j2 isn't helpful
> >for GM, as it starts the slowest encoder late only.
> 
> I could think of one more improvement for -j2: running the slowest
> encoder first (provided that can be achieved easily).

OK.  I did an unscientific comparison (best of three, GM package at
roughly 37M sources) of timings of the compressors we provide, sorted
by time:

lzma    34.0
xz      33.9
shar    14.9
bzip2   11.3
gzip     5.1
zip      2.2
compress 1.5

I'm merging this patch into master.

Thanks,
Ralf

    dist: allow running several compressors in parallel.
    
    * lib/am/distdir.am (am__post_remove_distdir): New internal
    variable.
    (DIST_TARGETS): New variable, set to list of chosen distribution
    formats.  Order formats by expected duration, slowest first, for
    better parallelism.
    (dist-gzip, dist-bzip2, dist-lzma, dist-xz, dist-tarZ)
    (dist-shar, dist-zip): Use $(am__post_remove_distdir).
    (dist, dist-all): Do not depend on distdir.  Instead of
    replicating each compression command, use a recursive invocation
    to allow running all $(DIST_TARGETS) in parallel.
    * NEWS: Update.
    Report by Peter Breitenlohner.

diff --git a/NEWS b/NEWS
index cd427fb..5ccb59e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,9 @@
 New in 1.11.0a:
 
+* Miscellaneous changes:
+
+  - The `dist' and `dist-all' targets now can run compressors in parallel.
+
 Bugs fixed in 1.11.0a:
 
 * Bugs introduced by 1.11:
diff --git a/lib/am/distdir.am b/lib/am/distdir.am
index e3294fb..79277bc 100644
--- a/lib/am/distdir.am
+++ b/lib/am/distdir.am
@@ -26,7 +26,7 @@ am__remove_distdir = \
   { test ! -d "$(distdir)" \
     || { find "$(distdir)" -type d ! -perm -200 -exec chmod u+w {} ';' \
          && rm -fr "$(distdir)"; }; }
-
+am__post_remove_distdir = $(am__remove_distdir)
 endif %?TOPDIR_P%
 
 if %?SUBDIRS%
@@ -331,6 +331,10 @@ endif %?TOPDIR_P%
 ## with tar 1.11.2).  We do not do anything specific w.r.t. this
 ## incompatibility since packages where empty directories need to be
 ## present in the archive are really unusual.
+##
+## We order DIST_TARGETS by expected duration of the compressors,
+## slowest first, for better parallelism in `make dist'.  Do not
+## reorder DIST_ARCHIVES, users may expect gzip to be first.
 
 if %?TOPDIR_P%
 
@@ -339,44 +343,52 @@ GZIP_ENV = --best
 .PHONY: dist-gzip
 dist-gzip: distdir
        tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c 
>$(distdir).tar.gz
-       $(am__remove_distdir)
+       $(am__post_remove_distdir)
 
 ?BZIP2?DIST_ARCHIVES += $(distdir).tar.bz2
 .PHONY: dist-bzip2
 dist-bzip2: distdir
        tardir=$(distdir) && $(am__tar) | bzip2 -9 -c >$(distdir).tar.bz2
-       $(am__remove_distdir)
+       $(am__post_remove_distdir)
 
 ?LZMA?DIST_ARCHIVES += $(distdir).tar.lzma
 .PHONY: dist-lzma
 dist-lzma: distdir
        tardir=$(distdir) && $(am__tar) | lzma -9 -c >$(distdir).tar.lzma
-       $(am__remove_distdir)
+       $(am__post_remove_distdir)
 
 ?XZ?DIST_ARCHIVES += $(distdir).tar.xz
 .PHONY: dist-xz
 dist-xz: distdir
        tardir=$(distdir) && $(am__tar) | xz -c >$(distdir).tar.xz
-       $(am__remove_distdir)
+       $(am__post_remove_distdir)
 
 ?COMPRESS?DIST_ARCHIVES += $(distdir).tar.Z
 .PHONY: dist-tarZ
 dist-tarZ: distdir
        tardir=$(distdir) && $(am__tar) | compress -c >$(distdir).tar.Z
-       $(am__remove_distdir)
+       $(am__post_remove_distdir)
 
 ?SHAR?DIST_ARCHIVES += $(distdir).shar.gz
 .PHONY: dist-shar
 dist-shar: distdir
        shar $(distdir) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).shar.gz
-       $(am__remove_distdir)
+       $(am__post_remove_distdir)
 
 ?ZIP?DIST_ARCHIVES += $(distdir).zip
 .PHONY: dist-zip
 dist-zip: distdir
        -rm -f $(distdir).zip
        zip -rq $(distdir).zip $(distdir)
-       $(am__remove_distdir)
+       $(am__post_remove_distdir)
+
+?LZMA?DIST_TARGETS += dist-lzma
+?XZ?DIST_TARGETS += dist-xz
+?SHAR?DIST_TARGETS += dist-shar
+?BZIP2?DIST_TARGETS += dist-bzip2
+?GZIP?DIST_TARGETS += dist-gzip
+?ZIP?DIST_TARGETS += dist-zip
+?COMPRESS?DIST_TARGETS += dist-tarZ
 
 endif %?TOPDIR_P%
 
@@ -396,16 +408,9 @@ if %?SUBDIRS%
 AM_RECURSIVE_TARGETS += dist dist-all
 endif %?SUBDIRS%
 
-dist dist-all: distdir
-?GZIP? tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c 
>$(distdir).tar.gz
-?BZIP2?        tardir=$(distdir) && $(am__tar) | bzip2 -9 -c 
>$(distdir).tar.bz2
-?LZMA? tardir=$(distdir) && $(am__tar) | lzma -9 -c >$(distdir).tar.lzma
-?XZ?   tardir=$(distdir) && $(am__tar) | xz -c >$(distdir).tar.xz
-?COMPRESS?     tardir=$(distdir) && $(am__tar) | compress -c >$(distdir).tar.Z
-?SHAR? shar $(distdir) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).shar.gz
-?ZIP?  -rm -f $(distdir).zip
-?ZIP?  zip -rq $(distdir).zip $(distdir)
-       $(am__remove_distdir)
+dist dist-all:
+       $(MAKE) $(AM_MAKEFLAGS) $(DIST_TARGETS) am__post_remove_distdir='@:'
+       $(am__post_remove_distdir)
 
 endif %?TOPDIR_P%
 
@@ -508,7 +513,7 @@ distcheck: dist
 ## Cater to parallel BSD make (see above).
          && cd "$$am__cwd" \
          || exit 1
-       $(am__remove_distdir)
+       $(am__post_remove_distdir)
        @(echo "$(distdir) archives ready for distribution: "; \
          list='$(DIST_ARCHIVES)'; for i in $$list; do echo $$i; done) | \
          sed -e 1h -e 1s/./=/g -e 1p -e 1x -e '$$p' -e '$$x'