>From d5a8d41cd30e78bb614a8324acc1e7220368185a Mon Sep 17 00:00:00 2001 From: Alexander Gramiak Date: Sat, 29 Dec 2018 11:54:24 -0600 Subject: [PATCH 5/5] Add Zstandard compression option for make install * lisp/jka-cmpr-hook.el (jka-compr-load-suffixes): Add .zst by default. * configure.ac (--with-compress-install): Add `zstd' option and deprecate `GZIP_PROG' in favour of the more general `COMPR_PROG'. * Makefile.in: Add and use Make variables `COMPR_PROG', `COMPR_ARGS', `COMPR_TYPE', `COMPR_EXT', `COMPR_INFO_PROG', `COMPR_INFO_ARGS', `COMPR_INFO_EXT'. Deprecate, but still support `GZIP_PROG'. * doc/emacs/Makefile.in: doc/lispintro/Makefile.in: doc/lispref/Makefile.in: doc/misc/Makefile.in: Same as Makefile.in, but without the `COMPR_INFO*' Make variables. * admin/admin.el (make-manuals-dist-output-variables): Use `COMPR_PROG' over `GZIP_PROG'. * INSTALL: Document new `COMPR_{PROG,TYPE,ARGS,EXT}' Make variables. --- INSTALL | 34 +++++++++++++++++++---- Makefile.in | 57 ++++++++++++++++++++++++++------------- admin/admin.el | 2 +- configure.ac | 36 ++++++++++++++++++------- doc/emacs/Makefile.in | 25 ++++++++++++----- doc/lispintro/Makefile.in | 25 ++++++++++++----- doc/lispref/Makefile.in | 25 ++++++++++++----- doc/misc/Makefile.in | 25 ++++++++++++----- etc/NEWS | 19 +++++++++++++ lisp/jka-cmpr-hook.el | 4 +-- 10 files changed, 192 insertions(+), 60 deletions(-) diff --git a/INSTALL b/INSTALL index 0c56fff6d4..ae9047d1ea 100644 --- a/INSTALL +++ b/INSTALL @@ -101,8 +101,9 @@ sections if you need to. make distclean Note that the install automatically saves space by compressing - (provided you have the 'gzip' program) those installed Lisp source (.el) - files that have corresponding .elc versions, as well as the Info files. + (provided you have 'gzip' or another compression program) those + installed Lisp source (.el) files that have corresponding .elc + versions, as well as the Info files. ADDITIONAL DISTRIBUTION FILES @@ -641,9 +642,32 @@ GNU software; the following variables are specific to Emacs. see), is '/usr/local/libexec/emacs/VERSION/CONFIGURATION-NAME' (where VERSION and CONFIGURATION-NAME are as described above). -'GZIP_PROG' is the name of the executable that compresses installed info, - manual, and .el files. It defaults to gzip. Setting it to - the empty string suppresses compression. +'COMPR_PROG' is the name of the executable that compresses various + installed files, such as info, manual, and .el files. It + defaults to gzip. Setting it to the empty string suppresses + compression. + +'GZIP_PROG' is a deprecated variable that behaves like 'COMPR_PROG', + but also forces 'COMPR_TYPE' to gzip if 'COMPR_TYPE' is + unspecified. + +'COMPR_TYPE' is the type of compression used for the above, which for + supported values (currently 'gzip' and 'zstd') determines the + default values of 'COMPR_ARGS' and 'COMPR_EXT'. Its default + value is based on the configure option + '--with-compress-install'; if that option is specified, then + 'COMPR_TYPE' is set to the specified value of that option, + otherwise 'gzip'. + +'COMPR_ARGS' is the list of arguments passed to 'COMPR_PROG', which is + set according to 'COMPR_TYPE'. The default arguments for + supported 'COMPR_TYPE' set the slowest (best) compression + level that is enabled by default, and signify that the + original files are deleted after compression. + +'COMPR_EXT' is the extension used by 'COMPR_PROG', which is set + according to 'COMPR_TYPE'. It defaults to the canonical + extension of supported 'COMPR_TYPE'. Remember that you must specify any variable values you need each time you run 'make' in the top directory. If you run 'make' once to build diff --git a/Makefile.in b/Makefile.in index bbb028a74a..5020abc494 100644 --- a/Makefile.in +++ b/Makefile.in @@ -295,8 +295,33 @@ MKDIR_P = # Create a link to a file in the same directory as the target. LN_S_FILEONLY = @LN_S_FILEONLY@ -# We use gzip to compress installed .el and some .txt files. -GZIP_PROG = @GZIP_PROG@ +# We compress many installed .el and other files. +# GZIP_PROG is used here for backwards-compatibility. +ifdef GZIP_PROG + COMPR_PROG = ${GZIP_PROG} + COMPR_TYPE = gzip +else + COMPR_PROG = @COMPR_PROG@ + COMPR_TYPE = @COMPR_TYPE@ +endif + +ifeq (${COMPR_TYPE}, gzip) + COMPR_ARGS = "-9n" + COMPR_EXT = ".gz" + + # Info/man don't support zstd at the time of writing, so provide an + # alternative for them that defaults to gzip. + COMPR_INFO_PROG = ${COMPR_PROG} + COMPR_INFO_ARGS = ${COMPR_ARGS} + COMPR_INFO_EXT = ${COMPR_EXT} +else ifeq (${COMPR_TYPE}, zstd) + COMPR_ARGS = "-19q --rm" + COMPR_EXT = ".zst" + + COMPR_INFO_PROG = @COMPR_INFO_PROG@ + COMPR_INFO_ARGS = "-9n" + COMPR_INFO_EXT = ".gz" +endif # ============================= Targets ============================== @@ -609,16 +634,16 @@ install-arch-indep: ${write_subdir} subdir="$(DESTDIR)${datadir}/emacs/site-lisp" ; \ ${write_subdir} || true - [ -z "${GZIP_PROG}" ] || { \ + [ -z "${COMPR_PROG}" ] || { \ echo "Compressing *.el etc. ..." && \ cd "$(DESTDIR)${lispdir}" && \ for f in `find . -name "*.elc" -print | sed 's/.elc$$/.el/'`; do \ - ${GZIP_PROG} -9n "$$f"; \ + eval ${COMPR_PROG} ${COMPR_ARGS} "\"$$f\""; \ done; \ cd "$(DESTDIR)${etcdir}" && \ for f in `find . -maxdepth 1 -name "*NEWS*" -not -name NEWS \ -not -name ORG-NEWS` `find refcards -name "*.ps"` "publicsuffix.txt"; do \ - ${GZIP_PROG} -9n "$$f"; \ + eval ${COMPR_PROG} ${COMPR_ARGS} "\"$$f\""; \ done; \ } -chmod -R a+r "$(DESTDIR)${datadir}/emacs/${version}" ${COPYDESTS} @@ -668,9 +693,9 @@ install-info: for f in `ls $$elt $$elt-[1-9] $$elt-[1-9][0-9] 2>/dev/null`; do \ (cd "$${thisdir}"; \ ${INSTALL_DATA} ${srcdir}/info/$$f "$(DESTDIR)${infodir}/$$f"); \ - [ -n "${GZIP_PROG}" ] || continue ; \ - rm -f "$(DESTDIR)${infodir}/$$f.gz"; \ - ${GZIP_PROG} -9n "$(DESTDIR)${infodir}/$$f"; \ + [ -n "${COMPR_INFO_PROG}" ] || continue ; \ + rm -f "$(DESTDIR)${infodir}/$${f}${COMPR_INFO_EXT}"; \ + eval ${COMPR_INFO_PROG} ${COMPR_INFO_ARGS} "\"$(DESTDIR)${infodir}/$$f\""; \ done; \ (cd "$${thisdir}"; \ ${INSTALL_INFO} --info-dir="$(DESTDIR)${infodir}" "$(DESTDIR)${infodir}/$$elt"); \ @@ -692,9 +717,9 @@ install-man: dest=`echo "$${page}" | sed -e 's/\.1$$//' -e '$(TRANSFORM)'`.1; \ (cd "$${thisdir}"; \ ${INSTALL_DATA} ${mansrcdir}/$${page} "$(DESTDIR)${man1dir}/$${dest}"); \ - [ -n "${GZIP_PROG}" ] || continue ; \ - rm -f "$(DESTDIR)${man1dir}/$${dest}.gz"; \ - ${GZIP_PROG} -9n "$(DESTDIR)${man1dir}/$${dest}" || true; \ + [ -n "${COMPR_INFO_PROG}" ] || continue ; \ + rm -f "$(DESTDIR)${man1dir}/$${dest}${COMPR_INFO_EXT}"; \ + eval ${COMPR_INFO_PROG} ${COMPR_INFO_ARGS} "\"$(DESTDIR)${man1dir}/$${dest}\"" || true; \ done ## Install those items from etc/ that need to end up elsewhere. @@ -774,16 +799,12 @@ uninstall: for elt in ${INFO_NONMISC} $${info_misc}; do \ (cd "$${thisdir}"; \ $(INSTALL_INFO) --remove --info-dir="$(DESTDIR)${infodir}" "$(DESTDIR)${infodir}/$$elt"); \ - if [ -n "${GZIP_PROG}" ]; then \ - ext=.gz; else ext=; fi; \ - rm -f $$elt$$ext $$elt-[1-9]$$ext $$elt-[1-9][0-9]$$ext; \ + rm -f $$elt${COMPR_INFO_EXT} $$elt-[1-9]${COMPR_INFO_EXT} $$elt-[1-9][0-9]${COMPR_INFO_EXT}; \ done; \ fi) - (if [ -n "${GZIP_PROG}" ]; then \ - ext=.gz; else ext=; fi; \ - if cd ${mansrcdir}; then \ + (if cd ${mansrcdir}; then \ for page in *.1; do \ - rm -f "$(DESTDIR)${man1dir}"/`echo "$${page}" | sed -e 's/\.1$$//' -e '$(TRANSFORM)'`.1$$ext; done; \ + rm -f "$(DESTDIR)${man1dir}"/`echo "$${page}" | sed -e 's/\.1$$//' -e '$(TRANSFORM)'`.1${COMPR_INFO_EXT}; done; \ fi) rm -f "$(DESTDIR)${bindir}/$(EMACS)" "$(DESTDIR)${bindir}/$(EMACSFULL)" (if cd "$(DESTDIR)${icondir}"; then \ diff --git a/admin/admin.el b/admin/admin.el index 3fc50afe9f..61792bea89 100644 --- a/admin/admin.el +++ b/admin/admin.el @@ -671,7 +671,7 @@ make-manuals-dist-output-variables ("@PACKAGE_TARNAME@" . "emacs") ("@docdir@" . "${datarootdir}/doc/${PACKAGE_TARNAME}") ("@\\(dvi\\|html\\|pdf\\|ps\\)dir@" . "${docdir}") - ("@GZIP_PROG@" . "gzip") + ("@COMPR_PROG@" . "gzip") ("@INSTALL@" . "install -c") ("@INSTALL_DATA@" . "${INSTALL} -m 644") ("@configure_input@" . "") diff --git a/configure.ac b/configure.ac index 8b34c3b658..d0061d9865 100644 --- a/configure.ac +++ b/configure.ac @@ -415,11 +415,32 @@ AC_DEFUN ## Makefile.in needs the cache file name. AC_SUBST(cache_file) -## This is an option because I do not know if all info/man support -## compressed files, nor how to test if they do so. -OPTION_DEFAULT_ON([compress-install], - [don't compress some files (.el, .info, etc.) when installing. Equivalent to: -make GZIP_PROG= install]) +AC_ARG_WITH([compress-install],dnl +[AS_HELP_STRING([--with-compress-install=PROGRAM], + [compression program used to compress some files + (.el, .info, etc.) when installing (PROGRAM one of: yes, gzip, + zstd, no; default 'yes'='gzip'). Note that Info/man files are + still compressed with gzip when 'zstd' is selected.])], +[ case "${withval}" in + yes|no|gzip|zstd) ;; + *) AC_MSG_ERROR(['--with-compress-install=$withval' is invalid; +this option's value should be 'yes', 'no', 'gzip', or 'zstd'.]) + ;; + esac +], +[with_compress_install=$with_features]) + +if test "${with_compress_install}" != no; then + if test "${with_compress_install}" = yes; then + with_compress_install=gzip; + fi + COMPR_TYPE=$with_compress_install + AC_PATH_PROG(COMPR_PROG, $with_compress_install) + + # Support separate compression for Info/man files for compatibility + AC_PATH_PROG(COMPR_INFO_PROG, gzip) +fi +AC_SUBST(COMPR_TYPE) AC_ARG_WITH(gameuser,dnl [AS_HELP_STRING([--with-gameuser=USER_OR_GROUP], @@ -1209,11 +1230,6 @@ AC_DEFUN AC_PATH_PROG(INSTALL_INFO, install-info, :, $PATH$PATH_SEPARATOR/usr/sbin$PATH_SEPARATOR/sbin) -dnl Don't use GZIP, which is used by gzip for additional parameters. -AC_PATH_PROG(GZIP_PROG, gzip) - -test $with_compress_install != yes && test -n "$GZIP_PROG" && \ - GZIP_PROG=" # $GZIP_PROG # (disabled by configure --without-compress-install)" AC_CACHE_CHECK([for 'find' args to delete a file], [emacs_cv_find_delete], diff --git a/doc/emacs/Makefile.in b/doc/emacs/Makefile.in index 54e173f8d6..81825c9f82 100644 --- a/doc/emacs/Makefile.in +++ b/doc/emacs/Makefile.in @@ -51,7 +51,21 @@ psdir = MKDIR_P = @MKDIR_P@ -GZIP_PROG = @GZIP_PROG@ +ifdef GZIP_PROG + COMPR_PROG = ${GZIP_PROG} + COMPR_TYPE = gzip +else + COMPR_PROG = @COMPR_PROG@ + COMPR_TYPE = @COMPR_TYPE@ +endif + +ifeq (${COMPR_TYPE}, gzip) + COMPR_ARGS = "-9n" + COMPR_EXT = ".gz" +else ifeq (${COMPR_TYPE}, zstd) + COMPR_ARGS = "-19q --rm" + COMPR_EXT = ".zst" +endif HTML_OPTS = --no-split --html @@ -241,9 +255,9 @@ install-ps: umask 022; $(MKDIR_P) "$(DESTDIR)$(psdir)" for file in $(PS_TARGETS); do \ $(INSTALL_DATA) $${file} "$(DESTDIR)$(psdir)"; \ - [ -n "${GZIP_PROG}" ] || continue; \ - rm -f "$(DESTDIR)$(psdir)/$${file}.gz"; \ - ${GZIP_PROG} -9n "$(DESTDIR)$(psdir)/$${file}"; \ + [ -n "${COMPR_PROG}" ] || continue; \ + rm -f "$(DESTDIR)$(psdir)/$${file}${COMPR_EXT}"; \ + eval ${COMPR_PROG} ${COMPR_ARGS} "\"$(DESTDIR)$(psdir)/$${file}\""; \ done ## Top-level Makefile installs the info pages. @@ -261,9 +275,8 @@ uninstall-html: rm -f "$(DESTDIR)$(htmldir)/$${file}"; \ done uninstall-ps: - ext= ; [ -n "${GZIP_PROG}" ] && ext=.gz; \ for file in $(PS_TARGETS); do \ - rm -f "$(DESTDIR)$(psdir)/$${file}$${ext}"; \ + rm -f "$(DESTDIR)$(psdir)/$${file}${COMPR_EXT}"; \ done uninstall-pdf: for file in $(PDF_TARGETS); do \ diff --git a/doc/lispintro/Makefile.in b/doc/lispintro/Makefile.in index e2a1229d5c..989163b7c6 100644 --- a/doc/lispintro/Makefile.in +++ b/doc/lispintro/Makefile.in @@ -39,7 +39,21 @@ psdir = MKDIR_P = @MKDIR_P@ -GZIP_PROG = @GZIP_PROG@ +ifdef GZIP_PROG + COMPR_PROG = ${GZIP_PROG} + COMPR_TYPE = gzip +else + COMPR_PROG = @COMPR_PROG@ + COMPR_TYPE = @COMPR_TYPE@ +endif + +ifeq (${COMPR_TYPE}, gzip) + COMPR_ARGS = "-9n" + COMPR_EXT = ".gz" +else ifeq (${COMPR_TYPE}, zstd) + COMPR_ARGS = "-19q --rm" + COMPR_EXT = ".zst" +endif HTML_OPTS = --no-split --html @@ -140,9 +154,9 @@ install-ps: umask 022; $(MKDIR_P) "$(DESTDIR)$(psdir)" for file in $(PS_TARGETS); do \ $(INSTALL_DATA) $${file} "$(DESTDIR)$(psdir)"; \ - [ -n "${GZIP_PROG}" ] || continue; \ - rm -f "$(DESTDIR)$(psdir)/$${file}.gz"; \ - ${GZIP_PROG} -9n "$(DESTDIR)$(psdir)/$${file}"; \ + [ -n "${COMPR_PROG}" ] || continue; \ + rm -f "$(DESTDIR)$(psdir)/$${file}${COMPR_EXT}"; \ + eval ${COMPR_PROG} ${COMPR_ARGS} "\"$(DESTDIR)$(psdir)/$${file}\""; \ done ## Top-level Makefile installs the info pages. @@ -160,9 +174,8 @@ uninstall-html: rm -f "$(DESTDIR)$(htmldir)/$${file}"; \ done uninstall-ps: - ext= ; [ -n "${GZIP_PROG}" ] && ext=.gz; \ for file in $(PS_TARGETS); do \ - rm -f "$(DESTDIR)$(psdir)/$${file}$${ext}"; \ + rm -f "$(DESTDIR)$(psdir)/$${file}${COMPR_EXT}"; \ done uninstall-pdf: for file in $(PDF_TARGETS); do \ diff --git a/doc/lispref/Makefile.in b/doc/lispref/Makefile.in index 221f4f97f5..352cbab258 100644 --- a/doc/lispref/Makefile.in +++ b/doc/lispref/Makefile.in @@ -43,7 +43,21 @@ psdir = MKDIR_P = @MKDIR_P@ -GZIP_PROG = @GZIP_PROG@ +ifdef GZIP_PROG + COMPR_PROG = ${GZIP_PROG} + COMPR_TYPE = gzip +else + COMPR_PROG = @COMPR_PROG@ + COMPR_TYPE = @COMPR_TYPE@ +endif + +ifeq (${COMPR_TYPE}, gzip) + COMPR_ARGS = "-9n" + COMPR_EXT = ".gz" +else ifeq (${COMPR_TYPE}, zstd) + COMPR_ARGS = "-19q --rm" + COMPR_EXT = ".zst" +endif HTML_OPTS = --no-split --html @@ -201,9 +215,9 @@ install-ps: umask 022; $(MKDIR_P) "$(DESTDIR)$(psdir)" for file in $(PS_TARGETS); do \ $(INSTALL_DATA) $${file} "$(DESTDIR)$(psdir)"; \ - [ -n "${GZIP_PROG}" ] || continue; \ - rm -f "$(DESTDIR)$(psdir)/$${file}.gz"; \ - ${GZIP_PROG} -9n "$(DESTDIR)$(psdir)/$${file}"; \ + [ -n "${COMPR_PROG}" ] || continue; \ + rm -f "$(DESTDIR)$(psdir)/$${file}${COMPR_EXT}"; \ + eval ${COMPR_PROG} ${COMPR_ARGS} "\"$(DESTDIR)$(psdir)/$${file}\""; \ done ## Top-level Makefile installs the info pages. @@ -221,9 +235,8 @@ uninstall-html: rm -f "$(DESTDIR)$(htmldir)/$${file}"; \ done uninstall-ps: - ext= ; [ -n "${GZIP_PROG}" ] && ext=.gz; \ for file in $(PS_TARGETS); do \ - rm -f "$(DESTDIR)$(psdir)/$${file}$${ext}"; \ + rm -f "$(DESTDIR)$(psdir)/$${file}${COMPR_EXT}"; \ done uninstall-pdf: for file in $(PDF_TARGETS); do \ diff --git a/doc/misc/Makefile.in b/doc/misc/Makefile.in index fd07ea4ca1..380efb8761 100644 --- a/doc/misc/Makefile.in +++ b/doc/misc/Makefile.in @@ -44,7 +44,21 @@ psdir = MKDIR_P = @MKDIR_P@ -GZIP_PROG = @GZIP_PROG@ +ifdef GZIP_PROG + COMPR_PROG = ${GZIP_PROG} + COMPR_TYPE = gzip +else + COMPR_PROG = @COMPR_PROG@ + COMPR_TYPE = @COMPR_TYPE@ +endif + +ifeq (${COMPR_TYPE}, gzip) + COMPR_ARGS = "-9n" + COMPR_EXT = ".gz" +else ifeq (${COMPR_TYPE}, zstd) + COMPR_ARGS = "-19q --rm" + COMPR_EXT = ".zst" +endif HTML_OPTS = --no-split --html @@ -262,9 +276,9 @@ install-ps: umask 022; $(MKDIR_P) "$(DESTDIR)$(psdir)" for file in $(PS_TARGETS); do \ $(INSTALL_DATA) $${file} "$(DESTDIR)$(psdir)"; \ - [ -n "${GZIP_PROG}" ] || continue; \ - rm -f "$(DESTDIR)$(psdir)/$${file}.gz"; \ - ${GZIP_PROG} -9n "$(DESTDIR)$(psdir)/$${file}"; \ + [ -n "${COMPR_PROG}" ] || continue; \ + rm -f "$(DESTDIR)$(psdir)/$${file}${COMPR_EXT}"; \ + eval ${COMPR_PROG} ${COMPR_ARGS} "\"$(DESTDIR)$(psdir)/$${file}\""; \ done ## Top-level Makefile installs the info pages. @@ -283,9 +297,8 @@ uninstall-html: rm -f "$(DESTDIR)$(htmldir)/$${file}"; \ done uninstall-ps: - ext= ; [ -n "${GZIP_PROG}" ] && ext=.gz; \ for file in $(PS_TARGETS); do \ - rm -f "$(DESTDIR)$(psdir)/$${file}$${ext}"; \ + rm -f "$(DESTDIR)$(psdir)/$${file}${COMPR_EXT}"; \ done uninstall-pdf: for file in $(PDF_TARGETS); do \ diff --git a/etc/NEWS b/etc/NEWS index e65823413b..633ddd9f62 100644 --- a/etc/NEWS +++ b/etc/NEWS @@ -40,6 +40,20 @@ counterparts from json.el. ** NEWS files for past Emacs versions and packages are now compressed by default. ++++ +** Emacs now supports Zstandard compression for most compressed +installation files using '--with-compress-install=zstd'. Info/man +files are still compressed with 'gzip' due to lack of support in +'info' and 'man'. + ++++ +** The make variable GZIP_PROG is now a deprecated alias for the new +variable COMPR_PROG. + +** New make variables COMPR_TYPE, COMPR_ARGS, and COMPR_EXT allow for +fine-tuning the compression type, arguments, and extension used when +(un)installing compressed files. + ** The etags program now uses the C library's regular expression matcher when possible, and a compatible regex substitute otherwise. This will let developers maintain Emacs's own regex code without having to also @@ -313,6 +327,11 @@ the node "(emacs) Directory Variables" of the user manual. * Changes in Specialized Modes and Packages in Emacs 27.1 +** Auto Compression mode +*** Zstandard files are now recognized as valid compressed +representative files by default. See 'load-file-rep-suffixes' for +details. + ** map.el *** Now also understands plists. *** Now defined via generic functions that can be extended via 'cl-defmethod'. diff --git a/lisp/jka-cmpr-hook.el b/lisp/jka-cmpr-hook.el index d800b60513..355a72df26 100644 --- a/lisp/jka-cmpr-hook.el +++ b/lisp/jka-cmpr-hook.el @@ -155,7 +155,7 @@ jka-compr-install (setq auto-mode-alist (append auto-mode-alist jka-compr-mode-alist-additions)) - ;; Make sure that (load "foo") will find /bla/foo.el.gz. + ;; Make sure that (load "foo") will find, e.g., /bla/foo.el.gz. (setq load-file-rep-suffixes (append load-file-rep-suffixes jka-compr-load-suffixes nil))) @@ -331,7 +331,7 @@ jka-compr-mode-alist-additions :set 'jka-compr-set :group 'jka-compr) -(defcustom jka-compr-load-suffixes (purecopy '(".gz")) +(defcustom jka-compr-load-suffixes (purecopy '(".gz" ".zst")) "List of compression related suffixes to try when loading files. Enabling Auto Compression mode appends this list to `load-file-rep-suffixes', which see. Disabling Auto Compression mode removes all suffixes -- 2.20.1