bug#27748: 26.0.50; doc strings should be in DOC file

[npostavs]

Ken Raeburn <raeburn@raeburn.org> writes:

>
> 1. defcustom doc strings from files compiled with lexical binding.
>
>    For example, files.el (lexical bindings) defines
>    delete-auto-save-files but it doesn't show up in the DOC file;
>    files.elc starts with an initial byte-code blob which includes the
>    symbol delete-auto-save-files and its doc string in the constants
>    array.

Actually, it's not only about lexical binding, the following file:

    ;;; -*- lexical-binding: nil -*-

    (defcustom custom-foo nil
      "a custom variable"
      :type 'boolean
      :group 'foo-group)

    ;; (defun foo ()
    ;;   t)

    (defcustom custom-bar nil
      "another custom variable"
      :type 'boolean
      :group 'foo-group)

produces (with prologue removed, and reformatted for readability)

    (byte-code 
"\300\301\302\303\304\305\306\307&\210\300\310\302\311\304\305\306\307&\207"
               [custom-declare-variable custom-foo nil "a custom variable" 
:type boolean :group foo-group
                                        custom-bar "another custom variable"]
               8)

Uncommenting the (defun foo...) produces:

    #@19 a custom variable
    (custom-declare-variable 'custom-foo nil '(#$ . 411) :type 'boolean :group 
'foo-group)
    (defalias 'foo #[nil "\300\207" [t] 1])
    #@25 another custom variable
    (custom-declare-variable 'custom-bar nil '(#$ . 562) :type 'boolean :group 
'foo-group)

Then changing to lexical binding produces:

    (byte-code "\300\301\302\303\304DD\305\306\307\310\311&\207"
               [custom-declare-variable
                custom-foo funcall function
                #[0 "\300\207" [nil] 1] "a custom variable"
                :type boolean :group foo-group]
               8)
    (defalias 'foo #[0 "\300\207" [t] 1])
    (byte-code "\300\301\302\303\304DD\305\306\307\310\311&\207"
               [custom-declare-variable
                custom-bar funcall function
                #[0 "\300\207" [nil] 1]
                "another custom variable" :type boolean :group foo-group]
               8)

As far as I can tell, the problem is that the
byte-compile-dynamic-docstrings feature (that's the #@19 thing) relies
on `byte-compile-out-toplevel' to decompile "trivial" functions back
into source code.  So having lexical binding set, or 2 defcustoms in a
row produces "non-trivial" code which is not decompiled, and therefore
not recognized in byte-compile-output-file-form as something which
should be used with byte-compile-output-docform.

    (defun byte-compile-out-toplevel (&optional for-effect output-type)
      ...
      ;; Decompile trivial functions:
      ...
        (cond
         ;; #### This should be split out into 
byte-compile-nontrivial-function-p.
         ((or (eq output-type 'lambda)
          (nthcdr (if (eq output-type 'file) 50 8) byte-compile-output)
          ...
            (while
                    (cond
                     ((memq (car (car rest)) '(byte-varref byte-constant))
                     ...
                     ((and maycall
                           ;; Allow a funcall if at most one atom follows it.
                      ...
                      (setq maycall nil)        ; Only allow one real function 
call.
                      ...
                      (or (eq output-type 'file)
                          (not (delq nil (mapcar 'consp (cdr (car body))))))))
            ...
        (list 'byte-code (byte-compile-lapcode byte-compile-output)
              byte-compile-vector byte-compile-maxdepth)))
         ;; it's a trivial function
         ((cdr body) (cons 'progn (nreverse body)))
         ((car body))))

    (defun byte-compile-output-file-form (form)
      ;; Write the given form to the output buffer, being careful of docstrings
      ;; in defvar, defvaralias, defconst, autoload and
      ;; custom-declare-variable because make-docfile is so amazingly stupid.
      ...
        (if (and (memq (car-safe form) '(defvar defvaralias defconst
                                          autoload custom-declare-variable))
                 (stringp (nth 3 form)))
            (byte-compile-output-docform nil nil '("\n(" 3 ")") form nil
                                         (memq (car form)
                                               '(defvaralias autoload
                                                  custom-declare-variable)))
          (princ "\n" byte-compile--outbuffer)
          (prin1 form byte-compile--outbuffer)
          nil)))

The following patch prevents custom-declare-variable from being compiled
and lets the docstrings get printed properly.  Probably needs a bit more
refinement though.

>From 4cb45936966de76f91b95971c886599a24361c5b Mon Sep 17 00:00:00 2001
From: Noam Postavsky <npostavs@gmail.com>
Date: Sat, 5 Aug 2017 20:02:19 -0400
Subject: [PATCH] * lisp/custom.el (custom-declare-variable): Don't compile
 (Bug#27748).

---
 lisp/custom.el | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lisp/custom.el b/lisp/custom.el
index ecfa34db5b..5876d3fd56 100644
--- a/lisp/custom.el
+++ b/lisp/custom.el
@@ -144,6 +144,9 @@ (defun custom-declare-variable (symbol default doc &rest 
args)
 DEFAULT is stored as SYMBOL's standard value, in SYMBOL's property
 `standard-value'.  At the same time, SYMBOL's property `force-value' is
 set to nil, as the value is no longer rogue."
+  (declare (compiler-macro
+            (lambda (form)
+              `(eval ',form lexical-binding))))
   (put symbol 'standard-value (purecopy (list default)))
   ;; Maybe this option was rogue in an earlier version.  It no longer is.
   (when (get symbol 'force-value)
-- 
2.11.1