Re: CC-mode highlight change between 24.5 and 25

[Alan Mackenzie]

Hello, Oleh.

On Fri, Sep 02, 2016 at 04:18:28PM +0200, Oleh Krehel wrote:
> Alan Mackenzie <address@hidden> writes:

> > Can you help me out here, please - by what criterion is the C++ compiler
> > deciding that `fout' is a variable initialised to `file_name' rather
> > than a function declaration?  The only syntactic thing I can see for
> > this is that `fout' is declared inside a function rather than at the top
> > level (or directly inside a class, struct, or namespace).

> I think it very uncommon to declare a function inside another function
> in C++: it's done in top-level instead.

> I would be happy enough with the heuristic of parsing all indented
> (i.e. not top-level) declarations/definitions as (variable) definitions.

OK.  Here's a first approximation to a solution, which I would be
grateful if you could try out on real code.  Please let me know how well
it works, and if it introduces any nasty looking bugs.

What I've done is to count nesting depth of braces inside a class or
namespace, etc.  When that depth is 1, we're at the top level, and
anything looking like a function is fontified as one.  When the depth is
more than 1, we're not at top level, and anything looking like a
function is fontified as a uniform initialisation.

The following patch should apply OK to the savannah master branch:



diff --git a/lisp/progmodes/cc-engine.el b/lisp/progmodes/cc-engine.el
index 2cad2d0..4622398 100644
--- a/lisp/progmodes/cc-engine.el
+++ b/lisp/progmodes/cc-engine.el
@@ -422,6 +422,15 @@ c-forward-to-cpp-define-body
 
 ;;; Basic utility functions.
 
+(defmacro c-pull-open-brace (ps)
+  ;; Pull the next open brace from PS (which has the form of paren-state),
+  ;; skipping over any brace pairs.  Returns NIL when PS is exhausted.
+  `(progn
+     (while (consp (car ,ps))
+       (setq ,ps (cdr ,ps)))
+     (prog1 (car ,ps)
+       (setq ,ps (cdr ,ps)))))
+
 (defun c-delq-from-dotted-list (elt dlist)
   ;; If ELT is a member of the (possibly dotted) list DLIST, remove all
   ;; occurrences of it (except for any in the last cdr of DLIST).
@@ -5109,6 +5118,123 @@ c-debug-remove-decl-spot-faces
        (c-debug-remove-face ,beg ,end 'c-debug-decl-spot-face)
        (c-debug-remove-face ,beg ,end 'c-debug-decl-sws-face))))
 
+;;;; NEW STOUGH, 2016-09-02, and ...-03
+(defconst c-bs-interval 5000)
+(defvar c-bs-cache nil)
+(make-variable-buffer-local 'c-bs-cache)
+(defvar c-bs-cache-limit 1)
+(make-variable-buffer-local 'c-bs-cache-limit)
+(defvar c-bs-prev-pos most-positive-fixnum)
+(make-variable-buffer-local 'c-bs-prev-pos)
+(defvar c-bs-prev-stack nil)
+(make-variable-buffer-local 'c-bs-prev-stack)
+
+(defun c-init-bs-cache ()
+  ;; Initialize the cache in `c-bs-cache' and related variables.
+  (setq c-bs-cache nil
+       c-bs-cache-limit 1
+       c-bs-prev-pos most-positive-fixnum
+       c-bs-prev-stack nil))
+
+(defun c-truncate-bs-cache (pos &rest _ignore)
+  ;; Truncate the upper bound of the cache `c-bs-cache' to POS, if it is
+  ;; higher than that position.  This is called as either a before- or
+  ;; after-change-function.
+  (setq c-bs-cache-limit
+       (min c-bs-cache-limit pos)))
+
+(defun c-update-brace-stack (stack from to)
+  ;; Give a brace-stack which has the value STACK at position FROM, update it
+  ;; to it's value at position TO, where TO is after (or equal to) FROM.
+  ;; Return this new value.
+  (let (match kwd-sym (prev-match-pos 1)
+             )
+    (save-excursion
+      (goto-char from)
+      (while (and (< (point) to)
+                 (c-syntactic-re-search-forward c-brace-stack-thing-key to 
'bound)
+                 (> (point) prev-match-pos))
+       (setq prev-match-pos (point))
+       (setq match (match-string-no-properties 1)
+             kwd-sym (c-keyword-sym match))
+       (cond
+        ((and (equal match "{")
+              (progn (backward-char)
+                     (prog1 (looking-at "\\s(")
+                       (forward-char)))                      
+         (setq stack (if stack
+                         (cons (1+ (car stack)) (cdr stack))
+                       (list 1)))))
+        ((and (equal match "}")
+              (progn (backward-char)
+                     (prog1 (looking-at "\\s)")
+                       (forward-char))))               
+         (setq stack
+               (cond
+                ((and stack (> (car stack) 1))
+                 (cons (1- (car stack)) (cdr stack)))
+                ((and (cdr stack) (eq (car stack) 1))
+                 (cdr stack))
+                (t stack))))
+        ((equal match ";")
+         (when (and stack (cdr stack) (eq (car stack) 0))
+           (setq stack (cdr stack))))
+        ((c-keyword-member kwd-sym 'c-flat-decl-block-kwds)
+         (push 0 stack))))
+      stack)))
+
+(defun c-brace-stack-at (here)
+  ;; Given a buffer position HERE, Return the value of the brace stack there.
+  (save-excursion
+    (save-restriction
+      (widen)
+      (let ((c c-bs-cache)
+           (can-use-prev (<= c-bs-prev-pos c-bs-cache-limit))
+           elt stack pos npos high-elt)
+       ;; Trim the cache to take account of buffer changes.
+       (while (and c
+                   (> (caar c) c-bs-cache-limit))
+         (setq c (cdr c)))
+       (setq c-bs-cache c)
+
+       (while (and c
+                   (> (caar c) here))
+         (setq high-elt (car c))
+         (setq c (cdr c)))
+       (setq pos (or (and c (caar c))
+                     (point-min)))
+
+       (setq elt (if c
+                     (car c)
+                   (cons (point-min) (list 1))))
+       (when (not high-elt)
+         (setq stack (cdr elt))
+         (while
+             ;; Add an element to `c-state-semi-nonlit-pos-cache' each 
iteration.
+             (<= (setq npos (+ pos c-bs-interval)) here)
+           (setq stack (c-update-brace-stack stack pos npos))
+           (setq elt (cons npos stack))
+           (setq c-bs-cache (cons elt c-bs-cache))
+           (setq pos npos)))
+
+       (if (> pos c-bs-cache-limit)
+           (setq c-bs-cache-limit pos))
+
+       ;; Can we just use the previous value?
+       (if (and can-use-prev
+                (<= c-bs-prev-pos here)
+                (> c-bs-prev-pos (car elt)))
+           (setq pos c-bs-prev-pos
+                 stack c-bs-prev-stack)
+         (setq pos (car elt)
+               stack (cdr elt)))
+       (if (> here c-bs-cache-limit)
+           (setq c-bs-cache-limit here))
+       (setq c-bs-prev-pos here
+             c-bs-prev-stack (c-update-brace-stack stack pos here))))))
+
+;;;; END OF NEW STOUGH, 2016-09-02, and ...-03, and ...-04
+
 (defmacro c-find-decl-prefix-search ()
   ;; Macro used inside `c-find-decl-spots'.  It ought to be a defun,
   ;; but it contains lots of free variables that refer to things
@@ -5202,6 +5328,9 @@ c-find-decl-prefix-search
               cfd-re-match nil)
        (setq cfd-match-pos cfd-prop-match
             cfd-prop-match nil))
+     (let ((stack (c-brace-stack-at cfd-match-pos)))
+       (setq cfd-top-level (or (null stack)
+                              (<= (car stack) 1))))
 
      (goto-char cfd-match-pos)
 
@@ -5300,8 +5429,12 @@ c-find-decl-spots
        ;; comments.
        (cfd-token-pos 0)
        ;; The end position of the last entered macro.
-       (cfd-macro-end 0))
-
+       (cfd-macro-end 0)
+       ;; Whether the last position returned from `c-find-decl-prefix-search'
+       ;; is at the top-level (including directly in a class or namespace,
+       ;; etc.).
+       cfd-top-level)
+    
     ;; Initialize by finding a syntactically relevant start position
     ;; before the point, and do the first `c-decl-prefix-or-start-re'
     ;; search unless we're at bob.
@@ -5608,7 +5741,7 @@ c-find-decl-spots
                   nil))))              ; end of when condition
 
        (c-debug-put-decl-spot-faces cfd-match-pos (point))
-       (if (funcall cfd-fun cfd-match-pos (/= cfd-macro-end 0))
+       (if (funcall cfd-fun cfd-match-pos (/= cfd-macro-end 0) cfd-top-level)
            (setq cfd-prop-match nil))
 
        (when (/= cfd-macro-end 0)
@@ -7349,15 +7482,6 @@ c-forward-annotation
            t))
      (progn (goto-char pos) nil))))
 
-(defmacro c-pull-open-brace (ps)
-  ;; Pull the next open brace from PS (which has the form of paren-state),
-  ;; skipping over any brace pairs.  Returns NIL when PS is exhausted.
-  `(progn
-     (while (consp (car ,ps))
-       (setq ,ps (cdr ,ps)))
-     (prog1 (car ,ps)
-       (setq ,ps (cdr ,ps)))))
-
 (defun c-back-over-compound-identifier ()
   ;; Point is putatively just after a "compound identifier", i.e. something
   ;; looking (in C++) like this "FQN::of::base::Class".  Move to the start of
@@ -7555,7 +7679,7 @@ c-forward-declarator
   ;; array/struct initialization) or "=" or terminating delimiter
   ;; (e.g. "," or ";" or "}").
   (let ((here (point))
-       id-start id-end brackets-after-id paren-depth)
+       id-start id-end brackets-after-id paren-depth paren-pos)
     (or limit (setq limit (point-max)))
     (if        (and
         (< (point) limit)
@@ -7597,7 +7721,8 @@ c-forward-declarator
                           t))
                    (if (eq (char-after) ?\()
                        (progn
-                         (setq paren-depth (1+ paren-depth))
+                         (setq paren-depth (1+ paren-depth)
+                               paren-pos (or paren-pos (point)))
                          (forward-char))
                      (goto-char (match-end 1)))
                    (c-forward-syntactic-ws)
@@ -7725,6 +7850,9 @@ c-forward-decl-or-cast-1
   ;;           inside a function declaration arglist).
   ;; '<>       In an angle bracket arglist.
   ;; 'arglist  Some other type of arglist.
+  ;; 'top      Some other context and point is at the top-level (either
+  ;;           outside any braces or directly inside a class or namespace,
+  ;;           etc.)
   ;; nil       Some other context or unknown context.  Includes
   ;;           within the parens of an if, for, ... construct.
   ;; 'not-decl This value is never supplied to this function.  It
@@ -8133,7 +8261,7 @@ c-forward-decl-or-cast-1
                              maybe-typeless
                              backup-maybe-typeless
                              (when c-recognize-typeless-decls
-                               (and (not context)
+                               (and (memq context '(nil top))
                                     ;; Deal with C++11's "copy-initialization"
                                     ;; where we have <type>(<constant>), by
                                     ;; contrasting with a typeless
@@ -8196,7 +8324,7 @@ c-forward-decl-or-cast-1
 
         (setq at-decl-end
               (looking-at (cond ((eq context '<>) "[,>]")
-                                (context "[,)]")
+                                ((not (memq context '(nil top))) "[,\)]")
                                 (t "[,;]"))))
 
         ;; Now we've collected info about various characteristics of
@@ -8325,7 +8453,7 @@ c-forward-decl-or-cast-1
 
           (if (and got-parens
                    (not got-prefix)
-                   (not context)
+                   (memq context '(nil top))
                    (not (eq at-type t))
                    (or backup-at-type
                        maybe-typeless
@@ -8375,6 +8503,18 @@ c-forward-decl-or-cast-1
               ;; instantiation expression).
               (throw 'at-decl-or-cast nil))))
 
+        ;; CASE 9.5
+        (when (and (not context)       ; i.e. not at top level.
+                   (c-major-mode-is 'c++-mode)
+                   (eq at-decl-or-cast 'ids)
+                   after-paren-pos)
+          ;; We've got something like "foo bar (...)" in C++ which isn't at
+          ;; the top level.  This is probably a uniform initialization of bar
+          ;; to the contents of the parens.  In this case the declarator ends
+          ;; at the open paren.
+          (goto-char (1- after-paren-pos))
+          (throw 'at-decl-or-cast t))
+
         ;; CASE 10
         (when at-decl-or-cast
           ;; By now we've located the type in the declaration that we know
@@ -8383,7 +8523,7 @@ c-forward-decl-or-cast-1
 
         ;; CASE 11
         (when (and got-identifier
-                   (not context)
+                   (memq context '(nil top))
                    (looking-at c-after-suffixed-type-decl-key)
                    (if (and got-parens
                             (not got-prefix)
@@ -8479,7 +8619,7 @@ c-forward-decl-or-cast-1
               (when (and got-prefix-before-parens
                          at-type
                          (or at-decl-end (looking-at "=[^=]"))
-                         (not context)
+                         (memq context '(nil top))
                          (or (not got-suffix)
                              at-decl-start))
                 ;; Got something like "foo * bar;".  Since we're not inside
@@ -8505,7 +8645,7 @@ c-forward-decl-or-cast-1
                 (throw 'at-decl-or-cast t)))
 
           ;; CASE 18
-          (when (and context
+          (when (and (not (memq context '(nil top)))
                      (or got-prefix
                          (and (eq context 'decl)
                               (not c-recognize-paren-inits)
diff --git a/lisp/progmodes/cc-fonts.el b/lisp/progmodes/cc-fonts.el
index 735108f..38362b2 100644
--- a/lisp/progmodes/cc-fonts.el
+++ b/lisp/progmodes/cc-fonts.el
@@ -991,7 +991,7 @@ c-font-lock-<>-arglists
          (goto-char pos)))))
   nil)
 
-(defun c-font-lock-declarators (limit list types)
+(defun c-font-lock-declarators (limit list types &optional not-top)
   ;; Assuming the point is at the start of a declarator in a declaration,
   ;; fontify the identifier it declares.  (If TYPES is set, it does this via
   ;; the macro `c-fontify-types-and-refs'.)
@@ -1001,7 +1001,9 @@ c-font-lock-declarators
   ;; additionally, mark the commas with c-type property 'c-decl-id-start or
   ;; 'c-decl-type-start (according to TYPES).  Stop at LIMIT.
   ;;
-  ;; If TYPES is non-nil, fontify all identifiers as types.
+  ;; If TYPES is non-nil, fontify all identifiers as types.  If NOT-TOP is
+  ;; non-nil, we are not at the top-level ("top-level" includes being directly
+  ;; inside a class or namespace, etc.).
   ;;
   ;; Nil is always returned.  The function leaves point at the delimiter after
   ;; the last declarator it processes.
@@ -1025,6 +1027,11 @@ c-font-lock-declarators
       (setq next-pos (point)
            id-start (car decl-res)
            id-face (if (and (eq (char-after) ?\()
+                            (or (not not-top)
+                                (save-excursion
+                                  (forward-char)
+                                  (c-forward-syntactic-ws)
+                                  (looking-at "[*&]")))
                             (not (car (cddr decl-res))) ; brackets-after-id
                             (or (not (c-major-mode-is 'c++-mode))
                                 (save-excursion
@@ -1199,7 +1206,7 @@ c-font-lock-declarations
        c-decl-start-re
        (eval c-maybe-decl-faces)
 
-       (lambda (match-pos inside-macro)
+       (lambda (match-pos inside-macro &optional toplev)
         ;; Note to maintainers: don't use `limit' inside this lambda form;
         ;; c-find-decl-spots sometimes narrows to less than `limit'.
         (setq start-pos (point))
@@ -1223,7 +1230,7 @@ c-font-lock-declarations
            (let ((type (and (> match-pos (point-min))
                             (c-get-char-property (1- match-pos) 'c-type))))
              (cond ((not (memq (char-before match-pos) '(?\( ?, ?\[ ?< ?{)))
-                    (setq context nil
+                    (setq context (and toplev 'top)
                           c-restricted-<>-arglists nil))
                    ;; A control flow expression or a decltype
                    ((and (eq (char-before match-pos) ?\()
@@ -1273,7 +1280,7 @@ c-font-lock-declarations
                                          'c-not-decl))
                    ;; We're inside an "ordinary" open brace.
                    ((eq (char-before match-pos) ?{)
-                    (setq context nil
+                    (setq context (and toplev 'top)
                           c-restricted-<>-arglists nil))
                    ;; Inside an angle bracket arglist.
                    ((or (eq type 'c-<>-arg-sep)
@@ -1437,7 +1444,8 @@ c-font-lock-declarations
                                                 'c-decl-id-start)))))
 
                    (c-font-lock-declarators
-                    (min limit (point-max)) decl-list (cadr decl-or-cast)))
+                    (min limit (point-max)) decl-list
+                    (cadr decl-or-cast) (not toplev)))
 
                  ;; A declaration has been successfully identified, so do all 
the
                  ;; fontification of types and refs that've been recorded.
@@ -2413,7 +2421,7 @@ c-font-lock-objc-methods
      limit
      "[-+]"
      nil
-     (lambda (match-pos inside-macro)
+     (lambda (match-pos inside-macro &optional top-level)
        (forward-char)
        (c-font-lock-objc-method))))
   nil)
diff --git a/lisp/progmodes/cc-langs.el b/lisp/progmodes/cc-langs.el
index ae6e6a3..b8c4420 100644
--- a/lisp/progmodes/cc-langs.el
+++ b/lisp/progmodes/cc-langs.el
@@ -479,10 +479,12 @@ c-populate-syntax-table
        c-before-change-check-<>-operators
        c-depropertize-CPP
        c-before-after-change-digit-quote
-       c-invalidate-macro-cache)
+       c-invalidate-macro-cache
+       c-truncate-bs-cache)
   (c objc) '(c-extend-region-for-CPP
             c-depropertize-CPP
-            c-invalidate-macro-cache)
+            c-invalidate-macro-cache
+            c-truncate-bs-cache)
   ;; java 'c-before-change-check-<>-operators
   awk 'c-awk-record-region-clear-NL)
 (c-lang-defvar c-get-state-before-change-functions
@@ -2588,6 +2590,26 @@ 'c-opt-op-identitier-prefix
 (c-lang-defvar c-opt-inexpr-brace-list-key
   (c-lang-const c-opt-inexpr-brace-list-key))
 
+(c-lang-defconst c-flat-decl-block-kwds
+  ;; Keywords that can introduce another declaration level, i.e. where a
+  ;; following "{" isn't a function block or brace list.  Note that, for
+  ;; historical reasons, `c-decl-block-key' is NOT constructed from this lang
+  ;; const.
+  t (c--delete-duplicates
+     (append (c-lang-const c-class-decl-kwds)
+            (c-lang-const c-other-block-decl-kwds)
+            (c-lang-const c-inexpr-class-kwds))
+     :test 'string-equal))
+
+(c-lang-defconst c-brace-stack-thing-key
+  ;; Regexp matching any keyword or operator relevant to the brace stack (see
+  ;; `c-update-brace-stack' in cc-engine.el).
+  t (c-make-keywords-re 'appendable
+      (append
+       (c-lang-const c-flat-decl-block-kwds)
+       '("{" "}" ";"))))
+(c-lang-defvar c-brace-stack-thing-key (c-lang-const c-brace-stack-thing-key))
+          
 (c-lang-defconst c-decl-block-key
   ;; Regexp matching keywords in any construct that contain another
   ;; declaration level, i.e. that isn't followed by a function block
diff --git a/lisp/progmodes/cc-mode.el b/lisp/progmodes/cc-mode.el
index f2c6256..5b0679a 100644
--- a/lisp/progmodes/cc-mode.el
+++ b/lisp/progmodes/cc-mode.el
@@ -557,6 +557,8 @@ c-basic-common-init
 
   ;; Initialize the cache of brace pairs, and opening braces/brackets/parens.
   (c-state-cache-init)
+  ;; Initialize the "brace stack" cache.
+  (c-init-bs-cache)
 
   (when (or c-recognize-<>-arglists
            (c-major-mode-is 'awk-mode)


>     Oleh

-- 
Alan Mackenzie (Nuremberg, Germany).