feature/tree-sitter eeeae5e9ee 1/8: Add an argument OVERRIDE to tree-sitter font-lock functions

[Yuan Fu]

branch: feature/tree-sitter
commit eeeae5e9ee34da5539046ff8dac4f4fffa19a5b9
Author: Yuan Fu <casouri@gmail.com>
Commit: Yuan Fu <casouri@gmail.com>

    Add an argument OVERRIDE to tree-sitter font-lock functions
    
    * doc/lispref/modes.texi (Parser-based Font Lock): Reflect this change
    in manual.
    * lisp/progmodes/js.el (js--fontify-template-string): Add _OVERRIDE
    argument.
    * lisp/progmodes/python.el (python--treesit-fontify-string): Add
    _OVERRIDE argument.
    * lisp/treesit.el (treesit-font-lock-rules): Update docstring.
    (treesit-fontify-with-override): New function.
    (treesit-font-lock-fontify-region): Extract out into
    treesit-fontify-with-override.
---
 doc/lispref/modes.texi   | 22 ++++++++++++++------
 lisp/progmodes/js.el     |  2 +-
 lisp/progmodes/python.el |  2 +-
 lisp/treesit.el          | 54 +++++++++++++++++++++++++++---------------------
 4 files changed, 49 insertions(+), 31 deletions(-)

diff --git a/doc/lispref/modes.texi b/doc/lispref/modes.texi
index d778636d6d..40d966ef17 100644
--- a/doc/lispref/modes.texi
+++ b/doc/lispref/modes.texi
@@ -3975,12 +3975,22 @@ that starts with @code{@@}), and tree-sitter will 
return matched nodes
 tagged with those same capture names.  For the purpose of
 fontification, capture names in @var{query} should be face names like
 @code{font-lock-keyword-face}.  The captured node will be fontified
-with that face.  Capture names can also be function names, in which
-case the function is called with 3 arguments: @var{start}, @var{end},
-and @var{node}, where @var{start} and @var{end} are the start and end
-position of the node in buffer, and @var{node} is the node itself.  If
-a capture name is both a face and a function, the face takes priority.
-If a capture name is neither a face nor a function, it is ignored.
+with that face.
+
+@findex treesit-fontify-with-override
+Capture names can also be function names, in which case the function
+is called with 4 arguments: @var{start}, @var{end}, @var{node}, and
+@var{override}, where @var{start} and @var{end} are the start and end
+position of the node in buffer, @var{node} is the node itself, and
+@var{override} is the override property of the rule which captured
+this node.  (If this function wants to respect the @var{override}
+argument, it can use @code{treesit-fontify-with-override}.)  Beyond
+the 4 arguments presented, this function should accept more arguments
+as optional arguments for future extensibility.
+
+If a capture name is both a face and a function, the face takes
+priority.  If a capture name is neither a face nor a function, it is
+ignored.
 @end defun
 
 @defvar treesit-font-lock-feature-list
diff --git a/lisp/progmodes/js.el b/lisp/progmodes/js.el
index 8d1cfbd3c0..2801a729eb 100644
--- a/lisp/progmodes/js.el
+++ b/lisp/progmodes/js.el
@@ -3573,7 +3573,7 @@ This function is intended for use in 
`after-change-functions'."
       @font-lock-constant-face)))
   "Tree-sitter font-lock settings.")
 
-(defun js--fontify-template-string (beg end node)
+(defun js--fontify-template-string (beg end node _override &rest _)
   "Fontify template string but not substitution inside it.
 BEG, END, NODE refers to the template_string node."
   (ignore end)
diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index a9aff16776..0b10058eeb 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -1015,7 +1015,7 @@ It makes underscores and dots word constituent chars.")
     "VMSError" "WindowsError"
     ))
 
-(defun python--treesit-fontify-string (_beg _end node)
+(defun python--treesit-fontify-string (_beg _end node _override &rest _)
   "Fontify string.
 NODE is the last quote in the string.  Do not fontify the initial
 f for f-strings."
diff --git a/lisp/treesit.el b/lisp/treesit.el
index 2bd71cdf5d..72c8186044 100644
--- a/lisp/treesit.el
+++ b/lisp/treesit.el
@@ -410,7 +410,7 @@ omitted, default END to BEG."
                return rng
                finally return nil))))
 
-;;; Font-lock
+;;; Fontification
 
 (define-error 'treesit-font-lock-error
               "Generic tree-sitter font-lock error"
@@ -500,12 +500,14 @@ Other keywords include:
 Capture names in QUERY should be face names like
 `font-lock-keyword-face'.  The captured node will be fontified
 with that face.  Capture names can also be function names, in
-which case the function is called with (START END NODE), where
-START and END are the start and end position of the node in
-buffer, and NODE is the tree-sitter node object.  If a capture
-name is both a face and a function, the face takes priority.  If
-a capture name is not a face name nor a function name, it is
-ignored.
+which case the function is called with (START END NODE OVERRIDE),
+where START and END are the start and end position of the node in
+buffer, NODE is the tree-sitter node object, and OVERRIDE is the
+override option of that rule.  This function should accept more
+arguments as optional arguments for future extensibility.  If a
+capture name is both a face and a function, the face takes
+priority.  If a capture name is not a face name nor a function
+name, it is ignored.
 
 \(fn :KEYWORD VALUE QUERY...)"
   ;; Other tree-sitter function don't tend to be called unless
@@ -600,6 +602,26 @@ Set the ENABLE flag for each setting in
              do (setf (nth 1 (nth idx treesit-font-lock-settings))
                       (if (memq feature features) t nil)))))
 
+(defun treesit-fontify-with-override (start end face override)
+  "Apply FACE to the region between START and END.
+OVERRIDE can be nil, t, `append', `prepend', or `keep'.
+See `treesit-font-lock-rules' for their semantic."
+  (pcase override
+    ('nil (unless (text-property-not-all
+                   start end 'face nil)
+            (put-text-property start end 'face face)))
+    ('t (put-text-property start end 'face face))
+    ('append (font-lock-append-text-property
+              start end 'face face))
+    ('prepend (font-lock-prepend-text-property
+               start end 'face face))
+    ('keep (font-lock-fillin-text-property
+            start end 'face face))
+    (_ (signal 'treesit-font-lock-error
+               (list
+                "Unrecognized value of :override option"
+                override)))))
+
 (defun treesit-font-lock-fontify-region
     (start end &optional loudly)
   "Fontify the region between START and END.
@@ -633,23 +655,9 @@ If LOUDLY is non-nil, display some debugging information."
                      (end (treesit-node-end node)))
                 (cond
                  ((facep face)
-                  (pcase override
-                    ('nil (unless (text-property-not-all
-                                   start end 'face nil)
-                            (put-text-property start end 'face face)))
-                    ('t (put-text-property start end 'face face))
-                    ('append (font-lock-append-text-property
-                              start end 'face face))
-                    ('prepend (font-lock-prepend-text-property
-                               start end 'face face))
-                    ('keep (font-lock-fillin-text-property
-                            start end 'face face))
-                    (_ (signal 'treesit-font-lock-error
-                               (list
-                                "Unrecognized value of :override option"
-                                override)))))
+                  (treesit-fontify-with-override start end face override))
                  ((functionp face)
-                  (funcall face start end node)))
+                  (funcall face start end node override)))
                 ;; Don't raise an error if FACE is neither a face nor
                 ;; a function.  This is to allow intermediate capture
                 ;; names used for #match and #eq.