bug#7378: 23.2; grep buffer misinterprets result if filename contains colon character

[npostavs]

tags 7378 + patch
quit

>> The only way to do this reliably with GNU grep is to ask it to output
>> escape sequences around file names and line numbers.

> Feel free to try, but not for 24.1.  And please try and quantify the
> performance impact (if any), because M-x grep is already too slow.

GNU grep also supports --null to output NUL bytes following file names,
I haven't measured but it should be no slower and possibly faster since
the regexp is simpler.  There is a bit of added complexity to still
support other greps which don't have --null:

>From 0042aa5616560657e867aa977dad732ab18e30dc Mon Sep 17 00:00:00 2001
From: Noam Postavsky <npostavs@gmail.com>
Date: Tue, 13 Sep 2016 20:48:09 -0400
Subject: [PATCH] Use grep's --null option (Bug#6843)

* lisp/progmodes/grep.el (grep-use-null-filename-separator): New option.
(grep--regexp-alist-column, grep--regexp-alist-bin-matcher)
(grep-with-null-regexp-alist, grep-fallback-regexp-alist): New
constants, replacing `grep-regexp-alist'.
(grep-regex-alist): Mark the variable obsolete, add a new function of
the same name to replace it.
(grep-compute-defaults): Compute default for
`grep-use-null-filename-separator'.
(grep-mode): Set compilation-error-regexp-alist (buffer locally) to the
value of `grep-with-null-regexp-alist' or `grep-fallback-regexp-alist'
according to `grep-use-null-filename-separator'.
* lisp/progmodes/xref.el (xref-collect-matches): Call
`grep-regex-alist' instead of the obsolete variable.  Don't hardcode
grep-regexp-alist match groups.
---
 lisp/progmodes/grep.el | 103 +++++++++++++++++++++++++++++++++++--------------
 lisp/progmodes/xref.el |  33 ++++++++--------
 2 files changed, 92 insertions(+), 44 deletions(-)

diff --git a/lisp/progmodes/grep.el b/lisp/progmodes/grep.el
index b3d8a51cee..a4b85bbbe4 100644
--- a/lisp/progmodes/grep.el
+++ b/lisp/progmodes/grep.el
@@ -47,8 +47,8 @@ (defvar grep-host-defaults-alist nil
 (defun grep-apply-setting (symbol value)
   "Set SYMBOL to VALUE, and update `grep-host-defaults-alist'.
 SYMBOL should be one of `grep-command', `grep-template',
-`grep-use-null-device', `grep-find-command',
-`grep-find-template', `grep-find-use-xargs', or
+`grep-use-null-device', `grep-find-command' `grep-find-template',
+`grep-find-use-xargs', `grep-use-null-filename-separator', or
 `grep-highlight-matches'."
   (when grep-host-defaults-alist
     (let* ((host-id
@@ -160,6 +160,15 @@ (defcustom grep-use-null-device 'auto-detect
   :set 'grep-apply-setting
   :group 'grep)
 
+(defcustom grep-use-null-filename-separator 'auto-detect
+  "If non-nil, use `grep's `--null' option.
+This is done to disambiguate file names in `grep's output."
+  :type '(choice (const :tag "Do Not Use `--null'" nil)
+                (const :tag "Use `--null'" t)
+                (other :tag "Not Set" auto-detect))
+  :set 'grep-apply-setting
+  :group 'grep)
+
 ;;;###autoload
 (defcustom grep-find-command nil
   "The default find command for \\[grep-find].
@@ -357,33 +366,53 @@ (defvar grep-last-buffer nil
 Notice that using \\[next-error] or \\[compile-goto-error] modifies
 `compilation-last-buffer' rather than `grep-last-buffer'.")
 
-;;;###autoload
-(defconst grep-regexp-alist
-  '(
-    ;; Use a tight regexp to handle weird file names (with colons
+(defconst grep--regexp-alist-column
+  ;; Calculate column positions (col . end-col) of first grep match on a line
+  (cons
+   (lambda ()
+     (when grep-highlight-matches
+       (let* ((beg (match-end 0))
+              (end (save-excursion (goto-char beg) (line-end-position)))
+              (mbeg (text-property-any beg end 'font-lock-face 
'grep-match-face)))
+         (when mbeg
+           (- mbeg beg)))))
+   (lambda ()
+     (when grep-highlight-matches
+       (let* ((beg (match-end 0))
+              (end (save-excursion (goto-char beg) (line-end-position)))
+              (mbeg (text-property-any beg end 'font-lock-face 
'grep-match-face))
+              (mend (and mbeg (next-single-property-change mbeg 
'font-lock-face nil end))))
+         (when mend
+           (- mend beg)))))))
+(defconst grep--regexp-alist-bin-matcher
+  '("^Binary file \\(.+\\) matches$" 1 nil nil 0 1))
+(defconst grep-with-null-regexp-alist
+  `(("^\\([^\0]+\\)\\(\0\\)\\([0-9]+\\):" 1 3 ,grep--regexp-alist-column nil 
nil
+     (2 '(face unspecified display ":")))
+    ,grep--regexp-alist-bin-matcher)
+  "Regexp used to match grep hits.
+See `compilation-error-regexp-alist'.")
+(defconst grep-fallback-regexp-alist
+  `(;; Use a tight regexp to handle weird file names (with colons
     ;; in them) as well as possible.  E.g., use [1-9][0-9]* rather
     ;; than [0-9]+ so as to accept ":034:" in file names.
     ("^\\(.*?[^/\n]\\):[ \t]*\\([1-9][0-9]*\\)[ \t]*:"
-     1 2
-     ;; Calculate column positions (col . end-col) of first grep match on a 
line
-     ((lambda ()
-       (when grep-highlight-matches
-         (let* ((beg (match-end 0))
-                (end (save-excursion (goto-char beg) (line-end-position)))
-                (mbeg (text-property-any beg end 'font-lock-face 
grep-match-face)))
-           (when mbeg
-             (- mbeg beg)))))
-      .
-      (lambda ()
-       (when grep-highlight-matches
-         (let* ((beg (match-end 0))
-                (end (save-excursion (goto-char beg) (line-end-position)))
-                (mbeg (text-property-any beg end 'font-lock-face 
grep-match-face))
-                (mend (and mbeg (next-single-property-change mbeg 
'font-lock-face nil end))))
-           (when mend
-             (- mend beg)))))))
-    ("^Binary file \\(.+\\) matches$" 1 nil nil 0 1))
-  "Regexp used to match grep hits.  See `compilation-error-regexp-alist'.")
+     1 2 ,grep--regexp-alist-column)
+    ,grep--regexp-alist-bin-matcher)
+  "Regexp used to match grep hits when `--null' is not supported.
+See `compilation-error-regexp-alist'.")
+
+(defvaralias 'grep-regex-alist 'grep-with-null-regexp-alist)
+(make-obsolete-variable
+ 'grep-regex-alist "Call `grep-regexp-alist' instead." "26.1")
+
+;;;###autoload
+(defun grep-regexp-alist ()
+  "Return a regexp alist to match grep hits.
+The regexp used depends on `grep-use-null-filename-separator'.
+See `compilation-error-regexp-alist' for format details."
+  (if grep-use-null-filename-separator
+      grep-with-null-regexp-alist grep-fallback-regexp-alist))
 
 (defvar grep-first-column 0            ; bug#10594
   "Value to use for `compilation-first-column' in grep buffers.")
@@ -538,6 +567,7 @@ (defun grep-compute-defaults ()
             (grep-use-null-device ,grep-use-null-device)
             (grep-find-command ,grep-find-command)
             (grep-find-template ,grep-find-template)
+             (grep-use-null-filename-separator 
,grep-use-null-filename-separator)
             (grep-find-use-xargs ,grep-find-use-xargs)
             (grep-highlight-matches ,grep-highlight-matches)))))
   (let* ((host-id
@@ -550,7 +580,8 @@ (defun grep-compute-defaults ()
     ;; computed for every host once.
     (dolist (setting '(grep-command grep-template
                       grep-use-null-device grep-find-command
-                      grep-find-template grep-find-use-xargs
+                      grep-use-null-filename-separator
+                       grep-find-template grep-find-use-xargs
                       grep-highlight-matches))
       (set setting
           (cadr (or (assq setting host-defaults)
@@ -576,6 +607,21 @@ (defun grep-compute-defaults ()
                         (concat (regexp-quote hello-file)
                                 ":[0-9]+:English")))))))))
 
+    (when (eq grep-use-null-filename-separator 'auto-detect)
+      (setq grep-use-null-filename-separator
+            (with-temp-buffer
+              (let* ((hello-file (expand-file-name "HELLO" data-directory))
+                     (args `("--null" "-ne" "^English" ,hello-file)))
+                (if grep-use-null-device
+                    (setq args (append args (list null-device)))
+                  (push "-H" args))
+                (and (grep-probe grep-program `(nil t nil ,@args))
+                     (progn
+                       (goto-char (point-min))
+                       (looking-at
+                        (concat (regexp-quote hello-file)
+                                "\0[0-9]+:English"))))))))
+
     (when (eq grep-highlight-matches 'auto-detect)
       (setq grep-highlight-matches
            (with-temp-buffer
@@ -591,6 +637,7 @@ (defun grep-compute-defaults ()
                 grep-template grep-find-template)
       (let ((grep-options
             (concat (if grep-use-null-device "-n" "-nH")
+                     (if grep-use-null-filename-separator " --null")
                     (if (grep-probe grep-program
                                     `(nil nil nil "-e" "foo" ,null-device)
                                     nil 1)
@@ -733,7 +780,7 @@ (define-compilation-mode grep-mode "Grep"
   (set (make-local-variable 'compilation-error-face)
        grep-hit-face)
   (set (make-local-variable 'compilation-error-regexp-alist)
-       grep-regexp-alist)
+       (grep-regexp-alist))
   ;; compilation-directory-matcher can't be nil, so we set it to a regexp that
   ;; can never match.
   (set (make-local-variable 'compilation-directory-matcher) '("\\`a\\`"))
diff --git a/lisp/progmodes/xref.el b/lisp/progmodes/xref.el
index b8ec50f14a..cc9b794c5a 100644
--- a/lisp/progmodes/xref.el
+++ b/lisp/progmodes/xref.el
@@ -917,20 +917,21 @@ (defun xref-collect-matches (regexp files dir ignores)
   (grep-compute-defaults)
   (defvar grep-find-template)
   (defvar grep-highlight-matches)
-  (let* ((grep-find-template (replace-regexp-in-string "<C>" "<C> -E"
-                                                       grep-find-template t t))
-         (grep-highlight-matches nil)
-         ;; TODO: Sanitize the regexp to remove Emacs-specific terms,
-         ;; so that Grep can search for the "relaxed" version.  Can we
-         ;; do that reliably enough, without creating false negatives?
-         (command (xref--rgrep-command (xref--regexp-to-extended regexp)
-                                       files
-                                       (expand-file-name dir)
-                                       ignores))
-         (buf (get-buffer-create " *xref-grep*"))
-         (grep-re (caar grep-regexp-alist))
-         status
-         hits)
+  (pcase-let*
+      ((grep-find-template (replace-regexp-in-string "<C>" "<C> -E"
+                                                     grep-find-template t t))
+       (grep-highlight-matches nil)
+       ;; TODO: Sanitize the regexp to remove Emacs-specific terms,
+       ;; so that Grep can search for the "relaxed" version.  Can we
+       ;; do that reliably enough, without creating false negatives?
+       (command (xref--rgrep-command (xref--regexp-to-extended regexp)
+                                     files
+                                     (expand-file-name dir)
+                                     ignores))
+       (buf (get-buffer-create " *xref-grep*"))
+       (`(,grep-re ,file-group ,line-group . ,_) (car (grep-regexp-alist)))
+       (status nil)
+       (hits nil))
     (with-current-buffer buf
       (erase-buffer)
       (setq status
@@ -944,8 +945,8 @@ (defun xref-collect-matches (regexp files dir ignores)
                  (not (looking-at grep-re)))
         (user-error "Search failed with status %d: %s" status (buffer-string)))
       (while (re-search-forward grep-re nil t)
-        (push (list (string-to-number (match-string 2))
-                    (match-string 1)
+        (push (list (string-to-number (match-string line-group))
+                    (match-string file-group)
                     (buffer-substring-no-properties (point) 
(line-end-position)))
               hits)))
     (xref--convert-hits (nreverse hits) regexp)))
-- 
2.11.1