[elpa] externals/phps-mode 9172bf2 35/62: Improved incremental lex-analyzer

[Christian Johansson]

branch: externals/phps-mode
commit 9172bf271aeaba4c4addde92fc41b904a71cd4a7
Author: Christian Johansson <christian@mediastrategi.se>
Commit: Christian Johansson <christian@mediastrategi.se>

    Improved incremental lex-analyzer
---
 phps-mode-lex-analyzer.el          | 51 ++++++++++++++++++++++++++------------
 phps-mode-lexer.el                 |  9 +++++++
 phps-mode-macros.el                |  4 ++-
 test/phps-mode-test-integration.el |  8 ++++++
 4 files changed, 55 insertions(+), 17 deletions(-)

diff --git a/phps-mode-lex-analyzer.el b/phps-mode-lex-analyzer.el
index f834f18..4a0dd5f 100644
--- a/phps-mode-lex-analyzer.el
+++ b/phps-mode-lex-analyzer.el
@@ -684,14 +684,31 @@
 
                 (catch 'quit
                   (dolist (token old-tokens)
-                    (let ((start (car (cdr token)))
+                    (let ((token-type (car token))
+                          (start (car (cdr token)))
                           (end (cdr (cdr token))))
                       (if (< end change-start)
                           (push token head-tokens)
                         (when (< start change-start)
+                          (when (equal token-type 'T_END_HEREDOC)
+                            ;; When incremental start is on a T_END_HEREDOC 
token
+                            ;; rewind another token to allow expansion of
+                            ;; T_ENCAPSED_AND_WHITESPACE
+                            (phps-mode-debug-message
+                             (message
+                              "Rewinding incremental start due to 
'T_END_HEREDOC token"))
+                            (let ((previous-token (pop head-tokens)))
+                              (setq
+                               start
+                               (car (cdr previous-token)))))
+
                           (phps-mode-debug-message
-                           (message "New incremental-start-new-buffer: %s" 
start))
-                          (setq incremental-start-new-buffer start))
+                           (message
+                            "New incremental-start-new-buffer: %s"
+                            start))
+                          (setq
+                           incremental-start-new-buffer
+                           start))
                         (throw 'quit "break")))))
 
                 (setq head-tokens (nreverse head-tokens))
@@ -705,23 +722,25 @@
                       (phps-mode-debug-message
                        (message "Found head tokens"))
 
-                      ;; TODO Change on ST_END_HEREDOC should start before it
-
                       ;; In old buffer:
                       ;; 1. Determine state (incremental-state) and 
state-stack (incremental-state-stack) heredoc label (incremental-heredoc-label) 
heredoc-label-stack (heredoc-label-stack) before incremental start
                       ;; 2. Build list of states before incremental start 
(head-states)
                       (catch 'quit
-                        (dolist (state-object (nreverse old-states))
-                          (let ((end (nth 1 state-object)))
-                            (if (< end change-start)
-                                (progn
-                                  (setq incremental-state (nth 2 state-object))
-                                  (setq incremental-state-stack (nth 3 
state-object))
-                                  (setq incremental-heredoc-label (nth 4 
state-object))
-                                  (setq incremental-heredoc-label-stack (nth 5 
state-object))
-                                  (setq incremental-nest-location-stack (nth 6 
state-object))
-                                  (push state-object head-states))
-                              (throw 'quit "break")))))
+                        (let ((previous-state))
+                          (dolist (state-object (nreverse old-states))
+                            (let ((end (nth 1 state-object)))
+                              (if (<= end incremental-start-new-buffer)
+                                  (progn
+                                    (setq incremental-state (nth 2 
state-object))
+                                    (setq incremental-state-stack (nth 3 
state-object))
+                                    (setq incremental-heredoc-label (nth 4 
state-object))
+                                    (setq incremental-heredoc-label-stack (nth 
5 state-object))
+                                    (setq incremental-nest-location-stack (nth 
6 state-object))
+                                    (push state-object head-states))
+                                (throw 'quit "break")))
+                            (setq
+                             previous-state
+                             state-object))))
 
                       (phps-mode-debug-message
                        (message "Head states: %s" head-states)
diff --git a/phps-mode-lexer.el b/phps-mode-lexer.el
index 47c3449..358d8b3 100644
--- a/phps-mode-lexer.el
+++ b/phps-mode-lexer.el
@@ -260,6 +260,15 @@
 
 (defun phps-mode-lexer--emit-token (token start end)
   "Emit TOKEN with START and END."
+  (when (= start end)
+    (signal
+     'phps-lexer-error
+     (list
+      (format "Empty token detected: %s %s %s" token start end)
+      start
+      end
+      token)))
+
   (semantic-lex-push-token (semantic-lex-token token start end))
   (push `(,token ,start . ,end) phps-mode-lexer--generated-tokens)
 
diff --git a/phps-mode-macros.el b/phps-mode-macros.el
index 9e505e7..5591e6f 100644
--- a/phps-mode-macros.el
+++ b/phps-mode-macros.el
@@ -7,7 +7,9 @@
 
 ;;; Code:
 
-(defconst phps-mode-macrotime-debug nil
+(defconst
+  phps-mode-macrotime-debug
+  nil
   "Debug messages during macro expansion time, default nil.")
 
 (defmacro phps-mode-debug-message (&rest message)
diff --git a/test/phps-mode-test-integration.el 
b/test/phps-mode-test-integration.el
index cf2ce03..0a8e90e 100644
--- a/test/phps-mode-test-integration.el
+++ b/test/phps-mode-test-integration.el
@@ -161,6 +161,14 @@
    (goto-char 85)
    (kill-line))
 
+  (phps-mode-test--incremental-vs-intial-buffer
+   "<?php\n$str = <<<EOD\nExample of string\nspanning multiple lines\nusing 
heredoc syntax.\nEOD;\n\n/* More complex example, with variables. */\nclass 
foo\n{\n    var $foo;\n    var $bar;\n\n    function __construct()\n    {\n     
   $this->foo = 'Foo';\n        $this->bar = array('Bar1', 'Bar2', 'Bar3');\n   
 }\n}\n\n$foo = new foo();\n$name = 'MyName';\n\necho <<<EOT\nMy name is 
\"$name\". I am printing some $foo->foo.\nNow, I am printing some 
{$foo->bar[1]}.\nThis should print a capi [...]
+   "Integration-test 14 complex HEREDOC with removed heredoc delimiter 
semicolon"
+   ;; (message "\nTokens: %s" phps-mode-lex-analyzer--tokens)
+   ;; (message "States: %s\n" phps-mode-lex-analyzer--states)
+   (goto-char 88)
+   (delete-char 1))
+
   )
 
 (defun phps-mode-test-integration--whitespace-modifications ()