[elpa] externals/phps-mode 41c4fbb 009/405: Lexer unit tests now working

[Stefan Monnier]

branch: externals/phps-mode
commit 41c4fbb46383378aa0e6a96ccbb9b9a9633c5da6
Author: Christian Johansson <address@hidden>
Commit: Christian Johansson <address@hidden>

    Lexer unit tests now working
---
 phps-lexer.el        | 104 ++++++++++++++++++++-------------------------------
 phps-syntax-table.el |   7 +++-
 phps-test-lexer.el   |  54 +++++++++++++-------------
 3 files changed, 73 insertions(+), 92 deletions(-)

diff --git a/phps-lexer.el b/phps-lexer.el
index 18179d2..2f104fc 100644
--- a/phps-lexer.el
+++ b/phps-lexer.el
@@ -44,6 +44,10 @@
 ;; Make sure `semantic-lex-syntax-modifications' is correct since lexer is 
dependent on Emacs syntax-table
 
 
+(defvar phps-mode/lexer-tokens nil
+  "Last lexer tokens.")
+
+
 ;; SETTINGS
 
 
@@ -158,7 +162,7 @@
 (defun phps-mode/BEGIN (state)
   "Begin STATE."
   (setq phps-mode/STATE state)
-  (message "Begun state %s" state)
+  ;; (message "Begun state %s" state)
   )
 
 ;; _yy_push_state
@@ -170,7 +174,7 @@
     (if (not phps-mode/state_stack)
         (setq phps-mode/state_stack (list old-state))
       (push old-state phps-mode/state_stack))
-    (message "Added state %s to stack" old-state)
+    ;; (message "Added state %s to stack" old-state)
     )
   (phps-mode/BEGIN state))
 
@@ -178,8 +182,8 @@
   "Pop current state from stack."
   (let* ((old-state (pop phps-mode/state_stack))
          (new-state (car phps-mode/state_stack)))
-    (message "Going back to poppped state %s" old-state)
-    (message "Ended state %s, going back to %s" old-state new-state)
+    ;; (message "Going back to poppped state %s" old-state)
+    ;; (message "Ended state %s, going back to %s" old-state new-state)
     (if old-state
         (phps-mode/BEGIN old-state)
       (display-warning "phps-mode" "PHPs Lexer Error - Going back to nil?"))
@@ -356,7 +360,7 @@
     (overlay-put (make-overlay start end) 'font-lock-face 
'font-lock-constant-face))
 
    ((string= token 'T_ERROR)
-    (overlay-put (make-overlay start end)  'font-lock-face 
'font-lock-warning-face))
+    (overlay-put (make-overlay start end) 'font-lock-face 
'font-lock-warning-face))
 
    ))
 
@@ -367,10 +371,12 @@
   (when (and
          phps-mode/prepend_trailing_brace
          (> end (- (point-max) 2)))
-    (message "Adding trailing brace")
+    ;; (message "Adding trailing brace")
     (setq phps-mode/prepend_trailing_brace nil)
     (phps-mode/RETURN_TOKEN "}" (- end 1) end))
 
+  ;; (message "Added token %s %s %s" token start end)
+
   (semantic-lex-push-token
    (semantic-lex-token token start end)))
 
@@ -392,7 +398,7 @@
     (let ((start (match-beginning 0))
           (end (match-end 0)))
       (phps-mode/BEGIN phps-mode/ST_IN_SCRIPTING)
-      (message "Starting scripting after <?=")
+      ;; (message "Starting scripting after <?=")
       (when phps-mode/PARSER_MODE
         (phps-mode/RETURN_TOKEN 'T_ECHO start end))
       (phps-mode/RETURN_TOKEN 'T_OPEN_TAG_WITH_ECHO start end)))
@@ -401,7 +407,7 @@
     (let ((start (match-beginning 0))
           (end (match-end 0)))
       (phps-mode/BEGIN phps-mode/ST_IN_SCRIPTING)
-      (message "Starting scripting after <?php")
+      ;; (message "Starting scripting after <?php")
       (when phps-mode/EXPECTED
         (phps-mode/SKIP_TOKEN 'T_OPEN_TAG start end))
       (phps-mode/RETURN_TOKEN 'T_OPEN_TAG start end)))
@@ -413,7 +419,7 @@
         (phps-mode/BEGIN phps-mode/ST_IN_SCRIPTING)
         (when phps-mode/EXPECTED
           (phps-mode/SKIP_TOKEN 'T_OPEN_TAG start end))
-        (message "Starting scripting after <?")
+        ;; (message "Starting scripting after <?")
         (phps-mode/RETURN_TOKEN 'T_OPEN_TAG start end))))
 
    ;; NOTE: mimics inline_char_handler
@@ -773,7 +779,9 @@
             position)
         (if string-start
             (setq position string-start)
-          (setq position (point-max)))
+          (progn
+            (setq position (point-max))
+            (phps-mode/MOVE_FORWARD (point-max))))
         (phps-mode/RETURN_TOKEN 'T_DOC_COMMENT start position)
         )))
 
@@ -785,7 +793,9 @@
             position)
         (if string-start
             (setq position string-start)
-          (setq position (point-max)))
+          (progn
+            (setq position (point-max))
+            (phps-mode/MOVE_FORWARD (point-max))))
         (phps-mode/RETURN_TOKEN 'T_COMMENT start position)
         )))
 
@@ -826,6 +836,7 @@
             ;; Unclosed single quotes
             ;; (message "Single quoted string never ends..")
             (phps-mode/RETURN_TOKEN 'T_ENCAPSED_AND_WHITESPACE start 
(point-max))
+            (phps-mode/MOVE_FORWARD (point-max))
             )))))
 
    ;; Double quoted string
@@ -861,6 +872,7 @@
           (progn
             ;; (message "Found no ending quote, skipping to end")
             (phps-mode/RETURN_TOKEN 'T_ERROR start (point-max))
+            (phps-mode/MOVE_FORWARD (point-max))
             )))))
 
    ((looking-at "[`]")
@@ -881,7 +893,8 @@
 
    ((looking-at phps-mode/ANY_CHAR)
     ;; Unexpected character
-    (phps-mode/RETURN_TOKEN 'T_ERROR (match-beginning 0) (point-max)))
+    (phps-mode/RETURN_TOKEN 'T_ERROR (match-beginning 0) (point-max))
+    (phps-mode/MOVE_FORWARD (point-max)))
 
    ))
 
@@ -919,7 +932,7 @@
           (end (match-end 0)))
       (phps-mode/yy_pop_state)
       ;; TODO goto restart here?
-      (message "Restart here")
+      ;; (message "Restart here")
       (phps-mode/MOVE_FORWARD end)
       ))
 
@@ -975,14 +988,15 @@
                   (progn
                     (let ((variable-start (+ start (match-beginning 0))))
                       (phps-mode/RETURN_TOKEN 'T_CONSTANT_ENCAPSED_STRING 
start variable-start)
-                    ))
+                      ))
                 (progn
                   (phps-mode/RETURN_TOKEN 'T_CONSTANT_ENCAPSED_STRING start 
end)
                   ;; (message "Found end of quote at %s-%s, moving ahead after 
'%s'" start end (buffer-substring start end))
                   )))
           (progn
             ;; "Found no end of double-quoted region
-            (phps-mode/RETURN_TOKEN 'T_ERROR start (point-max)))))))
+            (phps-mode/RETURN_TOKEN 'T_ERROR start (point-max))
+            (phps-mode/MOVE_FORWARD (point-max)))))))
 
    ))
 
@@ -1031,7 +1045,8 @@ ANY_CHAR'
                   )
               (progn
                 ;; (message "Found no end of backquote.. skipping to end from 
%s" (buffer-substring (point) (point-max)))
-                (phps-mode/RETURN_TOKEN 'T_ERROR old-start (point-max))))))
+                (phps-mode/RETURN_TOKEN 'T_ERROR old-start (point-max))
+                (phps-mode/MOVE_FORWARD (point-max))))))
 
          )))
 
@@ -1082,7 +1097,7 @@ ANY_CHAR'
               (cond
 
                ((string-match (concat "\n" heredoc_label ";?\n") data)
-                ;, (message "Found heredoc end at %s-%s" start end)
+                                        ;, (message "Found heredoc end at 
%s-%s" start end)
                 (phps-mode/BEGIN phps-mode/ST_END_HEREDOC)
                 (phps-mode/RETURN_TOKEN 'T_ENCAPSED_AND_WHITESPACE old-start 
start))
 
@@ -1095,6 +1110,7 @@ ANY_CHAR'
           (progn
             ;; (message "Found no ending of heredoc at %s '%s'" heredoc_label 
(buffer-substring (point) (point-max)))
             (phps-mode/RETURN_TOKEN 'T_ERROR old-start (point-max))
+            (phps-mode/MOVE_FORWARD (point-max))
             ))))
 
      )))
@@ -1122,6 +1138,7 @@ ANY_CHAR'
           (progn
             ;; (message "Found no ending of nowdoc at %s '%s'" heredoc_label 
(buffer-substring (point) (point-max)))
             (phps-mode/RETURN_TOKEN 'T_ERROR old-start (point-max))
+            (phps-mode/MOVE_FORWARD (point-max))
             ))))
      )))
 
@@ -1209,7 +1226,8 @@ ANY_CHAR'
 
    ((looking-at phps-mode/ANY_CHAR)
     ;; Unexpected character
-    (phps-mode/RETURN_TOKEN 'T_ERROR (match-beginning 0) (point-max)))
+    (phps-mode/RETURN_TOKEN 'T_ERROR (match-beginning 0) (point-max))
+    (phps-mode/MOVE_FORWARD (point-max)))
 
    ))
 
@@ -1232,53 +1250,11 @@ ANY_CHAR'
 
 (defun phps-mode/lexer-init ()
   "Initialize lexer."
-
-  (let ((phps-mode-syntax-table (make-syntax-table)))
-
-    ;; This is added so entity names with underscores can be more easily 
parsed as one word
-    (modify-syntax-entry ?_ "w" phps-mode-syntax-table)
-
-    ;; Comment styles are same as C++
-    (modify-syntax-entry ?/ ". 124b"  phps-mode-syntax-table)
-    (modify-syntax-entry ?* ". 23" phps-mode-syntax-table)
-    (modify-syntax-entry ?\n "> b" phps-mode-syntax-table)
-
-    (setq semantic-lex-syntax-table phps-mode-syntax-table))
-
+  (when (boundp 'phps-mode/syntax-table)
+    (setq semantic-lex-syntax-table phps-mode/syntax-table))
   (phps-mode/BEGIN phps-mode/ST_INITIAL)
-
-  (setq
-
-   ;; Syntax table modifications
-   ;; TODO Understand this
-   ;; semantic-lex-syntax-modifications
-   ;; '(
-   ;;   (?= ".")
-   ;;   (?& ".")
-   ;;   (?+ ".")
-   ;;   (?- ".")
-   ;;   (?| ".")
-   ;;   (?< ".")
-   ;;   (?> ".")
-   ;;   (?% ".")
-   ;;   (?' "\"")
-   ;;   (?\" "\"")
-   ;;   (?` "\"")
-   ;;   (?_ "w")
-   ;;   (?$ "_")
-   ;;   (?/ ". 124b")
-   ;;   (?* ". 23")
-   ;;   (?\n "> b")
-   ;;   (?# "< b"))
-
-   ;; Lexical analysis
-   semantic-lex-analyzer 'phps-mode/tags-lexer
-
-   )
-
-  (semantic-lex-buffer)
-
-  )
+  (setq semantic-lex-analyzer #'phps-mode/tags-lexer)
+  (setq phps-mode/lexer-tokens (semantic-lex-buffer)))
 
 (provide 'phps-mode/lexer)
 
diff --git a/phps-syntax-table.el b/phps-syntax-table.el
index cfe63ed..6cbdecd 100644
--- a/phps-syntax-table.el
+++ b/phps-syntax-table.el
@@ -43,7 +43,12 @@
   (let ((phps-mode/syntax-table (make-syntax-table)))
 
     ;; This is added so entity names with underscores can be more easily 
parsed as one word
-    (modify-syntax-entry ?_ "_" phps-mode/syntax-table)
+    (modify-syntax-entry ?_ "w" phps-mode/syntax-table)
+
+    ;; Improve parsing of <?php and ?> as words
+    (modify-syntax-entry ?? "w" phps-mode/syntax-table)
+    (modify-syntax-entry ?< "w" phps-mode/syntax-table)
+    (modify-syntax-entry ?> "w" phps-mode/syntax-table)
 
     ;; (modify-syntax-entry ?_ "w" phps-mode/syntax-table)
 
diff --git a/phps-test-lexer.el b/phps-test-lexer.el
index 50dcd56..314d47f 100644
--- a/phps-test-lexer.el
+++ b/phps-test-lexer.el
@@ -66,31 +66,31 @@
 
   (phps-mode/with-test-buffer
    "<?php\t$var=1; exit $var;\t?>"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_VARIABLE = T_LNUMBER ; T_EXIT 
T_VARIABLE ; ; T_CLOSE_TAG"))))
 
   (phps-mode/with-test-buffer
    "<?php\nexit;\n?>"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_EXIT ; ; T_CLOSE_TAG"))))
 
   (phps-mode/with-test-buffer
    "<?php exit; ?>"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_EXIT ; ; T_CLOSE_TAG"))))
 
   (phps-mode/with-test-buffer
    "<html><head>blabla</head<body>\n\n \t<?php\nexit;\n?>\n\n</body></html>"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_EXIT ; ; T_CLOSE_TAG"))))
 
   (phps-mode/with-test-buffer
    "\n\n \t<html><title>echo 
\"Blaha\";</title><?php\n\n\nexit?>\n\n<html><random /></html><?php exit ?>"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_EXIT ; T_CLOSE_TAG T_OPEN_TAG 
T_EXIT ; T_CLOSE_TAG"))))
 
@@ -101,7 +101,7 @@
 
   (phps-mode/with-test-buffer
    "<?php $var EXIT die function return yield from yield try catch finally 
throw if elseif endif else while endwhile do for endfor foreach endforeach 
declare enddeclare instanceof as switch endswitch case default break continue 
goto echo print class interface trait extends implements :: \\ ... ?? new clone 
var (int) (integer) (real) (double) (float) (string) (binary) (array) (object) 
(boolean) (bool) (unset) eval include include_once require require_once 
namespace use insteadof global is [...]
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_VARIABLE T_EXIT T_DIE 
T_FUNCTION T_RETURN T_YIELD_FROM T_YIELD T_TRY T_CATCH T_FINALLY T_THROW T_IF 
T_ELSEIF T_ENDIF T_ELSE T_WHILE T_ENDWHILE T_DO T_FOR T_ENDFOR T_FOREACH 
T_ENDFOREACH T_DECLARE T_ENDDECLARE T_INSTANCEOF T_AS T_SWITCH T_ENDSWITCH 
T_CASE T_DEFAULT T_BREAK T_CONTINUE T_GOTO T_ECHO T_PRINT T_CLASS T_INTERFACE 
T_TRAIT T_EXTENDS T_IMPLEMENTS T_PAAMAYIM_NEKUDOTAYIM T_NS_SEPARATOR T_ELLIPSIS 
T_COALESCE T_NEW T_CLONE T_VAR T_INT_ [...]
 
@@ -112,72 +112,72 @@
 
   (phps-mode/with-test-buffer
    "<?php $var->property;"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_VARIABLE T_OBJECT_OPERATOR 
T_STRING ;"))))
 
   ;; Double quoted strings with variables
   (phps-mode/with-test-buffer
    "<?php echo \"My $variable is here\"; echo \"you know\";"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO \" 
T_ENCAPSED_AND_WHITESPACE T_VARIABLE T_CONSTANT_ENCAPSED_STRING \" ; T_ECHO 
T_CONSTANT_ENCAPSED_STRING ;"))))
   (phps-mode/with-test-buffer
    "<?php echo \"My ${variable} is here 1\";"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO \" 
T_ENCAPSED_AND_WHITESPACE T_DOLLAR_OPEN_CURLY_BRACES T_STRING_VARNAME } 
T_CONSTANT_ENCAPSED_STRING \" ;"))))
   (phps-mode/with-test-buffer
    "<?php echo \"Mine {$first_variable} is here and my $second is there.\";"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO \" 
T_ENCAPSED_AND_WHITESPACE T_CURLY_OPEN T_VARIABLE } T_CONSTANT_ENCAPSED_STRING 
T_VARIABLE T_CONSTANT_ENCAPSED_STRING \" ;"))))
   (phps-mode/with-test-buffer
    "<?php echo \" Hello $variable[0], how are you?\";"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO \" 
T_ENCAPSED_AND_WHITESPACE T_VARIABLE T_NUM_STRING ] T_CONSTANT_ENCAPSED_STRING 
\" ;"))))
 
   ;; Heredoc
   (phps-mode/with-test-buffer
    "<?php echo <<<\"MYLABEL\"\nline 1\n line 2\nMYLABEL\n;"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO T_START_HEREDOC 
T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC ;"))))
   (phps-mode/with-test-buffer
    "<?php echo <<<MYLABEL\nline 1\n line 2\nMYLABEL\n;"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO T_START_HEREDOC 
T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC ;"))))
   (phps-mode/with-test-buffer
    "<?php echo <<<\"MYLABEL\"\nMYLABEL\n"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO T_START_HEREDOC 
T_END_HEREDOC"))))
 
   ;; Test heredoc with variables $, {$, ${ here
   (phps-mode/with-test-buffer
    "<?php echo <<<\"MYLABEL\"\nline 1 $variable1\n line 2\n${variable2} line 
3\n line {$variable3} here\nline 5 $variable[3] here\nMYLABEL;\n"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO T_START_HEREDOC 
T_ENCAPSED_AND_WHITESPACE T_VARIABLE T_ENCAPSED_AND_WHITESPACE 
T_DOLLAR_OPEN_CURLY_BRACES T_STRING_VARNAME } T_ENCAPSED_AND_WHITESPACE 
T_CURLY_OPEN T_VARIABLE } T_ENCAPSED_AND_WHITESPACE T_VARIABLE T_NUM_STRING ] 
T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC ;"))))
 
   ;; Nowdoc
   (phps-mode/with-test-buffer
    "<?php echo <<<'MYLABEL'\nline 1\n line 2\nMYLABEL;\n"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO T_START_HEREDOC 
T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC ;"))))
 
   ;; Backquotes
   (phps-mode/with-test-buffer
    "<?php `echo \"HELLO\"`;"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG ` T_CONSTANT_ENCAPSED_STRING ` 
;"))))
   (phps-mode/with-test-buffer
    "<?php `echo \"HELLO $variable or {$variable2} or ${variable3} or 
$variable[index][0] here\"`;"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG ` T_CONSTANT_ENCAPSED_STRING 
T_VARIABLE T_CONSTANT_ENCAPSED_STRING T_CURLY_OPEN T_VARIABLE } 
T_CONSTANT_ENCAPSED_STRING T_DOLLAR_OPEN_CURLY_BRACES T_STRING_VARNAME } 
T_CONSTANT_ENCAPSED_STRING T_VARIABLE T_STRING ] T_CONSTANT_ENCAPSED_STRING ` 
;"))))
 
@@ -188,13 +188,13 @@
 
   (phps-mode/with-test-buffer
    "<?php\nnamespace MyNameSpace{\n\tclass MyClass {\n\t\tpublic function 
__construct() {\n\t\t\texit;\n\t\t}\n\t}\n}\n"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_NAMESPACE T_STRING { T_CLASS 
T_STRING { T_PUBLIC T_FUNCTION T_STRING ( ) { T_EXIT ; } } }"))))
 
   (phps-mode/with-test-buffer
    "<?php\nNAMESPACE MyNameSpace;\nCLASS MyClass {\n\tpublic function 
__construct() {\n\t\texit;\n\t}\n}\n"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_NAMESPACE T_STRING { T_CLASS 
T_STRING { T_PUBLIC T_FUNCTION T_STRING ( ) { T_EXIT ; } } }"))))
   )
@@ -204,19 +204,19 @@
 
   (phps-mode/with-test-buffer
    "<?php\necho \"My neverending double quotation\n"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_ECHO T_ERROR"))))
 
   (phps-mode/with-test-buffer
    "<?php\n`My neverending backquotes\n"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG ` T_ERROR"))))
 
   (phps-mode/with-test-buffer
    "<?php\n<<<LABEL\nMy neverending heredoc\ngoes on forever\n"
-   (let* ((tokens (semantic-lex-buffer))
+   (let* ((tokens phps-mode/lexer-tokens)
           (string-tokens (phps-mode/token-stream-to-string tokens)))
      (should (equal string-tokens " T_OPEN_TAG T_START_HEREDOC T_ERROR"))))
 
@@ -226,11 +226,11 @@
   "Run test for lexer."
   ;; (message "-- Running all tests for lexer... --\n")
   ;; (setq debug-on-error t)
-  ;; (phps-mode/test-lexer--script-boundaries)
+  (phps-mode/test-lexer--script-boundaries)
   (phps-mode/test-lexer--simple-tokens)
-  ;; (phps-mode/test-lexer--complex-tokens)
-  ;; (phps-mode/test-lexer--namespaces)
-  ;; (phps-mode/test-lexer--errors)
+  (phps-mode/test-lexer--complex-tokens)
+  (phps-mode/test-lexer--namespaces)
+  (phps-mode/test-lexer--errors)
   ;; (message "\n-- Ran all tests for lexer. --")
   )