[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[elpa] externals/phps-mode 6b3a298 29/96: More work on parsers lex-analy
|
From: |
Christian Johansson |
|
Subject: |
[elpa] externals/phps-mode 6b3a298 29/96: More work on parsers lex-analyzer refactor |
|
Date: |
Fri, 29 Oct 2021 11:14:39 -0400 (EDT) |
branch: externals/phps-mode
commit 6b3a298ca48748f75b6863cfa07c7a28dcc20d0e
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>
More work on parsers lex-analyzer refactor
---
phps-mode-automation-grammar.el | 34 ++++----
phps-mode-parser.el | 166 ++++++++++++++++++++++++++++++++++++++--
test/phps-mode-test-parser.el | 23 ------
3 files changed, 180 insertions(+), 43 deletions(-)
diff --git a/phps-mode-automation-grammar.el b/phps-mode-automation-grammar.el
index 572cbdd..04121e3 100644
--- a/phps-mode-automation-grammar.el
+++ b/phps-mode-automation-grammar.el
@@ -1459,6 +1459,7 @@
'$
"The EOF-identifier of grammar.")
+;; TODO Need to use a separate reversed list of tokens
(defvar
phps-mode-automation-grammar--lex-analyzer-function
(lambda (buffer-index)
@@ -1510,7 +1511,11 @@
(phps-mode-lexer--re2c)
(setq
index
- semantic-lex-end-point)))
+ semantic-lex-end-point)
+ (goto-char index)))
+ (setq
+ phps-mode-lexer--generated-tokens
+ (reverse phps-mode-lexer--generated-tokens))
;; Reset buffer-index to token-list-index connections
(setq-local
@@ -1520,9 +1525,10 @@
(if (and
phps-mode-parser-position
(= (car (car phps-mode-parser-position)) buffer-index))
- (setq
+ (progn
+ (setq
token-list-index
- (cdr (car phps-mode-parser-position)))
+ (car (cdr (car phps-mode-parser-position)))))
;; Search from last requested index and forward until
;; we find a token starting at or after buffer-index and
@@ -1534,7 +1540,7 @@
(< (car (car phps-mode-parser-position)) buffer-index))
(setq
previous-token-list-index
- (cdr (car phps-mode-parser-position))))
+ (car (cdr (car phps-mode-parser-position)))))
(let ((temp-token-list-index
previous-token-list-index)
@@ -1556,6 +1562,11 @@
;; Save it
(when (= (car (cdr token)) buffer-index)
(let ((token-type (car token)))
+ (push
+ (list
+ buffer-index
+ temp-token-list-index)
+ phps-mode-parser-position)
(unless (or
(equal token-type 'T_OPEN_TAG)
(equal token-type 'T_CLOSE_TAG)
@@ -1564,11 +1575,6 @@
(setq
token-list-index
temp-token-list-index)
- (push
- (list
- buffer-index
- temp-token-list-index)
- phps-mode-parser-position)
(setq
continue
nil))))
@@ -1577,6 +1583,11 @@
;; Save it
(when (> (car (cdr token)) buffer-index)
(let ((token-type (car token)))
+ (push
+ (list
+ (car (cdr token))
+ temp-token-list-index)
+ phps-mode-parser-position)
(unless (or
(equal token-type 'T_OPEN_TAG)
(equal token-type 'T_CLOSE_TAG)
@@ -1585,11 +1596,6 @@
(setq-local
phps-mode-parser-lex-analyzer--move-to-index-flag
(car (cdr token)))
- (push
- (list
- (car (cdr token))
- temp-token-list-index)
- phps-mode-parser-position)
(setq
continue
nil))))
diff --git a/phps-mode-parser.el b/phps-mode-parser.el
index c02fb65..4964fd6 100644
--- a/phps-mode-parser.el
+++ b/phps-mode-parser.el
@@ -10,6 +10,11 @@
(require 'semantic)
(require 'semantic/lex)
+(defvar-local
+ phps-mode-parser-position
+ nil
+ "Position of parser.")
+
;;; Constants:
@@ -65,7 +70,160 @@
(defvar
phps-mode-parser-lex-analyzer--function
- (lambda (index) (save-current-buffer (set-buffer "*PHPs Lexer*") (if (=
(point) index) nil (goto-char index)) (if (< index (point-max)) (progn (if (and
phps-mode-lexer--generated-new-tokens-index (=
phps-mode-lexer--generated-new-tokens-index index)) nil
(phps-mode-lexer--re2c)) (let ((first (car (reverse
phps-mode-lexer--generated-new-tokens)))) (cond ((and (not first) (not (equal
index semantic-lex-end-point))) (set (make-local-variable
'phps-mode-parser-lex-analyzer--move-to-index-f [...]
+ (lambda (buffer-index)
+
+ ;; Create lexer buffer if none exists
+ (unless (get-buffer "*PHPs Lexer*")
+ (generate-new-buffer "*PHPs Lexer*")
+ (let ((old-buffer
+ (buffer-substring-no-properties
+ (point-min)
+ (point-max))))
+ (with-current-buffer "*PHPs Lexer*"
+ (insert old-buffer))))
+
+ (with-current-buffer "*PHPs Lexer*"
+ (let ((token-list-index))
+
+ ;; Unless we have lexed the buffer
+ (unless phps-mode-lexer--generated-tokens
+
+ ;; Reset lexer
+ (setq-local
+ phps-mode-lexer--generated-tokens
+ nil)
+ (setq-local
+ phps-mode-lexer--state
+ 'ST_INITIAL)
+ (setq-local
+ phps-mode-lexer--states
+ nil)
+ (setq-local
+ phps-mode-lexer--state-stack
+ nil)
+ (setq-local
+ phps-mode-lexer--heredoc-label
+ nil)
+ (setq-local
+ phps-mode-lexer--heredoc-label-stack
+ nil)
+ (setq-local
+ phps-mode-lexer--nest-location-stack
+ nil)
+ (goto-char (point-min))
+
+ ;; Run lexer on entire buffer here
+ (let ((index (point))
+ (max-index (point-max)))
+ (while (< index max-index)
+ (phps-mode-lexer--re2c)
+ (setq
+ index
+ semantic-lex-end-point)
+ (goto-char index)))
+ (setq
+ phps-mode-lexer--generated-tokens
+ (reverse phps-mode-lexer--generated-tokens))
+
+ ;; Reset buffer-index to token-list-index connections
+ (setq-local
+ phps-mode-parser-position
+ nil))
+
+ (if (and
+ phps-mode-parser-position
+ (= (car (car phps-mode-parser-position)) buffer-index))
+ (progn
+ (setq
+ token-list-index
+ (car (cdr (car phps-mode-parser-position)))))
+
+ ;; Search from last requested index and forward until
+ ;; we find a token starting at or after buffer-index and
+ ;; use this as buffer-index, save buffer-index to
+ ;; token-list-index connection
+ (let ((previous-token-list-index 0))
+ (when (and
+ phps-mode-parser-position
+ (< (car (car phps-mode-parser-position)) buffer-index))
+ (setq
+ previous-token-list-index
+ (car (cdr (car phps-mode-parser-position)))))
+
+ (let ((temp-token-list-index
+ previous-token-list-index)
+ (token-list-size
+ (length
+ phps-mode-lexer--generated-tokens))
+ (continue t))
+ (while (and
+ continue
+ (<
+ temp-token-list-index
+ token-list-size))
+ (let ((token
+ (nth
+ temp-token-list-index
+ phps-mode-lexer--generated-tokens)))
+
+ ;; When token starts at cursor we found correct index
+ ;; Save it
+ (when (= (car (cdr token)) buffer-index)
+ (let ((token-type (car token)))
+ (push
+ (list
+ buffer-index
+ temp-token-list-index)
+ phps-mode-parser-position)
+ (unless (or
+ (equal token-type 'T_OPEN_TAG)
+ (equal token-type 'T_CLOSE_TAG)
+ (equal token-type 'T_DOC_COMMENT)
+ (equal token-type 'T_COMMENT))
+ (setq
+ token-list-index
+ temp-token-list-index)
+ (setq
+ continue
+ nil))))
+
+ ;; When token starts after cursor, flag move of cursor
+ ;; Save it
+ (when (> (car (cdr token)) buffer-index)
+ (let ((token-type (car token)))
+ (push
+ (list
+ (car (cdr token))
+ temp-token-list-index)
+ phps-mode-parser-position)
+ (unless (or
+ (equal token-type 'T_OPEN_TAG)
+ (equal token-type 'T_CLOSE_TAG)
+ (equal token-type 'T_DOC_COMMENT)
+ (equal token-type 'T_COMMENT))
+ (setq-local
+ phps-mode-parser-lex-analyzer--move-to-index-flag
+ (car (cdr token)))
+ (setq
+ continue
+ nil))))
+
+ (setq
+ temp-token-list-index
+ (1+ temp-token-list-index))
+ )))))
+
+ (when
+ token-list-index
+ (let ((token
+ (nth
+ token-list-index
+ phps-mode-lexer--generated-tokens)))
+ (when (equal (car token) 'T_OPEN_TAG_WITH_ECHO)
+ (setf
+ (car token)
+ 'T_ECHO))
+ token)))))
"The lex-analyzer function.")
(defvar
@@ -162,13 +320,9 @@
phps-mode-parser-lex-analyzer--function
index)))
(if phps-mode-parser-lex-analyzer--move-to-index-flag
- (progn
- (setq-local
- phps-mode-parser-lex-analyzer--index
- phps-mode-parser-lex-analyzer--move-to-index-flag)
(setq
index
- phps-mode-parser-lex-analyzer--index))
+ phps-mode-parser-lex-analyzer--move-to-index-flag)
(if next-look-ahead
(progn
(unless (listp (car next-look-ahead))
diff --git a/test/phps-mode-test-parser.el b/test/phps-mode-test-parser.el
index fd00015..2e5dd59 100644
--- a/test/phps-mode-test-parser.el
+++ b/test/phps-mode-test-parser.el
@@ -40,29 +40,6 @@
name
(buffer-substring-no-properties (point-min) (point-max)))
- ;; Reset lexer
- (setq-local
- phps-mode-lexer--generated-tokens
- nil)
- (setq-local
- phps-mode-lexer--state
- 'ST_INITIAL)
- (setq-local
- phps-mode-lexer--states
- nil)
- (setq-local
- phps-mode-lexer--state-stack
- nil)
- (setq-local
- phps-mode-lexer--heredoc-label
- nil)
- (setq-local
- phps-mode-lexer--heredoc-label-stack
- nil)
- (setq-local
- phps-mode-lexer--nest-location-stack
- nil)
-
(funcall logic)
(message "Passed %s" name)
- [elpa] externals/phps-mode b0ad204 44/96: Improved YACC grammar parser for productions, (continued)
- [elpa] externals/phps-mode b0ad204 44/96: Improved YACC grammar parser for productions, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 3824d7d 46/96: More work on automatically parsing YACC grammar, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 42c4e0e 48/96: Work on automatically generating terminals, non-terminals and start from YACC grammar, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode f56ba64 49/96: Terminals, non-terminals and start is automatically generated from YACC, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 7323416 31/96: Added failing test for parser, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 7c66145 30/96: Refactor of parsers lex-analyzer complete, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 0e89b6e 35/96: Fixed bug in automation grammar, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 65152f4 36/96: Added TODO item, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode add17a7 24/96: T_COMMENT and T_DOC_COMMENT are excluded from parser, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode f46b98f 25/96: More work on integrating parser, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 6b3a298 29/96: More work on parsers lex-analyzer refactor,
Christian Johansson <=
- [elpa] externals/phps-mode 45508e1 32/96: Reverted integration of parser into other files due to immaturity, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 37dda7a 34/96: Added another failing test for parser, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 339e39b 37/96: Started on YACC grammar parser generator, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 244a62b 39/96: Parser generator now can parse PHP YACC productions, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode a449b5b 42/96: Improved formatting of generated productions, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode e3d973d 43/96: Improved support for semi-colon in grammar delimiter, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 962e7c8 52/96: Context-sensitive precedence attribute is automatically parsed from YACC, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode b94cbbf 58/96: Set GB collection in automation, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode bb4858b 61/96: Defined lex-analyzer reset function for parser, Christian Johansson, 2021/10/29
- [elpa] externals/phps-mode 22f0a47 62/96: Parser generator passing byte-compilation, Christian Johansson, 2021/10/29