[elpa] externals/parser-generator fa7089e 144/434: Re-factored lex analyzer function to not use length argument

[ELPA Syncer]

branch: externals/parser-generator
commit fa7089ef2dbdcd7eb199d4740e2a02d2f5a3eda8
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    Re-factored lex analyzer function to not use length argument
---
 parser-generator-lex-analyzer.el           | 41 ++++++++++++++++++++++--------
 parser-generator-lr.el                     | 10 +-------
 test/parser-generator-lex-analyzer-test.el | 11 ++++----
 test/parser-generator-lr-test.el           | 16 ++++++------
 4 files changed, 45 insertions(+), 33 deletions(-)

diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index b7f6274..307d497 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -37,10 +37,27 @@
     (error "Missing lex-analyzer function!"))
   (unless parser-generator--look-ahead-number
     (error "Missing look-ahead number!"))
-  (funcall
-   parser-generator-lex-analyzer--function
-   parser-generator-lex-analyzer--index
-   parser-generator--look-ahead-number))
+  (let ((look-ahead)
+        (look-ahead-length 0)
+        (index parser-generator-lex-analyzer--index))
+    (while (< look-ahead-length parser-generator--look-ahead-number)
+      (let ((next-look-ahead
+             (funcall
+              parser-generator-lex-analyzer--function
+              index)))
+        (if next-look-ahead
+            (progn
+              (unless (listp (car next-look-ahead))
+                (setq next-look-ahead (list next-look-ahead)))
+              (dolist (next-look-ahead-item next-look-ahead)
+                (when (< look-ahead-length parser-generator--look-ahead-number)
+                  (push next-look-ahead-item look-ahead)
+                  (setq look-ahead-length (1+ look-ahead-length))
+                  (setq index (1+ index)))))
+          (push (list parser-generator--e-identifier) look-ahead)
+          (setq look-ahead-length (1+ look-ahead-length))
+          (setq index (1+ index)))))
+    (nreverse look-ahead)))
 
 (defun parser-generator-lex-analyzer--pop-token ()
   "Pop next token via lex-analyzer."
@@ -48,13 +65,15 @@
     (error "Missing lex-analyzer index!"))
   (unless parser-generator-lex-analyzer--function
     (error "Missing lex-analyzer function!"))
-  (let ((token (car (funcall
-                parser-generator-lex-analyzer--function
-                parser-generator-lex-analyzer--index
-                1))))
-    (setq parser-generator-lex-analyzer--index
-          (1+ parser-generator-lex-analyzer--index))
-    token))
+  (let ((token (funcall
+                     parser-generator-lex-analyzer--function
+                     parser-generator-lex-analyzer--index)))
+    (unless (listp (car token))
+      (setq token (list token)))
+    (let ((first-token (car token)))
+      (setq parser-generator-lex-analyzer--index
+            (1+ parser-generator-lex-analyzer--index))
+      first-token)))
 
 (defun parser-generator-lex-analyzer--reset ()
   "Reset lex-analyzer."
diff --git a/parser-generator-lr.el b/parser-generator-lr.el
index 09fd2c4..b1f55f3 100644
--- a/parser-generator-lr.el
+++ b/parser-generator-lr.el
@@ -514,19 +514,11 @@
 
       ;; (1) The lookahead string u, consisting of the next k input symbols, 
is determined.
       (let ((look-ahead (parser-generator-lex-analyzer--peek-next-look-ahead))
-            (look-ahead-length 0)
             (look-ahead-full))
-
-        (setq look-ahead-length (length look-ahead))
-
-        ;; If we reached end of input-tape and look-ahead is too small, append 
e-identifiers
-        (while (< look-ahead-length parser-generator--look-ahead-number)
-          (push parser-generator--e-identifier look-ahead)
-          (setq look-ahead-length (1+ look-ahead-length)))
-
         ;; Save token stream indexes in separate variable if needed later
         (setq look-ahead-full (nreverse look-ahead))
 
+        ;; Create simplified look-ahead for logic below
         (setq look-ahead nil)
         (dolist (look-ahead-item look-ahead-full)
           (if (listp look-ahead-item)
diff --git a/test/parser-generator-lex-analyzer-test.el 
b/test/parser-generator-lex-analyzer-test.el
index 03d4a05..ad4df4c 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -22,10 +22,10 @@
    (parser-generator-lex-analyzer--peek-next-look-ahead))
   (setq
    parser-generator-lex-analyzer--function
-   (lambda (index length)
+   (lambda (index)
      (let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5)))
             (string-length (length string))
-            (max-index (+ index length))
+            (max-index (1+ index))
             (tokens))
        (while (and
                (< index string-length)
@@ -54,7 +54,7 @@
   (setq parser-generator--look-ahead-number 10)
   (should
    (equal
-    '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5))
+    '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) (e) (e) (e) (e) (e) (e))
     (parser-generator-lex-analyzer--peek-next-look-ahead)))
 
   (message "Ended tests for 
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
@@ -71,10 +71,10 @@
    (parser-generator-lex-analyzer--pop-token))
   (setq
    parser-generator-lex-analyzer--function
-   (lambda (index length)
+   (lambda (index)
      (let* ((string '(("a" 1 . 2) ("b" 2 . 3)))
             (string-length (length string))
-            (max-index (+ index length))
+            (max-index (1+ index))
             (tokens))
        (while (and
                (< index string-length)
@@ -103,6 +103,7 @@
 
 (defun parser-generator-lex-analyzer-test ()
   "Run test."
+  ;; (setq debug-on-error t)
   (parser-generator-lex-analyzer-test--peek-next-look-ahead)
   (parser-generator-lex-analyzer-test--pop-token))
 
diff --git a/test/parser-generator-lr-test.el b/test/parser-generator-lr-test.el
index 71017fa..82e9d3a 100644
--- a/test/parser-generator-lr-test.el
+++ b/test/parser-generator-lr-test.el
@@ -266,10 +266,10 @@
 
   (setq
    parser-generator-lex-analyzer--function
-   (lambda (index length)
+   (lambda (index)
      (let* ((string '((a 1 . 2) (a 2 . 3) (b 3 . 4) (b 4 . 5)))
             (string-length (length string))
-            (max-index (+ index length))
+            (max-index (1+ index))
             (tokens))
        (while (and
                (< index string-length)
@@ -285,10 +285,10 @@
 
   (setq
    parser-generator-lex-analyzer--function
-   (lambda (index length)
+   (lambda (index)
      (let* ((string '((a 1 . 2) (a 2 . 3) (b 3 . 4) (b 4 . 5) (b 5 . 6)))
             (string-length (length string))
-            (max-index (+ index length))
+            (max-index (1+ index))
             (tokens))
        (while (and
                (< index string-length)
@@ -309,10 +309,10 @@
 
   (setq
    parser-generator-lex-analyzer--function
-   (lambda (index length)
+   (lambda (index)
      (let* ((string '(("a" 1 . 2) ("a" 2 . 3) ("b" 3 . 4) ("b" 4 . 5)))
             (string-length (length string))
-            (max-index (+ index length))
+            (max-index (1+ index))
             (tokens))
        (while (and
                (< index string-length)
@@ -328,10 +328,10 @@
 
   (setq
    parser-generator-lex-analyzer--function
-   (lambda (index length)
+   (lambda (index)
      (let* ((string '(("a" 1 . 2) ("a" 2 . 3) ("b" 3 . 4) ("b" 4 . 5) ("b" 5 . 
6)))
             (string-length (length string))
-            (max-index (+ index length))
+            (max-index (1+ index))
             (tokens))
        (while (and
                (< index string-length)