[elpa] externals/parser-generator 0416ca9 134/434: Added information about lex-analyzer in README

[ELPA Syncer]

branch: externals/parser-generator
commit 0416ca9ed4549cfd2c1b281763878e940fc21698
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    Added information about lex-analyzer in README
---
 README.md                                  | 70 +++++++++++++++++++++++++++++-
 test/parser-generator-lex-analyzer-test.el | 48 +++++++++++++++-----
 2 files changed, 107 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index cbff6e6..7a87dbe 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,75 @@ This is just started, so most stuff are *WIP*.
 
 ## Lexical Analysis
 
-We use a regular-language based lexical analyzer that can be implemented by a 
finite-state-machine (FSM).
+We use a regular-language based lexical analyzer that can be implemented by a 
finite-state-machine (FSM). Set lexical analysis function by setting variable 
`parser-generator-lex-analyzer--function`. Optionally set reset function by 
setting variable `parser-generator-lex-analyzer--reset-function`. The lexical 
analysis is indexed on variable `parser-generator-lex-analyzer--index`. All 
parsers expect a list of tokens as response from lexical-analysis.
+
+### Peek next look-ahead
+
+Returns the look-ahead number of next terminals in stream.
+
+``` emacs-lisp
+(require 'ert)
+(setq
+   parser-generator-lex-analyzer--function
+   (lambda (index length)
+     (let* ((string '(a a b b b))
+            (string-length (length string))
+            (max-index (+ index length))
+            (tokens))
+       (while (and
+               (< index string-length)
+               (< index max-index))
+         (push (nth index string) tokens)
+         (setq index (1+ index)))
+       (nreverse tokens))))
+(parser-generator-lex-analyzer--reset)
+(setq parser-generator--look-ahead-number 1)
+  (should
+   (equal
+    '(a)
+    (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+  (setq parser-generator--look-ahead-number 2)
+  (should
+   (equal
+    '(a b)
+    (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+```
+
+### Pop token
+
+Returns the next token in stream and moves index one forward.
+
+``` emacs-lisp
+(require 'ert)
+(setq
+   parser-generator-lex-analyzer--function
+   (lambda (index length)
+     (let* ((string '(a b))
+            (string-length (length string))
+            (max-index (+ index length))
+            (tokens))
+       (while (and
+               (< index string-length)
+               (< index max-index))
+         (push (nth index string) tokens)
+         (setq index (1+ index)))
+       (nreverse tokens))))
+(parser-generator-lex-analyzer--reset)
+(should
+   (equal
+    '(a)
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    '(b)
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    nil
+    (parser-generator-lex-analyzer--pop-token)))
+```
 
 ## Syntax Analysis / Parsing
 
diff --git a/test/parser-generator-lex-analyzer-test.el 
b/test/parser-generator-lex-analyzer-test.el
index 0638e47..f68722c 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -20,7 +20,19 @@
 
   (should-error
    (parser-generator-lex-analyzer--peek-next-look-ahead))
-  (setq parser-generator-lex-analyzer--function (lambda (index length) 
(substring "abcdefghijklmnopqrstuvxz" index (+ index length))))
+  (setq
+   parser-generator-lex-analyzer--function
+   (lambda (index length)
+     (let* ((string '(a b c d))
+            (string-length (length string))
+            (max-index (+ index length))
+            (tokens))
+       (while (and
+               (< index string-length)
+               (< index max-index))
+         (push (nth index string) tokens)
+         (setq index (1+ index)))
+       (nreverse tokens))))
   (should-error
    (parser-generator-lex-analyzer--peek-next-look-ahead))
   (parser-generator-lex-analyzer--reset)
@@ -30,7 +42,19 @@
   (setq parser-generator--look-ahead-number 1)
   (should
    (equal
-    "a"
+    '(a)
+    (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+  (setq parser-generator--look-ahead-number 2)
+  (should
+   (equal
+    '(a b)
+    (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+  (setq parser-generator--look-ahead-number 10)
+  (should
+   (equal
+    '(a b c d)
     (parser-generator-lex-analyzer--peek-next-look-ahead)))
 
   (message "Ended tests for 
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
@@ -48,23 +72,27 @@
   (setq
    parser-generator-lex-analyzer--function
    (lambda (index length)
-     (let* ((string "ab")
-            (string-length (length string)))
-       (when (> string-length index)
-         (if (>= string-length (+ index length))
-             (substring string index (+ index length))
-           (substring string index (1- string-length)))))))
+     (let* ((string '(a b))
+            (string-length (length string))
+            (max-index (+ index length))
+            (tokens))
+       (while (and
+               (< index string-length)
+               (< index max-index))
+         (push (nth index string) tokens)
+         (setq index (1+ index)))
+       (nreverse tokens))))
   (should-error
    (parser-generator-lex-analyzer--pop-token))
   (parser-generator-lex-analyzer--reset)
 
   (should
    (equal
-    "a"
+    '(a)
     (parser-generator-lex-analyzer--pop-token)))
   (should
    (equal
-    "b"
+    '(b)
     (parser-generator-lex-analyzer--pop-token)))
   (should
    (equal