[elpa] externals/parser-generator 3563720 266/434: Updated docs about lexical analysis

[ELPA Syncer]

branch: externals/parser-generator
commit 356372084758433e92baeb6537a6b3aff59dab64
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    Updated docs about lexical analysis
---
 docs/Lexical-Analysis.md | 92 +++++++++++++++++++++++++++++-------------------
 1 file changed, 55 insertions(+), 37 deletions(-)

diff --git a/docs/Lexical-Analysis.md b/docs/Lexical-Analysis.md
index 45f4b8c..a356016 100644
--- a/docs/Lexical-Analysis.md
+++ b/docs/Lexical-Analysis.md
@@ -2,74 +2,92 @@
 
 Set lexical analysis function by setting variable 
`parser-generator-lex-analyzer--function`. Optionally set reset function by 
setting variable `parser-generator-lex-analyzer--reset-function`. The lexical 
analysis is indexed on variable `parser-generator-lex-analyzer--index`. All 
parsers expect a list of tokens as response from lexical-analysis.
 
-### Peek next look-ahead
+## Token
 
-Returns the look-ahead number of next terminals in stream.
+A token is defined as a list with 3 elements, first is a string or symbol, 
second is the start index of token in stream and third is the end index of 
token in stream, second and third element have a dot between them, this 
structure is to be compatible with Emacs Semantic system. Example token
+
+``` emacs-lisp
+'(("a" 1 . 2))
+```
+
+## Peek next look-ahead
+
+Returns the look-ahead number of next terminals in stream, if end of stream is 
reached a EOF-identifier is returned.
 
 ``` emacs-lisp
 (require 'ert)
 (setq
    parser-generator-lex-analyzer--function
-   (lambda (index length)
-     (let* ((string '(a a b b b))
+   (lambda (index)
+     (let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5)))
             (string-length (length string))
-            (max-index (+ index length))
-            (tokens))
+            (max-index index)
+            (tokens)
+            (next-token))
        (while (and
-               (< index string-length)
-               (< index max-index))
-         (push (nth index string) tokens)
+               (< (1- index) string-length)
+               (< (1- index) max-index))
+         (setq next-token (nth (1- index) string))
+         (push next-token tokens)
          (setq index (1+ index)))
        (nreverse tokens))))
 (parser-generator-lex-analyzer--reset)
 
 (setq parser-generator--look-ahead-number 1)
-  (should
-   (equal
-    '(a)
-    (parser-generator-lex-analyzer--peek-next-look-ahead)))
+(setq parser-generator--look-ahead-number 1)
+(should
+ (equal
+  '(("a" 1 . 2))
+  (parser-generator-lex-analyzer--peek-next-look-ahead)))
 
-  (setq parser-generator--look-ahead-number 2)
-  (should
-   (equal
-    '(a b)
-    (parser-generator-lex-analyzer--peek-next-look-ahead)))
+(setq parser-generator--look-ahead-number 2)
+(should
+ (equal
+  '(("a" 1 . 2) ("b" 2 . 3))
+  (parser-generator-lex-analyzer--peek-next-look-ahead)))
 
+(setq parser-generator--look-ahead-number 10)
+(should
+ (equal
+  '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) ($) ($) ($) ($) ($) ($))
+  (parser-generator-lex-analyzer--peek-next-look-ahead)))
 ```
 
-### Pop token
+## Pop token
 
-Returns the next token in stream and moves the index one point forward.
+Returns the next token in stream and moves the lexical analyzer index one 
point forward. If end of stream is reached return nil.
 
 ``` emacs-lisp
 (require 'ert)
 (setq
    parser-generator-lex-analyzer--function
-   (lambda (index length)
-     (let* ((string '(a b))
+   (lambda (index)
+     (let* ((string '(("a" 1 . 2) ("b" 2 . 3)))
             (string-length (length string))
-            (max-index (+ index length))
+            (max-index index)
             (tokens))
        (while (and
-               (< index string-length)
-               (< index max-index))
-         (push (nth index string) tokens)
+               (< (1- index) string-length)
+               (< (1- index) max-index))
+         (push (nth (1- index) string) tokens)
          (setq index (1+ index)))
        (nreverse tokens))))
 (parser-generator-lex-analyzer--reset)
 
+(setq parser-generator--look-ahead-number 1)
+
+(should
+ (equal
+  '(("a" 1 . 2))
+  (parser-generator-lex-analyzer--pop-token)))
+(should
+ (equal
+  '(("b" 2 . 3))
+  (parser-generator-lex-analyzer--pop-token)))
 (should
-   (equal
-    '(a)
-    (parser-generator-lex-analyzer--pop-token)))
-  (should
-   (equal
-    '(b)
-    (parser-generator-lex-analyzer--pop-token)))
-  (should
-   (equal
-    nil
-    (parser-generator-lex-analyzer--pop-token)))
+ (equal
+  nil
+  (parser-generator-lex-analyzer--pop-token)))
 ```
 
 [Back to start](../../../)