[PATCH 01/13] traces: write the "Reading a token" alone on its line

[Akim Demaille]

The Java skeleton displays

    Reading a token:
    Next token is token "number" (1)

while the other display

    Reading a token: Next token is token "number" (1)

When generating logs in the scanner, the first part is separated from
the second, and the end of the scanner logs have the second part
pasted in.  So let's propagate the Java way, but with the colon.

* data/skeletons/glr.c, data/skeletons/lalr1.cc, data/skeletons/lalr1.d,
* data/skeletons/lalr1.java, data/skeletons/yacc.c: Do it.
Adjust test cases and doc.
---
 data/skeletons/glr.c      |  2 +-
 data/skeletons/lalr1.cc   |  2 +-
 data/skeletons/lalr1.d    |  2 +-
 data/skeletons/lalr1.java |  6 ++--
 data/skeletons/yacc.c     |  2 +-
 doc/bison.texi            | 24 ++++++++++------
 tests/actions.at          | 58 +++++++++++++++++++++++++--------------
 tests/c++.at              |  9 ++++--
 tests/calc.at             |  6 ++--
 tests/glr-regression.at   | 12 +++++---
 tests/regression.at       |  9 ++++--
 11 files changed, 85 insertions(+), 47 deletions(-)

diff --git a/data/skeletons/glr.c b/data/skeletons/glr.c
index 8cf56c31..a40d239f 100644
--- a/data/skeletons/glr.c
+++ b/data/skeletons/glr.c
@@ -824,7 +824,7 @@ yygetToken (int *yycharp][]b4_pure_if([, yyGLRStack* 
yystackp])[]b4_user_formals
 ]b4_parse_param_use()dnl
 [  if (*yycharp == YYEMPTY)
     {
-      YY_DPRINTF ((stderr, "Reading a token: "));]b4_glr_cc_if([[
+      YY_DPRINTF ((stderr, "Reading a token\n"));]b4_glr_cc_if([[
 #if YY_EXCEPTIONS
       try
         {
diff --git a/data/skeletons/lalr1.cc b/data/skeletons/lalr1.cc
index 3a4b7a91..80a2ad4b 100644
--- a/data/skeletons/lalr1.cc
+++ b/data/skeletons/lalr1.cc
@@ -874,7 +874,7 @@ b4_dollar_popdef])[]dnl
     // Read a lookahead token.
     if (yyla.empty ())
       {
-        YYCDEBUG << "Reading a token: ";
+        YYCDEBUG << "Reading a token\n";
 #if YY_EXCEPTIONS
         try
 #endif // YY_EXCEPTIONS
diff --git a/data/skeletons/lalr1.d b/data/skeletons/lalr1.d
index 6bae65f4..9eb71958 100644
--- a/data/skeletons/lalr1.d
+++ b/data/skeletons/lalr1.d
@@ -493,7 +493,7 @@ m4_popdef([b4_at_dollar])])dnl
         /* Read a lookahead token.  */
         if (yychar == yyempty_)
         {]b4_parse_trace_if([[
-          yycdebug ("Reading a token: ");]])[
+          yycdebugln ("Reading a token");]])[
           yychar = yylex ();]b4_locations_if([[
           static if (yy_location_is_class) {
             yylloc = new ]b4_location_type[(yylexer.startPos, yylexer.endPos);
diff --git a/data/skeletons/lalr1.java b/data/skeletons/lalr1.java
index e9da8cc5..b28e7bbd 100644
--- a/data/skeletons/lalr1.java
+++ b/data/skeletons/lalr1.java
@@ -586,7 +586,7 @@ b4_dollar_pushdef([yylval], [], [], [yylloc])dnl
     b4_user_initial_action
 b4_dollar_popdef[]dnl
 ])[]b4_parse_trace_if([[
-        yycdebug ("Starting parse\n");]])[
+        yycdebug ("Starting parse");]])[
         yyerrstatus_ = 0;
       } else
         label = YYGETTOKEN;
@@ -624,12 +624,12 @@ b4_dollar_popdef[]dnl
 ]b4_push_if([[
             if (!push_token_consumed)
               return YYPUSH_MORE;]b4_parse_trace_if([[
-            yycdebug ("Reading a token: ");]])[
+            yycdebug ("Reading a token");]])[
             yychar = yylextoken;
             yylval = yylexval;]b4_locations_if([
             yylloc = yylexloc;])[
             push_token_consumed = false;]], [b4_parse_trace_if([[
-            yycdebug ("Reading a token: ");]])[
+            yycdebug ("Reading a token");]])[
             yychar = yylexer.yylex ();
             yylval = yylexer.getLVal ();]b4_locations_if([
             yylloc = new b4_location_type (yylexer.getStartPos (),
diff --git a/data/skeletons/yacc.c b/data/skeletons/yacc.c
index c3c50da6..6c7c0732 100644
--- a/data/skeletons/yacc.c
+++ b/data/skeletons/yacc.c
@@ -1702,7 +1702,7 @@ yybackup:
       yylval = yypushed_val;]b4_locations_if([[
       yylloc = yypushed_loc;]])])[
 yyread_pushed_token:]])[
-      YYDPRINTF ((stderr, "Reading a token: "));]b4_push_if([b4_pure_if([[
+      YYDPRINTF ((stderr, "Reading a token\n"));]b4_push_if([b4_pure_if([[
       yychar = yypushed_char;
       if (yypushed_val)
         yylval = *yypushed_val;]b4_locations_if([[
diff --git a/doc/bison.texi b/doc/bison.texi
index 9daa4d43..35dbaff4 100644
--- a/doc/bison.texi
+++ b/doc/bison.texi
@@ -10177,7 +10177,8 @@ a valueless (@samp{()}) @code{input} nonterminal 
(@code{nterm}).
 
 Then the parser calls the scanner.
 @example
-Reading a token: Next token is token FUN (sin())
+Reading a token
+Next token is token FUN (sin())
 Shifting token FUN (sin())
 Entering state 6
 @end example
@@ -10189,10 +10190,12 @@ The parser stores (@code{Shifting}) that token, and 
others, until it can do
 something about it.
 
 @example
-Reading a token: Next token is token '(' ()
+Reading a token
+Next token is token '(' ()
 Shifting token '(' ()
 Entering state 14
-Reading a token: Next token is token NUM (1.000000)
+Reading a token
+Next token is token NUM (1.000000)
 Shifting token NUM (1.000000)
 Entering state 4
 Reducing stack by rule 6 (line 44):
@@ -10208,10 +10211,12 @@ The previous reduction demonstrates the 
@code{%printer} directive for
 @code{exp} have @samp{1} as value.
 
 @example
-Reading a token: Next token is token '-' ()
+Reading a token
+Next token is token '-' ()
 Shifting token '-' ()
 Entering state 17
-Reading a token: Next token is token NUM (1.000000)
+Reading a token
+Next token is token NUM (1.000000)
 Shifting token NUM (1.000000)
 Entering state 4
 Reducing stack by rule 6 (line 44):
@@ -10219,7 +10224,8 @@ Reducing stack by rule 6 (line 44):
 -> $$ = nterm exp (1.000000)
 Stack now 0 1 6 14 24 17
 Entering state 26
-Reading a token: Next token is token ')' ()
+Reading a token
+Next token is token ')' ()
 Reducing stack by rule 11 (line 49):
    $1 = nterm exp (1.000000)
    $2 = token '-' ()
@@ -10252,7 +10258,8 @@ Finally, the end-of-line allow the parser to complete 
the computation, and
 display its result.
 
 @example
-Reading a token: Next token is token '\n' ()
+Reading a token
+Next token is token '\n' ()
 Shifting token '\n' ()
 Entering state 22
 Reducing stack by rule 4 (line 40):
@@ -10275,7 +10282,8 @@ expression to evaluate, or for the end-of-file token, 
which causes the
 completion of the parsing.
 
 @example
-Reading a token: Now at end of input.
+Reading a token
+Now at end of input.
 Shifting token $end ()
 Entering state 2
 Stack now 0 1 2
diff --git a/tests/actions.at b/tests/actions.at
index 2e9e2374..e0e325e4 100644
--- a/tests/actions.at
+++ b/tests/actions.at
@@ -1123,19 +1123,24 @@ AT_PARSER_CHECK([input --debug], 1,
 ]],
 [[Starting parse
 Entering state 0
-Reading a token: Next token is token 'a' (1.1: <> printer for 'a' @ 1)
+Reading a token
+Next token is token 'a' (1.1: <> printer for 'a' @ 1)
 Shifting token 'a' (1.1: <> printer for 'a' @ 1)
 Entering state 1
-Reading a token: Next token is token 'b' (1.2: 'b'/'c' printer for 'b' @ 2)
+Reading a token
+Next token is token 'b' (1.2: 'b'/'c' printer for 'b' @ 2)
 Shifting token 'b' (1.2: 'b'/'c' printer for 'b' @ 2)
 Entering state 3
-Reading a token: Next token is token 'c' (1.3: 'b'/'c' printer for 'c' @ 3)
+Reading a token
+Next token is token 'c' (1.3: 'b'/'c' printer for 'c' @ 3)
 Shifting token 'c' (1.3: 'b'/'c' printer for 'c' @ 3)
 Entering state 5
-Reading a token: Next token is token 'd' (1.4: <> printer for 'd' @ 4)
+Reading a token
+Next token is token 'd' (1.4: <> printer for 'd' @ 4)
 Shifting token 'd' (1.4: <> printer for 'd' @ 4)
 Entering state 6
-Reading a token: Now at end of input.
+Reading a token
+Now at end of input.
 1.5: syntax error, unexpected $end, expecting 'e'
 Error: popping token 'd' (1.4: <> printer for 'd' @ 4)
 Stack now 0 1 3 5
@@ -1232,25 +1237,32 @@ AT_PARSER_CHECK([input --debug], 1,
 ]],
 [[Starting parse
 Entering state 0
-Reading a token: Next token is token 'a' (<*>/<field2>/e printer)
+Reading a token
+Next token is token 'a' (<*>/<field2>/e printer)
 Shifting token 'a' (<*>/<field2>/e printer)
 Entering state 1
-Reading a token: Next token is token 'b' (<field1> printer)
+Reading a token
+Next token is token 'b' (<field1> printer)
 Shifting token 'b' (<field1> printer)
 Entering state 3
-Reading a token: Next token is token 'c' ('c' printer)
+Reading a token
+Next token is token 'c' ('c' printer)
 Shifting token 'c' ('c' printer)
 Entering state 5
-Reading a token: Next token is token 'd' ('d' printer)
+Reading a token
+Next token is token 'd' ('d' printer)
 Shifting token 'd' ('d' printer)
 Entering state 6
-Reading a token: Next token is token 'e' (<*>/<field2>/e printer)
+Reading a token
+Next token is token 'e' (<*>/<field2>/e printer)
 Shifting token 'e' (<*>/<field2>/e printer)
 Entering state 7
-Reading a token: Next token is token 'f' (<*>/<field2>/e printer)
+Reading a token
+Next token is token 'f' (<*>/<field2>/e printer)
 Shifting token 'f' (<*>/<field2>/e printer)
 Entering state 8
-Reading a token: Now at end of input.
+Reading a token
+Now at end of input.
 syntax error, unexpected $end, expecting 'g'
 Error: popping token 'f' (<*>/<field2>/e printer)
 Stack now 0 1 3 5 6 7
@@ -1362,7 +1374,8 @@ Reducing stack by rule 1 (line 49):
 -> $$ = nterm start (1.1: <]]kind[[> for 'S' @ 1)
 Stack now 0
 Entering state 1
-Reading a token: Now at end of input.
+Reading a token
+Now at end of input.
 Shifting token END (1.1: <]]kind[[> for 'E' @ 1)
 Entering state 2
 Stack now 0 1 2
@@ -1440,17 +1453,20 @@ AT_COMPILE([input])
 AT_PARSER_CHECK([input --debug], [1], [],
 [[Starting parse
 Entering state 0
-Reading a token: Next token is token 'a' ('a')
+Reading a token
+Next token is token 'a' ('a')
 Shifting token 'a' ('a')
 Entering state 1
-Reading a token: Next token is token 'b' ('b')
+Reading a token
+Next token is token 'b' ('b')
 syntax error
 Shifting token error ()
 Entering state 3
 Next token is token 'b' ('b')
 Shifting token 'b' ('b')
 Entering state 5
-Reading a token: Next token is token $undefined ()
+Reading a token
+Next token is token $undefined ()
 Error: popping token 'b' ('b')
 DESTROY 'b'
 Stack now 0 1 3
@@ -1464,7 +1480,8 @@ Error: popping token error ()
 Stack now 0 1
 Shifting token error ()
 Entering state 3
-Reading a token: Now at end of input.
+Reading a token
+Now at end of input.
 Cleanup: discarding lookahead token $end ()
 Stack now 0 1 3
 Cleanup: popping token error ()
@@ -1624,7 +1641,8 @@ Reducing stack by rule 4 (line 33):
 -> $$ = nterm @4 (: 4)
 Stack now 0 2 4 5
 Entering state 6
-Reading a token: Now at end of input.
+Reading a token
+Now at end of input.
 syntax error
 Error: popping nterm @4 (: 4)
 DESTROY 4
@@ -1790,9 +1808,9 @@ AT_PARSER_CHECK([input --debug], 0, [], [stderr])
 # Don't be too picky on the traces, GLR is not exactly the same.  Keep
 # only the lines from the printer.
 AT_CHECK([[sed -ne '/ival:/p' stderr]], 0,
-[[Reading a token: Next token is token UNTYPED (ival: 10, fval: 0.1)
+[[Next token is token UNTYPED (ival: 10, fval: 0.1)
 Shifting token UNTYPED (ival: 10, fval: 0.1)
-Reading a token: Next token is token INT (ival: 20, fval: 0.2)
+Next token is token INT (ival: 20, fval: 0.2)
 Shifting token INT (ival: 20, fval: 0.2)
    $][1 = token UNTYPED (ival: 10, fval: 0.1)
    $][2 = token INT (ival: 20, fval: 0.2)
diff --git a/tests/c++.at b/tests/c++.at
index d896af19..52b959fb 100644
--- a/tests/c++.at
+++ b/tests/c++.at
@@ -648,7 +648,8 @@ AT_FULL_COMPILE([[input]])
 AT_PARSER_CHECK([[input]], [[0]], [[]],
 [[Starting parse
 Entering state 0
-Reading a token: Next token is token NUMBER (1)
+Reading a token
+Next token is token NUMBER (1)
 Shifting token NUMBER (1)
 Entering state 1
 Reducing stack by rule 1 (line 34):
@@ -657,7 +658,8 @@ Reducing stack by rule 1 (line 34):
 destroy: 1
 Stack now 0
 Entering state 2
-Reading a token: Next token is token NUMBER (30)
+Reading a token
+Next token is token NUMBER (30)
 Reducing stack by rule 2 (line 35):
 -> $$ = nterm @1 (20)
 Stack now 0 2
@@ -676,7 +678,8 @@ destroy: 20
 destroy: 10
 Stack now 0
 Entering state 2
-Reading a token: Next token is token EOI ()
+Reading a token
+Next token is token EOI ()
 Shifting token EOI ()
 Entering state 3
 Stack now 0 2 3
diff --git a/tests/calc.at b/tests/calc.at
index 837a4d52..1392d1bb 100644
--- a/tests/calc.at
+++ b/tests/calc.at
@@ -780,7 +780,9 @@ AT_JAVA_IF(
   [AT_PARSER_CHECK([calc input],        0, [AT_PARAM_IF([m4_n([$3])])], 
[stderr])])
 AT_LANG_MATCH([c\|c++],
   [AT_GLR_IF([],
-    [AT_CHECK([cat stderr | wc -l], [0], [m4_n([AT_DEBUG_IF([$4], [0])])])])])
+    [AT_CHECK([grep -v 'Return for a new token:' stderr | wc -l],
+              [0],
+              [m4_n([AT_DEBUG_IF([$4], [0])])])])])
 ])
 
 
@@ -920,7 +922,7 @@ _AT_CHECK_CALC([$1],
 2^2^3 = 256
 (2^2)^3 = 64],
 [[final: 64 12 0]],
-               [AT_PUSH_IF([930], [846])])
+               [931])
 
 # Some syntax errors.
 _AT_CHECK_CALC_ERROR([$1], [1], [1 2],
diff --git a/tests/glr-regression.at b/tests/glr-regression.at
index 1ad9f5a7..7a8ca76f 100644
--- a/tests/glr-regression.at
+++ b/tests/glr-regression.at
@@ -1709,23 +1709,27 @@ AT_COMPILE([input])
 AT_PARSER_CHECK([[input --debug]], 1, [],
 [Starting parse
 Entering state 0
-Reading a token: Next token is token 'a' ()
+Reading a token
+Next token is token 'a' ()
 Shifting token 'a' ()
 Entering state 1
-Reading a token: Next token is token 'b' ()
+Reading a token
+Next token is token 'b' ()
 Shifting token 'b' ()
 Entering state 3
 Reducing stack 0 by rule 3 (line 27):
    $1 = token 'b' ()
 -> $$ = nterm b ()
 Entering state 4
-Reading a token: Next token is token 'c' ()
+Reading a token
+Next token is token 'c' ()
 Shifting token 'c' ()
 Entering state 6
 Reducing stack 0 by rule 4 (line 28):
 -> $$ = nterm d ()
 Entering state 7
-Reading a token: Now at end of input.
+Reading a token
+Now at end of input.
 Stack 0 Entering state 7
 Now at end of input.
 Splitting off stack 1 from 0.
diff --git a/tests/regression.at b/tests/regression.at
index 41cef311..9e52cd7e 100644
--- a/tests/regression.at
+++ b/tests/regression.at
@@ -964,7 +964,8 @@ Reducing stack by rule 1 (line 20):
 -> $$ = nterm start ()
 Stack now 0
 Entering state 1
-Reading a token: Next token is token 'a' (PRINTER)
+Reading a token
+Next token is token 'a' (PRINTER)
 syntax error, unexpected 'a', expecting $end
 Error: popping nterm start ()
 Stack now 0
@@ -1440,7 +1441,8 @@ AT_LAC_CHECK([])
 AT_PARSER_CHECK([[input --debug]], [[2]], [],
 [[Starting parse
 Entering state 0
-Reading a token: Now at end of input.
+Reading a token
+Now at end of input.
 LAC: initial context established for $end
 LAC: checking lookahead $end: R2 G3 R2 G5 R2 G6 R2 G7 R2 G8 R2 G9 R2 G10 R2 
G11 R2 (max size exceeded)
 memory exhausted
@@ -1454,7 +1456,8 @@ AT_LAC_CHECK([z], [[0]])
 AT_PARSER_CHECK([[input --debug]], [[2]], [],
 [[Starting parse
 Entering state 0
-Reading a token: Next token is token $undefined ()
+Reading a token
+Next token is token $undefined ()
 LAC: initial context established for $undefined
 LAC: checking lookahead $undefined: Always Err
 Constructing syntax error message
-- 
2.25.0