yacc: restructure and fix yysyntax_error

[Akim Demaille]

commit f983d00e77868baca64eaeceb05ee0b1e28a1f68
Author: Akim Demaille <address@hidden>
Date:   Tue Dec 31 07:28:59 2019 +0100

   yacc: restructure and fix yysyntax_error

   I would like to offer new ways to build the error message.  As a first
   step, let's simplify yysyntax_error whose first loop does two things
   at the same time: (i) collect the tokens to be reported in the error
   message, and (ii) accumulate their sizes and possibly return
   "overflow".  Let's pull (ii) in a second step.

   Then test 525 (regression.at:1193: parse.error=verbose overflow)
   failed.  This test checks that we correctly report "memory overflow"
   when the error message is too large.  However the test is mistaken: it
   is triggered in a place where there are five (large) expected tokens,
   so anyway we would not display them, so there is no (memory) overflow
   here!  Transform this test to (i) check that indeed there is no
   overflow, and (ii) create syntax_error3 which does check the intended
   behavior, but with four expected tokens.

   * data/skeletons/yacc.c (yysyntax_error): First compute the list of
   arguments, then compute yysize.
   * tests/regression.at (parse.error=verbose overflow): Enhance and fix.

diff --git a/data/skeletons/yacc.c b/data/skeletons/yacc.c
index 0fb9db0f..1f16a733 100644
--- a/data/skeletons/yacc.c
+++ b/data/skeletons/yacc.c
@@ -1184,15 +1184,12 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
  */
  if (yytoken != YYEMPTY)
    {
-      int yyn = yypact[+*yyssp];
-      YYPTRDIFF_T yysize0 = yytnamerr (YY_NULLPTR, yytname[yytoken]);
-      yysize = yysize0;]b4_lac_if([[
+      int yyn = yypact[+*yyssp];]b4_lac_if([[
      YYDPRINTF ((stderr, "Constructing syntax error message\n"));]])[
      yyarg[yycount++] = yytname[yytoken];
      if (!yypact_value_is_default (yyn))
        {]b4_lac_if([[
          int yyx;
-
          for (yyx = 0; yyx < YYNTOKENS; ++yyx)
            if (yyx != YYTERROR && yyx != YYUNDEFTOK)
              {
@@ -1212,7 +1209,6 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
          int yychecklim = YYLAST - yyn + 1;
          int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
          int yyx;
-
          for (yyx = yyxbegin; yyx < yyxend; ++yyx)
            if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR
                && !yytable_value_is_error (yytable[yyx + yyn]))
@@ -1220,18 +1216,9 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
                if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
                  {
                    yycount = 1;
-                    yysize = yysize0;
                    break;
                  }
                yyarg[yycount++] = yytname[yyx];
-                {
-                  YYPTRDIFF_T yysize1
-                    = yysize + yytnamerr (YY_NULLPTR, yytname[yyx]);
-                  if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)
-                    yysize = yysize1;
-                  else
-                    return 2;
-                }
              }
        }]b4_lac_if([[
# if ]b4_api_PREFIX[DEBUG
@@ -1256,14 +1243,19 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
# undef YYCASE_
    }

+  /* Compute error message size.  Don't count the "%s"s, but reserve
+     room for the terminator.  */
+  yysize = (yystrlen (yyformat) - 2 * yycount) + 1;
  {
-    /* Don't count the "%s"s in the final size, but reserve room for
-       the terminator.  */
-    YYPTRDIFF_T yysize1 = yysize + (yystrlen (yyformat) - 2 * yycount) + 1;
-    if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)
-      yysize = yysize1;
-    else
-      return 2;
+    int yyi;
+    for (yyi = 0; yyi < yycount; ++yyi)
+      {
+        YYPTRDIFF_T yysize1 = yysize + yytnamerr (YY_NULLPTR, yyarg[yyi]);
+        if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)
+          yysize = yysize1;
+        else
+          return 2;
+      }
  }

  if (*yymsg_alloc < yysize)
diff --git a/tests/regression.at b/tests/regression.at
index a1a85e1f..bbb5634a 100644
--- a/tests/regression.at
+++ b/tests/regression.at
@@ -1219,7 +1219,7 @@ AT_DATA_GRAMMAR([input.y],

%%

-start: syntax_error1 check syntax_error2 ;
+start: syntax_error1 check syntax_error2 check syntax_error3;

// Induce a syntax error message whose total length causes yymsg in
// yyparse to be reallocated to size YYSTACK_ALLOC_MAXIMUM, which
@@ -1251,13 +1251,26 @@ check:
}
;

-// Now overflow.
+// We used to overflow memory here because the first four "expected"
+// tokens plus rest of the error message is greater that 255 bytes.
+// However there are *five* expected tokens here, so anyway we will
+// *not* display these tokens.  So the message fits, no overflow.
syntax_error2:
  "123456789112345678921234567893123456789412345678951234567896123A"
| "123456789112345678921234567893123456789412345678951234567896123B"
| "123456789112345678921234567893123456789412345678951234567896123C"
| "123456789112345678921234567893123456789412345678951234567896123D"
| "123456789112345678921234567893123456789412345678951234567896123E"
+| error 'd' 'e' 'f'
+;
+
+
+// Now overflow.
+syntax_error3:
+  "123456789112345678921234567893123456789412345678951234567896123A"
+| "123456789112345678921234567893123456789412345678951234567896123B"
+| "123456789112345678921234567893123456789412345678951234567896123C"
+| "123456789112345678921234567893123456789412345678951234567896123D"
;

%%
@@ -1265,7 +1278,7 @@ syntax_error2:
]AT_YYERROR_DEFINE[
  /* Induce two syntax error messages (which requires full error
     recovery by shifting 3 tokens).  */
-]AT_YYLEX_DEFINE(["abc"])[
+]AT_YYLEX_DEFINE(["abcdef"])[
int
main (void)
{
@@ -1288,6 +1301,7 @@ AT_COMPILE([[input]])

AT_PARSER_CHECK([[input]], [[2]], [],
[[syntax error, unexpected 'a', expecting 
123456789112345678921234567893123456789412345678951234567896123A or 
123456789112345678921234567893123456789412345678951234567896123B or 
123456789112345678921234567893123456789412345678951234567896123C
+syntax error, unexpected 'd'
syntax error
memory exhausted
]])