bug#24358: 25.1.50; re-search-forward errors with "Variable binding depth exceeds max-specpdl-size"

[Sam Halliday]

Sorry Eli, this patch doesn't apply cleanly for me. I'm on the tip of
emacs-25 and I get failures in dired.c, regex.c and search.c that
aren't easy for me to resolve.

I'm happy to keep being guinea pig so you can fix this, but as a
workaround so that I can have a more stable Emacs to get my main work
done, is there a way for me to force the use of a less jumpy malloc
implementation? It sounds like whatever happened in ArchLinux has
meant that Emacs now prefers ralloc whereas previously it may not.

On 23 October 2016 at 20:32, Eli Zaretskii <eliz@gnu.org> wrote:
>> Date: Sun, 23 Oct 2016 22:10:55 +0300
>> From: Eli Zaretskii <eliz@gnu.org>
>> Cc: 24358@debbugs.gnu.org, npostavs@users.sourceforge.net
>>
>> The problem is to find the code which causes this.  Hmm...
>
> Sam, the patch below removes the changes done by Noam in an attempt to
> solve the original report of this bug.  Please apply it, and see if
> doing so allows you to complete your bootstrap without any errors.
>
> Noam, do you configure Emacs with --enable-check-lisp-object-type?
> If not, please do, because that's what Sam does; perhaps that is the
> reason why he gets the problem and you don't.
>
> One other difference is that Sam is running a bootstrap, so his
> bootstrap-emacs used to build ja-dic has the necessary Lisp files
> loaded as *.el files, not *.elc.  Therefore another thing to try
> (perhaps even before reconfiguring) is a full bootstrap, preferably in
> a separate clean directory, maybe you will see this problem then.
>
> Your systems seem to be similar, so I'd expect you both to see the
> same problems.
>
> Here's the patch to remove Noam's changes and Paul's follow-up
> changes:
>
> diff --git a/src/dired.c b/src/dired.c
> index 006f74c..dba575c 100644
> --- a/src/dired.c
> +++ b/src/dired.c
> @@ -259,11 +259,9 @@ directory_files_internal (Lisp_Object directory, 
> Lisp_Object full,
>        QUIT;
>
>        bool wanted = (NILP (match)
> -                    || (re_match_object = name,
> -                         re_search (bufp, SSDATA (name), len, 0, len, 0) >= 
> 0));
> +                    || re_search (bufp, SSDATA (name), len, 0, len, 0) >= 0);
>
>        immediate_quit = 0;
> -      re_match_object = Qnil;   /* Stop protecting name from GC.  */
>
>        if (wanted)
>         {
> diff --git a/src/regex.c b/src/regex.c
> index b12e95b..164eb46 100644
> --- a/src/regex.c
> +++ b/src/regex.c
> @@ -152,8 +152,6 @@
>
>  /* Converts the pointer to the char to BEG-based offset from the start.  */
>  # define PTR_TO_OFFSET(d) POS_AS_IN_BUFFER (POINTER_TO_OFFSET (d))
> -/* Strings are 0-indexed, buffers are 1-indexed; we pun on the boolean
> -   result to get the right base index.  */
>  # define POS_AS_IN_BUFFER(p) ((p) + (NILP (re_match_object) || BUFFERP 
> (re_match_object)))
>
>  # define RE_MULTIBYTE_P(bufp) ((bufp)->multibyte)
> @@ -1438,62 +1436,11 @@ typedef struct
>  #define NEXT_FAILURE_HANDLE(h) fail_stack.stack[(h) - 3].integer
>  #define TOP_FAILURE_HANDLE() fail_stack.frame
>
> -#ifdef emacs
> -# define STR_BASE_PTR(obj)                      \
> -   (NILP (obj) ? current_buffer->text->beg      \
> -    : STRINGP (obj) ? SDATA (obj)               \
> -    : NULL)
> -#else
> -# define STR_BASE_PTR(obj) NULL
> -#endif
>
>  #define ENSURE_FAIL_STACK(space)                                       \
>  while (REMAINING_AVAIL_SLOTS <= space) {                               \
> -  re_char *orig_base = STR_BASE_PTR (re_match_object);                  \
> -  bool might_relocate = orig_base != NULL;                             \
> -  ptrdiff_t string1_off, end1_off, end_match_1_off;                     \
> -  ptrdiff_t string2_off, end2_off, end_match_2_off;                     \
> -  ptrdiff_t d_off, dend_off, dfail_off;                                 \
> -  if (might_relocate)                                                  \
> -    {                                                                   \
> -      if (string1)                                                      \
> -        {                                                               \
> -          string1_off = string1 - orig_base;                            \
> -          end1_off = end1 - orig_base;                                  \
> -          end_match_1_off = end_match_1 - orig_base;                    \
> -        }                                                               \
> -      if (string2)                                                      \
> -        {                                                               \
> -          string2_off = string2 - orig_base;                            \
> -          end2_off = end2 - orig_base;                                  \
> -          end_match_2_off = end_match_2 - orig_base;                    \
> -        }                                                               \
> -      d_off = d - orig_base;                                            \
> -      dend_off = dend - orig_base;                                      \
> -      dfail_off = dfail - orig_base;                                    \
> -    }                                                                   \
>    if (!GROW_FAIL_STACK (fail_stack))                                   \
>      return -2;                                                         \
> -  /* In Emacs, GROW_FAIL_STACK might relocate string pointers.  */     \
> -  if (might_relocate)                                                  \
> -    {                                                                   \
> -      re_char *new_base = STR_BASE_PTR (re_match_object);              \
> -      if (string1)                                                      \
> -        {                                                               \
> -          string1 = new_base + string1_off;                             \
> -          end1 = new_base + end1_off;                                   \
> -          end_match_1 = new_base + end_match_1_off;                     \
> -        }                                                               \
> -      if (string2)                                                      \
> -        {                                                               \
> -          string2 = new_base + string2_off;                             \
> -          end2 = new_base + end2_off;                                   \
> -          end_match_2 = new_base + end_match_2_off;                     \
> -        }                                                               \
> -      d = new_base + d_off;                                             \
> -      dend = new_base + dend_off;                                       \
> -      dfail = new_base + dfail_off;                                     \
> -    }                                                                   \
>    DEBUG_PRINT ("\n  Doubled stack; size now: %zd\n", (fail_stack).size);\
>    DEBUG_PRINT ("        slots available: %zd\n", REMAINING_AVAIL_SLOTS);\
>  }
> @@ -4380,10 +4327,6 @@ re_search_2 (struct re_pattern_buffer *bufp, const 
> char *str1, size_t size1,
>    /* Loop through the string, looking for a place to start matching.  */
>    for (;;)
>      {
> -      ptrdiff_t offset1, offset2;
> -      re_char *orig_base;
> -      bool might_relocate;
> -
>        /* If the pattern is anchored,
>          skip quickly past places we cannot match.
>          We don't bother to treat startpos == 0 specially
> @@ -4500,17 +4443,6 @@ re_search_2 (struct re_pattern_buffer *bufp, const 
> char *str1, size_t size1,
>           && !bufp->can_be_null)
>         return -1;
>
> -      /* re_match_2_internal may allocate, relocating the Lisp text
> -        object that we're searching.  */
> -      IF_LINT (offset2 = 0);  /* Work around GCC bug 78081.  */
> -      orig_base = STR_BASE_PTR (re_match_object);
> -      might_relocate = orig_base != NULL;
> -      if (might_relocate)
> -        {
> -          if (string1) offset1 = string1 - orig_base;
> -          if (string2) offset2 = string2 - orig_base;
> -        }
> -
>        val = re_match_2_internal (bufp, string1, size1, string2, size2,
>                                  startpos, regs, stop);
>
> @@ -4520,13 +4452,6 @@ re_search_2 (struct re_pattern_buffer *bufp, const 
> char *str1, size_t size1,
>        if (val == -2)
>         return -2;
>
> -      if (might_relocate)
> -        {
> -         re_char *new_base = STR_BASE_PTR (re_match_object);
> -          if (string1) string1 = offset1 + new_base;
> -          if (string2) string2 = offset2 + new_base;
> -        }
> -
>      advance:
>        if (!range)
>         break;
> @@ -4962,8 +4887,8 @@ WEAK_ALIAS (__re_match, re_match)
>  #endif /* not emacs */
>
>  #ifdef emacs
> -/* In Emacs, this is the string or buffer in which we are matching.
> -   See the declaration in regex.h for details.  */
> +/* In Emacs, this is the string or buffer in which we
> +   are matching.  It is used for looking up syntax properties.  */
>  Lisp_Object re_match_object;
>  #endif
>
> diff --git a/src/search.c b/src/search.c
> index 5c04916..dc7e2d8 100644
> --- a/src/search.c
> +++ b/src/search.c
> @@ -287,10 +287,8 @@ looking_at_1 (Lisp_Object string, bool posix)
>    immediate_quit = 1;
>    QUIT;                        /* Do a pending quit right away, to avoid 
> paradoxical behavior */
>
> -  /* Get pointers and sizes of the two strings that make up the
> -     visible portion of the buffer.  Note that we can use pointers
> -     here, unlike in search_buffer, because we only call re_match_2
> -     once, after which we never use the pointers again.  */
> +  /* Get pointers and sizes of the two strings
> +     that make up the visible portion of the buffer. */
>
>    p1 = BEGV_ADDR;
>    s1 = GPT_BYTE - BEGV_BYTE;
> @@ -409,7 +407,6 @@ string_match_1 (Lisp_Object regexp, Lisp_Object string, 
> Lisp_Object start,
>                    (NILP (Vinhibit_changing_match_data)
>                     ? &search_regs : NULL));
>    immediate_quit = 0;
> -  re_match_object = Qnil;       /* Stop protecting string from GC.  */
>
>    /* Set last_thing_searched only when match data is changed.  */
>    if (NILP (Vinhibit_changing_match_data))
> @@ -480,7 +477,6 @@ fast_string_match_internal (Lisp_Object regexp, 
> Lisp_Object string,
>                    SBYTES (string), 0,
>                    SBYTES (string), 0);
>    immediate_quit = 0;
> -  re_match_object = Qnil;       /* Stop protecting string from GC.  */
>    return val;
>  }
>
> @@ -568,7 +564,6 @@ fast_looking_at (Lisp_Object regexp, ptrdiff_t pos, 
> ptrdiff_t pos_byte,
>    len = re_match_2 (buf, (char *) p1, s1, (char *) p2, s2,
>                     pos_byte, NULL, limit_byte);
>    immediate_quit = 0;
> -  re_match_object = Qnil;       /* Stop protecting string from GC.  */
>
>    return len;
>  }
> @@ -1183,8 +1178,8 @@ search_buffer (Lisp_Object string, ptrdiff_t pos, 
> ptrdiff_t pos_byte,
>
>    if (RE && !(trivial_regexp_p (string) && NILP (Vsearch_spaces_regexp)))
>      {
> -      unsigned char *base;
> -      ptrdiff_t off1, off2, s1, s2;
> +      unsigned char *p1, *p2;
> +      ptrdiff_t s1, s2;
>        struct re_pattern_buffer *bufp;
>
>        bufp = compile_pattern (string,
> @@ -1198,19 +1193,16 @@ search_buffer (Lisp_Object string, ptrdiff_t pos, 
> ptrdiff_t pos_byte,
>                                    can take too long. */
>        QUIT;                    /* Do a pending quit right away,
>                                    to avoid paradoxical behavior */
> -      /* Get offsets and sizes of the two strings that make up the
> -         visible portion of the buffer.  We compute offsets instead of
> -         pointers because re_search_2 may call malloc and therefore
> -         change the buffer text address.  */
> +      /* Get pointers and sizes of the two strings
> +        that make up the visible portion of the buffer. */
>
> -      base = current_buffer->text->beg;
> -      off1 = BEGV_ADDR - base;
> +      p1 = BEGV_ADDR;
>        s1 = GPT_BYTE - BEGV_BYTE;
> -      off2 = GAP_END_ADDR - base;
> +      p2 = GAP_END_ADDR;
>        s2 = ZV_BYTE - GPT_BYTE;
>        if (s1 < 0)
>         {
> -          off2 = off1;
> +         p2 = p1;
>           s2 = ZV_BYTE - BEGV_BYTE;
>           s1 = 0;
>         }
> @@ -1225,16 +1217,12 @@ search_buffer (Lisp_Object string, ptrdiff_t pos, 
> ptrdiff_t pos_byte,
>         {
>           ptrdiff_t val;
>
> -          val = re_search_2 (bufp,
> -                             (char*) (base + off1), s1,
> -                             (char*) (base + off2), s2,
> +         val = re_search_2 (bufp, (char *) p1, s1, (char *) p2, s2,
>                              pos_byte - BEGV_BYTE, lim_byte - pos_byte,
>                              (NILP (Vinhibit_changing_match_data)
>                               ? &search_regs : &search_regs_1),
>                              /* Don't allow match past current point */
>                              pos_byte - BEGV_BYTE);
> -         /* Update 'base' due to possible relocation inside re_search_2.  */
> -         base = current_buffer->text->beg;
>           if (val == -2)
>             {
>               matcher_overflow ();
> @@ -1274,15 +1262,11 @@ search_buffer (Lisp_Object string, ptrdiff_t pos, 
> ptrdiff_t pos_byte,
>         {
>           ptrdiff_t val;
>
> -          val = re_search_2 (bufp,
> -                             (char*) (base + off1), s1,
> -                             (char*) (base + off2), s2,
> -                             pos_byte - BEGV_BYTE, lim_byte - pos_byte,
> +         val = re_search_2 (bufp, (char *) p1, s1, (char *) p2, s2,
> +                            pos_byte - BEGV_BYTE, lim_byte - pos_byte,
>                              (NILP (Vinhibit_changing_match_data)
>                               ? &search_regs : &search_regs_1),
>                              lim_byte - BEGV_BYTE);
> -         /* Update 'base' due to possible relocation inside re_search_2.  */
> -         base = current_buffer->text->beg;
>           if (val == -2)
>             {
>               matcher_overflow ();