[Bug-gnulib] xalloc.h proposed fix to detect potential ptrdiff_t overflow

[Paul Eggert]

Here is a proposed fix to xalloc.h etc. to detect potential ptrdiff_t
overflow.  I originally coded this to not depend on ptrdiff_max.m4 and
size_max.m4, but the resulting code proved to be a bit hard to read,
so this patch bites the bullet and adds the dependency.

2003-11-18  Paul Eggert  <address@hidden>

        When checking for size_t overflow, check for ptrdiff_t overflow too.
        This resurrects some of Bruno Haible's 2003-11-11 patch.

        * modules/xalloc: Add m4/ptrdiff_max.m4, m4/size_max.m4.
        Depend on minmax.
        * lib/xalloc.h: Include "minmax.h", <limits.h>, <stdint.h>.
        (xalloc_oversized): Reject PTRDIFF_MAX allocations too.
        * m4/xalloc.m4 (gl_PREREQ_XALLOC): Require gl_PTRDIFF_MAX,
        gl_SIZE_MAX.

Index: modules/xalloc
===================================================================
RCS file: /cvsroot/gnulib/gnulib/modules/xalloc,v
retrieving revision 1.9
diff -p -u -r1.9 xalloc
--- modules/xalloc      13 Nov 2003 07:19:10 -0000      1.9
+++ modules/xalloc      18 Nov 2003 21:03:46 -0000
@@ -6,6 +6,8 @@ lib/xalloc.h
 lib/xmalloc.c
 lib/xstrdup.c
 m4/xalloc.m4
+m4/ptrdiff_max.m4
+m4/size_max.m4
 
 Depends-on:
 malloc
@@ -14,6 +16,7 @@ error
 gettext
 exitfail
 stdbool
+minmax
 
 configure.ac:
 gl_XALLOC
Index: lib/xalloc.h
===================================================================
RCS file: /cvsroot/gnulib/gnulib/lib/xalloc.h,v
retrieving revision 1.24
diff -p -u -r1.24 xalloc.h
--- lib/xalloc.h        13 Nov 2003 07:19:09 -0000      1.24
+++ lib/xalloc.h        18 Nov 2003 21:03:46 -0000
@@ -21,6 +21,12 @@
 # define XALLOC_H_
 
 # include <stddef.h>
+# include "minmax.h"
+
+# include <limits.h>
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
 
 # ifndef __attribute__
 #  if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 8) || __STRICT_ANSI__
@@ -59,20 +65,24 @@ void *x2nrealloc (void *p, size_t *pn, s
 void *xclone (void const *p, size_t s);
 char *xstrdup (const char *str);
 
-/* Return 1 if an array of N objects, each of size S, cannot exist due
-   to size arithmetic overflow.  S must be positive and N must be
-   nonnegative.  This is a macro, not an inline function, so that it
-   works correctly even when SIZE_MAX < N.
-
-   By gnulib convention, SIZE_MAX represents overflow in size
-   calculations, so the conservative dividend to use here is
-   SIZE_MAX - 1, since SIZE_MAX might represent an overflowed value.
-   However, malloc (SIZE_MAX) fails on all known hosts where
-   sizeof (ptrdiff_t) <= sizeof (size_t), so do not bother to test for
-   exactly-SIZE_MAX allocations on such hosts; this avoids a test and
-   branch when S is known to be 1.  */
+/* Return 1 if attempting to allocate an array of N objects, each of
+   size S, would cause problems due to arithmetic overflow.  S must be
+   positive and N must be nonnegative.
+
+   Check that the array contains no more than PTRDIFF_MAX - 1 bytes,
+   to avoid undefined behavior when subtracting pointers into the
+   resulting array.
+
+   Check also that the array contains no more than SIZE_MAX - 1 bytes.
+   By gnulib convention, SIZE_MAX represents overflow in size_t
+   calculations, so an N equal to SIZE_MAX might represent an
+   overflowed value.
+
+   This is a macro, not an inline function, so that it works correctly
+   even when SIZE_MAX < N.  */
+
 # define xalloc_oversized(n, s) \
-    ((size_t) (sizeof (ptrdiff_t) <= sizeof (size_t) ? -1 : -2) / (s) < (n))
+    ((size_t) (MIN (PTRDIFF_MAX, SIZE_MAX) - 1) / (s) < (n))
 
 /* These macros are deprecated; they will go away soon, and are retained
    temporarily only to ease conversion to the functions described above.  */
Index: m4/xalloc.m4
===================================================================
RCS file: /cvsroot/gnulib/gnulib/m4/xalloc.m4,v
retrieving revision 1.8
diff -p -u -r1.8 xalloc.m4
--- m4/xalloc.m4        13 Nov 2003 07:19:10 -0000      1.8
+++ m4/xalloc.m4        18 Nov 2003 21:03:46 -0000
@@ -1,4 +1,4 @@
-# xalloc.m4 serial 8
+# xalloc.m4 serial 9
 dnl Copyright (C) 2002-2003 Free Software Foundation, Inc.
 dnl This file is free software, distributed under the terms of the GNU
 dnl General Public License.  As a special exception to the GNU General
@@ -15,6 +15,8 @@ AC_DEFUN([gl_XALLOC],
 
 # Prerequisites of lib/xalloc.h.
 AC_DEFUN([gl_PREREQ_XALLOC], [
+  AC_REQUIRE([gl_PTRDIFF_MAX])
+  AC_REQUIRE([gl_SIZE_MAX])
   :
 ])