Re: ttyname_r on Solaris

[Bruno Haible]

> 2010-04-25  Bruno Haible  <address@hidden>
> 
>       ttyname_r: Make it work on MacOS X 10.4 and Solaris 10.

On Solaris, there are two additional complications:
  1) When combined with the 'extensions' module, _POSIX_PTHREAD_SEMANTICS gets
     defined, and the system function that returns 'int' becomes visible. This
     causes ttyname_r.m4 to determine that the function needs not be replaced.
     But then the unit test fails, due to:
  2) The system ttyname_r functions (both of them!) fail if the buffer is less
     than 128 bytes large. I.e. even if the result would be only 10 bytes long,
     the function refuses to produce the result if the provided buffer is
     smaller than 128 bytes.

This fixes both problems:


2010-04-25  Bruno Haible  <address@hidden>

        ttyname_r: Make it work on Solaris 10.
        * m4/ttyname_r.m4 (gl_FUNC_TTYNAME_R): Define HAVE_POSIXDECL_TTYNAME_R
        if the system function has the POSIX declaration. Test whether the
        function fails if the buffer is less than 128 bytes large.
        * lib/ttyname_r.c (ttyname_r): Handle both possible declarations of the
        system's ttyname_r function. Provide a reasonably large buffer.
        * modules/ttyname_r (Depends-on): Add extensions.
        * doc/posix-functions/ttyname_r.texi: Mention the Solaris problem.

--- doc/posix-functions/ttyname_r.texi.orig     Mon Apr 26 00:02:07 2010
+++ doc/posix-functions/ttyname_r.texi  Sun Apr 25 23:36:52 2010
@@ -14,6 +14,10 @@
 @item
 This function has an incompatible declaration on some platforms:
 MacOS X 10.4, Solaris 10 (when @code{_POSIX_PTHREAD_SEMANTICS} is not defined).
address@hidden
+This function refuses to do anything when the output buffer is less than 128
+bytes large, on some platforms:
+Solaris 10.
 @end itemize
 
 Portability problems not fixed by Gnulib:
--- lib/ttyname_r.c.orig        Mon Apr 26 00:02:07 2010
+++ lib/ttyname_r.c     Sun Apr 25 23:55:03 2010
@@ -29,12 +29,31 @@
 ttyname_r (int fd, char *buf, size_t buflen)
 #undef ttyname_r
 {
-  /* When ttyname_r exists and works, use it.
-     But on Solaris 10, ttyname_r is broken: it returns NULL in situations
-     when ttyname finds the result.  */
-#if HAVE_TTYNAME_R && !defined __sun
+  /* When ttyname_r exists, use it.  */
+#if HAVE_TTYNAME_R
   /* This code is multithread-safe.  */
-  char *name = ttyname_r (fd, buf, buflen <= INT_MAX ? buflen : INT_MAX);
+  /* On Solaris, ttyname_r always fails if buflen < 128.  So provide a buffer
+     that is large enough.  */
+  char largerbuf[512];
+# if HAVE_POSIXDECL_TTYNAME_R
+  int err =
+    (buflen < sizeof (largerbuf)
+     ? ttyname_r (fd, largerbuf, sizeof (largerbuf))
+     : ttyname_r (fd, buf, buflen <= INT_MAX ? buflen : INT_MAX));
+  if (err != 0)
+    return err;
+  if (buflen < sizeof (largerbuf))
+    {
+      size_t namelen = strlen (largerbuf);
+      if (namelen > buflen)
+        return ERANGE;
+      memcpy (buf, largerbuf, namelen);
+    }
+# else
+  char *name =
+    (buflen < sizeof (largerbuf)
+     ? ttyname_r (fd, largerbuf, sizeof (largerbuf))
+     : ttyname_r (fd, buf, buflen <= INT_MAX ? buflen : INT_MAX));
   if (name == NULL)
     return errno;
   if (name != buf)
@@ -44,6 +63,7 @@
         return ERANGE;
       memmove (buf, name, namelen);
     }
+# endif
   return 0;
 #elif HAVE_TTYNAME
   /* Note: This is not multithread-safe.  */
--- m4/ttyname_r.m4.orig        Mon Apr 26 00:02:07 2010
+++ m4/ttyname_r.m4     Sun Apr 25 23:46:58 2010
@@ -1,4 +1,4 @@
-# ttyname_r.m4 serial 2
+# ttyname_r.m4 serial 3
 dnl Copyright (C) 2010 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -8,11 +8,16 @@
 [
   AC_REQUIRE([gl_UNISTD_H_DEFAULTS])
 
+  dnl Persuade Solaris <unistd.h> to provide the POSIX compliant declaration of
+  dnl ttyname_r().
+  AC_REQUIRE([gl_USE_SYSTEM_EXTENSIONS])
+
   AC_CHECK_FUNCS([ttyname_r])
   if test $ac_cv_func_ttyname_r = no; then
     HAVE_TTYNAME_R=0
   else
-    dnl On MacOS X 10.4 and Solaris 10 the return type is 'char *', not 'int'.
+    dnl On MacOS X 10.4 (and Solaris 10 without gl_USE_SYSTEM_EXTENSIONS)
+    dnl the return type is 'char *', not 'int'.
     AC_CACHE_CHECK([whether ttyname_r is compatible with its POSIX signature],
       [gl_cv_func_ttyname_r_posix],
       [AC_COMPILE_IFELSE(
@@ -26,6 +31,47 @@
       ])
     if test $gl_cv_func_ttyname_r_posix = no; then
       REPLACE_TTYNAME_R=1
+    else
+      AC_DEFINE([HAVE_POSIXDECL_TTYNAME_R], [1],
+        [Define if the ttyname_r function has a POSIX compliant declaration.])
+      dnl On Solaris 10, both ttyname_r functions (the one with the non-POSIX
+      dnl declaration and the one with the POSIX declaration) refuse to do
+      dnl anything when the output buffer is less than 128 bytes large.
+      AC_REQUIRE([AC_CANONICAL_HOST])
+      AC_CACHE_CHECK([whether ttyname_r works with small buffers],
+        [gl_cv_func_ttyname_r_works],
+        [
+          dnl Initial guess, used when cross-compiling or when /dev/tty cannot
+          dnl be opened.
+changequote(,)dnl
+          case "$host_os" in
+                      # Guess no on Solaris.
+            solaris*) gl_cv_func_ttyname_r_works="guessing no" ;;
+                      # Guess yes otherwise.
+            *)        gl_cv_func_ttyname_r_works="guessing yes" ;;
+          esac
+changequote([,])dnl
+          AC_TRY_RUN([
+#include <fcntl.h>
+#include <unistd.h>
+int
+main (void)
+{
+  int fd;
+  char buf[31]; /* use any size < 128 here */
+
+  fd = open ("/dev/tty", O_RDONLY);
+  if (fd < 0)
+    return 1;
+  if (ttyname_r (fd, buf, sizeof (buf)) != 0)
+    return 1;
+  return 0;
+}], [gl_cv_func_ttyname_r_works=yes], [:], [:])
+        ])
+      case "$gl_cv_func_ttyname_r_works" in
+        *yes) ;;
+        *) REPLACE_TTYNAME_R=1 ;;
+      esac
     fi
   fi
   if test $HAVE_TTYNAME_R = 0 || test $REPLACE_TTYNAME_R = 1; then
--- modules/ttyname_r.orig      Mon Apr 26 00:02:07 2010
+++ modules/ttyname_r   Sun Apr 25 22:41:05 2010
@@ -7,6 +7,7 @@
 
 Depends-on:
 unistd
+extensions
 
 configure.ac:
 gl_FUNC_TTYNAME_R