another dup2 mingw failure

[Simon Josefsson]

There is another dup2 failure due to Wine, see:

http://bugs.winehq.org/show_bug.cgi?id=21291

The patch below works around it.  Thoughts?

/Simon

2010-01-09  Simon Josefsson  <address@hidden>

        * lib/dup2.c (rpl_dup2): Restore text mode when needed, to work
        around Wine bug #21291.

diff --git a/lib/dup2.c b/lib/dup2.c
index a4422bf..946acdb 100644
--- a/lib/dup2.c
+++ b/lib/dup2.c
@@ -35,11 +35,17 @@
 
 # undef dup2
 
+#if !O_BINARY
+# define setmode(f,m) zero ()
+static int zero (void) { return 0; }
+#endif
+
 int
 rpl_dup2 (int fd, int desired_fd)
 {
   int result;
 # if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
+  int fd_mode = -1;
   /* If fd is closed, mingw hangs on dup2 (fd, fd).  If fd is open,
      dup2 (fd, fd) returns 0, but all further attempts to use fd in
      future dup2 calls will hang.  */
@@ -59,6 +65,14 @@ rpl_dup2 (int fd, int desired_fd)
       errno = EBADF;
       return -1;
     }
+  /* Wine 1.0.1 puts desired_fd into binary mode when fd is in text
+     mode, so we save the old mode here.
+     http://bugs.winehq.org/show_bug.cgi?id=21291 */
+  if ((HANDLE) _get_osfhandle (fd) != (HANDLE) -1)
+    {
+      fd_mode = setmode (fd, O_BINARY);
+      setmode (fd, fd_mode);
+    }
 # endif
   result = dup2 (fd, desired_fd);
 # ifdef __linux__
@@ -80,6 +94,12 @@ rpl_dup2 (int fd, int desired_fd)
   if (fd != desired_fd && result != -1)
     result = _gl_register_dup (fd, result);
 # endif
+# if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
+  /* Restore text mode if needed.
+     http://bugs.winehq.org/show_bug.cgi?id=21291 */
+  if (result != -1 && fd_mode != -1)
+    setmode (desired_fd, fd_mode);
+# endif
   return result;
 }