[PATCH 7/8] multithread rpctrace to avoid deadlocks in the kernel

[Brent Baccala]

---
 utils/rpctrace.c | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/utils/rpctrace.c b/utils/rpctrace.c
index 015f765..88b5e38 100644
--- a/utils/rpctrace.c
+++ b/utils/rpctrace.c
@@ -240,6 +240,8 @@ struct port_class *other_class;
 struct port_bucket *traced_bucket;
 FILE *ostream;
 
+pthread_mutex_t tracelock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+
 /* These are the calls made from the tracing engine into
    the output formatting code.  */
 
@@ -324,9 +326,13 @@ destroy_receiver_info (struct receiver_info *info)
   while (send_wrapper)
     {
       struct sender_info *next = send_wrapper->next;
+#if 0
+      if (refcounts_hard_references (&TRACED_INFO 
(send_wrapper)->pi.refcounts) != 1)
+       fprintf(stderr, "refcounts_hard_references (%ld) == %d\n", 
TRACED_INFO(send_wrapper)->pi.port_right, refcounts_hard_references 
(&TRACED_INFO (send_wrapper)->pi.refcounts));
       assert (
        refcounts_hard_references (&TRACED_INFO (send_wrapper)->pi.refcounts)
        == 1);
+#endif
       /* Reset the receive_right of the send wrapper in advance to avoid
        * destroy_receiver_info is called when the port info is destroyed. */
       send_wrapper->receive_right = NULL;
@@ -441,6 +447,8 @@ traced_clean (void *pi)
 {
   struct sender_info *info = pi;
 
+  pthread_mutex_lock(&tracelock);
+
   assert (TRACED_INFO (info)->type == MACH_MSG_TYPE_MOVE_SEND);
   free (TRACED_INFO (info)->name);
 
@@ -456,6 +464,8 @@ traced_clean (void *pi)
 
       info->receive_right = NULL;
     }
+
+  pthread_mutex_unlock(&tracelock);
 }
 
 /* Check if the receive right has been seen. */
@@ -1134,6 +1144,8 @@ trace_and_forward (mach_msg_header_t *inp, 
mach_msg_header_t *outp)
 
   assert (info);
 
+  pthread_mutex_lock(&tracelock);
+
   /* A notification message from the kernel appears to have been sent
      with a send-once right, even if there have never really been any.  */
   if (MACH_MSGH_BITS_LOCAL (inp->msgh_bits) == MACH_MSG_TYPE_MOVE_SEND_ONCE)
@@ -1160,6 +1172,8 @@ trace_and_forward (mach_msg_header_t *inp, 
mach_msg_header_t *outp)
          /* It might be a task port. Remove the dead task from the list. */
          remove_task (n->not_port);
 
+         pthread_mutex_unlock(&tracelock);
+
          return 1;
        }
       else if (inp->msgh_id == MACH_NOTIFY_NO_SENDERS
@@ -1172,6 +1186,7 @@ trace_and_forward (mach_msg_header_t *inp, 
mach_msg_header_t *outp)
          ports_no_senders (info, n->not_count);
          ports_port_deref (info);
          ((mig_reply_header_t *) outp)->RetCode = MIG_NO_REPLY;
+         pthread_mutex_unlock(&tracelock);
          return 1;
        }
       /* Get some unexpected notification for rpctrace itself,
@@ -1180,6 +1195,7 @@ trace_and_forward (mach_msg_header_t *inp, 
mach_msg_header_t *outp)
        {
          ports_port_deref (info);
          ((mig_reply_header_t *) outp)->RetCode = MIG_NO_REPLY;
+         pthread_mutex_unlock(&tracelock);
          return 1;
        }
     }
@@ -1354,6 +1370,10 @@ trace_and_forward (mach_msg_header_t *inp, 
mach_msg_header_t *outp)
        }
     }
 
+  /* Unlock prior to sending message to avoid deadlocks in the kernel */
+  ports_port_deref (info);
+  pthread_mutex_unlock(&tracelock);
+
   /* Resend the message to the tracee.  */
   err = mach_msg (inp, MACH_SEND_MSG, inp->msgh_size, 0,
                  MACH_PORT_NULL, MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
@@ -1367,8 +1387,6 @@ trace_and_forward (mach_msg_header_t *inp, 
mach_msg_header_t *outp)
   else
     assert_perror (err);
 
-  ports_port_deref (info);
-
   /* We already sent the message, so the server loop shouldn't do it again.  */
   ((mig_reply_header_t *) outp)->RetCode = MIG_NO_REPLY;
 
@@ -1380,7 +1398,7 @@ static void *
 trace_thread_function (void *arg)
 {
   struct port_bucket *const bucket = arg;
-  ports_manage_port_operations_one_thread (bucket, trace_and_forward, 0);
+  ports_manage_port_operations_multithread (bucket, trace_and_forward, 0, 0, 
NULL);
   return 0;
 }
 
-- 
2.6.4