[GNUnet-SVN] [gnunet] branch master updated: Implement asynchronous scope identifiers.

[gnunet]

This is an automated email from the git hooks/post-receive script.

dold pushed a commit to branch master
in repository gnunet.

The following commit(s) were added to refs/heads/master by this push:
     new c1f06ac28 Implement asynchronous scope identifiers.
c1f06ac28 is described below

commit c1f06ac28fef5498952994e0399312b1fc567f22
Author: Florian Dold <address@hidden>
AuthorDate: Fri May 10 20:08:01 2019 +0200

    Implement asynchronous scope identifiers.
    
    Without entering an asynchronous scope, logs are the same before.  When
    entering an asynchronous scope (either thread-based of
    scheduler/task-based), all log lines within an asynchronous scope
    contain its ID.
    
    Currently this is only used in GNU Taler, for debugging requests across
    multiple services.  This allows us to get all log lines pertaining to a
    particular request for a user or another service.
---
 src/curl/curl.c                    | 89 ++++++++++++++++++++++++++++++++------
 src/include/gnunet_common.h        | 64 +++++++++++++++++++++++++++
 src/include/gnunet_curl_lib.h      | 10 +++++
 src/include/gnunet_scheduler_lib.h | 20 +++++++++
 src/util/common_logging.c          | 83 +++++++++++++++++++++++++++++++++++
 src/util/scheduler.c               | 43 +++++++++++++++++-
 6 files changed, 295 insertions(+), 14 deletions(-)

diff --git a/src/curl/curl.c b/src/curl/curl.c
index e13c1478f..871401c60 100644
--- a/src/curl/curl.c
+++ b/src/curl/curl.c
@@ -105,6 +105,18 @@ struct GNUNET_CURL_Job
    * Buffer for response received from CURL.
    */
   struct GNUNET_CURL_DownloadBuffer db;
+
+  /**
+   * Headers used for this job, the list needs to be freed
+   * after the job has finished.
+   */
+  struct curl_slist *job_headers;
+
+  /**
+   * Header for the async scope id or NULL.
+   */
+  char *aid_header;
+
 };
 
 
@@ -134,10 +146,15 @@ struct GNUNET_CURL_Context
   struct GNUNET_CURL_Job *jobs_tail;
 
   /**
-   * HTTP header "application/json", created once and used
-   * for all requests that need it.
+   * Headers common for all requests in the context.
+   */
+  struct curl_slist *common_headers;
+
+  /**
+   * If non-NULL, the async scope ID is sent in a request
+   * header of this name.
    */
-  struct curl_slist *json_header;
+  const char *async_scope_id_header;
 
   /**
    * Function we need to call whenever the event loop's
@@ -149,6 +166,7 @@ struct GNUNET_CURL_Context
    * Closure for @e cb.
    */
   void *cb_cls;
+
 };
 
 
@@ -189,13 +207,23 @@ GNUNET_CURL_init (GNUNET_CURL_RescheduleCallback cb, void 
*cb_cls)
   ctx->cb_cls = cb_cls;
   ctx->multi = multi;
   ctx->share = share;
-  GNUNET_assert (
-    NULL != (ctx->json_header =
-               curl_slist_append (NULL, "Content-Type: application/json")));
   return ctx;
 }
 
 
+/**
+ * Enable sending the async scope ID as a header.
+ *
+ * @param ctx the context to enable this for
+ * @param header_name name of the header to send.
+ */
+void
+GNUNET_CURL_enable_async_scope_header (struct GNUNET_CURL_Context *ctx, const 
char *header_name)
+{
+  ctx->async_scope_id_header = header_name;
+}
+
+
 /**
  * Callback used when downloading the reply to an HTTP request.
  * Just appends all of the data to the `buf` in the
@@ -259,16 +287,49 @@ GNUNET_CURL_job_add (struct GNUNET_CURL_Context *ctx,
                      void *jcc_cls)
 {
   struct GNUNET_CURL_Job *job;
+  struct curl_slist *all_headers = NULL;
+  char *aid_header = NULL;
 
   if (GNUNET_YES == add_json)
-    if (CURLE_OK != curl_easy_setopt (eh, CURLOPT_HTTPHEADER, 
ctx->json_header))
+  {
+    GNUNET_assert (
+      NULL != (all_headers =
+                 curl_slist_append (NULL, "Content-Type: application/json")));
+  }
+
+  for (struct curl_slist *curr = ctx->common_headers;
+       curr != NULL;
+       curr = curr->next)
+  {
+    GNUNET_assert (
+      NULL != (all_headers =
+                 curl_slist_append (all_headers, "Content-Type: 
application/json")));
+  }
+
+  if (NULL != ctx->async_scope_id_header)
+  {
+    struct GNUNET_AsyncScopeSave scope;
+
+    GNUNET_async_scope_get (&scope);
+    if (GNUNET_YES == scope.have_scope)
     {
-      GNUNET_break (0);
-      curl_easy_cleanup (eh);
-      return NULL;
+      aid_header = GNUNET_STRINGS_data_to_string_alloc (&scope.scope_id,
+                                                        sizeof (struct 
GNUNET_AsyncScopeId));
+      GNUNET_assert (NULL != curl_slist_append(all_headers, aid_header));
     }
+  }
+
+  if (CURLE_OK != curl_easy_setopt (eh, CURLOPT_HTTPHEADER, all_headers))
+  {
+    GNUNET_break (0);
+    curl_easy_cleanup (eh);
+    return NULL;
+  }
 
   job = GNUNET_new (struct GNUNET_CURL_Job);
+  job->job_headers = all_headers;
+  job->aid_header = aid_header;
+
   if ((CURLE_OK != curl_easy_setopt (eh, CURLOPT_PRIVATE, job)) ||
       (CURLE_OK !=
        curl_easy_setopt (eh, CURLOPT_WRITEFUNCTION, &download_cb)) ||
@@ -308,6 +369,8 @@ GNUNET_CURL_job_cancel (struct GNUNET_CURL_Job *job)
                 curl_multi_remove_handle (ctx->multi, job->easy_handle));
   curl_easy_cleanup (job->easy_handle);
   GNUNET_free_non_null (job->db.buf);
+  GNUNET_free_non_null (job->aid_header);
+  curl_slist_free_all (job->job_headers);
   GNUNET_free (job);
 }
 
@@ -401,8 +464,8 @@ download_get_result (struct GNUNET_CURL_DownloadBuffer *db,
 int
 GNUNET_CURL_append_header (struct GNUNET_CURL_Context *ctx, const char *header)
 {
-  ctx->json_header = curl_slist_append (ctx->json_header, header);
-  if (NULL == ctx->json_header)
+  ctx->common_headers = curl_slist_append (ctx->common_headers, header);
+  if (NULL == ctx->common_headers)
     return GNUNET_SYSERR;
 
   return GNUNET_OK;
@@ -599,7 +662,7 @@ GNUNET_CURL_fini (struct GNUNET_CURL_Context *ctx)
   GNUNET_assert (NULL == ctx->jobs_head);
   curl_share_cleanup (ctx->share);
   curl_multi_cleanup (ctx->multi);
-  curl_slist_free_all (ctx->json_header);
+  curl_slist_free_all (ctx->common_headers);
   GNUNET_free (ctx);
 }
 
diff --git a/src/include/gnunet_common.h b/src/include/gnunet_common.h
index 2c40aef79..1f5600381 100644
--- a/src/include/gnunet_common.h
+++ b/src/include/gnunet_common.h
@@ -319,8 +319,34 @@ struct GNUNET_OperationResultMessage
   /* Followed by data. */
 };
 
+
+/**
+ * Identifier for an asynchronous execution context.
+ */
+struct GNUNET_AsyncScopeId
+{
+  uint32_t bits[16 / sizeof (uint32_t)];  /* = 16 bytes */
+};
+
 GNUNET_NETWORK_STRUCT_END
 
+
+/**
+ * Saved async scope identifier or root scope.
+ */
+struct GNUNET_AsyncScopeSave {
+  /**
+   * Saved scope.  Unused if 'have_scope==GNUNET_NO'.
+   */
+  struct GNUNET_AsyncScopeId scope_id;
+
+  /**
+   * GNUNET_YES unless this saved scope is the unnamed root scope.
+   */
+  int have_scope;
+};
+
+
 /**
  * Function called with a filename.
  *
@@ -1331,6 +1357,44 @@ struct GNUNET_MessageHeader *
 GNUNET_copy_message (const struct GNUNET_MessageHeader *msg);
 
 
+/**
+ * Set the async scope for the current thread.
+ *
+ * @param aid the async scope identifier
+ * @param old_scope[out] location to save the old scope
+ */
+void
+GNUNET_async_scope_enter (const struct GNUNET_AsyncScopeId *aid,
+                          struct GNUNET_AsyncScopeSave *old_scope);
+
+
+/**
+ * Clear the current thread's async scope.
+ *
+ * @param old_scope scope to restore
+ */
+void
+GNUNET_async_scope_restore (struct GNUNET_AsyncScopeSave *old_scope);
+
+
+/**
+ * Get the current async scope.
+ *
+ * @param[out] scope_ret pointer to where the result is stored
+ */
+void
+GNUNET_async_scope_get (struct GNUNET_AsyncScopeSave *scope_ret);
+
+
+/**
+ * Generate a fresh async scope identifier.
+ *
+ * @param[out] aid_ret pointer to where the result is stored
+ */
+void
+GNUNET_async_scope_fresh (struct GNUNET_AsyncScopeId *aid_ret);
+
+
 #if __STDC_VERSION__ < 199901L
 #if __GNUC__ >= 2
 #define __func__ __FUNCTION__
diff --git a/src/include/gnunet_curl_lib.h b/src/include/gnunet_curl_lib.h
index 86cb54f6b..84c8d59f8 100644
--- a/src/include/gnunet_curl_lib.h
+++ b/src/include/gnunet_curl_lib.h
@@ -294,6 +294,16 @@ void
 GNUNET_CURL_gnunet_scheduler_reschedule (void *cls);
 
 
+/**
+ * Enable sending the async scope ID as a header.
+ *
+ * @param ctx the context to enable this for
+ * @param header_name name of the header to send.
+ */
+void
+GNUNET_CURL_enable_async_scope_header (struct GNUNET_CURL_Context *ctx, const 
char *header_name);
+
+
 #endif
 /** @} */  /* end of group */
 
diff --git a/src/include/gnunet_scheduler_lib.h 
b/src/include/gnunet_scheduler_lib.h
index 3f4e3c9a8..3f0350f54 100644
--- a/src/include/gnunet_scheduler_lib.h
+++ b/src/include/gnunet_scheduler_lib.h
@@ -214,6 +214,8 @@ struct GNUNET_SCHEDULER_TaskContext
 };
 
 
+
+
 /**
  * Function used by event-loop implementations to signal the scheduler
  * that a particular @a task is ready due to an event specified in the
@@ -904,6 +906,24 @@ GNUNET_SCHEDULER_set_select (GNUNET_SCHEDULER_select 
new_select,
                              void *new_select_cls);
 
 
+
+/**
+ * Change the async scope for the currently executing task and (transitively)
+ * for all tasks scheduled by the current task after calling this function.
+ * Nested tasks can begin their own nested async scope.
+ *
+ * Once the current task is finished, the async scope ID is reset to
+ * its previous value.
+ *
+ * Must only be called from a running task.
+ *
+ * @param aid the asynchronous scope id to enter
+ */
+void
+GNUNET_SCHEDULER_begin_async_scope (struct GNUNET_AsyncScopeId *aid);
+
+
+
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
 #endif
diff --git a/src/util/common_logging.c b/src/util/common_logging.c
index 77a595878..5052134f8 100644
--- a/src/util/common_logging.c
+++ b/src/util/common_logging.c
@@ -95,6 +95,13 @@ struct CustomLogger
   void *logger_cls;
 };
 
+
+/**
+ * Asynchronous scope of the current thread, or NULL if we have not
+ * entered an async scope yet.
+ */
+static __thread struct GNUNET_AsyncScopeSave current_async_scope;
+
 /**
  * The last "bulk" error message that we have been logging.
  * Note that this message maybe truncated to the first BULK_TRACK_SIZE
@@ -838,6 +845,28 @@ output_message (enum GNUNET_ErrorType kind,
                "* %s",
                msg);
     }
+    else if (GNUNET_YES == current_async_scope.have_scope)
+    {
+      static GNUNET_THREAD_LOCAL char id_buf[27];
+      char *end;
+
+      /* We're logging, so skip_log must be currently 0. */
+      skip_log = 100;
+      end = GNUNET_STRINGS_data_to_string (&current_async_scope.scope_id,
+                                           sizeof (struct GNUNET_AsyncScopeId),
+                                           id_buf,
+                                           sizeof (id_buf) - 1);
+      GNUNET_assert (NULL != end);
+      *end = '\0';
+      skip_log = 0;
+      FPRINTF (GNUNET_stderr,
+               "%s %s(%s) %s %s",
+               datestr,
+               comp,
+               id_buf,
+               GNUNET_error_type_to_string (kind),
+               msg);
+    }
     else
     {
       FPRINTF (GNUNET_stderr,
@@ -1547,6 +1576,60 @@ GNUNET_log_config_invalid (enum GNUNET_ErrorType kind,
 }
 
 
+/**
+ * Set the async scope for the current thread.
+ *
+ * @param aid the async scope identifier
+ * @param old_scope[out] location to save the old scope
+ */
+void
+GNUNET_async_scope_enter (const struct GNUNET_AsyncScopeId *aid,
+                          struct GNUNET_AsyncScopeSave *old_scope)
+{
+  *old_scope = current_async_scope;
+  current_async_scope.have_scope = GNUNET_YES;
+  current_async_scope.scope_id = *aid;
+}
+
+
+/**
+ * Clear the current thread's async scope.
+ *
+ * @param old_scope scope to restore
+ */
+void
+GNUNET_async_scope_restore (struct GNUNET_AsyncScopeSave *old_scope)
+{
+  current_async_scope = *old_scope;
+}
+
+
+/**
+ * Generate a fresh async scope identifier.
+ *
+ * @param[out] aid_ret pointer to where the result is stored
+ */
+void
+GNUNET_async_scope_fresh (struct GNUNET_AsyncScopeId *aid_ret)
+{
+  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK,
+                              aid_ret,
+                              sizeof (struct GNUNET_AsyncScopeId));
+}
+
+
+/**
+ * Get the current async scope.
+ *
+ * @param[out] scope_ret pointer to where the result is stored
+ */
+void
+GNUNET_async_scope_get (struct GNUNET_AsyncScopeSave *scope_ret)
+{
+  *scope_ret = current_async_scope;
+}
+
+
 /**
  * Initializer
  */
diff --git a/src/util/scheduler.c b/src/util/scheduler.c
index 2ddbb8c60..c818e91ee 100644
--- a/src/util/scheduler.c
+++ b/src/util/scheduler.c
@@ -241,6 +241,11 @@ struct GNUNET_SCHEDULER_Task
   int num_backtrace_strings;
 #endif
 
+  /**
+   * Asynchronous scope of the task that scheduled this scope,
+   */
+  struct GNUNET_AsyncScopeSave scope;
+
 };
 
 
@@ -1105,6 +1110,7 @@ GNUNET_SCHEDULER_add_at_with_priority (struct 
GNUNET_TIME_Absolute at,
   GNUNET_assert (NULL != scheduler_driver);
   GNUNET_assert (NULL != task);
   t = GNUNET_new (struct GNUNET_SCHEDULER_Task);
+  GNUNET_async_scope_get (&t->scope);
   t->callback = task;
   t->callback_cls = task_cls;
   t->read_fd = -1;
@@ -1293,6 +1299,7 @@ GNUNET_SCHEDULER_add_shutdown 
(GNUNET_SCHEDULER_TaskCallback task,
   GNUNET_assert (NULL != scheduler_driver);
   GNUNET_assert (NULL != task);
   t = GNUNET_new (struct GNUNET_SCHEDULER_Task);
+  GNUNET_async_scope_get (&t->scope);
   t->callback = task;
   t->callback_cls = task_cls;
   t->read_fd = -1;
@@ -1411,6 +1418,7 @@ add_without_sets (struct GNUNET_TIME_Relative delay,
   GNUNET_assert (NULL != scheduler_driver);
   GNUNET_assert (NULL != task);
   t = GNUNET_new (struct GNUNET_SCHEDULER_Task);
+  GNUNET_async_scope_get (&t->scope);
   init_fd_info (t,
                 &read_nh,
                 read_nh ? 1 : 0,
@@ -1882,6 +1890,7 @@ GNUNET_SCHEDULER_add_select (enum 
GNUNET_SCHEDULER_Priority prio,
                                                        task,
                                                        task_cls);
   t = GNUNET_new (struct GNUNET_SCHEDULER_Task);
+  GNUNET_async_scope_get (&t->scope);
   init_fd_info (t,
                 read_nhandles,
                 read_nhandles_len,
@@ -2114,7 +2123,15 @@ GNUNET_SCHEDULER_do_work (struct GNUNET_SCHEDULER_Handle 
*sh)
            "Running task %p\n",
            pos);
       GNUNET_assert (NULL != pos->callback);
-      pos->callback (pos->callback_cls);
+      {
+        struct GNUNET_AsyncScopeSave old_scope;
+        if (pos->scope.have_scope)
+          GNUNET_async_scope_enter (&pos->scope.scope_id, &old_scope);
+        else
+          GNUNET_async_scope_get (&old_scope);
+        pos->callback (pos->callback_cls);
+        GNUNET_async_scope_restore (&old_scope);
+      }
       if (NULL != pos->fds)
       {
         int del_result = scheduler_driver->del (scheduler_driver->cls, pos);
@@ -2513,4 +2530,28 @@ GNUNET_SCHEDULER_driver_select ()
 }
 
 
+/**
+ * Change the async scope for the currently executing task and (transitively)
+ * for all tasks scheduled by the current task after calling this function.
+ * Nested tasks can begin their own nested async scope.
+ *
+ * Once the current task is finished, the async scope ID is reset to
+ * its previous value.
+ *
+ * Must only be called from a running task.
+ *
+ * @param aid the asynchronous scope id to enter
+ */
+void
+GNUNET_SCHEDULER_begin_async_scope (struct GNUNET_AsyncScopeId *aid)
+{
+  struct GNUNET_AsyncScopeSave dummy_old_scope;
+
+  GNUNET_assert (NULL != active_task);
+  /* Since we're in a task, the context will be automatically
+     restored by the scheduler. */
+  GNUNET_async_scope_enter (aid, &dummy_old_scope);
+}
+
+
 /* end of scheduler.c */

-- 
To stop receiving notification emails like this one, please contact
address@hidden