gnunet-svn
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[GNUnet-SVN] r34769 - gnunet/src/datastore


From: gnunet
Subject: [GNUnet-SVN] r34769 - gnunet/src/datastore
Date: Tue, 23 Dec 2014 21:37:23 +0100

Author: grothoff
Date: 2014-12-23 21:37:22 +0100 (Tue, 23 Dec 2014)
New Revision: 34769

Modified:
   gnunet/src/datastore/gnunet-service-datastore.c
Log:
fixing datastore-statistics interaction to ensure stats are properly written to 
disk on exit

Modified: gnunet/src/datastore/gnunet-service-datastore.c
===================================================================
--- gnunet/src/datastore/gnunet-service-datastore.c     2014-12-23 05:07:02 UTC 
(rev 34768)
+++ gnunet/src/datastore/gnunet-service-datastore.c     2014-12-23 20:37:22 UTC 
(rev 34769)
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet
-     (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other 
contributing authors)
+     (C) 2004-2014 Christian Grothoff (and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -142,19 +142,19 @@
 static struct GNUNET_CONTAINER_BloomFilter *filter;
 
 /**
- * How much space are we allowed to use?
+ * Name of our plugin.
  */
-static unsigned long long quota;
+static char *plugin_name;
 
 /**
- * Should the database be dropped on exit?
+ * Our configuration.
  */
-static int do_drop;
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
 /**
- * Name of our plugin.
+ * Handle for reporting statistics.
  */
-static char *plugin_name;
+static struct GNUNET_STATISTICS_Handle *stats;
 
 /**
  * How much space are we using for the cache?  (space available for
@@ -176,42 +176,47 @@
 static unsigned long long payload;
 
 /**
- * Number of updates that were made to the
- * payload value since we last synchronized
- * it with the statistics service.
+ * Identity of the task that is used to delete
+ * expired content.
  */
-static unsigned int lastSync;
+static GNUNET_SCHEDULER_TaskIdentifier expired_kill_task;
 
 /**
- * Did we get an answer from statistics?
+ * Minimum time that content should have to not be discarded instantly
+ * (time stamp of any content that we've been discarding recently to
+ * stay below the quota).  FOREVER if we had to expire content with
+ * non-zero priority.
  */
-static int stats_worked;
+static struct GNUNET_TIME_Absolute min_expiration;
 
 /**
- * Identity of the task that is used to delete
- * expired content.
+ * How much space are we allowed to use?
  */
-static GNUNET_SCHEDULER_TaskIdentifier expired_kill_task;
+static unsigned long long quota;
 
 /**
- * Our configuration.
+ * Should the database be dropped on exit?
  */
-const struct GNUNET_CONFIGURATION_Handle *cfg;
+static int do_drop;
 
 /**
- * Minimum time that content should have to not be discarded instantly
- * (time stamp of any content that we've been discarding recently to
- * stay below the quota).  FOREVER if we had to expire content with
- * non-zero priority.
+ * Should we refresh the BF when the DB is loaded?
  */
-static struct GNUNET_TIME_Absolute min_expiration;
+static int refresh_bf;
 
 /**
- * Handle for reporting statistics.
+ * Number of updates that were made to the
+ * payload value since we last synchronized
+ * it with the statistics service.
  */
-static struct GNUNET_STATISTICS_Handle *stats;
+static unsigned int last_sync;
 
+/**
+ * Did we get an answer from statistics?
+ */
+static int stats_worked;
 
+
 /**
  * Synchronize our utilization statistics with the
  * statistics service.
@@ -221,11 +226,10 @@
 {
   GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES);
   GNUNET_STATISTICS_set (stats, "# utilization by current datastore", payload, 
GNUNET_NO);
-  lastSync = 0;
+  last_sync = 0;
 }
 
 
-
 /**
  * Context for transmitting replies to clients.
  */
@@ -292,7 +296,8 @@
  * @param tc task context
  */
 static void
-delete_expired (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+delete_expired (void *cls,
+                const struct GNUNET_SCHEDULER_TaskContext *tc);
 
 
 /**
@@ -311,15 +316,20 @@
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
  *
- * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue
+ * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
  *         (continue on call to "next", of course),
- *         GNUNET_NO to delete the item and continue (if supported)
+ *         #GNUNET_NO to delete the item and continue (if supported)
  */
 static int
-expired_processor (void *cls, const struct GNUNET_HashCode * key, uint32_t 
size,
-                   const void *data, enum GNUNET_BLOCK_Type type,
-                   uint32_t priority, uint32_t anonymity,
-                   struct GNUNET_TIME_Absolute expiration, uint64_t uid)
+expired_processor (void *cls,
+                   const struct GNUNET_HashCode *key,
+                   uint32_t size,
+                   const void *data,
+                   enum GNUNET_BLOCK_Type type,
+                   uint32_t priority,
+                   uint32_t anonymity,
+                   struct GNUNET_TIME_Absolute expiration,
+                   uint64_t uid)
 {
   struct GNUNET_TIME_Absolute now;
 
@@ -564,7 +574,6 @@
 }
 
 
-
 /**
  * Function that will transmit the given datastore entry
  * to the client.
@@ -719,7 +728,8 @@
  * @param message the actual message
  */
 static void
-handle_release_reserve (void *cls, struct GNUNET_SERVER_Client *client,
+handle_release_reserve (void *cls,
+                        struct GNUNET_SERVER_Client *client,
                         const struct GNUNET_MessageHeader *message)
 {
   const struct ReleaseReserveMessage *msg =
@@ -730,7 +740,8 @@
   int rid = ntohl (msg->rid);
   unsigned long long rem;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n",
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Processing `%s' request\n",
               "RELEASE_RESERVE");
   next = reservations;
   prev = NULL;
@@ -748,7 +759,8 @@
           ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * 
pos->entries;
       GNUNET_assert (reserved >= rem);
       reserved -= rem;
-      GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved,
+      GNUNET_STATISTICS_set (stats,
+                             gettext_noop ("# reserved"), reserved,
                              GNUNET_NO);
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Returning %llu remaining reserved bytes to storage pool\n",
@@ -835,7 +847,8 @@
                         GNUNET_TIME_absolute_ntoh (dm->expiration), &msg);
   if (GNUNET_OK == ret)
   {
-    GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes stored"), size,
+    GNUNET_STATISTICS_update (stats,
+                              gettext_noop ("# bytes stored"), size,
                               GNUNET_YES);
     GNUNET_CONTAINER_bloomfilter_add (filter, &dm->key);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -961,7 +974,8 @@
       pos->entries--;
       pos->amount -= size;
       reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
-      GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved,
+      GNUNET_STATISTICS_set (stats,
+                             gettext_noop ("# reserved"), reserved,
                              GNUNET_NO);
     }
   }
@@ -1007,7 +1021,8 @@
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Processing `%s' request for `%s' of type %u\n", "GET",
               GNUNET_h2s (&msg->key), ntohl (msg->type));
-  GNUNET_STATISTICS_update (stats, gettext_noop ("# GET requests received"), 1,
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# GET requests received"), 1,
                             GNUNET_NO);
   GNUNET_SERVER_client_keep (client);
   if ((size == sizeof (struct GetMessage)) &&
@@ -1196,7 +1211,9 @@
 handle_drop (void *cls, struct GNUNET_SERVER_Client *client,
              const struct GNUNET_MessageHeader *message)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", "DROP");
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Processing `%s' request\n",
+              "DROP");
   do_drop = GNUNET_YES;
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
 }
@@ -1211,21 +1228,25 @@
  *        0 for "reset to empty"
  */
 static void
-disk_utilization_change_cb (void *cls, int delta)
+disk_utilization_change_cb (void *cls,
+                            int delta)
 {
   if ((delta < 0) && (payload < -delta))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _
-                ("Datastore payload inaccurate (%lld < %lld).  Trying to 
fix.\n"),
-                (long long) payload, (long long) -delta);
+                _("Datastore payload must have been inaccurate (%lld < %lld). 
Recomputing it.\n"),
+                (long long) payload,
+                (long long) -delta);
     payload = plugin->api->estimate_size (plugin->api->cls);
-    sync_stats ();
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _("New payload: %lld\n"),
+                (long long) payload);
+     sync_stats ();
     return;
   }
   payload += delta;
-  lastSync++;
-  if (lastSync >= MAX_STAT_SYNC_LAG)
+  last_sync++;
+  if (last_sync >= MAX_STAT_SYNC_LAG)
     sync_stats ();
 }
 
@@ -1237,14 +1258,17 @@
  * @param subsystem name of subsystem that created the statistic
  * @param name the name of the datum
  * @param value the current value
- * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
- * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if 
not
+ * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
  */
 static int
-process_stat_in (void *cls, const char *subsystem, const char *name,
-                 uint64_t value, int is_persistent)
+process_stat_in (void *cls,
+                 const char *subsystem,
+                 const char *name,
+                 uint64_t value,
+                 int is_persistent)
 {
-  GNUNET_assert (stats_worked == GNUNET_NO);
+  GNUNET_assert (GNUNET_NO == stats_worked);
   stats_worked = GNUNET_YES;
   payload += value;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1254,17 +1278,6 @@
 }
 
 
-static void
-process_stat_done (void *cls, int success)
-{
-  struct DatastorePlugin *plugin = cls;
-
-  stat_get = NULL;
-  if (stats_worked == GNUNET_NO)
-    payload = plugin->api->estimate_size (plugin->api->cls);
-}
-
-
 /**
  * Load the datastore plugin.
  */
@@ -1278,16 +1291,20 @@
   ret->env.cfg = cfg;
   ret->env.duc = &disk_utilization_change_cb;
   ret->env.cls = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"),
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Loading `%s' datastore plugin\n"),
               plugin_name);
-  GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", plugin_name);
+  GNUNET_asprintf (&libname,
+                   "libgnunet_plugin_datastore_%s",
+                   plugin_name);
   ret->short_name = GNUNET_strdup (plugin_name);
   ret->lib_name = libname;
   ret->api = GNUNET_PLUGIN_load (libname, &ret->env);
-  if (ret->api == NULL)
+  if (NULL == ret->api)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                _("Failed to load datastore plugin for `%s'\n"), plugin_name);
+                _("Failed to load datastore plugin for `%s'\n"),
+                plugin_name);
     GNUNET_free (ret->short_name);
     GNUNET_free (libname);
     GNUNET_free (ret);
@@ -1312,50 +1329,85 @@
   GNUNET_free (plug->lib_name);
   GNUNET_free (plug->short_name);
   GNUNET_free (plug);
-  GNUNET_free (quota_stat_name);
-  quota_stat_name = NULL;
 }
 
 
 /**
- * Final task run after shutdown.  Unloads plugins and disconnects us from
- * statistics.
+ * Adds a given @a key to the bloomfilter in @a cls @a count times.
+ *
+ * @param cls the bloomfilter
+ * @param key key to add
+ * @param count number of times to add key
  */
 static void
-unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+add_key_to_bloomfilter (void *cls,
+                       const struct GNUNET_HashCode *key,
+                       unsigned int count)
 {
-  if (lastSync > 0)
-    sync_stats ();
-  if (GNUNET_YES == do_drop)
-    plugin->api->drop (plugin->api->cls);
-  unload_plugin (plugin);
-  plugin = NULL;
-  if (filter != NULL)
+  struct GNUNET_CONTAINER_BloomFilter *bf = cls;
+
+  while (0 < count--)
+    GNUNET_CONTAINER_bloomfilter_add (bf, key);
+}
+
+
+/**
+ * We finished receiving the statistic.  Initialize the plugin; if
+ * loading the statistic failed, run the estimator.
+ *
+ * @param cls NULL
+ * @param success #GNUNET_NO if we failed to read the stat
+ */
+static void
+process_stat_done (void *cls, int success)
+{
+  stat_get = NULL;
+  plugin = load_plugin ();
+  if (NULL == plugin)
   {
     GNUNET_CONTAINER_bloomfilter_free (filter);
     filter = NULL;
+    if (NULL != stats)
+    {
+      GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
+      stats = NULL;
+    }
+    return;
   }
-  if (stat_get != NULL)
+  if (GNUNET_NO == stats_worked)
   {
-    GNUNET_STATISTICS_get_cancel (stat_get);
-    stat_get = NULL;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Failed to obtain value from statistics service, recomputing 
it\n");
+    payload = plugin->api->estimate_size (plugin->api->cls);
   }
-  if (stats != NULL)
+  if (GNUNET_YES == refresh_bf)
   {
-    GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
-    stats = NULL;
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+               _("Rebuilding bloomfilter.  Please be patient.\n"));
+    if (NULL != plugin->api->get_keys)
+      plugin->api->get_keys (plugin->api->cls,
+                             &add_key_to_bloomfilter,
+                             filter);
+    else
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("Plugin does not support get_keys function. Please 
fix!\n"));
+
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+               _("Bloomfilter construction complete.\n"));
   }
-  GNUNET_free_non_null (plugin_name);
-  plugin_name = NULL;
+  expired_kill_task
+    = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                          &delete_expired,
+                                          NULL);
 }
 
 
 /**
- * Last task run during shutdown.  Disconnects us from
- * the transport and core.
+ * Task run during shutdown.
  */
 static void
-cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+cleaning_task (void *cls,
+               const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct TransmitCallbackContext *tcc;
 
@@ -1376,8 +1428,31 @@
     GNUNET_SCHEDULER_cancel (expired_kill_task);
     expired_kill_task = GNUNET_SCHEDULER_NO_TASK;
   }
-  GNUNET_SCHEDULER_add_continuation (&unload_task, NULL,
-                                     GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+  if (GNUNET_YES == do_drop)
+    plugin->api->drop (plugin->api->cls);
+  unload_plugin (plugin);
+  plugin = NULL;
+  if (NULL != filter)
+  {
+    GNUNET_CONTAINER_bloomfilter_free (filter);
+    filter = NULL;
+  }
+  if (NULL != stat_get)
+  {
+    GNUNET_STATISTICS_get_cancel (stat_get);
+    stat_get = NULL;
+  }
+  GNUNET_free_non_null (plugin_name);
+  plugin_name = NULL;
+  if (last_sync > 0)
+    sync_stats ();
+  if (NULL != stats)
+  {
+    GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
+    stats = NULL;
+  }
+  GNUNET_free (quota_stat_name);
+  quota_stat_name = NULL;
 }
 
 
@@ -1390,13 +1465,14 @@
  * @param client identification of the client
  */
 static void
-cleanup_reservations (void *cls, struct GNUNET_SERVER_Client *client)
+cleanup_reservations (void *cls,
+                      struct GNUNET_SERVER_Client *client)
 {
   struct ReservationList *pos;
   struct ReservationList *prev;
   struct ReservationList *next;
 
-  if (client == NULL)
+  if (NULL == client)
     return;
   prev = NULL;
   pos = reservations;
@@ -1424,24 +1500,6 @@
 
 
 /**
- * Adds a given key to the bloomfilter 'count' times.
- *
- * @param cls the bloomfilter
- * @param key key to add
- * @param count number of times to add key
- */
-static void
-add_key_to_bloomfilter (void *cls,
-                       const struct GNUNET_HashCode *key,
-                       unsigned int count)
-{
-  struct GNUNET_CONTAINER_BloomFilter *bf = cls;
-  while (0 < count--)
-    GNUNET_CONTAINER_bloomfilter_add (bf, key);
-}
-
-
-/**
  * Process datastore requests.
  *
  * @param cls closure
@@ -1449,7 +1507,8 @@
  * @param c configuration to use
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
+run (void *cls,
+     struct GNUNET_SERVER_Handle *server,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
   static const struct GNUNET_SERVER_MessageHandler handlers[] = {
@@ -1476,7 +1535,6 @@
   char *fn;
   char *pfn;
   unsigned int bf_size;
-  int refresh_bf;
 
   cfg = c;
   if (GNUNET_OK !=
@@ -1516,11 +1574,11 @@
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Could not use specified filename `%s' for bloomfilter.\n"),
-                fn != NULL ? fn : "");
+                NULL != fn ? fn : "");
     GNUNET_free_non_null (fn);
     fn = NULL;
   }
-  if (fn != NULL)
+  if (NULL != fn)
   {
     GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name);
     if (GNUNET_YES == GNUNET_DISK_file_test (pfn))
@@ -1576,52 +1634,31 @@
     refresh_bf = GNUNET_YES;
   }
   GNUNET_free_non_null (fn);
-  if (filter == NULL)
+  if (NULL == filter)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 _("Failed to initialize bloomfilter.\n"));
-    if (stats != NULL)
+    if (NULL != stats)
     {
       GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
       stats = NULL;
     }
     return;
   }
-  plugin = load_plugin ();
-  if (NULL == plugin)
-  {
-    GNUNET_CONTAINER_bloomfilter_free (filter);
-    filter = NULL;
-    if (stats != NULL)
-    {
-      GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
-      stats = NULL;
-    }
-    return;
-  }
   stat_get =
-      GNUNET_STATISTICS_get (stats, "datastore", quota_stat_name,
-                             GNUNET_TIME_UNIT_SECONDS, &process_stat_done,
-                             &process_stat_in, plugin);
-  GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
+      GNUNET_STATISTICS_get (stats,
+                             "datastore",
+                             quota_stat_name,
+                             GNUNET_TIME_UNIT_SECONDS,
+                             &process_stat_done,
+                             &process_stat_in,
+                             NULL);
+  GNUNET_SERVER_disconnect_notify (server,
+                                   &cleanup_reservations,
+                                   NULL);
   GNUNET_SERVER_add_handlers (server, handlers);
-  if (GNUNET_YES == refresh_bf)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-               _("Rebuilding bloomfilter.  Please be patient.\n"));
-    if (NULL != plugin->api->get_keys)
-      plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, 
filter);
-    else
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                 _("Plugin does not support get_keys function. Please 
fix!\n"));
-
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-               _("Bloomfilter construction complete.\n"));
-  }
-  expired_kill_task =
-      GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
-                                          &delete_expired, NULL);
-  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &cleaning_task,
+  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
+                                &cleaning_task,
                                 NULL);
 }
 
@@ -1634,13 +1671,15 @@
  * @return 0 ok, 1 on error
  */
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
   int ret;
 
   ret =
       (GNUNET_OK ==
-       GNUNET_SERVICE_run (argc, argv, "datastore", GNUNET_SERVICE_OPTION_NONE,
+       GNUNET_SERVICE_run (argc, argv, "datastore",
+                           GNUNET_SERVICE_OPTION_NONE,
                            &run, NULL)) ? 0 : 1;
   return ret;
 }




reply via email to

[Prev in Thread] Current Thread [Next in Thread]