[GNUnet-SVN] r14854 - gnunet/src/datastore

[gnunet]

Author: grothoff
Date: 2011-04-04 20:23:50 +0200 (Mon, 04 Apr 2011)
New Revision: 14854

Added:
   gnunet/src/datastore/test_plugin_datastore.c
   gnunet/src/datastore/test_plugin_datastore_data_mysql.conf
   gnunet/src/datastore/test_plugin_datastore_data_postgres.conf
   gnunet/src/datastore/test_plugin_datastore_data_sqlite.conf
Modified:
   gnunet/src/datastore/Makefile.am
   gnunet/src/datastore/gnunet-service-datastore.c
Log:
test

Modified: gnunet/src/datastore/Makefile.am
===================================================================
--- gnunet/src/datastore/Makefile.am    2011-04-04 18:11:53 UTC (rev 14853)
+++ gnunet/src/datastore/Makefile.am    2011-04-04 18:23:50 UTC (rev 14854)
@@ -42,6 +42,7 @@
  MYSQL_TESTS = \
   test_datastore_api_mysql \
   test_datastore_api_management_mysql \
+  test_plugin_datastore_mysql \
   perf_datastore_api_mysql \
   perf_plugin_datastore_mysql
 endif
@@ -51,6 +52,7 @@
  SQLITE_TESTS = \
   test_datastore_api_sqlite \
   test_datastore_api_management_sqlite \
+  test_plugin_datastore_sqlite \
   perf_datastore_api_sqlite \
   perf_plugin_datastore_sqlite
 endif
@@ -60,6 +62,7 @@
  POSTGRES_TESTS = \
   test_datastore_api_postgres \
   test_datastore_api_management_postgres \
+  test_plugin_datastore_postgres \
   perf_datastore_api_postgres \
   perf_plugin_datastore_postgres
 endif
@@ -141,7 +144,12 @@
 perf_plugin_datastore_sqlite_LDADD = \
  $(top_builddir)/src/util/libgnunetutil.la  
 
+test_plugin_datastore_sqlite_SOURCES = \
+ test_plugin_datastore.c
+test_plugin_datastore_sqlite_LDADD = \
+ $(top_builddir)/src/util/libgnunetutil.la  
 
+
 test_datastore_api_mysql_SOURCES = \
  test_datastore_api.c
 test_datastore_api_mysql_LDADD = \
@@ -160,6 +168,11 @@
  $(top_builddir)/src/datastore/libgnunetdatastore.la \
  $(top_builddir)/src/util/libgnunetutil.la  
 
+test_plugin_datastore_mysql_SOURCES = \
+ test_plugin_datastore.c
+test_plugin_datastore_mysql_LDADD = \
+ $(top_builddir)/src/util/libgnunetutil.la  
+
 perf_plugin_datastore_mysql_SOURCES = \
  perf_plugin_datastore.c
 perf_plugin_datastore_mysql_LDADD = \
@@ -184,6 +197,11 @@
  $(top_builddir)/src/datastore/libgnunetdatastore.la \
  $(top_builddir)/src/util/libgnunetutil.la  
 
+test_plugin_datastore_postgres_SOURCES = \
+ test_plugin_datastore.c
+test_plugin_datastore_postgres_LDADD = \
+ $(top_builddir)/src/util/libgnunetutil.la  
+
 perf_plugin_datastore_postgres_SOURCES = \
  perf_plugin_datastore.c
 perf_plugin_datastore_postgres_LDADD = \

Modified: gnunet/src/datastore/gnunet-service-datastore.c
===================================================================
--- gnunet/src/datastore/gnunet-service-datastore.c     2011-04-04 18:11:53 UTC 
(rev 14853)
+++ gnunet/src/datastore/gnunet-service-datastore.c     2011-04-04 18:23:50 UTC 
(rev 14854)
@@ -999,7 +999,6 @@
 }
 
 
-
 /**
  * Function that will check if the given datastore entry
  * matches the put and if none match executes the put.
@@ -1266,6 +1265,7 @@
                                client);  
 }
 
+
 /**
  * Handle GET_ZERO_ANONYMITY-message.
  *
@@ -1585,7 +1585,8 @@
  * statistics.
  */
 static void
-unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+unload_task (void *cls,
+            const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   unload_plugin (plugin);
   plugin = NULL;
@@ -1614,7 +1615,8 @@
  * the transport and core.
  */
 static void
-cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+cleaning_task (void *cls,
+              const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct TransmitCallbackContext *tcc;
 
@@ -1655,8 +1657,7 @@
  */
 static void
 cleanup_reservations (void *cls,
-                     struct GNUNET_SERVER_Client
-                     * client)
+                     struct GNUNET_SERVER_Client *client)
 {
   struct ReservationList *pos;
   struct ReservationList *prev;

Added: gnunet/src/datastore/test_plugin_datastore.c
===================================================================
--- gnunet/src/datastore/test_plugin_datastore.c                                
(rev 0)
+++ gnunet/src/datastore/test_plugin_datastore.c        2011-04-04 18:23:50 UTC 
(rev 14854)
@@ -0,0 +1,467 @@
+/*
+     This file is part of GNUnet.
+     (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other 
contributing authors)
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+     Boston, MA 02111-1307, USA.
+*/
+/*
+ * @file test_plugin_datastore.c
+ * @brief Profile database plugin directly, focusing on iterators.
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_protocols.h"
+#include "gnunet_datastore_plugin.h"
+
+#define VERBOSE GNUNET_NO
+
+/**
+ * Target datastore size (in bytes).  Realistic sizes are
+ * more like 16 GB (not the default of 16 MB); however,
+ * those take too long to run them in the usual "make check"
+ * sequence.  Hence the value used for shipping is tiny.
+ */
+#define MAX_SIZE 1024LL * 1024 * 128
+
+#define ITERATIONS 10
+
+/**
+ * Number of put operations equivalent to 1/10th of MAX_SIZE
+ */
+#define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS)
+
+static unsigned long long stored_bytes;
+
+static unsigned long long stored_entries;
+
+static unsigned long long stored_ops;
+
+static const char *plugin_name;
+
+static int ok;
+
+enum RunPhase
+  {
+    RP_DONE = 0,
+    RP_PUT,
+    RP_LP_GET,
+    RP_AE_GET,
+    RP_ZA_GET
+  };
+
+
+struct CpsRunContext
+{
+  unsigned int i;
+  struct GNUNET_TIME_Absolute start;
+  struct GNUNET_TIME_Absolute end;
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+  struct GNUNET_DATASTORE_PluginFunctions * api;
+  const char *msg;
+  enum RunPhase phase;
+  unsigned int cnt;
+};
+
+
+/**
+ * Function called by plugins to notify us about a
+ * change in their disk utilization.
+ *
+ * @param cls closure (NULL)
+ * @param delta change in disk utilization, 
+ *        0 for "reset to empty"
+ */
+static void
+disk_utilization_change_cb (void *cls,
+                           int delta)
+{
+}
+
+            
+static void
+putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k)
+{
+  char value[65536];
+  size_t size;
+  static GNUNET_HashCode key;
+  static int ic;
+  char *msg;
+  unsigned int prio;
+
+  /* most content is 32k */
+  size = 32 * 1024;
+
+  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0)  /* but 
some of it is less! */
+    size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
+  size = size - (size & 7);     /* always multiple of 8 */
+
+  /* generate random key */
+  key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value;
+  GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key);
+  memset (value, i, size);
+  if (i > 255)
+    memset (value, i - 255, size / 2);
+  value[0] = k;
+  msg = NULL;
+  prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
+  if (GNUNET_OK != api->put (api->cls,
+                            &key, 
+                            size,
+                            value,
+                            i /* type */,
+                            prio,
+                            i /* anonymity */,
+                            0 /* replication */,
+                            GNUNET_TIME_relative_to_absolute 
+                            (GNUNET_TIME_relative_multiply 
(GNUNET_TIME_UNIT_MILLISECONDS,
+                                                            60 * 60 * 60 * 
1000 +
+                                                            
GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
+                            &msg))
+    {
+      fprintf (stderr, "ERROR: `%s'\n", msg);
+      GNUNET_free_non_null (msg);
+      return;
+    }
+  ic++;
+  stored_bytes += size;
+  stored_ops++;
+  stored_entries++;
+}
+
+static void
+test (void *cls,
+      const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+
+static int
+iterateDummy (void *cls,
+             void *next_cls,
+             const GNUNET_HashCode * key,
+             uint32_t size,
+             const void *data,
+             enum GNUNET_BLOCK_Type type,
+             uint32_t priority,
+             uint32_t anonymity,
+             struct GNUNET_TIME_Absolute
+             expiration, 
+             uint64_t uid)
+{
+  struct CpsRunContext *crc = cls;
+  
+  if (key == NULL)
+    {
+      crc->end = GNUNET_TIME_absolute_get();
+      printf (crc->msg,
+             crc->i,
+             (unsigned long long) (crc->end.abs_value - crc->start.abs_value),
+             crc->cnt);
+      if (crc->phase != RP_ZA_GET)
+       {
+         crc->phase++;
+       }
+      else
+       {
+         if (crc->i == ITERATIONS)
+           crc->phase = RP_DONE;
+         else
+           crc->phase = RP_PUT;
+       }
+      crc->cnt = 0;
+      crc->start = GNUNET_TIME_absolute_get ();      
+      GNUNET_SCHEDULER_add_now (&test, crc);
+      return GNUNET_OK;
+    }
+#if VERBOSE
+  fprintf (stderr, "Found result type=%u, priority=%u, size=%u, expire=%llu\n",
+          type, priority, size,
+          (unsigned long long) expiration.abs_value);
+#endif
+  crc->cnt++;
+  crc->api->next_request (next_cls,
+                         GNUNET_NO);
+  return GNUNET_OK;
+}
+
+
+
+static int
+dummy_get (void *cls,
+          void *next_cls,
+          const GNUNET_HashCode * key,
+          uint32_t size,
+          const void *data,
+          enum GNUNET_BLOCK_Type type,
+          uint32_t priority,
+          uint32_t anonymity,
+          struct GNUNET_TIME_Absolute
+          expiration, 
+          uint64_t uid)
+{
+  struct CpsRunContext *crc = cls;
+
+  crc->cnt++;
+  if (1000 == crc->cnt)
+    {
+      crc->end = GNUNET_TIME_absolute_get();
+      printf (crc->msg,
+             crc->i,
+             (unsigned long long) (crc->end.abs_value - crc->start.abs_value),
+             crc->cnt);
+      crc->phase++;
+      crc->cnt = 0;
+      crc->start = GNUNET_TIME_absolute_get ();      
+    }
+  GNUNET_SCHEDULER_add_now (&test, crc);
+  return GNUNET_OK;
+}
+
+/**
+ * Function called when the service shuts
+ * down.  Unloads our datastore plugin.
+ *
+ * @param api api to unload
+ * @param cfg configuration to use
+ */
+static void
+unload_plugin (struct GNUNET_DATASTORE_PluginFunctions * api,
+              const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  char *name;
+  char *libname;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (cfg,
+                                             "DATASTORE", "DATABASE", &name))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("No `%s' specified for `%s' in configuration!\n"),
+                 "DATABASE",
+                 "DATASTORE");
+      return;
+    }
+  GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
+  GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
+  GNUNET_free (libname);
+  GNUNET_free (name);
+}
+
+
+
+/**
+ * Last task run during shutdown.  Disconnects us from
+ * the transport and core.
+ */
+static void
+cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct CpsRunContext *crc = cls;
+
+  unload_plugin (crc->api, crc->cfg);
+  GNUNET_free (crc);
+}
+
+
+static void
+test (void *cls,
+      const struct GNUNET_SCHEDULER_TaskContext *tc)
+{  
+  struct CpsRunContext *crc = cls;
+  int j;
+
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    {
+      crc->phase = RP_DONE;
+      ok = 1;
+    }
+  switch (crc->phase)
+    {
+    case RP_PUT:      
+      crc->start = GNUNET_TIME_absolute_get ();
+      for (j=0;j<PUT_10;j++)
+       putValue (crc->api, j, crc->i);
+      crc->end = GNUNET_TIME_absolute_get ();
+      printf ("%3u insertion took                      %20llums for %u\n",
+             crc->i,
+             (unsigned long long) (crc->end.abs_value - crc->start.abs_value),
+             (unsigned int) PUT_10);
+      crc->i++;
+      crc->start = GNUNET_TIME_absolute_get ();      
+      crc->phase = RP_LP_GET;
+      GNUNET_SCHEDULER_add_after (GNUNET_SCHEDULER_NO_TASK,
+                                 &test, crc);
+      break;
+    case RP_LP_GET:
+      crc->msg = "%3u replication iteration took %20llums for %u\n";
+      crc->api->replication_get (crc->api->cls, 
+                                &dummy_get,
+                                crc);
+      break;
+    case RP_AE_GET:
+      crc->msg = "%3u expiration iteration took %20llums for %u\n";
+      crc->api->expiration_get (crc->api->cls, 
+                               &dummy_get,
+                               crc);
+      break;
+    case RP_ZA_GET:
+      crc->msg = "%3u zero anonymity iteration took %20llums for %u\n";
+      crc->api->iter_zero_anonymity (crc->api->cls, 0, 
+                                    &iterateDummy,
+                                    crc);
+      break;
+    case RP_DONE:
+      crc->api->drop (crc->api->cls);
+      GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                         &cleaning_task, crc);
+      break;
+    }
+}
+
+
+/**
+ * Load the datastore plugin.
+ */
+static struct GNUNET_DATASTORE_PluginFunctions *
+load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  static struct GNUNET_DATASTORE_PluginEnvironment env;
+  struct GNUNET_DATASTORE_PluginFunctions * ret; 
+  char *name;
+  char *libname;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (cfg,
+                                             "DATASTORE", "DATABASE", &name))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("No `%s' specified for `%s' in configuration!\n"),
+                 "DATABASE",
+                 "DATASTORE");
+      return NULL;
+    }
+  env.cfg = cfg;
+  env.duc = &disk_utilization_change_cb;
+  env.cls = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Loading `%s' datastore plugin\n"), name);
+  GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
+  if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
+    {
+      fprintf (stderr,
+              "Failed to load plugin `%s'!\n",
+              name);
+      return NULL;
+    }
+  GNUNET_free (libname);
+  GNUNET_free (name);
+  return ret;
+}
+
+
+static void
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
+     const struct GNUNET_CONFIGURATION_Handle *c)
+{
+  struct GNUNET_DATASTORE_PluginFunctions *api;
+  struct CpsRunContext *crc;
+
+  api = load_plugin (c);
+  if (api == NULL)
+    {
+      fprintf (stderr, 
+              "Could not initialize plugin, assuming database not configured. 
Test not run!\n");
+      return;
+    }
+  crc = GNUNET_malloc(sizeof(struct CpsRunContext));
+  crc->api = api;
+  crc->cfg = c;
+  crc->phase = RP_PUT;
+  GNUNET_SCHEDULER_add_now (&test, crc);
+}
+
+
+static int
+check ()
+{
+  char cfg_name[128];
+  char *const argv[] = { 
+    "test-plugin-datastore",
+    "-c",
+    cfg_name,
+#if VERBOSE
+    "-L", "DEBUG",
+#endif
+    NULL
+  };
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_END
+  };
+
+  GNUNET_snprintf (cfg_name,
+                  sizeof (cfg_name),
+                  "test_plugin_datastore_data_%s.conf",
+                  plugin_name);
+  GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
+                      argv, "test-plugin-datastore", "nohelp",
+                      options, &run, NULL);
+  if (ok != 0)
+    fprintf (stderr, "Missed some testcases: %u\n", ok);
+  return ok;
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  int ret;
+  char *pos;
+  char dir_name[128];
+
+  /* determine name of plugin to use */
+  plugin_name = argv[0];
+  while (NULL != (pos = strstr(plugin_name, "_")))
+    plugin_name = pos+1;
+  if (NULL != (pos = strstr(plugin_name, ".")))
+    pos[0] = 0;
+  else
+    pos = (char *) plugin_name;
+
+  GNUNET_snprintf (dir_name,
+                  sizeof (dir_name),
+                  "/tmp/test-gnunet-datastore-%s",
+                  plugin_name);
+  GNUNET_DISK_directory_remove (dir_name);
+  GNUNET_log_setup ("test-plugin-datastore",
+#if VERBOSE
+                    "DEBUG",
+#else
+                    "WARNING",
+#endif
+                    NULL);
+  ret = check ();
+  if (pos != plugin_name)
+    pos[0] = '.';
+  GNUNET_DISK_directory_remove (dir_name);
+
+  return ret;
+}
+
+/* end of test_plugin_datastore.c */
+
+

Added: gnunet/src/datastore/test_plugin_datastore_data_mysql.conf
===================================================================
--- gnunet/src/datastore/test_plugin_datastore_data_mysql.conf                  
        (rev 0)
+++ gnunet/src/datastore/test_plugin_datastore_data_mysql.conf  2011-04-04 
18:23:50 UTC (rev 14854)
@@ -0,0 +1,34 @@
+[PATHS]
+SERVICEHOME = /tmp/perf-gnunet-datastore-mysql/
+
+[datastore]
+PORT = 22654
+HOSTNAME = localhost
+HOME = $SERVICEHOME
+CONFIG = $DEFAULTCONFIG
+BINARY = gnunet-service-datastore
+ACCEPT_FROM = 127.0.0.1;
+ACCEPT_FROM6 = ::1;
+QUOTA = 1000000
+BLOOMFILTER = $SERVICEHOME/fs/bloomfilter
+DATABASE = mysql
+# USERNAME = 
+# MAXBUF =
+# TIMEOUT =
+# DISABLEV6 =
+# BINDTO =
+# REJECT_FROM =
+# REJECT_FROM6 =
+# PREFIX =
+
+
+[dht]
+AUTOSTART = NO
+
+[datastore-mysql]
+DATABASE = gnunetcheck
+# CONFIG = ~/.my.cnf
+# USER =
+# PASSWORD =
+# HOST = 
+# PORT = 

Added: gnunet/src/datastore/test_plugin_datastore_data_postgres.conf
===================================================================
--- gnunet/src/datastore/test_plugin_datastore_data_postgres.conf               
                (rev 0)
+++ gnunet/src/datastore/test_plugin_datastore_data_postgres.conf       
2011-04-04 18:23:50 UTC (rev 14854)
@@ -0,0 +1,29 @@
+[PATHS]
+SERVICEHOME = /tmp/perf-gnunet-datastore-postgres/
+
+[datastore]
+PORT = 22654
+HOSTNAME = localhost
+HOME = $SERVICEHOME
+CONFIG = $DEFAULTCONFIG
+BINARY = gnunet-service-datastore
+ACCEPT_FROM = 127.0.0.1;
+ACCEPT_FROM6 = ::1;
+QUOTA = 1000000
+BLOOMFILTER = $SERVICEHOME/fs/bloomfilter
+DATABASE = postgres
+# USERNAME = 
+# MAXBUF =
+# TIMEOUT =
+# DISABLEV6 =
+# BINDTO =
+# REJECT_FROM =
+# REJECT_FROM6 =
+# PREFIX =
+
+
+[dht]
+AUTOSTART = NO
+
+[datastore-postgres]
+CONFIG = dbname=gnunetcheck

Added: gnunet/src/datastore/test_plugin_datastore_data_sqlite.conf
===================================================================
--- gnunet/src/datastore/test_plugin_datastore_data_sqlite.conf                 
        (rev 0)
+++ gnunet/src/datastore/test_plugin_datastore_data_sqlite.conf 2011-04-04 
18:23:50 UTC (rev 14854)
@@ -0,0 +1,27 @@
+[PATHS]
+SERVICEHOME = /tmp/perf-gnunet-datastore-sqlite/
+
+[datastore]
+PORT = 22654
+HOSTNAME = localhost
+HOME = $SERVICEHOME
+CONFIG = $DEFAULTCONFIG
+BINARY = gnunet-service-datastore
+ACCEPT_FROM = 127.0.0.1;
+ACCEPT_FROM6 = ::1;
+QUOTA = 1000000
+BLOOMFILTER = $SERVICEHOME/fs/bloomfilter
+DATABASE = sqlite
+# USERNAME = 
+# MAXBUF =
+# TIMEOUT =
+# DISABLEV6 =
+# BINDTO =
+# REJECT_FROM =
+# REJECT_FROM6 =
+# PREFIX =
+
+[dht]
+AUTOSTART = NO
+
+