[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [gnunet] branch master updated: add perf_ logic for namesto
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [gnunet] branch master updated: add perf_ logic for namestore iterations, improve namestore insertion performance by 30 percent |
|
Date: |
Tue, 01 May 2018 11:53:10 +0200 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a commit to branch master
in repository gnunet.
The following commit(s) were added to refs/heads/master by this push:
new 8f0e0c9df add perf_ logic for namestore iterations, improve namestore
insertion performance by 30 percent
8f0e0c9df is described below
commit 8f0e0c9dfd1020c5a202cd1927054291c034ca20
Author: Christian Grothoff <address@hidden>
AuthorDate: Tue May 1 11:53:00 2018 +0200
add perf_ logic for namestore iterations, improve namestore insertion
performance by 30
percent
---
.gitignore | 3 +
src/namestore/Makefile.am | 41 ++-
src/namestore/gnunet-service-namestore.c | 21 +-
src/namestore/perf_namestore_api_zone_iteration.c | 389 ++++++++++++++++++++++
src/namestore/test_namestore_api_flat.conf | 7 +
src/namestore/test_namestore_api_postgres.conf | 7 +
src/namestore/test_namestore_api_sqlite.conf | 4 +
src/namestore/test_plugin_namestore.c | 2 -
8 files changed, 454 insertions(+), 20 deletions(-)
diff --git a/.gitignore b/.gitignore
index 3c739546f..b7dd58739 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,6 @@ src/credential/gnunet-credential
src/credential/gnunet-service-credential
src/identity-provider/gnunet-idp
*.patch
+src/namestore/perf_namestore_api_zone_iteration_flat
+src/namestore/perf_namestore_api_zone_iteration_postgres
+src/namestore/perf_namestore_api_zone_iteration_sqlite
diff --git a/src/namestore/Makefile.am b/src/namestore/Makefile.am
index 5f18506a7..b925261b7 100644
--- a/src/namestore/Makefile.am
+++ b/src/namestore/Makefile.am
@@ -20,29 +20,28 @@ if USE_COVERAGE
XLIBS = -lgcov
endif
-if HAVE_EXPERIMENTAL
FLAT_PLUGIN = libgnunet_plugin_namestore_flat.la
if HAVE_TESTING
-FLAT_TESTS = test_plugin_namestore_flat
-endif
+FLAT_TESTS = test_plugin_namestore_flat \
+ perf_namestore_api_zone_iteration_flat
endif
if HAVE_SQLITE
SQLITE_PLUGIN = libgnunet_plugin_namestore_sqlite.la
if HAVE_TESTING
-SQLITE_TESTS = test_plugin_namestore_sqlite
+SQLITE_TESTS = test_plugin_namestore_sqlite \
+ perf_namestore_api_zone_iteration_sqlite
endif
endif
if HAVE_POSTGRESQL
-# postgres doesn't even build yet; thus: experimental!
POSTGRES_PLUGIN = libgnunet_plugin_namestore_postgres.la
if HAVE_TESTING
-POSTGRES_TESTS = test_plugin_namestore_postgres
+POSTGRES_TESTS = test_plugin_namestore_postgres \
+ perf_namestore_api_zone_iteration_postgres
endif
endif
-# testcases do not even build yet; thus: experimental!
if HAVE_TESTING
TESTING_TESTS = \
test_namestore_api_store.nc \
@@ -81,7 +80,6 @@ test_namestore_api_zone_iteration_specific_zone.log:
test_namestore_api_zone_ite
test_namestore_api_zone_iteration_stop.log: test_namestore_api_monitoring.log
test_namestore_api_monitoring.log: test_namestore_api_monitoring_existing.log
-
if HAVE_SQLITE
check_PROGRAMS = \
$(SQLITE_TESTS) \
@@ -341,6 +339,30 @@ test_namestore_api_zone_iteration_nc_LDADD = \
$(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
libgnunetnamestore.la
+perf_namestore_api_zone_iteration_postgres_SOURCES = \
+ perf_namestore_api_zone_iteration.c
+perf_namestore_api_zone_iteration_postgres_LDADD = \
+ $(top_builddir)/src/testing/libgnunettesting.la \
+ $(top_builddir)/src/util/libgnunetutil.la \
+ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
+ libgnunetnamestore.la
+
+perf_namestore_api_zone_iteration_sqlite_SOURCES = \
+ perf_namestore_api_zone_iteration.c
+perf_namestore_api_zone_iteration_sqlite_LDADD = \
+ $(top_builddir)/src/testing/libgnunettesting.la \
+ $(top_builddir)/src/util/libgnunetutil.la \
+ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
+ libgnunetnamestore.la
+
+perf_namestore_api_zone_iteration_flat_SOURCES = \
+ perf_namestore_api_zone_iteration.c
+perf_namestore_api_zone_iteration_flat_LDADD = \
+ $(top_builddir)/src/testing/libgnunettesting.la \
+ $(top_builddir)/src/util/libgnunetutil.la \
+ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
+ libgnunetnamestore.la
+
test_namestore_api_zone_iteration_nick_nc_SOURCES = \
test_namestore_api_zone_iteration_nick.c
test_namestore_api_zone_iteration_nick_nc_LDADD = \
@@ -391,6 +413,9 @@ check_SCRIPTS = \
EXTRA_DIST = \
test_namestore_api.conf \
+ test_namestore_api_postgres.conf \
+ test_namestore_api_sqlite.conf \
+ test_namestore_api_flat.conf \
test_plugin_namestore_sqlite.conf \
test_plugin_namestore_postgres.conf \
test_plugin_namestore_flat.conf \
diff --git a/src/namestore/gnunet-service-namestore.c
b/src/namestore/gnunet-service-namestore.c
index e3936a7cc..0456e5c15 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -1032,7 +1032,8 @@ check_record_store (void *cls,
GNUNET_break (0);
return GNUNET_SYSERR;
}
- if ((0 == name_len) || (name_len > MAX_NAME_LEN))
+ if ( (0 == name_len) ||
+ (name_len > MAX_NAME_LEN) )
{
GNUNET_break (0);
return GNUNET_SYSERR;
@@ -1066,7 +1067,6 @@ handle_record_store (void *cls,
const char *rd_ser;
unsigned int rd_count;
int res;
- struct GNUNET_CRYPTO_EcdsaPublicKey pubkey;
struct ZoneMonitor *zm;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1093,8 +1093,6 @@ handle_record_store (void *cls,
}
/* Extracting and converting private key */
- GNUNET_CRYPTO_ecdsa_key_get_public (&rp_msg->private_key,
- &pubkey);
conv_name = GNUNET_GNSRECORD_string_to_lowercase (name_tmp);
if (NULL == conv_name)
{
@@ -1105,11 +1103,9 @@ handle_record_store (void *cls,
return;
}
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Creating %u records for name `%s' in zone `%s'\n",
+ "Creating %u records for name `%s'\n",
(unsigned int) rd_count,
- conv_name,
- GNUNET_GNSRECORD_z2s (&pubkey));
-
+ conv_name);
if ( (0 == rd_count) &&
(GNUNET_NO ==
GSN_database->iterate_records (GSN_database->cls,
@@ -1150,7 +1146,8 @@ handle_record_store (void *cls,
{
for (zm = monitor_head; NULL != zm; zm = zm->next)
{
- if ( (0 == memcmp (&rp_msg->private_key, &zm->zone,
+ if ( (0 == memcmp (&rp_msg->private_key,
+ &zm->zone,
sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) ||
(0 == memcmp (&zm->zone,
&zero,
@@ -1363,7 +1360,7 @@ struct ZoneIterationProcResult
/**
* Process results for zone iteration from database
- *
+ *
* @param cls struct ZoneIterationProcResult
* @param seq sequence number of the record
* @param zone_key the zone key
@@ -1446,6 +1443,10 @@ run_zone_iteration_round (struct ZoneIteration *zi,
memset (&proc,
0,
sizeof (proc));
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Asked to return up to %llu records at position %llu\n",
+ (unsigned long long) limit,
+ (unsigned long long) zi->seq);
proc.zi = zi;
proc.limit = limit;
start = GNUNET_TIME_absolute_get ();
diff --git a/src/namestore/perf_namestore_api_zone_iteration.c
b/src/namestore/perf_namestore_api_zone_iteration.c
new file mode 100644
index 000000000..8b5c53576
--- /dev/null
+++ b/src/namestore/perf_namestore_api_zone_iteration.c
@@ -0,0 +1,389 @@
+/*
+ This file is part of GNUnet.
+ Copyright (C) 2013, 2018 GNUnet e.V.
+
+ GNUnet is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version.
+
+ GNUnet is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with GNUnet; see the file COPYING. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+*/
+/**
+ * @file namestore/perf_namestore_api_zone_iteration.c
+ * @brief testcase for zone iteration functionality: iterate all zones
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_namestore_service.h"
+#include "gnunet_testing_lib.h"
+#include "namestore.h"
+
+/**
+ * A #BENCHMARK_SIZE of 1000 takes less than a minute on a reasonably
+ * modern system, so 30 minutes should be OK even for very, very
+ * slow systems.
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)
+
+/**
+ * The runtime of the benchmark is expected to be linear
+ * for the iteration phase with a *good* database. The FLAT
+ * database uses a quadratic retrieval algorithm,
+ * hence it should be quadratic in the size.
+ */
+#define BENCHMARK_SIZE 1000
+
+/**
+ * Maximum record size
+ */
+#define MAX_REC_SIZE 500
+
+/**
+ * How big are the blocks we fetch? Note that the first block is
+ * always just 1 record set per current API. Smaller block
+ * sizes will make quadratic iteration-by-offset penalties
+ * more pronounced.
+ */
+#define BLOCK_SIZE 100
+
+static struct GNUNET_NAMESTORE_Handle *nsh;
+
+static struct GNUNET_SCHEDULER_Task *timeout_task;
+
+static struct GNUNET_SCHEDULER_Task *t;
+
+static struct GNUNET_CRYPTO_EcdsaPrivateKey *privkey;
+
+static struct GNUNET_NAMESTORE_ZoneIterator *zi;
+
+static struct GNUNET_NAMESTORE_QueueEntry *qe;
+
+static int res;
+
+static char *directory;
+
+static unsigned int off;
+
+static unsigned int left_until_next;
+
+static uint8_t seen[1 + BENCHMARK_SIZE / 8];
+
+static struct GNUNET_TIME_Absolute start;
+
+
+/**
+ * Terminate everything
+ *
+ * @param cls NULL
+ */
+static void
+end (void *cls)
+{
+ (void) cls;
+ if (NULL != qe)
+ {
+ GNUNET_NAMESTORE_cancel (qe);
+ qe = NULL;
+ }
+ if (NULL != zi)
+ {
+ GNUNET_NAMESTORE_zone_iteration_stop (zi);
+ zi = NULL;
+ }
+ if (NULL != nsh)
+ {
+ GNUNET_NAMESTORE_disconnect (nsh);
+ nsh = NULL;
+ }
+ if (NULL != t)
+ {
+ GNUNET_SCHEDULER_cancel (t);
+ t = NULL;
+ }
+ if (NULL != timeout_task)
+ {
+ GNUNET_SCHEDULER_cancel (timeout_task);
+ timeout_task = NULL;
+ }
+ if (NULL != privkey)
+ {
+ GNUNET_free (privkey);
+ privkey = NULL;
+ }
+ res = 1;
+}
+
+
+/**
+ * End with timeout. As this is a benchmark, we do not
+ * fail hard but return "skipped".
+ */
+static void
+timeout (void *cls)
+{
+ (void) cls;
+ timeout_task = NULL;
+ GNUNET_SCHEDULER_shutdown ();
+ res = 77;
+}
+
+
+static struct GNUNET_GNSRECORD_Data *
+create_record (unsigned int count)
+{
+ struct GNUNET_GNSRECORD_Data *rd;
+
+ rd = GNUNET_malloc (count + sizeof (struct GNUNET_GNSRECORD_Data));
+ rd->expiration_time = GNUNET_TIME_relative_to_absolute
(GNUNET_TIME_UNIT_HOURS).abs_value_us;
+ rd->record_type = count;
+ rd->data_size = count;
+ rd->data = (void *) &rd[1];
+ rd->flags = 0;
+ memset (&rd[1],
+ 'a',
+ count);
+ return rd;
+}
+
+
+static void
+zone_end (void *cls)
+{
+ struct GNUNET_TIME_Relative delay;
+
+ zi = NULL;
+ delay = GNUNET_TIME_absolute_get_duration (start);
+ fprintf (stdout,
+ "Iterating over %u records took %s\n",
+ off,
+ GNUNET_STRINGS_relative_time_to_string (delay,
+ GNUNET_YES));
+ if (BENCHMARK_SIZE == off)
+ {
+ res = 0;
+ }
+ else
+ {
+ GNUNET_break (0);
+ res = 1;
+ }
+ GNUNET_SCHEDULER_shutdown ();
+}
+
+
+static void
+fail_cb (void *cls)
+{
+ zi = NULL;
+ res = 2;
+ GNUNET_break (0);
+ GNUNET_SCHEDULER_shutdown ();
+}
+
+
+static void
+zone_proc (void *cls,
+ const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
+ const char *label,
+ unsigned int rd_count,
+ const struct GNUNET_GNSRECORD_Data *rd)
+{
+ struct GNUNET_GNSRECORD_Data *wrd;
+ unsigned int xoff;
+
+ GNUNET_assert (NULL != zone);
+ if (1 != sscanf (label,
+ "l%u",
+ &xoff))
+ {
+ res = 3;
+ GNUNET_break (0);
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ if ( (xoff > BENCHMARK_SIZE) ||
+ (0 != (seen[xoff / 8] & (1U << (xoff % 8)))) )
+ {
+ res = 3;
+ GNUNET_break (0);
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ seen[xoff / 8] |= (1U << (xoff % 8));
+ wrd = create_record (xoff % MAX_REC_SIZE);
+ if ( (rd->record_type != wrd->record_type) ||
+ (rd->data_size != wrd->data_size) ||
+ (rd->flags != wrd->flags) )
+ {
+ res = 4;
+ GNUNET_break (0);
+ GNUNET_SCHEDULER_shutdown ();
+ GNUNET_free (wrd);
+ return;
+ }
+ if (0 != memcmp (rd->data,
+ wrd->data,
+ wrd->data_size))
+ {
+ res = 4;
+ GNUNET_break (0);
+ GNUNET_SCHEDULER_shutdown ();
+ GNUNET_free (wrd);
+ return;
+ }
+ GNUNET_free (wrd);
+ if (0 != memcmp (zone,
+ privkey,
+ sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey)))
+ {
+ res = 5;
+ GNUNET_break (0);
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ off++;
+ left_until_next--;
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Obtained record %u, expecting %u more until asking for mor
explicitly\n",
+ off,
+ left_until_next);
+ if (0 == left_until_next)
+ {
+ left_until_next = BLOCK_SIZE;
+ GNUNET_NAMESTORE_zone_iterator_next (zi,
+ left_until_next);
+ }
+}
+
+
+static void
+publish_record (void *cls);
+
+
+static void
+put_cont (void *cls,
+ int32_t success,
+ const char *emsg)
+{
+ (void) cls;
+ qe = NULL;
+ GNUNET_assert (GNUNET_OK == success);
+ t = GNUNET_SCHEDULER_add_now (&publish_record,
+ NULL);
+}
+
+
+static void
+publish_record (void *cls)
+{
+ struct GNUNET_GNSRECORD_Data *rd;
+ char *label;
+
+ (void) cls;
+ t = NULL;
+ if (BENCHMARK_SIZE == off)
+ {
+ struct GNUNET_TIME_Relative delay;
+
+ delay = GNUNET_TIME_absolute_get_duration (start);
+ fprintf (stdout,
+ "Inserting %u records took %s\n",
+ off,
+ GNUNET_STRINGS_relative_time_to_string (delay,
+ GNUNET_YES));
+ start = GNUNET_TIME_absolute_get ();
+ off = 0;
+ left_until_next = 1;
+ zi = GNUNET_NAMESTORE_zone_iteration_start (nsh,
+ NULL,
+ &fail_cb,
+ NULL,
+ &zone_proc,
+ NULL,
+ &zone_end,
+ NULL);
+ GNUNET_assert (NULL != zi);
+ return;
+ }
+ rd = create_record ((++off) % MAX_REC_SIZE);
+ GNUNET_asprintf (&label,
+ "l%u",
+ off);
+ qe = GNUNET_NAMESTORE_records_store (nsh,
+ privkey,
+ label,
+ 1, rd,
+ &put_cont,
+ NULL);
+ GNUNET_free (label);
+ GNUNET_free (rd);
+}
+
+
+static void
+run (void *cls,
+ const struct GNUNET_CONFIGURATION_Handle *cfg,
+ struct GNUNET_TESTING_Peer *peer)
+{
+ directory = NULL;
+ GNUNET_assert (GNUNET_OK ==
+ GNUNET_CONFIGURATION_get_value_string(cfg,
+ "PATHS",
+ "GNUNET_TEST_HOME",
+ &directory));
+ GNUNET_DISK_directory_remove (directory);
+ GNUNET_SCHEDULER_add_shutdown (&end,
+ NULL);
+ timeout_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
+ &timeout,
+ NULL);
+ nsh = GNUNET_NAMESTORE_connect (cfg);
+ GNUNET_assert (NULL != nsh);
+ privkey = GNUNET_CRYPTO_ecdsa_key_create ();
+ GNUNET_assert (NULL != privkey);
+ start = GNUNET_TIME_absolute_get ();
+ t = GNUNET_SCHEDULER_add_now (&publish_record,
+ NULL);
+}
+
+
+int
+main (int argc,
+ char *argv[])
+{
+ const char *plugin_name;
+ char *cfg_name;
+
+ plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
+ GNUNET_asprintf (&cfg_name,
+ "test_namestore_api_%s.conf",
+ plugin_name);
+ res = 1;
+ if (0 !=
+ GNUNET_TESTING_peer_run ("perf-namestore-api-zone-iteration",
+ cfg_name,
+ &run,
+ NULL))
+ {
+ res = 1;
+ }
+ GNUNET_free (cfg_name);
+ if (NULL != directory)
+ {
+ GNUNET_DISK_directory_remove (directory);
+ GNUNET_free (directory);
+ }
+ return res;
+}
+
+
+/* end of perf_namestore_api_zone_iteration.c */
diff --git a/src/namestore/test_namestore_api_flat.conf
b/src/namestore/test_namestore_api_flat.conf
new file mode 100644
index 000000000..26e2f2c51
--- /dev/null
+++ b/src/namestore/test_namestore_api_flat.conf
@@ -0,0 +1,7 @@
address@hidden@ test_namestore_api.conf
+
+[namestore]
+DATABASE = flat
+
+[namecache]
+DISABLE = YES
diff --git a/src/namestore/test_namestore_api_postgres.conf
b/src/namestore/test_namestore_api_postgres.conf
new file mode 100644
index 000000000..259ce35e7
--- /dev/null
+++ b/src/namestore/test_namestore_api_postgres.conf
@@ -0,0 +1,7 @@
address@hidden@ test_namestore_api.conf
+
+[namestore]
+DATABASE = postgres
+
+[namecache]
+DISABLE = YES
diff --git a/src/namestore/test_namestore_api_sqlite.conf
b/src/namestore/test_namestore_api_sqlite.conf
new file mode 100644
index 000000000..72b609226
--- /dev/null
+++ b/src/namestore/test_namestore_api_sqlite.conf
@@ -0,0 +1,4 @@
address@hidden@ test_namestore_api.conf
+
+[namecache]
+DISABLE = YES
diff --git a/src/namestore/test_plugin_namestore.c
b/src/namestore/test_plugin_namestore.c
index ac2a0bba9..6bccd1706 100644
--- a/src/namestore/test_plugin_namestore.c
+++ b/src/namestore/test_plugin_namestore.c
@@ -207,7 +207,6 @@ main (int argc,
GNUNET_GETOPT_OPTION_END
};
- //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
GNUNET_log_setup ("test-plugin-namestore",
"WARNING",
NULL);
@@ -227,7 +226,6 @@ main (int argc,
FPRINTF (stderr,
"Missed some testcases: %d\n",
ok);
- //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
return ok;
}
--
To stop receiving notification emails like this one, please contact
address@hidden
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] [gnunet] branch master updated: add perf_ logic for namestore iterations, improve namestore insertion performance by 30 percent,
gnunet <=