[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [gnunet] branch master updated (c15e79511 -> 823215c97)
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [gnunet] branch master updated (c15e79511 -> 823215c97) |
|
Date: |
Thu, 26 Apr 2018 16:40:12 +0200 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a change to branch master
in repository gnunet.
from c15e79511 clean up zone iteration logic
new 1b21e0b64 misc bugfixes
new 823215c97 misc zoneimporter fixes
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/gnsrecord/gnsrecord_crypto.c | 2 +-
src/namecache/plugin_namecache_sqlite.c | 75 +++++++-----
src/namestore/gnunet-service-namestore.c | 14 ++-
src/namestore/gnunet-zoneimport.c | 187 ++++++++++++++++-------------
src/namestore/namestore_api.c | 3 +-
src/zonemaster/gnunet-service-zonemaster.c | 29 ++++-
6 files changed, 186 insertions(+), 124 deletions(-)
diff --git a/src/gnsrecord/gnsrecord_crypto.c b/src/gnsrecord/gnsrecord_crypto.c
index 27c83b90e..d9fe2ae40 100644
--- a/src/gnsrecord/gnsrecord_crypto.c
+++ b/src/gnsrecord/gnsrecord_crypto.c
@@ -236,7 +236,7 @@ GNUNET_GNSRECORD_block_create2 (const struct
GNUNET_CRYPTO_EcdsaPrivateKey *key,
static struct KeyCacheLine cache[CSIZE];
struct KeyCacheLine *line;
- line = &cache[(*(unsigned int *) key) ^ CSIZE];
+ line = &cache[(*(unsigned int *) key) % CSIZE];
if (0 != memcmp (&line->key,
key,
sizeof (*key)))
diff --git a/src/namecache/plugin_namecache_sqlite.c
b/src/namecache/plugin_namecache_sqlite.c
index 6f5f2d952..e0f64a6b0 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -136,13 +136,16 @@ create_indices (sqlite3 * dbh)
{
/* create indices */
if ( (SQLITE_OK !=
- sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_query_hash ON
ns096blocks (query,expiration_time)",
+ sqlite3_exec (dbh,
+ "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks
(query,expiration_time)",
NULL, NULL, NULL)) ||
(SQLITE_OK !=
- sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_block_expiration ON
ns096blocks (expiration_time)",
+ sqlite3_exec (dbh,
+ "CREATE INDEX IF NOT EXISTS ir_block_expiration ON
ns096blocks (expiration_time)",
NULL, NULL, NULL)) )
LOG (GNUNET_ERROR_TYPE_ERROR,
- "Failed to create indices: %s\n", sqlite3_errmsg (dbh));
+ "Failed to create indices: %s\n",
+ sqlite3_errmsg (dbh));
}
@@ -208,28 +211,44 @@ database_setup (struct Plugin *plugin)
return GNUNET_SYSERR;
}
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA temp_store=MEMORY",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA synchronous=NORMAL", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA synchronous=NORMAL",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL,
NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA legacy_file_format=OFF",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
- NULL, ENULL));
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA auto_vacuum=INCREMENTAL",
+ NULL, NULL,
+ ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA encoding=\"UTF-8\"", NULL,
- NULL, ENULL));
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA encoding=\"UTF-8\"",
+ NULL, NULL,
+ ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL,
NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA locking_mode=EXCLUSIVE",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA page_size=4092",
+ NULL, NULL,
ENULL));
- CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
+ CHECK (SQLITE_OK ==
+ sqlite3_busy_timeout (plugin->dbh,
+ BUSY_TIMEOUT_MS));
/* Create tables */
@@ -237,17 +256,19 @@ database_setup (struct Plugin *plugin)
sq_prepare (plugin->dbh,
"SELECT 1 FROM sqlite_master WHERE tbl_name =
'ns096blocks'",
&stmt));
- if ((sqlite3_step (stmt) == SQLITE_DONE) &&
- (sqlite3_exec
- (plugin->dbh,
- "CREATE TABLE ns096blocks ("
- " query BLOB NOT NULL,"
- " block BLOB NOT NULL,"
- " expiration_time INT8 NOT NULL"
- ")",
- NULL, NULL, NULL) != SQLITE_OK))
+ if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
+ (SQLITE_OK !=
+ sqlite3_exec (plugin->dbh,
+ "CREATE TABLE ns096blocks ("
+ " query BLOB NOT NULL,"
+ " block BLOB NOT NULL,"
+ " expiration_time INT8 NOT NULL"
+ ")",
+ NULL, NULL, NULL)) )
{
- LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
+ LOG_SQLITE (plugin,
+ GNUNET_ERROR_TYPE_ERROR,
+ "sqlite3_exec");
sqlite3_finalize (stmt);
return GNUNET_SYSERR;
}
@@ -411,11 +432,11 @@ namecache_sqlite_cache_block (void *cls,
GNUNET_CRYPTO_hash (&block->derived_key,
sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
&query);
- fprintf (stderr, // GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Caching new version of block %s (expires %s)\n",
- GNUNET_h2s (&query),
- GNUNET_STRINGS_absolute_time_to_string (expiration));
expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Caching new version of block %s (expires %s)\n",
+ GNUNET_h2s (&query),
+ GNUNET_STRINGS_absolute_time_to_string (expiration));
if (block_size > 64 * 65536)
{
GNUNET_break (0);
diff --git a/src/namestore/gnunet-service-namestore.c
b/src/namestore/gnunet-service-namestore.c
index 994eae2c8..06a50132b 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -708,11 +708,15 @@ refresh_block (struct NamestoreClient *nc,
&res);
GNUNET_free (nick);
}
-
- exp_time = (0 == res_count)
- ? GNUNET_TIME_UNIT_ZERO_ABS
- : GNUNET_GNSRECORD_record_get_expiration_time (res_count,
- res);
+ if (0 == res_count)
+ {
+ send_store_response (nc,
+ GNUNET_OK,
+ rid);
+ return; /* no data, no need to update cache */
+ }
+ exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count,
+ res);
if (cache_keys)
block = GNUNET_GNSRECORD_block_create2 (zone_key,
exp_time,
diff --git a/src/namestore/gnunet-zoneimport.c
b/src/namestore/gnunet-zoneimport.c
index 779df695b..503262487 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -48,6 +48,16 @@
#define MAX_RETRIES 5
/**
+ * How many DNS requests do we at most issue in rapid series?
+ */
+#define MAX_SERIES 10
+
+/**
+ * How long do we wait at least between series of requests?
+ */
+#define SERIES_DELAY GNUNET_TIME_relative_multiply
(GNUNET_TIME_UNIT_MICROSECONDS, 10)
+
+/**
* How many requests do we request from NAMESTORE in one batch
* during our initial iteration?
*/
@@ -200,11 +210,16 @@ static struct GNUNET_NAMESTORE_Handle *ns;
static struct GNUNET_DNSSTUB_Context *ctx;
/**
- * The number of queries that are outstanding
+ * The number of DNS queries that are outstanding
*/
static unsigned int pending;
/**
+ * The number of NAMESTORE record store operations that are outstanding
+ */
+static unsigned int pending_rs;
+
+/**
* Number of lookups we performed overall.
*/
static unsigned int lookups;
@@ -692,7 +707,7 @@ process_record (void *cls,
req->hostname))
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "DNS returned record for `%s' of type %u while resolving
`%s'\n",
+ "DNS returned record from zone `%s' of type %u while resolving
`%s'\n",
rec->name,
(unsigned int) rec->type,
req->hostname);
@@ -910,7 +925,11 @@ store_completed_cb (void *cls,
struct Request *req = cls;
req->qe = NULL;
- pending--;
+ pending_rs--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
if (GNUNET_SYSERR == success)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -967,6 +986,10 @@ process_result (void *cls,
req_tail,
req);
pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Stub gave up on DNS reply for `%s'\n",
req->hostname);
@@ -981,12 +1004,17 @@ process_result (void *cls,
return;
}
if (req->id != dns->id)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "DNS ID did not match request, ignoring reply\n");
return;
+ }
GNUNET_CONTAINER_DLL_remove (req_head,
req_tail,
req);
GNUNET_DNSSTUB_resolve_cancel (req->rs);
req->rs = NULL;
+ pending--;
p = GNUNET_DNSPARSER_parse ((const char *) dns,
dns_len);
if (NULL == p)
@@ -998,11 +1026,17 @@ process_result (void *cls,
{
failures++;
insert_sorted (req);
- pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
return;
}
insert_sorted (req);
- pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
return;
}
/* import new records */
@@ -1047,6 +1081,7 @@ process_result (void *cls,
/* convert linked list into array */
for (rec = req->rec_head; NULL != rec; rec =rec->next)
rd[off++] = rec->grd;
+ pending_rs++;
req->qe = GNUNET_NAMESTORE_records_store (ns,
&req->zone->key,
get_label (req),
@@ -1054,69 +1089,13 @@ process_result (void *cls,
rd,
&store_completed_cb,
req);
+ GNUNET_assert (NULL != req->qe);
}
insert_sorted (req);
}
/**
- * Submit a request to DNS unless we need to slow down because
- * we are at the rate limit.
- *
- * @param req request to submit
- * @return #GNUNET_OK if request was submitted
- * #GNUNET_NO if request was already submitted
- * #GNUNET_SYSERR if we are at the rate limit
- */
-static int
-submit_req (struct Request *req)
-{
- static struct GNUNET_TIME_Absolute last_request;
- struct GNUNET_TIME_Absolute now;
- void *raw;
- size_t raw_size;
-
- if (NULL != req->qe)
- return GNUNET_NO; /* namestore op still pending */
- if (NULL != req->rs)
- {
- GNUNET_break (0);
- return GNUNET_NO; /* already submitted */
- }
- now = GNUNET_TIME_absolute_get ();
- if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) ||
- (pending >= THRESH) )
- return GNUNET_SYSERR;
- GNUNET_CONTAINER_DLL_insert (req_head,
- req_tail,
- req);
- GNUNET_assert (NULL == req->rs);
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Requesting resolution for `%s'\n",
- req->hostname);
- raw = build_dns_query (req,
- &raw_size);
- if (NULL == raw)
- {
- GNUNET_break (0);
- free_request (req);
- return GNUNET_SYSERR;
- }
- req->rs = GNUNET_DNSSTUB_resolve (ctx,
- raw,
- raw_size,
- &process_result,
- req);
- GNUNET_assert (NULL != req->rs);
- req->issue_num++;
- last_request = now;
- lookups++;
- pending++;
- return GNUNET_OK;
-}
-
-
-/**
* Process as many requests as possible from the queue.
*
* @param cls NULL
@@ -1125,23 +1104,60 @@ static void
process_queue (void *cls)
{
struct Request *req;
+ unsigned int series;
+ void *raw;
+ size_t raw_size;
(void) cls;
+ series = 0;
t = NULL;
- while (1)
+ while (pending + pending_rs < THRESH)
{
req = GNUNET_CONTAINER_heap_peek (req_heap);
if (NULL == req)
break;
+ if (NULL != req->qe)
+ return; /* namestore op still pending */
+ if (NULL != req->rs)
+ {
+ GNUNET_break (0);
+ return; /* already submitted */
+ }
if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
break;
- if (GNUNET_OK != submit_req (req))
- break;
GNUNET_assert (req ==
GNUNET_CONTAINER_heap_remove_root (req_heap));
req->hn = NULL;
+ GNUNET_CONTAINER_DLL_insert (req_head,
+ req_tail,
+ req);
+ GNUNET_assert (NULL == req->rs);
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Requesting resolution for `%s'\n",
+ req->hostname);
+ raw = build_dns_query (req,
+ &raw_size);
+ if (NULL == raw)
+ {
+ GNUNET_break (0);
+ free_request (req);
+ continue;
+ }
+ req->rs = GNUNET_DNSSTUB_resolve (ctx,
+ raw,
+ raw_size,
+ &process_result,
+ req);
+ GNUNET_assert (NULL != req->rs);
+ req->issue_num++;
+ lookups++;
+ pending++;
+ series++;
+ if (series > MAX_SERIES)
+ break;
}
-
+ if (pending + pending_rs >= THRESH)
+ return; /* wait for replies */
req = GNUNET_CONTAINER_heap_peek (req_heap);
if (NULL == req)
return;
@@ -1156,17 +1172,15 @@ process_queue (void *cls)
t = GNUNET_SCHEDULER_add_at (req->expires,
&process_queue,
NULL);
+ return;
}
- else
- {
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Throttling for 1ms\n");
- if (NULL != t)
- GNUNET_SCHEDULER_cancel (t);
- t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS,
- &process_queue,
- NULL);
- }
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Throttling\n");
+ if (NULL != t)
+ GNUNET_SCHEDULER_cancel (t);
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
}
@@ -1438,7 +1452,6 @@ queue (const char *hostname)
return;
}
- pending++;
hlen = strlen (hostname) + 1;
req = GNUNET_malloc (sizeof (struct Request) + hlen);
req->zone = zone;
@@ -1503,11 +1516,16 @@ iterate_zones (void *cls)
static struct Zone *last;
(void) cls;
- zone_it = NULL;
+ if (NULL != zone_it)
+ {
+ zone_it = NULL;
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Finished iteration over zone `%s'!\n",
+ last->domain);
+ }
GNUNET_assert (NULL != zone_tail);
if (zone_tail == last)
{
- GNUNET_assert (NULL == t);
/* Done iterating over relevant zones in NAMESTORE, move
rest of hash map to work queue as well. */
GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
@@ -1515,15 +1533,15 @@ iterate_zones (void *cls)
NULL);
GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
ns_pending = NULL;
- GNUNET_assert (NULL == t);
- t = GNUNET_SCHEDULER_add_now (&process_queue,
- NULL);
return;
}
if (NULL == last)
last = zone_head;
else
last = last->next;
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Starting iteration over zone `%s'!\n",
+ last->domain);
ns_iterator_trigger_next = 1;
zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
&last->key,
@@ -1745,12 +1763,13 @@ main (int argc,
NULL);
GNUNET_free ((void*) argv);
fprintf (stderr,
- "Rejected %u names, did %u lookups, found %u records, %u lookups
failed, %u pending on shutdown\n",
+ "Rejected %u names, did %u lookups, found %u records, %u lookups
failed, %u/%u pending on shutdown\n",
rejects,
lookups,
records,
failures,
- pending);
+ pending,
+ pending_rs);
return 0;
}
diff --git a/src/namestore/namestore_api.c b/src/namestore/namestore_api.c
index 65d3d75d0..ab356838b 100644
--- a/src/namestore/namestore_api.c
+++ b/src/namestore/namestore_api.c
@@ -1293,7 +1293,8 @@ GNUNET_NAMESTORE_zone_iterator_next (struct
GNUNET_NAMESTORE_ZoneIterator *it,
struct GNUNET_MQ_Envelope *env;
LOG (GNUNET_ERROR_TYPE_DEBUG,
- "Sending ZONE_ITERATION_NEXT message\n");
+ "Sending ZONE_ITERATION_NEXT message with limit %llu\n",
+ (unsigned long long) limit);
env = GNUNET_MQ_msg (msg,
GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT);
msg->gns_header.r_id = htonl (it->op_id);
diff --git a/src/zonemaster/gnunet-service-zonemaster.c
b/src/zonemaster/gnunet-service-zonemaster.c
index f40f25c3f..518d5f572 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -181,6 +181,13 @@ static struct GNUNET_SCHEDULER_Task *zone_publish_task;
static int first_zone_iteration;
/**
+ * Optimize block insertion by caching map of private keys to
+ * public keys in memory?
+ */
+static int cache_keys;
+
+
+/**
* Task run during shutdown.
*
* @param cls unused
@@ -388,11 +395,18 @@ perform_dht_put (const struct
GNUNET_CRYPTO_EcdsaPrivateKey *key,
expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
rd_public);
- block = GNUNET_GNSRECORD_block_create (key,
- expire,
- label,
- rd_public,
- rd_public_count);
+ if (cache_keys)
+ block = GNUNET_GNSRECORD_block_create2 (key,
+ expire,
+ label,
+ rd_public,
+ rd_public_count);
+ else
+ block = GNUNET_GNSRECORD_block_create (key,
+ expire,
+ label,
+ rd_public,
+ rd_public_count);
if (NULL == block)
{
GNUNET_break (0);
@@ -585,6 +599,7 @@ put_gns_record (void *cls,
static void
publish_zone_dht_start (void *cls)
{
+ (void) cls;
zone_publish_task = NULL;
GNUNET_STATISTICS_update (statistics,
"Full zone iterations launched",
@@ -736,7 +751,9 @@ run (void *cls,
GNUNET_SCHEDULER_shutdown ();
return;
}
-
+ cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
+ "namestore",
+ "CACHE_KEYS");
put_interval = INITIAL_PUT_INTERVAL;
zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
if (GNUNET_OK ==
--
To stop receiving notification emails like this one, please contact
address@hidden
- [GNUnet-SVN] [gnunet] branch master updated (c15e79511 -> 823215c97),
gnunet <=