[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [gnunet] branch master updated (e1f46c052 -> 08795cc18)
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [gnunet] branch master updated (e1f46c052 -> 08795cc18) |
|
Date: |
Mon, 23 Apr 2018 15:59:04 +0200 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a change to branch master
in repository gnunet.
from e1f46c052 move to @ for empty label, fix empty-label resolution bug in
GNS API, install gnunet-zoneimport and add man page
new d6e1cea9a more incremental processing of zoneimporter
new 3e2b9dc6d merge conflict resolution
new 08795cc18 fix transition to multi-server in cmd line for
gnunet-zoneimport
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
doc/man/gnunet-zoneimport.1 | 7 +-
src/gnsrecord/gnsrecord_misc.c | 18 ++---
src/namecache/plugin_namecache_sqlite.c | 8 +--
src/namestore/gnunet-service-namestore.c | 68 +++++++++++-------
src/namestore/gnunet-zoneimport.c | 110 ++++++++++++++++++++++--------
src/namestore/plugin_namestore_postgres.c | 2 +
6 files changed, 140 insertions(+), 73 deletions(-)
diff --git a/doc/man/gnunet-zoneimport.1 b/doc/man/gnunet-zoneimport.1
index d268bf9ff..afe3354e6 100644
--- a/doc/man/gnunet-zoneimport.1
+++ b/doc/man/gnunet-zoneimport.1
@@ -4,12 +4,14 @@
gnunet\-zoneimport \- import DNS zone into GNS zone
.SH SYNOPSIS
-.B gnunet\-zoneimport -s IP
+.B gnunet\-zoneimport [IP]+
.br
.SH DESCRIPTION
\fBgnunet\-zoneimport\fP reads a list of domain names (FQDN) from stdin and
issues DNS queries for each of the domain names given. It then checks if a
local ego with a name matching the domain exists. Specifically, if the domain
name is "example.fr", it will check if an ego "fr" exists, while for a domain
"example.com.fr" it will look for an ego called "com.fr"). If so, it will
convert the DNS records into GNS records (in particular converting NS records
and glue records to GNS2DNS reco [...]
+The arguments given to gnunet\-zoneimport is a list of IP addresses of DNS
servers to query.
+
gnunet\-zoneimport will usually never terminate: it will check when DNS
records expire, and re-issue requests when the old DNS records have expired so
that GNS always has the latest data.
gnunet\-zoneimport will issue many DNS queries in parallel, but is
rate-limited in various ways, so most DNS servers should easily handle the
load. gnunet\-zoneimport will perform a limited number of retries if queries
fail.
@@ -23,9 +25,6 @@ Finally, gnunet\-zoneimport keeps information for each domain
name in memory. T
.IP "\-c FILENAME, \-\-config=FILENAME"
Use the configuration file FILENAME.
.B
-.IP "\-s IP, \-\-server IP"
-IP address of the DNS server to query. Should be the authoritative server for
the domain (lookup the IP address of the NS server!) and not some recursive
resolver (which would be both slow and is more likely to return only partial
information).
-.B
.IP "\-h, \-\-help"
Print short help on options.
diff --git a/src/gnsrecord/gnsrecord_misc.c b/src/gnsrecord/gnsrecord_misc.c
index 3c1ead437..05f56cf70 100644
--- a/src/gnsrecord/gnsrecord_misc.c
+++ b/src/gnsrecord/gnsrecord_misc.c
@@ -152,8 +152,6 @@ struct GNUNET_TIME_Absolute
GNUNET_GNSRECORD_record_get_expiration_time (unsigned int rd_count,
const struct GNUNET_GNSRECORD_Data
*rd)
{
- unsigned int c;
- unsigned int c2;
struct GNUNET_TIME_Absolute expire;
struct GNUNET_TIME_Absolute at;
struct GNUNET_TIME_Relative rt;
@@ -163,7 +161,7 @@ GNUNET_GNSRECORD_record_get_expiration_time (unsigned int
rd_count,
if (NULL == rd)
return GNUNET_TIME_UNIT_ZERO_ABS;
expire = GNUNET_TIME_UNIT_FOREVER_ABS;
- for (c = 0; c < rd_count; c++)
+ for (unsigned int c = 0; c < rd_count; c++)
{
if (0 != (rd[c].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
{
@@ -175,12 +173,12 @@ GNUNET_GNSRECORD_record_get_expiration_time (unsigned int
rd_count,
at.abs_value_us = rd[c].expiration_time;
}
- for (c2 = 0; c2 < rd_count; c2++)
+ for (unsigned int c2 = 0; c2 < rd_count; c2++)
{
/* Check for shadow record */
- if ((c == c2) ||
- (rd[c].record_type != rd[c2].record_type) ||
- (0 == (rd[c2].flags & GNUNET_GNSRECORD_RF_SHADOW_RECORD)))
+ if ( (c == c2) ||
+ (rd[c].record_type != rd[c2].record_type) ||
+ (0 == (rd[c2].flags & GNUNET_GNSRECORD_RF_SHADOW_RECORD)) )
continue;
/* We have a shadow record */
if (0 != (rd[c2].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
@@ -192,9 +190,11 @@ GNUNET_GNSRECORD_record_get_expiration_time (unsigned int
rd_count,
{
at_shadow.abs_value_us = rd[c2].expiration_time;
}
- at = GNUNET_TIME_absolute_max (at, at_shadow);
+ at = GNUNET_TIME_absolute_max (at,
+ at_shadow);
}
- expire = GNUNET_TIME_absolute_min (at, expire);
+ expire = GNUNET_TIME_absolute_min (at,
+ expire);
}
LOG (GNUNET_ERROR_TYPE_DEBUG,
"Determined expiration time for block with %u records to be %s\n",
diff --git a/src/namecache/plugin_namecache_sqlite.c
b/src/namecache/plugin_namecache_sqlite.c
index e8a61a232..6f5f2d952 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -411,10 +411,10 @@ namecache_sqlite_cache_block (void *cls,
GNUNET_CRYPTO_hash (&block->derived_key,
sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
&query);
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Caching new version of block %s (expires %llu)\n",
- GNUNET_h2s (&query),
- (unsigned long long) expiration.abs_value_us);
+ fprintf (stderr, // GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Caching new version of block %s (expires %s)\n",
+ GNUNET_h2s (&query),
+ GNUNET_STRINGS_absolute_time_to_string (expiration));
expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
if (block_size > 64 * 65536)
{
diff --git a/src/namestore/gnunet-service-namestore.c
b/src/namestore/gnunet-service-namestore.c
index 3685c93af..3ba4f9ec3 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -439,9 +439,24 @@ get_nick_record (const struct
GNUNET_CRYPTO_EcdsaPrivateKey *zone)
}
+/**
+ * Merge the nick record @a nick_rd with the rest of the
+ * record set given in @a rd2. Store the result in @a rdc_res
+ * and @a rd_res. The @a nick_rd's expiration time is set to
+ * the maximum expiration time of all of the records in @a rd2.
+ *
+ * @param nick_rd the nick record to integrate
+ * @param rd2_length length of the @a rd2 array
+ * @param rd2 array of records
+ * @param rdc_res[out] length of the resulting @a rd_res array
+ * @param rd_res[out] set to an array of records,
+ * including @a nick_rd and @a rd2;
+ * all of the variable-size 'data' fields in @a rd2 are
+ * allocated in the same chunk of memory!
+ */
static void
merge_with_nick_records (const struct GNUNET_GNSRECORD_Data *nick_rd,
- unsigned int rdc2,
+ unsigned int rd2_length,
const struct GNUNET_GNSRECORD_Data *rd2,
unsigned int *rdc_res,
struct GNUNET_GNSRECORD_Data **rd_res)
@@ -452,24 +467,22 @@ merge_with_nick_records (const struct
GNUNET_GNSRECORD_Data *nick_rd,
int record_offset;
size_t data_offset;
- (*rdc_res) = 1 + rdc2;
- if (0 == 1 + rdc2)
+ (*rdc_res) = 1 + rd2_length;
+ if (0 == 1 + rd2_length)
{
(*rd_res) = NULL;
return;
}
-
req = 0;
for (unsigned int c=0; c< 1; c++)
req += sizeof (struct GNUNET_GNSRECORD_Data) + nick_rd[c].data_size;
- for (unsigned int c=0; c< rdc2; c++)
+ for (unsigned int c=0; c< rd2_length; c++)
req += sizeof (struct GNUNET_GNSRECORD_Data) + rd2[c].data_size;
(*rd_res) = GNUNET_malloc (req);
- data = (char *) &(*rd_res)[1 + rdc2];
+ data = (char *) &(*rd_res)[1 + rd2_length];
data_offset = 0;
latest_expiration = 0;
-
- for (unsigned int c=0; c< rdc2; c++)
+ for (unsigned int c=0; c< rd2_length; c++)
{
if (0 != (rd2[c].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
{
@@ -486,17 +499,15 @@ merge_with_nick_records (const struct
GNUNET_GNSRECORD_Data *nick_rd,
rd2[c].data_size);
data_offset += (*rd_res)[c].data_size;
}
- record_offset = rdc2;
- for (unsigned int c=0; c< 1; c++)
- {
- (*rd_res)[c+record_offset] = nick_rd[c];
- (*rd_res)[c+record_offset].expiration_time = latest_expiration;
- (*rd_res)[c+record_offset].data = (void *) &data[data_offset];
- GNUNET_memcpy ((void *) (*rd_res)[c+record_offset].data,
- nick_rd[c].data,
- nick_rd[c].data_size);
- data_offset += (*rd_res)[c+record_offset].data_size;
- }
+ /* append nick */
+ record_offset = rd2_length;
+ (*rd_res)[record_offset] = *nick_rd;
+ (*rd_res)[record_offset].expiration_time = latest_expiration;
+ (*rd_res)[record_offset].data = (void *) &data[data_offset];
+ GNUNET_memcpy ((void *) (*rd_res)[record_offset].data,
+ nick_rd->data,
+ nick_rd->data_size);
+ data_offset += (*rd_res)[record_offset].data_size;
GNUNET_assert (req == (sizeof (struct GNUNET_GNSRECORD_Data)) * (*rdc_res) +
data_offset);
}
@@ -531,7 +542,9 @@ send_lookup_response (struct NamestoreClient *nc,
char *rd_ser;
nick = get_nick_record (zone_key);
- if ((NULL != nick) && (0 != strcmp(name, GNUNET_GNS_EMPTY_LABEL_AT)))
+ if ( (NULL != nick) &&
+ (0 != strcmp (name,
+ GNUNET_GNS_EMPTY_LABEL_AT)))
{
nick->flags = (nick->flags | GNUNET_GNSRECORD_RF_PRIVATE) ^
GNUNET_GNSRECORD_RF_PRIVATE;
merge_with_nick_records (nick,
@@ -677,13 +690,15 @@ refresh_block (struct NamestoreClient *nc,
block = GNUNET_GNSRECORD_block_create (zone_key,
GNUNET_TIME_UNIT_ZERO_ABS,
name,
- res, rd_count);
+ res,
+ rd_count);
else
block = GNUNET_GNSRECORD_block_create (zone_key,
GNUNET_GNSRECORD_record_get_expiration_time (res_count,
- res),
+
res),
name,
- res, res_count);
+ res,
+ res_count);
GNUNET_assert (NULL != block);
GNUNET_CRYPTO_ecdsa_key_get_public (zone_key,
&pkey);
@@ -773,9 +788,10 @@ lookup_it (void *cls,
rdc_res = 0;
rlc->nick->flags = (rlc->nick->flags | GNUNET_GNSRECORD_RF_PRIVATE) ^
GNUNET_GNSRECORD_RF_PRIVATE;
merge_with_nick_records (rlc->nick,
- rd_count, rd,
- &rdc_res, &rd_res);
-
+ rd_count,
+ rd,
+ &rdc_res,
+ &rd_res);
rlc->rd_ser_len = GNUNET_GNSRECORD_records_get_size (rdc_res,
rd_res);
rlc->res_rd_count = rdc_res;
diff --git a/src/namestore/gnunet-zoneimport.c
b/src/namestore/gnunet-zoneimport.c
index 89afeeeb1..4fd0f3aef 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -34,7 +34,7 @@
/**
* Maximum number of queries pending at the same time.
*/
-#define THRESH 20
+#define THRESH 100
/**
* TIME_THRESH is in usecs. How quickly do we submit fresh queries.
@@ -242,6 +242,11 @@ static unsigned int failures;
static unsigned int records;
/**
+ * #GNUNET_YES if we have more work to be read from `stdin`.
+ */
+static int stdin_waiting;
+
+/**
* Heap of all requests to perform, sorted by
* the time we should next do the request (i.e. by expires).
*/
@@ -263,11 +268,6 @@ static struct Request *req_tail;
static struct GNUNET_SCHEDULER_Task *t;
/**
- * Which DNS server do we use for queries?
- */
-static char *dns_server;
-
-/**
* Head of list of zones we are managing.
*/
static struct Zone *zone_head;
@@ -786,6 +786,7 @@ store_completed_cb (void *cls,
int32_t success,
const char *emsg)
{
+ static unsigned int pdot;
struct Request *req = cls;
struct Record *rec;
@@ -803,6 +804,9 @@ store_completed_cb (void *cls,
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Stored records under `%s'\n",
req->label);
+ pdot++;
+ if (0 == pdot % 1000)
+ fprintf (stderr, ".");
}
/* Free records */
while (NULL != (rec = req->rec_head))
@@ -1051,28 +1055,32 @@ do_shutdown (void *cls)
GNUNET_SCHEDULER_cancel (t);
t = NULL;
}
- if (NULL != ns)
- {
- GNUNET_NAMESTORE_disconnect (ns);
- ns = NULL;
- }
- if (NULL != ctx)
- {
- GNUNET_DNSSTUB_stop (ctx);
- ctx = NULL;
- }
while (NULL != (req = req_head))
{
GNUNET_CONTAINER_DLL_remove (req_head,
req_tail,
req);
+ if (NULL != req->qe)
+ GNUNET_NAMESTORE_cancel (req->qe);
free_request (req);
}
while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
{
req->hn = NULL;
+ if (NULL != req->qe)
+ GNUNET_NAMESTORE_cancel (req->qe);
free_request (req);
}
+ if (NULL != ns)
+ {
+ GNUNET_NAMESTORE_disconnect (ns);
+ ns = NULL;
+ }
+ if (NULL != ctx)
+ {
+ GNUNET_DNSSTUB_stop (ctx);
+ ctx = NULL;
+ }
if (NULL != req_heap)
{
GNUNET_CONTAINER_heap_destroy (req_heap);
@@ -1090,6 +1098,32 @@ do_shutdown (void *cls)
/**
+ * Begin processing hostnames from stdin.
+ *
+ * @param cls NULL
+ */
+static void
+process_stdin (void *cls);
+
+
+/**
+ * If applicable, continue processing from stdin.
+ */
+static void
+continue_stdin ()
+{
+ if ( (pending < THRESH) &&
+ (stdin_waiting) )
+ {
+ if (NULL != t)
+ GNUNET_SCHEDULER_cancel (t);
+ t = GNUNET_SCHEDULER_add_now (&process_stdin,
+ NULL);
+ }
+}
+
+
+/**
* Function called if #GNUNET_NAMESTORE_records_lookup() failed.
* Continues resolution based on assumption namestore has no data.
*
@@ -1105,6 +1139,8 @@ ns_lookup_error_cb (void *cls)
"Failed to load data from namestore for `%s'\n",
req->label);
insert_sorted (req);
+ pending--;
+ continue_stdin ();
}
@@ -1127,6 +1163,7 @@ ns_lookup_result_cb (void *cls,
struct Request *req = cls;
req->qe = NULL;
+ pending--;
GNUNET_break (0 == memcmp (zone,
&req->zone->key,
sizeof (*zone)));
@@ -1177,6 +1214,7 @@ ns_lookup_result_cb (void *cls,
req->hostname,
GNUNET_STRINGS_absolute_time_to_string (req->expires));
insert_sorted (req);
+ continue_stdin ();
}
@@ -1203,6 +1241,7 @@ queue (const char *hostname)
"Refusing invalid hostname `%s'\n",
hostname);
rejects++;
+ continue_stdin ();
return;
}
dot = strchr (hostname,
@@ -1213,6 +1252,7 @@ queue (const char *hostname)
"Refusing invalid hostname `%s' (lacks '.')\n",
hostname);
rejects++;
+ continue_stdin ();
return;
}
for (zone = zone_head; NULL != zone; zone = zone->next)
@@ -1225,6 +1265,7 @@ queue (const char *hostname)
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Domain name `%s' not in ego list!\n",
dot + 1);
+ continue_stdin ();
return;
}
q.name = (char *) hostname;
@@ -1249,9 +1290,11 @@ queue (const char *hostname)
"Failed to pack query for hostname `%s'\n",
hostname);
rejects++;
+ continue_stdin ();
return;
}
+ pending++;
req = GNUNET_new (struct Request);
req->zone = zone;
req->hostname = GNUNET_strdup (hostname);
@@ -1278,21 +1321,33 @@ queue (const char *hostname)
static void
process_stdin (void *cls)
{
+ static unsigned int pdot;
char hn[256];
(void) cls;
t = NULL;
- GNUNET_IDENTITY_disconnect (id);
- id = NULL;
- while (NULL !=
- fgets (hn,
- sizeof (hn),
- stdin))
+ if (NULL != id)
+ {
+ GNUNET_IDENTITY_disconnect (id);
+ id = NULL;
+ }
+ if (NULL !=
+ fgets (hn,
+ sizeof (hn),
+ stdin))
{
if (strlen(hn) > 0)
hn[strlen(hn)-1] = '\0'; /* eat newline */
+ pdot++;
+ if (0 == pdot % 1000)
+ fprintf (stderr, ".");
queue (hn);
+ return;
}
+ stdin_waiting = GNUNET_NO;
+ fprintf (stderr, "\n");
+ t = GNUNET_SCHEDULER_add_now (&process_queue,
+ NULL);
}
@@ -1342,6 +1397,7 @@ identity_cb (void *cls,
{
if (NULL != zone_head)
{
+ stdin_waiting = GNUNET_YES;
t = GNUNET_SCHEDULER_add_now (&process_stdin,
NULL);
}
@@ -1393,13 +1449,13 @@ run (void *cls,
"Failed to initialize GNUnet DNS STUB\n");
return;
}
- if (NULL == args[1])
+ if (NULL == args[0])
{
fprintf (stderr,
"You must provide a list of DNS resolvers on the command line\n");
return;
}
- for (unsigned int i=1;NULL != args[i];i++)
+ for (unsigned int i=0;NULL != args[i];i++)
{
if (GNUNET_OK !=
GNUNET_DNSSTUB_add_dns_ip (ctx,
@@ -1439,12 +1495,6 @@ main (int argc,
char *const*argv)
{
struct GNUNET_GETOPT_CommandLineOption options[] = {
- GNUNET_GETOPT_option_mandatory
- (GNUNET_GETOPT_option_string ('s',
- "server",
- "IP",
- "which DNS server should be used",
- &dns_server)),
GNUNET_GETOPT_OPTION_END
};
diff --git a/src/namestore/plugin_namestore_postgres.c
b/src/namestore/plugin_namestore_postgres.c
index a9c19d517..e38fcafb1 100644
--- a/src/namestore/plugin_namestore_postgres.c
+++ b/src/namestore/plugin_namestore_postgres.c
@@ -296,6 +296,8 @@ parse_result_call_iterator (void *cls,
{
struct ParserContext *pc = cls;
+ if (NULL == pc->iter)
+ return; /* no need to do more work */
for (unsigned int i=0;i<num_results;i++)
{
void *data;
--
To stop receiving notification emails like this one, please contact
address@hidden
- [GNUnet-SVN] [gnunet] branch master updated (e1f46c052 -> 08795cc18),
gnunet <=