[GNUnet-SVN] r13324 - gnunet/src/fs

[gnunet]

Author: grothoff
Date: 2010-10-20 15:55:28 +0200 (Wed, 20 Oct 2010)
New Revision: 13324

Modified:
   gnunet/src/fs/
   gnunet/src/fs/fs_download.c
   gnunet/src/fs/fs_test_lib_data.conf
   gnunet/src/fs/gnunet-service-fs.c
   gnunet/src/fs/perf_gnunet_service_fs_p2p.c
Log:
finding causes


Property changes on: gnunet/src/fs
___________________________________________________________________
Modified: svn:ignore
   - test_fs_namespace_list_updateable
test_gnunet_service_fs_migration
test_fs_download_indexed
test_fs_unindex_persistence
test_fs_search_persistence
test_fs_publish_persistence
test_fs_download_persistence
test_gnunet_fs_rec.py
test_gnunet_fs_psd.py
test_gnunet_fs_ns.py
test_gnunet_fs_idx.py
test_gnunet_service_fs_p2p
test_fs_test_lib
gnunet-service-fs.gcda
test_fs_unindex.gcno
test_fs_unindex.gcda
test_fs_start_stop.gcno
test_fs_start_stop.gcda
test_fs_search.gcno
test_fs_search.gcda
test_fs_publish.gcno
test_fs_publish.gcda
test_fs_namespace.gcno
test_fs_list_indexed.gcno
test_fs_list_indexed.gcda
test_fs_file_information.gcno
test_fs_file_information.gcda
test_fs_download.gcno
test_fs_download.gcda
test_fs_publish
test_fs_list_indexed
test_fs_file_information
test_fs_unindex
test_fs_namespace
test_fs_start_stop
test_fs_search
test_fs_download
test_fs_uri.gcno
test_fs_uri.gcda
test_fs_getopt.gcno
test_fs_directory.gcno
test_fs_directory.gcda
test_fs_collection.gcno
gnunet-unindex.gcno
gnunet-service-fs.gcno
gnunet-search.gcno
gnunet-publish.gcno
gnunet-pseudonym.gcno
gnunet-download.gcno
gnunet-directory.gcno
fs_uri.gcno
fs_uri.gcda
fs_unindex.gcno
fs_unindex.gcda
fs_tree.gcno
fs_tree.gcda
fs_search.gcno
fs_search.gcda
fs_publish.gcno
fs_publish.gcda
fs_namespace.gcno
fs_namespace.gcda
fs_list_indexed.gcno
fs_list_indexed.gcda
fs_getopt.gcno
fs_getopt.gcda
fs_file_information.gcno
fs_file_information.gcda
fs_download.gcno
fs_download.gcda
fs_directory.gcno
fs_directory.gcda
fs_collection.gcno
fs_collection.gcda
fs.gcno
fs.gcda
test_fs_uri
test_fs_getopt
test_fs_directory
test_fs_collection
gnunet-service-fs
gnunet-pseudonym
gnunet-directory
gnunet-unindex
gnunet-search
gnunet-download
gnunet-publish
Makefile.in
Makefile
.deps

   + perf_gnunet_service_fs_p2p
test_fs_namespace_list_updateable
test_gnunet_service_fs_migration
test_fs_download_indexed
test_fs_unindex_persistence
test_fs_search_persistence
test_fs_publish_persistence
test_fs_download_persistence
test_gnunet_fs_rec.py
test_gnunet_fs_psd.py
test_gnunet_fs_ns.py
test_gnunet_fs_idx.py
test_gnunet_service_fs_p2p
test_fs_test_lib
gnunet-service-fs.gcda
test_fs_unindex.gcno
test_fs_unindex.gcda
test_fs_start_stop.gcno
test_fs_start_stop.gcda
test_fs_search.gcno
test_fs_search.gcda
test_fs_publish.gcno
test_fs_publish.gcda
test_fs_namespace.gcno
test_fs_list_indexed.gcno
test_fs_list_indexed.gcda
test_fs_file_information.gcno
test_fs_file_information.gcda
test_fs_download.gcno
test_fs_download.gcda
test_fs_publish
test_fs_list_indexed
test_fs_file_information
test_fs_unindex
test_fs_namespace
test_fs_start_stop
test_fs_search
test_fs_download
test_fs_uri.gcno
test_fs_uri.gcda
test_fs_getopt.gcno
test_fs_directory.gcno
test_fs_directory.gcda
test_fs_collection.gcno
gnunet-unindex.gcno
gnunet-service-fs.gcno
gnunet-search.gcno
gnunet-publish.gcno
gnunet-pseudonym.gcno
gnunet-download.gcno
gnunet-directory.gcno
fs_uri.gcno
fs_uri.gcda
fs_unindex.gcno
fs_unindex.gcda
fs_tree.gcno
fs_tree.gcda
fs_search.gcno
fs_search.gcda
fs_publish.gcno
fs_publish.gcda
fs_namespace.gcno
fs_namespace.gcda
fs_list_indexed.gcno
fs_list_indexed.gcda
fs_getopt.gcno
fs_getopt.gcda
fs_file_information.gcno
fs_file_information.gcda
fs_download.gcno
fs_download.gcda
fs_directory.gcno
fs_directory.gcda
fs_collection.gcno
fs_collection.gcda
fs.gcno
fs.gcda
test_fs_uri
test_fs_getopt
test_fs_directory
test_fs_collection
gnunet-service-fs
gnunet-pseudonym
gnunet-directory
gnunet-unindex
gnunet-search
gnunet-download
gnunet-publish
Makefile.in
Makefile
.deps


Modified: gnunet/src/fs/fs_download.c
===================================================================
--- gnunet/src/fs/fs_download.c 2010-10-20 13:19:53 UTC (rev 13323)
+++ gnunet/src/fs/fs_download.c 2010-10-20 13:55:28 UTC (rev 13324)
@@ -32,7 +32,7 @@
 #include "fs.h"
 #include "fs_tree.h"
 
-#define DEBUG_DOWNLOAD GNUNET_NO
+#define DEBUG_DOWNLOAD GNUNET_YES
 
 /**
  * Determine if the given download (options and meta data) should cause

Modified: gnunet/src/fs/fs_test_lib_data.conf
===================================================================
--- gnunet/src/fs/fs_test_lib_data.conf 2010-10-20 13:19:53 UTC (rev 13323)
+++ gnunet/src/fs/fs_test_lib_data.conf 2010-10-20 13:55:28 UTC (rev 13324)
@@ -20,7 +20,7 @@
 DEFAULTSERVICES = fs
 
 [datastore]
-#DEBUG = YES
+DEBUG = YES
 #PREFIX = valgrind --tool=memcheck --leak-check=yes
 QUOTA = 2000000000
 

Modified: gnunet/src/fs/gnunet-service-fs.c
===================================================================
--- gnunet/src/fs/gnunet-service-fs.c   2010-10-20 13:19:53 UTC (rev 13323)
+++ gnunet/src/fs/gnunet-service-fs.c   2010-10-20 13:55:28 UTC (rev 13324)
@@ -47,8 +47,15 @@
  * Should we introduce random latency in processing?  Required for proper
  * implementation of GAP, but can be disabled for performance evaluation of
  * the basic routing algorithm.
+ *
+ * Note that with delays enabled, performance can be significantly lower
+ * (several orders of magnitude in 2-peer test runs); if you want to
+ * measure throughput of other components, set this to NO.  Also, you
+ * might want to consider changing 'RETRY_PROBABILITY_INV' to 1 for
+ * a rather wasteful mode of operation (that might still get the highest
+ * throughput overall).
  */
-#define SUPPORT_DELAYS GNUNET_NO
+#define SUPPORT_DELAYS GNUNET_YES
 
 /**
  * Size for the hash map for DHT requests from the FS
@@ -73,6 +80,15 @@
  * repeatedly recently, the probability is multiplied by the inverse
  * of this number each time.  Note that we only try about every TTL_DECREMENT/2
  * plus MAX_CORK_DELAY (so roughly every 3.5s).
+ *
+ * Note that this factor is a key influence to performance in small
+ * networks (especially test networks of 2 peers) because if there is
+ * only a single peer with the data, this value will determine how
+ * soon we might re-try.  For example, a value of 3 can result in 
+ * 1.7 MB/s transfer rates for a 10 MB file when a value of 1 would
+ * give us 5 MB/s.  OTOH, obviously re-trying the same peer can be
+ * rather inefficient in larger networks, hence picking 1 is in 
+ * general not the best choice.
  */
 #define RETRY_PROBABILITY_INV 3
 
@@ -86,8 +102,9 @@
 #define MAX_TRANSMIT_DELAY GNUNET_TIME_relative_multiply 
(GNUNET_TIME_UNIT_SECONDS, 45)
 
 /**
- * Maximum number of requests (from other peers) that we're
- * willing to have pending at any given point in time.
+ * Maximum number of requests (from other peers, overall) that we're
+ * willing to have pending at any given point in time.  Can be changed
+ * via the configuration file (32k is just the default).
  */
 static unsigned long long max_pending_requests = (32 * 1024);
 
@@ -3704,7 +3721,11 @@
              return;
            }
        }
-
+      if (pr->local_only == GNUNET_YES)
+       {
+         destroy_pending_request (pr);
+         return;
+       }
       /* no more results */
       if (pr->task == GNUNET_SCHEDULER_NO_TASK)
        pr->task = GNUNET_SCHEDULER_add_now (sched,
@@ -3775,6 +3796,7 @@
     return; /* done here */
   if (prq.eval == GNUNET_BLOCK_EVALUATION_OK_LAST)
     {
+      pr->local_only = GNUNET_YES; /* do not forward */
       GNUNET_DATASTORE_get_next (dsh, GNUNET_NO);
       return;
     }
@@ -3790,8 +3812,12 @@
                                gettext_noop ("# processing result set cut 
short due to load"),
                                1,
                                GNUNET_NO);
+      /* FIXME: if this is activated, we might stall large downloads
+        indefinitely since (presumably) the load can never go down again! */
+#if 0
       GNUNET_DATASTORE_get_next (dsh, GNUNET_NO);
       return;
+#endif
     }
   GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
 }
@@ -4015,7 +4041,10 @@
       /* don't have BW to send to peer, or would likely take longer than we 
have for it,
         so at best indirect the query */
       priority = 0;
-      pr->forward_only = GNUNET_YES;
+      /* FIXME: if this line is enabled, the 'perf' test for larger files 
simply "hangs";
+        the cause seems to be that the load goes up (to the point where we do 
this)
+        and then never goes down again... (outch) */
+      // pr->forward_only = GNUNET_YES;
     }
   pr->type = type;
   pr->mingle = ntohl (gm->filter_mutator);
@@ -4166,7 +4195,13 @@
     case GNUNET_BLOCK_TYPE_FS_IBLOCK:
       /* only one result, wait for datastore */
       if (GNUNET_YES != pr->forward_only)
-       break;
+       {
+         GNUNET_STATISTICS_update (stats,
+                                   gettext_noop ("# requests not instantly 
forwarded (waiting for datastore)"),
+                                   1,
+                                   GNUNET_NO);
+         break;
+       }
     default:
       if (pr->task == GNUNET_SCHEDULER_NO_TASK)
        pr->task = GNUNET_SCHEDULER_add_now (sched,

Modified: gnunet/src/fs/perf_gnunet_service_fs_p2p.c
===================================================================
--- gnunet/src/fs/perf_gnunet_service_fs_p2p.c  2010-10-20 13:19:53 UTC (rev 
13323)
+++ gnunet/src/fs/perf_gnunet_service_fs_p2p.c  2010-10-20 13:55:28 UTC (rev 
13324)
@@ -37,7 +37,7 @@
 /**
  * How long until we give up on transmitting the message?
  */
-#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 3)
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 300)
 
 #define NUM_DAEMONS 2