[GNUnet-SVN] r11165 - in gnunet: . src/fs

[gnunet]

Author: grothoff
Date: 2010-05-03 18:39:34 +0200 (Mon, 03 May 2010)
New Revision: 11165

Modified:
   gnunet/TODO
   gnunet/src/fs/fs.c
   gnunet/src/fs/fs.h
   gnunet/src/fs/fs_download.c
Log:
more fs stuff

Modified: gnunet/TODO
===================================================================
--- gnunet/TODO 2010-05-03 15:18:51 UTC (rev 11164)
+++ gnunet/TODO 2010-05-03 16:39:34 UTC (rev 11165)
@@ -1,11 +1,24 @@
 0.9.0pre1:
 * FS: [CG]
+  - resume signalling for search/download must be recursive!
   - deserialization code (download)
   - serialization code (download)
-  - linking of downloads to searches 
+  - linking of downloads to searches (expose opaque struct SearchResult;
+    allow starting download based on search result (new API):
+    => have meta data for instant completion!
+    => have URI
+    => linking of download to search
+    => expose link to search result in download events (including search 
result's
+       client-info pointer!)
   - generate SUSPEND events (publish, unindex, search, download)
   - actually call 'sync' functions (publish, unindex, search, download)
-  - persistence testing (publish, unindex)
+  - code review:
+    => refactor fs.c to join common code segments!
+    => document directory structure
+    => ensure all files & dirs are cleaned up! (at least during 'clean' runs)
+  - persistence testing (publish, unindex, search, download):
+    => need driver!
+    => schedule suspending tasks DURING event handler => good coverage!
   - gnunet-service-fs (hot-path routing, load-based routing, nitpicks)  
   - [gnunet-service-fs.c:208]: member 'LocalGetContext::results_bf_size' is 
never used
   - [gnunet-service-fs.c:501]: member 'PendingRequest::used_pids_size' is 
never used

Modified: gnunet/src/fs/fs.c
===================================================================
--- gnunet/src/fs/fs.c  2010-05-03 15:18:51 UTC (rev 11164)
+++ gnunet/src/fs/fs.c  2010-05-03 16:39:34 UTC (rev 11165)
@@ -1244,6 +1244,21 @@
 
 
 /**
+ * Synchronize this download struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ * 
+ * @param dc the struct to sync
+ */
+void
+GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc)
+{
+  /* FIXME */
+}
+
+
+/**
  * Synchronize this search result with its mirror
  * on disk.  Note that all internal FS-operations that change
  * publishing structs should already call "sync" internally,
@@ -1700,6 +1715,167 @@
 
 
 /**
+ * Deserialize a download.
+ *
+ * @param h overall context
+ * @param rh file to deserialize from
+ * @param parent parent download
+ * @param serialization name under which the search was serialized
+ */
+static void
+deserialize_download (struct GNUNET_FS_Handle *h,
+                     struct GNUNET_BIO_ReadHandle *rh,
+                     struct GNUNET_FS_DownloadContext *parent,
+                     const char *serialization);
+
+
+/**
+ * Function called with a filename of serialized sub-download
+ * to deserialize.
+ *
+ * @param cls the 'struct GNUNET_FS_DownloadContext*' (parent)
+ * @param filename complete filename (absolute path)
+ * @return GNUNET_OK (continue to iterate)
+ */
+static int
+deserialize_subdownload (void *cls,
+                        const char *filename)
+{
+  struct GNUNET_FS_DownloadContext *parent = cls;
+  char *ser;
+  char *emsg;
+  struct GNUNET_BIO_ReadHandle *rh;
+
+  ser = get_serialization_short_name (filename);
+  rh = GNUNET_BIO_read_open (filename);
+  deserialize_download (parent->h,
+                       rh,
+                       parent,
+                       ser);
+  if (GNUNET_OK !=
+      GNUNET_BIO_read_close (rh, &emsg))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                 _("Failed to resume sub-download `%s': %s\n"),
+                 ser,
+                 emsg);
+      GNUNET_free (emsg);
+    }
+  GNUNET_free (ser);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Deserialize a download.
+ *
+ * @param h overall context
+ * @param rh file to deserialize from
+ * @param parent parent download
+ * @param serialization name under which the search was serialized
+ */
+static void
+deserialize_download (struct GNUNET_FS_Handle *h,
+                     struct GNUNET_BIO_ReadHandle *rh,
+                     struct GNUNET_FS_DownloadContext *parent,
+                     const char *serialization)
+{
+  struct GNUNET_FS_DownloadContext *dc;
+  struct GNUNET_FS_DownloadContext *dcc;
+  char pbuf[32];
+  struct GNUNET_FS_ProgressInfo pi;
+  char *emsg;
+  char *uris;
+  char *dn;
+
+  uris = NULL;
+  emsg = NULL;
+  dc = GNUNET_malloc (sizeof (struct GNUNET_FS_DownloadContext));
+  dc->parent = parent;
+  dc->h = h;
+  dc->serialization = GNUNET_strdup (serialization);
+#if 0
+  /* FIXME */
+  if ( (GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "-uri", &uris, 10*1024)) ||
+       (NULL == (sc->uri = GNUNET_FS_uri_parse (uris, &emsg))) ||       
+       ( (GNUNET_YES != GNUNET_FS_uri_test_ksk (sc->uri)) &&
+        (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri)) ) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_read_int64 (rh, &sc->start_time.value)) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10*1024)) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_read_int32 (rh, &options)) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_read (rh, "search-pause", &in_pause, sizeof (in_pause))) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_read_int32 (rh, &sc->anonymity)) )
+    goto cleanup;          
+  /* FIXME: adjust start_time.value */
+  sc->options = (enum GNUNET_FS_SearchOptions) options;
+  sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16);
+#endif
+  GNUNET_snprintf (pbuf,
+                  sizeof (pbuf),
+                  "%s%s%s",
+                  "subdownloads",
+                  DIR_SEPARATOR_STR,
+                  dc->serialization);
+  dn = get_serialization_file_name (h, pbuf, "");
+  if (dn != NULL)
+    {
+      GNUNET_DISK_directory_scan (dn, &deserialize_subdownload, dc);
+      GNUNET_free (dn);
+    }
+#if 0
+  if ('\0' == in_pause)
+    {
+      if (GNUNET_OK !=
+         GNUNET_FS_search_start_searching_ (sc))
+       goto cleanup;
+    }
+#endif
+  if (0)
+    goto cleanup;
+  if (parent != NULL)
+    GNUNET_CONTAINER_DLL_insert (parent->child_head,
+                                parent->child_tail,
+                                dc);
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME;
+#if 0
+  pi.value.search.specifics.resume.message = sc->emsg;
+  pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO 
: GNUNET_YES;
+#endif
+  GNUNET_FS_download_make_status_ (&pi,
+                                  dc);
+  dcc = dc->child_head;
+  while (NULL != dcc)
+    {
+      /* FIXME: wrong, need recursion! */
+      pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME;
+#if 0
+      pi.value.search.specifics.resume.message = scc->emsg;
+      pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? 
GNUNET_NO : GNUNET_YES;
+#endif
+      GNUNET_FS_download_make_status_ (&pi,
+                                      dcc);
+      dcc = dcc->next;
+    }
+#if 0
+  GNUNET_free (uris);
+#endif
+  return;
+ cleanup:
+#if 0
+  GNUNET_free_non_null (emsg);
+  free_search_context (sc);
+#endif
+  GNUNET_free_non_null (uris);
+}
+
+
+/**
  * Deserialize a search. 
  *
  * @param h overall context
@@ -1808,6 +1984,7 @@
   scc = sc->child_head;
   while (NULL != scc)
     {
+      /* FIXME: wrong, need recursion! */
       pi.status = GNUNET_FS_STATUS_SEARCH_RESUME;
       pi.value.search.specifics.resume.message = scc->emsg;
       pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? 
GNUNET_NO : GNUNET_YES;
@@ -1864,7 +2041,7 @@
       GNUNET_BIO_read_close (rh, &emsg))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                 _("Failure while resuming unindexing operation `%s': %s\n"),
+                 _("Failure while resuming search operation `%s': %s\n"),
                  filename,
                  emsg);
       GNUNET_free (emsg);
@@ -1892,6 +2069,67 @@
 
 
 /**
+ * Function called with a filename of serialized download operation
+ * to deserialize.
+ *
+ * @param cls the 'struct GNUNET_FS_Handle*'
+ * @param filename complete filename (absolute path)
+ * @return GNUNET_OK (continue to iterate)
+ */
+static int
+deserialize_download_file (void *cls,
+                          const char *filename)
+{
+  struct GNUNET_FS_Handle *h = cls;
+  char *ser;
+  char *emsg;
+  struct GNUNET_BIO_ReadHandle *rh;
+
+  ser = get_serialization_short_name (filename);
+  rh = GNUNET_BIO_read_open (filename);
+  if (rh == NULL)
+    {
+      if (ser != NULL)
+       {
+         GNUNET_FS_remove_sync_file_ (h, "download", ser);
+         GNUNET_free (ser);
+       }
+      return GNUNET_OK;
+    }
+  deserialize_download (h, rh, NULL, ser);
+  GNUNET_free (ser);
+  if (GNUNET_OK !=
+      GNUNET_BIO_read_close (rh, &emsg))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                 _("Failure while resuming download operation `%s': %s\n"),
+                 filename,
+                 emsg);
+      GNUNET_free (emsg);
+    }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Deserialize information about pending download operations.
+ *
+ * @param h master context
+ */
+static void
+deserialize_download_master (struct GNUNET_FS_Handle *h)
+{
+  char *dn;
+
+  dn = get_serialization_file_name (h, "download", "");
+  if (dn == NULL)
+    return;
+  GNUNET_DISK_directory_scan (dn, &deserialize_download_file, h);
+  GNUNET_free (dn);
+}
+
+
+/**
  * Setup a connection to the file-sharing service.
  *
  * @param sched scheduler to use
@@ -1960,7 +2198,7 @@
         function instead of these four... */
       deserialize_publish (ret);
       deserialize_search_master (ret);
-      /* FIXME: deserialize downloads that are NOT part of searches */
+      deserialize_download_master (ret);
       deserialize_unindex (ret);
     }
   return ret;

Modified: gnunet/src/fs/fs.h
===================================================================
--- gnunet/src/fs/fs.h  2010-05-03 15:18:51 UTC (rev 11164)
+++ gnunet/src/fs/fs.h  2010-05-03 16:39:34 UTC (rev 11165)
@@ -765,7 +765,20 @@
                                const struct GNUNET_FS_FileInformation *p,
                                uint64_t offset);
 
+
 /**
+ * Fill in all of the generic fields for a download event and call the
+ * callback.
+ *
+ * @param pi structure to fill in
+ * @param dc overall download context
+ */
+void
+GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                struct GNUNET_FS_DownloadContext *dc);
+
+
+/**
  * Fill in all of the generic fields for 
  * an unindex event and call the callback.
  *
@@ -887,6 +900,16 @@
 GNUNET_FS_search_result_sync_ (const GNUNET_HashCode *key,
                               struct SearchResult *sr);
 
+/**
+ * Synchronize this download struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ * 
+ * @param dc the struct to sync
+ */
+void
+GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc);
 
 /**
  * Master context for most FS operations.
@@ -1433,6 +1456,12 @@
   char *emsg;
 
   /**
+   * Random portion of filename we use for syncing state of this
+   * download.
+   */
+  char *serialization;
+
+  /**
    * Where are we writing the data (name of the
    * file, can be NULL!).
    */

Modified: gnunet/src/fs/fs_download.c
===================================================================
--- gnunet/src/fs/fs_download.c 2010-05-03 15:18:51 UTC (rev 11164)
+++ gnunet/src/fs/fs_download.c 2010-05-03 16:39:34 UTC (rev 11165)
@@ -144,9 +144,9 @@
  * @param pi structure to fill in
  * @param dc overall download context
  */
-static void
-make_download_status (struct GNUNET_FS_ProgressInfo *pi,
-                     struct GNUNET_FS_DownloadContext *dc)
+void
+GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                struct GNUNET_FS_DownloadContext *dc)
 {
   pi->value.download.dc = dc;
   pi->value.download.cctx
@@ -631,7 +631,7 @@
   dc->has_finished = GNUNET_YES;
   /* signal completion */
   pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
-  make_download_status (&pi, dc);
+  GNUNET_FS_download_make_status_ (&pi, dc);
   if (dc->parent != NULL)
     check_completed (dc->parent);  
 }
@@ -856,7 +856,7 @@
       /* signal error */
       pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
       pi.value.download.specifics.error.message = dc->emsg;
-      make_download_status (&pi, dc);
+      GNUNET_FS_download_make_status_ (&pi, dc);
       /* abort all pending requests */
       if (NULL != dc->th)
        {
@@ -936,7 +936,7 @@
          /* signal error */
          pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
          pi.value.download.specifics.error.message = emsg;
-         make_download_status (&pi, dc);
+         GNUNET_FS_download_make_status_ (&pi, dc);
          /* abort all pending requests */
          if (NULL != dc->th)
            {
@@ -984,7 +984,7 @@
   pi.value.download.specifics.progress.offset = sm->offset;
   pi.value.download.specifics.progress.data_len = prc->size;
   pi.value.download.specifics.progress.depth = sm->depth;
-  make_download_status (&pi, dc);
+  GNUNET_FS_download_make_status_ (&pi, dc);
   GNUNET_assert (dc->completed <= dc->length);
   if (dc->completed == dc->length)
     {
@@ -1015,7 +1015,7 @@
        {
          /* signal completion */
          pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
-         make_download_status (&pi, dc);
+         GNUNET_FS_download_make_status_ (&pi, dc);
          if (dc->parent != NULL)
            check_completed (dc->parent);
        }
@@ -1314,7 +1314,7 @@
                         dc,
                         GNUNET_TIME_UNIT_FOREVER_REL);
   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
-  make_download_status (&pi, dc);
+  GNUNET_FS_download_make_status_ (&pi, dc);
   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
                                         &retry_entry,
                                         dc);
@@ -1351,7 +1351,7 @@
       dc->client = NULL;
     }
   pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
-  make_download_status (&pi, dc);
+  GNUNET_FS_download_make_status_ (&pi, dc);
 }
 
 
@@ -1479,7 +1479,7 @@
   // FIXME: make persistent
   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
   pi.value.download.specifics.start.meta = meta;
-  make_download_status (&pi, dc);
+  GNUNET_FS_download_make_status_ (&pi, dc);
   schedule_block_download (dc, 
                           &dc->uri->data.chk.chk,
                           0, 
@@ -1538,7 +1538,7 @@
                                 dc);
   
   pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
-  make_download_status (&pi, dc);
+  GNUNET_FS_download_make_status_ (&pi, dc);
   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
     GNUNET_SCHEDULER_cancel (dc->h->sched,
                             dc->task);
@@ -1570,6 +1570,8 @@
                                  dc->temp_filename);
       GNUNET_free (dc->temp_filename);
     }
+  /* FIXME: clean up serialization file itself! */
+  GNUNET_free_non_null (dc->serialization);
   GNUNET_free (dc);
 }