[GNUnet-SVN] r450 - in GNUnet: . src/applications/fs/ecrs src/applications/fs/fsui src/include src/util

[grothoff]

Author: grothoff
Date: 2005-03-19 17:50:03 -0800 (Sat, 19 Mar 2005)
New Revision: 450

Modified:
   GNUnet/src/applications/fs/ecrs/ecrs.h
   GNUnet/src/applications/fs/fsui/namespace_info.c
   GNUnet/src/include/gnunet_ecrs_lib.h
   GNUnet/src/include/gnunet_fsui_lib.h
   GNUnet/src/include/gnunet_util.h
   GNUnet/src/util/storage.c
   GNUnet/todo
Log:
work on libfsui

Modified: GNUnet/src/applications/fs/ecrs/ecrs.h
===================================================================
--- GNUnet/src/applications/fs/ecrs/ecrs.h      2005-03-19 23:05:13 UTC (rev 
449)
+++ GNUnet/src/applications/fs/ecrs/ecrs.h      2005-03-20 01:50:03 UTC (rev 
450)
@@ -39,8 +39,8 @@
  * are interpreted as durations (in seconds) for periodical 
  * updates.
  */
-#define SBLOCK_UPDATE_SPORADIC  -1 
-#define SBLOCK_UPDATE_NONE       0
+#define SBLOCK_UPDATE_SPORADIC ECRS_SBLOCK_UPDATE_SPORADIC
+#define SBLOCK_UPDATE_NONE     ECRS_SBLOCK_UPDATE_NONE
 
 
 #define BLOCK_ALIGN_SIZE (DBLOCK_SIZE)

Modified: GNUnet/src/applications/fs/fsui/namespace_info.c
===================================================================
--- GNUnet/src/applications/fs/fsui/namespace_info.c    2005-03-19 23:05:13 UTC 
(rev 449)
+++ GNUnet/src/applications/fs/fsui/namespace_info.c    2005-03-20 01:50:03 UTC 
(rev 450)
@@ -30,7 +30,8 @@
 #include "gnunet_fsui_lib.h"
 #include "fsui.h"
 
-#define NS_DIR "data/namespaces/"
+#define NS_DIR "data" DIR_SEPARATOR_STR "namespaces" DIR_SEPARATOR_STR
+#define NS_UPDATE_DIR "data" DIR_SEPARATOR_STR "namespace-updates" 
DIR_SEPARATOR_STR
 
 static void writeNamespaceInfo(const char * namespaceName,
                               const struct ECRS_MetaData * meta,
@@ -119,9 +120,9 @@
   
   size = tag - sizeof(int);  
   *ranking = ntohl(((int *) buf)[0]);  
-  if(OK != ECRS_deserializeMetaData(meta,
-                                   &buf[sizeof(int)],
-                                   size)) {
+  if (OK != ECRS_deserializeMetaData(meta,
+                                    &buf[sizeof(int)],
+                                    size)) {
     /* invalid data! remove! */
     BREAK();
     UNLINK(fn);
@@ -310,13 +311,170 @@
 }
 
 /**
+ * Get the filename (or directory name) for the given
+ * namespace and content identifier.
+ * @param lastId maybe NULL
+ */
+static char * getUpdateDataFilename(const char * nsname,
+                                   const HashCode512 * lastId) {
+  char * tmp;
+  char * ret;
+
+  ret = getConfigurationString(NULL, "GNUNET_HOME");
+  tmp = expandFileName(ret);
+  FREE(ret);
+  ret = MALLOC(strlen(tmp) + strlen(NS_UPDATE_DIR) +
+              strlen(nsname) + sizeof(EncName) + 20);
+  strcpy(ret, tmp);
+  FREE(tmp);
+  strcat(ret, DIR_SEPARATOR_STR);
+  strcat(ret, NS_UPDATE_DIR);
+  strcat(ret, nsname);
+  strcat(ret, DIR_SEPARATOR_STR);
+  mkdirp(ret);
+  if (lastId != NULL) {
+    EncName enc;
+
+    hash2enc(lastId, &enc);
+    strcat(ret, (char*) &enc);
+  }  
+  return ret;
+}
+
+struct UpdateData {
+  cron_t updateInterval;
+  cron_t lastPubTime;
+  HashCode512 nextId;
+  HashCode512 thisId;  
+};
+
+/**
+ * Read content update information about content
+ * published in the given namespace under 'lastId'.
+ *
+ * @param fi maybe NULL
+ * @return OK if update data was found, SYSERR if not.
+ */
+static int readUpdateData(const char * nsname,
+                         const HashCode512 * lastId,
+                         HashCode512 * nextId,
+                         ECRS_FileInfo * fi,
+                         cron_t * updateInterval,
+                         cron_t * lastPubTime) {
+  char * fn;
+  struct UpdateData * buf;
+  char * uri;
+  size_t size;
+  size_t pos;
+
+  fn = getUpdateDataFilename(nsname,
+                            lastId);
+  size = getFileSize(fn);
+  if ( (size == 0) ||
+       (size <= sizeof(struct UpdateData)) || 
+       (size > 1024 * 1024 * 16) )
+    return SYSERR;
+  
+  buf = MALLOC(size);
+  if (size != readFile(fn,
+                      size,    
+                      buf)) {
+    FREE(buf);
+    return SYSERR;
+  }
+  FREE(fn);
+  if ( ! equalsHashCode512(lastId,
+                          &buf->thisId)) {
+    FREE(buf);
+    return SYSERR;
+  }
+  uri = (char*) &buf[1];
+  size -= sizeof(struct UpdateData);
+  pos = 0;
+  while ( (pos < size) &&
+         (uri[pos] != '\0') )
+    pos++;
+  size -= pos;
+  if (size == 0) {
+    FREE(buf);
+    BREAK();
+    return SYSERR;
+  }
+  if (OK != ECRS_deserializeMetaData(&fi->meta,
+                                    &uri[pos],
+                                    size)) {
+    FREE(buf);
+    BREAK();
+    return SYSERR;
+  }
+  fi->uri = ECRS_stringToUri(uri);
+  if (fi->uri == NULL) {
+    ECRS_freeMetaData(fi->meta);
+    fi->meta = NULL;
+    FREE(buf);
+    BREAK();
+    return SYSERR;
+  }
+  *updateInterval = ntohll(buf->updateInterval);
+  *lastPubTime = ntohll(buf->lastPubTime);
+  *nextId = buf->nextId;
+  FREE(buf);
+  return OK;
+}
+
+/**
+ * Write content update information.
+ */
+static int writeUpdateData(const char * nsname,
+                          const HashCode512 * thisId,
+                          const HashCode512 * nextId,
+                          const ECRS_FileInfo * fi,
+                          const cron_t updateInterval,
+                          const cron_t lastPubTime) {
+  char * fn;
+  char * uri;
+  size_t metaSize;
+  size_t size;
+  struct UpdateData * buf;
+
+  uri = ECRS_uriToString(fi->uri);
+  metaSize = ECRS_sizeofMetaData(fi->meta);
+  size = sizeof(struct UpdateData) + metaSize + strlen(uri) + 1;
+  buf = MALLOC(size);
+  buf->nextId = *nextId;
+  buf->thisId = *thisId;
+  buf->updateInterval = htonll(updateInterval);
+  buf->lastPubTime = htonll(lastPubTime);
+  memcpy(&buf[1],
+        uri,
+        strlen(uri)+1);
+  FREE(uri);
+  GNUNET_ASSERT(OK == 
+               ECRS_serializeMetaData(fi->meta,
+                                      &((char*)&buf[1])[strlen(uri)+1],
+                                      metaSize,
+                                      NO));  
+  fn = getUpdateDataFilename(nsname,
+                            thisId);
+  writeFile(fn,
+           buf,
+           size,
+           "400"); /* no editing, just deletion */
+  FREE(fn);
+  FREE(buf);
+  return OK;
+}
+                         
+
+
+/**
  * Add an entry into a namespace (also for publishing
- * updates).
+ * updates).  
  *
  * @param name in which namespace to publish
  * @param updateInterval the desired frequency for updates
  * @param lastId the ID of the last value (maybe NULL)
- * @param thisId the ID of the update
+ * @param thisId the ID of the update (maybe NULL)
  * @param nextId the ID of the next update (maybe NULL)
  * @param dst to which URI should the namespace entry refer?
  * @param md what meta-data should be associated with the
@@ -337,23 +495,175 @@
   cron_t creationTime;
   HashCode512 nid;
   HashCode512 tid;
+  cron_t now;
+  cron_t lastTime;
+  cron_t lastInterval;
+  ECRS_FileInfo fi;
+  char * old;
 
+  /* computation of IDs of update(s).  Not as terrible as
+     it looks, just enumerating all of the possible cases
+     of periodic/sporadic updates and how IDs are computed. */
+  creationTime = cronTime(&now);
+  if (updateInterval != ECRS_SBLOCK_UPDATE_NONE) {
+    if ( (lastId != NULL) &&
+        (OK == readUpdateData(name,
+                              lastId,
+                              &tid,
+                              NULL,
+                              &lastInterval,
+                              &lastTime)) ) {
+      if (lastInterval != updateInterval) {
+       LOG(LOG_WARNING,
+           _("Publication interval for periodic publication changed."));
+      }      
+      /* try to compute tid and/or
+        nid based on information read from lastId */
 
+      if (updateInterval != ECRS_SBLOCK_UPDATE_SPORADIC) {
+       HashCode512 delta;
+
+       deltaId(lastId,
+               &tid,
+               &delta);        
+
+       creationTime = lastTime + updateInterval;
+       while (creationTime < now - updateInterval) {
+         creationTime += updateInterval;
+         addHashCodes(&tid,
+                      &delta,
+                      &tid);
+       }
+       if (creationTime > cronTime(NULL) + 7 * cronDAYS) {
+         LOG(LOG_WARNING,
+             _("Publishing update for periodically updated content more than a 
week ahead of schedule.\n"));
+       }
+       if (thisId != NULL)
+         tid = *thisId; /* allow override! */
+       addHashCodes(&tid,
+                    &delta,
+                    &nid);
+       if (nextId != NULL)
+         nid = *nextId; /* again, allow override */
+      } else {
+       /* sporadic ones are unpredictable,
+          tid has been obtained from IO, pick random nid if
+          not specified */
+       if (thisId != NULL) 
+         tid = *thisId; /* allow user override */      
+       if (nextId == NULL) {
+         makeRandomId(&nid);
+       } else {
+         nid = *nextId;
+       }
+      }      
+    } else { /* no previous ID found or given */
+      if (nextId == NULL) {
+       /* no previous block found and nextId not specified;
+          pick random nid */
+       makeRandomId(&nid);
+      } else {
+       nid = *nextId;
+      }
+      if (thisId != NULL) {
+       tid = *thisId;
+      } else {
+       makeRandomId(&tid);  
+      }
+    }
+  } else {
+    if (thisId != NULL) {
+      nid = tid = *thisId;
+    } else {
+      makeRandomId(&tid);
+      nid = tid;
+    }
+  }
   ret = ECRS_addToNamespace(name,
                            anonymityLevel,
                            getConfigurationInt("FS", "INSERT-PRIORITY"),
                            getConfigurationInt("FS", "INSERT-EXPIRATION") * 
cronYEARS + cronTime(NULL),
                            creationTime,
                            updateInterval,
-                           thisId,
-                           nextId,
+                           &tid,
+                           &nid,
                            dst,
                            md,
-                           uri);
-  
+                           uri);  
+  if (updateInterval != ECRS_SBLOCK_UPDATE_NONE) {
+    fi.uri = (struct ECRS_URI*) uri;
+    fi.meta = (struct ECRS_MetaData*) md;
+    writeUpdateData(name,
+                   &tid,
+                   &nid,
+                   &fi,
+                   updateInterval,
+                   creationTime);
+  }
+  if (lastId != NULL) {
+    old = getUpdateDataFilename(name,
+                               lastId);
+    UNLINK(old);
+    FREE(old);
+  }
   return ret;
 }
 
+struct lNCC {
+  const char * name;
+  FSUI_UpdateIterator it;
+  void * closure;
+  int cnt;
+};
+
+void lNCHelper(const char * fil,
+              const char * dir,
+              struct lNCC * cls) {
+  ECRS_FileInfo fi;
+  HashCode512 lastId;
+  HashCode512 nextId;
+  cron_t pubFreq;
+  cron_t lastTime;
+  cron_t nextTime;
+  cron_t now;
+  
+  if (cls->cnt == SYSERR)
+    return;
+  if (OK != enc2hash(fil,
+                    &lastId))
+    return;
+  fi.uri = NULL;
+  fi.meta = NULL;
+  if (OK != readUpdateData(cls->name,
+                          &lastId,
+                          &nextId,
+                          &fi,
+                          &pubFreq,
+                          &lastTime))
+    return;
+  cls->cnt++;
+  if (pubFreq == ECRS_SBLOCK_UPDATE_SPORADIC) {
+    nextTime = 0;
+  } else {
+    cronTime(&now);
+    nextTime = lastTime;
+    while (nextTime + pubFreq < now)
+      nextTime += pubFreq;
+  }
+  if (cls->it != NULL) {
+    if (OK != cls->it(cls->closure,
+                     &fi,
+                     &lastId,
+                     &nextId,
+                     pubFreq,
+                     nextTime)) {
+      cls->cnt = SYSERR;
+    }
+  }
+  ECRS_freeUri(fi.uri);
+  ECRS_freeMetaData(fi.meta);
+}
+
 /**
  * List all updateable content in a given namespace.
  */
@@ -361,7 +671,20 @@
                              const char * name,
                              FSUI_UpdateIterator iterator,
                              void * closure) {
-  return SYSERR;
+  struct lNCC cls;
+  char * dirName;
+
+  cls.name = name;
+  cls.it = iterator;
+  cls.closure = closure;
+  cls.cnt = 0;
+  dirName = getUpdateDataFilename(name,
+                                 NULL);
+  scanDirectory(dirName,
+               (DirectoryEntryCallback)&lNCHelper,
+               &cls);
+  FREE(dirName);
+  return cls.cnt;
 }
 
 static int mergeMeta(EXTRACTOR_KeywordType type,

Modified: GNUnet/src/include/gnunet_ecrs_lib.h
===================================================================
--- GNUnet/src/include/gnunet_ecrs_lib.h        2005-03-19 23:05:13 UTC (rev 
449)
+++ GNUnet/src/include/gnunet_ecrs_lib.h        2005-03-20 01:50:03 UTC (rev 
450)
@@ -57,6 +57,16 @@
 #define ECRS_LOCATION_INFIX  "loc/"
 
 
+/**
+ * Fixed SBlock updateInterval codes. Positive values 
+ * are interpreted as durations (in seconds) for periodical 
+ * updates.
+ */
+#define ECRS_SBLOCK_UPDATE_SPORADIC  -1 
+#define ECRS_SBLOCK_UPDATE_NONE       0
+
+
+
 /* ***************** metadata API (meta.c) ******************** */
 
 /**

Modified: GNUnet/src/include/gnunet_fsui_lib.h
===================================================================
--- GNUnet/src/include/gnunet_fsui_lib.h        2005-03-19 23:05:13 UTC (rev 
449)
+++ GNUnet/src/include/gnunet_fsui_lib.h        2005-03-20 01:50:03 UTC (rev 
450)
@@ -299,8 +299,10 @@
 /**
  * Iterator over all updateable content.
  *
+ * @param uri URI of the last content published
  * @param lastId the ID of the last publication
  * @param nextId the ID of the next update
+ * @param publicationFrequency how often are updates scheduled?
  * @param nextPublicationTime the scheduled time for the
  *  next update (0 for sporadic updates)
  * @return OK to continue iteration, SYSERR to abort
@@ -309,6 +311,7 @@
                                   const ECRS_FileInfo * uri,
                                   const HashCode512 * lastId,
                                   const HashCode512 * nextId,
+                                  cron_t publicationFrequency,
                                   cron_t nextPublicationTime); 
 
 /**
@@ -607,7 +610,27 @@
 
 /**
  * Add an entry into a namespace (also for publishing
- * updates).
+ * updates).  Typical uses are (all others would be odd):
+ * <ul>
+ *  <li>updateInterval NONE, thisId some user-specified value
+ *      or NULL if user wants system to pick random value;
+ *      nextId and lastId NULL (irrelevant)</li>
+ *  <li>updateInterval SPORADIC, thisId given (initial
+ *      submission), nextId maybe given or NULL, 
+ *      lastId NULL</li>
+ *  <li>updateInterval SPORADIC, lastId given (either
+ *      user-provided or from listNamespaceContent 
+ *      iterator); thisId NULL or given (from lNC);
+ *      nextId maybe given or NULL, depending on user preference</li>
+ *  <li>updateInterval non-NULL, non-SPORADIC; lastId
+ *      is NULL (inital submission), thisId non-NULL or
+ *      rarely NULL (if user does not care about name of
+ *      starting entry), nextId maybe NULL or not</li>
+ *  <li>updateInterval non-NULL, non-SPORADIC; lastId
+ *      is non-NULL (periodic update), thisId NULL (computed!)
+ *      nextID NULL (computed)</li>
+ * </ul> 
+ * And yes, reading the ECRS paper maybe a good idea.
  *
  * @param name in which namespace to publish
  * @param updateInterval the desired frequency for updates

Modified: GNUnet/src/include/gnunet_util.h
===================================================================
--- GNUnet/src/include/gnunet_util.h    2005-03-19 23:05:13 UTC (rev 449)
+++ GNUnet/src/include/gnunet_util.h    2005-03-20 01:50:03 UTC (rev 450)
@@ -1889,7 +1889,7 @@
  */ 
 void writeFile(const char * fileName, 
               const void * buffer,
-              int n,
+              unsigned int n,
               const char * mode);
 
 /**

Modified: GNUnet/src/util/storage.c
===================================================================
--- GNUnet/src/util/storage.c   2005-03-19 23:05:13 UTC (rev 449)
+++ GNUnet/src/util/storage.c   2005-03-20 01:50:03 UTC (rev 450)
@@ -470,7 +470,7 @@
  */ 
 void writeFile(const char * fileName, 
               const void * buffer,
-              int n,
+              unsigned int n,
               const char *mode) {
   int handle;
   /* open file, open with 600, create if not 

Modified: GNUnet/todo
===================================================================
--- GNUnet/todo 2005-03-19 23:05:13 UTC (rev 449)
+++ GNUnet/todo 2005-03-20 01:50:03 UTC (rev 450)
@@ -13,7 +13,6 @@
   * gap
   * dht / gnunet-dht-join and gnunet-dht-query 
 - FSUI:
-  * namespace updates
   * collections
   * testing
 - transport refactoring: