[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r8863 - in gnunet: . doc/man src/datacache src/datastore sr
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] r8863 - in gnunet: . doc/man src/datacache src/datastore src/fs src/include src/testing src/transport src/upnp src/util |
|
Date: |
Thu, 27 Aug 2009 05:14:19 -0600 |
Author: grothoff
Date: 2009-08-27 05:14:19 -0600 (Thu, 27 Aug 2009)
New Revision: 8863
Modified:
gnunet/TODO
gnunet/doc/man/
gnunet/src/datacache/
gnunet/src/datastore/
gnunet/src/fs/
gnunet/src/fs/fs.h
gnunet/src/fs/fs_directory.c
gnunet/src/fs/fs_file_information.c
gnunet/src/fs/fs_publish.c
gnunet/src/include/gnunet_constants.h
gnunet/src/include/gnunet_container_lib.h
gnunet/src/include/gnunet_crypto_lib.h
gnunet/src/include/gnunet_datastore_service.h
gnunet/src/include/gnunet_fs_service.h
gnunet/src/testing/
gnunet/src/transport/
gnunet/src/upnp/
gnunet/src/util/container_meta_data.c
gnunet/src/util/crypto_hash.c
Log:
syn
Modified: gnunet/TODO
===================================================================
--- gnunet/TODO 2009-08-26 20:47:59 UTC (rev 8862)
+++ gnunet/TODO 2009-08-27 11:14:19 UTC (rev 8863)
@@ -43,14 +43,14 @@
+ getopt API -- DONE (but do more testing)
+ persistence mechanism (design done)
+ sharing API
- ~ file-information (almost done, needs testing)
- ~ insert
+ ~ file-information (needs testing)
+ ~ directory: implement new directory builder!
+ ~ insert: close, need directory builder first!
+ ~ unindex & list indexed!!!
~ search
~ download
- ~ unindex & list indexed!!!
~ namespaces
~ collection
- + directory API (builder API unclear)
- design network structs (P2P)
- datastore request queueing mechanism
- implement FS service (needs DHT)
Property changes on: gnunet/doc/man
___________________________________________________________________
Added: svn:ignore
+ Makefile.in
Property changes on: gnunet/src/datacache
___________________________________________________________________
Modified: svn:ignore
- Makefile.in
Makefile
.deps
+ test_datacache_quota
test_datacache_api_quota.gcno
test_datacache_api_quota.gcda
test_datacache_api_quota
test_datacache_api.gcno
test_datacache_api.gcda
test_datacache_api
test_datacache
perf_datacache_api.gcno
perf_datacache_api.gcda
perf_datacache_api
perf_datacache
Makefile.in
Makefile
.deps
Property changes on: gnunet/src/datastore
___________________________________________________________________
Modified: svn:ignore
- gnunet-service-datastore.gcno
perf_plugin_datastore
test_datastore_api
perf_datastore_api_iterators
perf_datastore_api
Makefile.in
Makefile
gnunet-service-datastore
.deps
+ test_datastore_api_management.gcno
test_datastore_api_management.gcda
test_datastore_api_management
test_datastore_api.gcno
test_datastore_api.gcda
plugin_datastore_template.gcno
plugin_datastore_sqlite.gcno
perf_plugin_datastore.gcno
perf_plugin_datastore.gcda
perf_datastore_api.gcno
perf_datastore_api.gcda
gnunet-service-datastore.gcda
datastore_api.gcno
gnunet-service-datastore.gcno
perf_plugin_datastore
test_datastore_api
perf_datastore_api_iterators
perf_datastore_api
Makefile.in
Makefile
gnunet-service-datastore
.deps
Property changes on: gnunet/src/fs
___________________________________________________________________
Added: svn:ignore
+ Makefile.in
Makefile
.deps
Modified: gnunet/src/fs/fs.h
===================================================================
--- gnunet/src/fs/fs.h 2009-08-26 20:47:59 UTC (rev 8862)
+++ gnunet/src/fs/fs.h 2009-08-27 11:14:19 UTC (rev 8863)
@@ -26,12 +26,34 @@
#ifndef FS_H
#define FS_H
+#include "gnunet_datastore_service.h"
+#include "gnunet_fs_service.h"
+
/**
* Size of the individual blocks used for file-sharing.
*/
#define GNUNET_FS_DBLOCK_SIZE (32*1024)
+
/**
+ * Pick a multiple of 2 here to achive 8-byte alignment!
+ * We also probably want DBlocks to have (roughly) the
+ * same size as IBlocks. With SHA-512, the optimal
+ * value is 32768 byte / 128 byte = 256
+ * (128 byte = 2 * 512 bits). DO NOT CHANGE!
+ */
+#define GNUNET_FS_CHK_PER_INODE 256
+
+
+/**
+ * Maximum size for a file to be considered for
+ * inlining in a directory.
+ */
+#define GNUNET_FS_MAX_INLINE_SIZE 65536
+
+
+
+/**
* @brief content hash key
*/
struct ContentHashKey
@@ -51,7 +73,7 @@
/**
* Total size of the file in bytes. (network byte order (!))
*/
- unsigned long long file_length;
+ uint64_t file_length;
/**
* Query and key of the top GNUNET_EC_IBlock.
@@ -187,6 +209,12 @@
struct GNUNET_FS_Uri *keywords;
/**
+ * CHK for this file or directory. NULL if
+ * we have not yet computed it.
+ */
+ struct GNUNET_FS_Uri *chk_uri;
+
+ /**
* At what time should the content expire?
*/
struct GNUNET_TIME_Absolute expirationTime;
@@ -198,6 +226,31 @@
char *serialization;
/**
+ * In-memory cache of the current CHK tree.
+ * This struct will contain the CHK values
+ * from the root to the currently processed
+ * node in the tree as identified by
+ * "current_depth" and "publish_offset".
+ * The "chktree" will be initially NULL,
+ * then allocated to a sufficient number of
+ * entries for the size of the file and
+ * finally freed once the upload is complete.
+ */
+ struct ContentHashKey *chk_tree;
+
+ /**
+ * Number of entries in "chk_tree".
+ */
+ unsigned int chk_tree_depth;
+
+ /**
+ * Depth in the CHK-tree at which we are
+ * currently publishing. 0 is the root
+ * of the tree.
+ */
+ unsigned int current_depth;
+
+ /**
* How many bytes of this file or directory have been
* published so far?
*/
@@ -257,6 +310,12 @@
*/
uint64_t dir_size;
+ /**
+ * Pointer to the data for the directory (or NULL if not
+ * available).
+ */
+ char *dir_data;
+
} dir;
} data;
@@ -358,6 +417,23 @@
* if the upload has completed.
*/
GNUNET_SCHEDULER_TaskIdentifier upload_task;
+
+ /**
+ * Current position in the file-tree for the
+ * upload.
+ */
+ struct GNUNET_FS_FileInformation *fi_pos;
+
+ /**
+ * Connection to the datastore service.
+ */
+ struct GNUNET_DATASTORE_Handle *dsh;
+
+ /**
+ * Space reservation ID with datastore service
+ * for this upload.
+ */
+ int rid;
};
Modified: gnunet/src/fs/fs_directory.c
===================================================================
--- gnunet/src/fs/fs_directory.c 2009-08-26 20:47:59 UTC (rev 8862)
+++ gnunet/src/fs/fs_directory.c 2009-08-27 11:14:19 UTC (rev 8863)
@@ -25,7 +25,6 @@
*
* TODO:
* - add support for embedded file data (use padding room!)
- * - add directory builder API to gnunet_fs_service
* - modify directory builder API to support incremental
* generation of directories (to allow directories that
* would not fit into memory to be created)
@@ -38,6 +37,9 @@
#include "gnunet_fs_service.h"
#include "fs.h"
+#ifndef EXTRACTOR_GNUNET_FULL_DATA
+#define EXTRACTOR_GNUNET_FULL_DATA 137
+#endif
/**
* Does the meta-data claim that this is a directory?
@@ -215,6 +217,7 @@
return; /* malformed ! */
}
pos += mdSize;
+ // EXTRACTOR_GNUNET_FULL_DATA
/* FIXME: add support for embedded data */
filename = GNUNET_CONTAINER_meta_data_get_by_type (md,
EXTRACTOR_FILENAME);
@@ -231,26 +234,162 @@
}
}
+/**
+ * Entries in the directory (builder).
+ */
+struct BuilderEntry
+{
+ /**
+ * This is a linked list.
+ */
+ struct BuilderEntry *next;
+
+ /**
+ * Length of this entry.
+ */
+ size_t len;
+};
-void
-GNUNET_FS_directory_create ()
+/**
+ * Internal state of a directory builder.
+ */
+struct GNUNET_FS_DirectoryBuilder
{
+ /**
+ * Meta-data for the directory itself.
+ */
+ struct GNUNET_CONTAINER_MetaData *meta;
+
+ /**
+ * Head of linked list of entries.
+ */
+ struct BuilderEntry *head;
+
+ /**
+ * Number of entires in the directory.
+ */
+ unsigned int count;
+};
+
+
+/**
+ * Create a directory builder.
+ *
+ * @param mdir metadata for the directory
+ */
+struct GNUNET_FS_DirectoryBuilder *
+GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData
*mdir)
+{
+ struct GNUNET_FS_DirectoryBuilder *ret;
+
+ ret = GNUNET_malloc(sizeof(struct GNUNET_FS_DirectoryBuilder));
+ ret->meta = GNUNET_CONTAINER_meta_data_duplicate (mdir);
+ GNUNET_FS_meta_data_make_directory (ret->meta);
+ return ret;
}
-#if 0
+/**
+ * Add an entry to a directory.
+ *
+ * @param bld directory to extend
+ * @param uri uri of the entry (must not be a KSK)
+ * @param md metadata of the entry
+ * @param data raw data of the entry, can be NULL, otherwise
+ * data must point to exactly the number of bytes specified
+ * by the uri which must be of type LOC or CHK
+ */
+void
+GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
+ const struct GNUNET_FS_Uri *uri,
+ const struct GNUNET_CONTAINER_MetaData *md,
+ const void *data)
+{
+ struct BuilderEntry *e;
+ uint64_t fsize;
+ uint32_t big;
+ size_t mds;
+ size_t mdxs;
+ char *uris;
+ char *ser;
+ size_t slen;
+ struct GNUNET_CONTAINER_MetaData *meta;
+ const struct GNUNET_CONTAINER_MetaData *meta_use;
+ GNUNET_assert (! GNUNET_FS_uri_ksk_test (uri));
+ if (NULL != data)
+ if (GNUNET_FS_uri_chk_test (uri))
+ fsize = GNUNET_FS_uri_chk_get_size (uri);
+ else
+ fsize = GNUNET_FS_uri_chk_get_size (GNUNET_FS_uri_loc_get_uri (uri));
+ else
+ fsize = 0; /* not given */
+ if (fsize > GNUNET_FS_MAX_INLINE_SIZE)
+ fsize = 0; /* too large */
+ if (memchr (data, fsize, '\0')) // FIXME: check memchr args!
+ fsize = 0; /* must not have 0's in data! */
+ uris = GNUNET_FS_uri_to_string (uri);
+ slen = strlen (uris) + 1;
+ mds =
+ GNUNET_CONTAINER_meta_data_get_serialized_size (md,
+
GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
+ meta_use = md;
+ meta = NULL;
+ if (fsize > 0)
+ {
+ meta = GNUNET_CONTAINER_meta_data_duplicate (md);
+ GNUNET_CONTAINER_meta_data_insert (meta,
+ EXTRACTOR_GNUNET_FULL_DATA,
+ data);
+ mdxs =
+ GNUNET_CONTAINER_meta_data_get_serialized_size (meta,
+
GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
+ if ( (slen + sizeof (uint32_t) + mdxs - 1) / GNUNET_FS_DBLOCK_SIZE ==
+ (slen + sizeof (uint32_t) + mds - 1) / GNUNET_FS_DBLOCK_SIZE)
+ {
+ /* adding full data would not cause us to cross
+ additional blocks, so add it! */
+ meta_use = meta;
+ mds = mdxs;
+ }
+ }
+ if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
+ mds = GNUNET_MAX_MALLOC_CHECKED / 2;
+ e = GNUNET_malloc (sizeof(struct BuilderEntry) +
+ slen + mds + sizeof (uint32_t));
+ ser = (char*) &e[1];
+ memcpy (ser, uris, slen);
+ GNUNET_free (uris);
+ ret = GNUNET_CONTAINER_meta_data_serialize (meta_use,
+ &ser[slen + sizeof(uint32_t)],
+ mds,
+
GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
+ if (NULL != meta)
+ GNUNET_CONTAINER_meta_data_destroy (meta);
+ if (ret == -1)
+ mds = 0;
+ else
+ mds = ret;
+ big = htonl (mds);
+ memcpy (&ser[slen], &big, sizeof (uint32_t));
+ e->len = slen + sizeof (uint32_t) + mds;
+ e->next = bld->head;
+ bld->head = e;
+ bld->count++;
+}
+
+
/**
* Given the start and end position of a block of
* data, return the end position of that data
* after alignment to the GNUNET_FS_DBLOCK_SIZE.
*/
-static uint64_t
-do_align (uint64_t start_position,
- uint64_t end_position)
+static size_t
+do_align (size_t start_position,
+ size_t end_position)
{
- uint64_t align;
+ size_t align;
align = (end_position / GNUNET_FS_DBLOCK_SIZE) * GNUNET_FS_DBLOCK_SIZE;
if ((start_position < align) && (end_position > align))
@@ -269,19 +408,19 @@
* @param perm the permutation of the blocks (updated)
*/
static void
-block_align (uint64_t start,
+block_align (size_t start,
unsigned int count,
- const uint64_t *sizes,
+ const size_t *sizes,
unsigned int *perm)
{
unsigned int i;
unsigned int j;
unsigned int tmp;
unsigned int best;
- int64_t badness;
- uint64_t cpos;
- uint64_t cend;
- int64_t cbad;
+ ssize_t badness;
+ size_t cpos;
+ size_t cend;
+ ssize_t cbad;
unsigned int cval;
cpos = start;
@@ -334,135 +473,94 @@
/**
- * Create a directory. We allow packing more than one variable
- * size entry into one block (and an entry could also span more
- * than one block), but an entry that is smaller than a single
- * block will never cross the block boundary. This is done to
- * allow processing entries of a directory already even if the
- * download is still partial.<p>
+ * Finish building the directory. Frees the
+ * builder context and returns the directory
+ * in-memory.
*
- * The first block begins with the directories MAGIC signature,
- * followed by the meta-data about the directory itself.<p>
- *
- * After that, the directory consists of block-aligned pairs
- * of URIs (0-terminated strings) and serialized meta-data.
- *
- * @param data pointer set to the beginning of the directory
- * @param len set to number of bytes in data
- * @param count number of entries in uris and mds
- * @param uris URIs of the files in the directory
- * @param mds meta-data for the files (must match
- * respective values at same offset in in uris)
- * @param mdir meta-data for the directory
- * @return GNUNET_OK on success, GNUNET_SYSERR on error
+ * @param bld directory to finish
+ * @param rsize set to the number of bytes needed
+ * @param rdata set to the encoded directory
*/
-int
-GNUNET_FS_directory_create (char **data,
- size_t *len,
- unsigned int count,
- const struct GNUNET_FS_Uri **uris,
- const struct GNUNET_CONTAINER_MetaData **mds,
- const struct GNUNET_CONTAINER_MetaData *mdir)
+void
+GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
+ size_t *rsize,
+ void **rdata)
{
+ char *data;
+ size_t *sizes;
+ unsigned int *perm;
unsigned int i;
unsigned int j;
- uint64_t psize;
- uint64_t size;
- uint64_t pos;
- char **ucs;
- int ret;
- uint64_t *sizes;
- unsigned int *perm;
+ struct BuilderEntry *pos;
+ struct BuilderEntry **bes;
+ size_t size;
+ size_t psize;
+ size_t off;
+ ssize_t ret;
+ uint32_t big;
- for (i = 0; i < count; i++)
+ size = 8 + sizeof (uint32_t);
+ size += GNUNET_meta_data_get_serialized_size (bld->meta,
+ GNUNET_SERIALIZE_FULL);
+ if (bld->count > 0)
{
- if (GNUNET_FS_uri_test_ksk (fis[i].uri))
- {
- GNUNET_break (0);
- return GNUNET_SYSERR; /* illegal in directory! */
- }
- }
- ucs = GNUNET_malloc (sizeof (char *) * count);
- size = 8 + sizeof (unsigned int);
- size += GNUNET_meta_data_get_serialized_size (meta, GNUNET_SERIALIZE_FULL);
- sizes = GNUNET_malloc (count * sizeof (unsigned long long));
- perm = GNUNET_malloc (count * sizeof (int));
- for (i = 0; i < count; i++)
- {
- perm[i] = i;
- ucs[i] = GNUNET_FS_uri_to_string (fis[i].uri);
- GNUNET_assert (ucs[i] != NULL);
- psize =
- GNUNET_meta_data_get_serialized_size (fis[i].meta,
- GNUNET_SERIALIZE_FULL);
- if (psize == -1)
- {
- GNUNET_break (0);
- GNUNET_free (sizes);
- GNUNET_free (perm);
- while (i >= 0)
- GNUNET_free (ucs[i--]);
- GNUNET_free (ucs);
- return GNUNET_SYSERR;
- }
- sizes[i] = psize + sizeof (unsigned int) + strlen (ucs[i]) + 1;
- }
- /* permutate entries to minimize alignment cost */
- block_align (size, count, sizes, perm);
+ sizes = GNUNET_malloc (bld->count * sizeof (size_t));
+ perm = GNUNET_malloc (bld->count * sizeof (unsigned int));
+ bes = GNUNET_malloc (bld->count * sizeof (struct BuilderEntry *));
+ pos = bld->head;
+ for (i = 0; i < bld->count; i++)
+ {
+ perm[i] = i;
+ bes[i] = pos;
+ sizes[i] = pos->size;
+ pos = pos->next;
+ }
+ }
+ block_align (size,
+ bld->count,
+ sizes,
+ perm);
/* compute final size with alignment */
- for (i = 0; i < count; i++)
+ for (i = 0; i < bld->count; i++)
{
psize = size;
size += sizes[perm[i]];
size = do_align (psize, size);
}
- *len = size;
- *data = GNUNET_malloc (size);
- memset (*data, 0, size);
+ *rsize = size;
+ data = GNUNET_malloc (size);
+ *rdata = data;
+ memcpy (data, GNUNET_DIRECTORY_MAGIC, 8);
+ off = 8;
- pos = 8;
- memcpy (*data, GNUNET_DIRECTORY_MAGIC, 8);
-
ret = GNUNET_CONTAINER_meta_data_serialize (meta,
- &(*data)[pos +
- sizeof (unsigned int)],
- size - pos - sizeof (unsigned
int),
+ &(*data)[off +
+ sizeof (uint32_t)],
+ size - pos - sizeof (uint32_t),
GNUNET_SERIALIZE_FULL);
- GNUNET_assert (ret != GNUNET_SYSERR);
- ret = htonl (ret);
- memcpy (&(*data)[pos], &ret, sizeof (unsigned int));
- pos += ntohl (ret) + sizeof (unsigned int);
-
+ GNUNET_assert (ret != -1);
+ big = htonl (ret);
+ memcpy (&(*data)[8], &big, sizeof (uint32_t));
+ pos += sizeof (uint32_t) + ret;
for (j = 0; j < count; j++)
{
i = perm[j];
psize = pos;
pos += sizes[i];
pos = do_align (psize, pos);
- pos -= sizes[i]; /* go back to beginning */
- memcpy (&(*data)[pos], ucs[i], strlen (ucs[i]) + 1);
- pos += strlen (ucs[i]) + 1;
- GNUNET_free (ucs[i]);
- ret = GNUNET_CONTAINER_meta_data_serialize (mds[i],
- &(*data)[pos +
- sizeof (unsigned
int)],
- size - pos -
- sizeof (unsigned int),
- GNUNET_SERIALIZE_FULL);
- GNUNET_assert (ret != GNUNET_SYSERR);
- ret = htonl (ret);
- memcpy (&(*data)[pos], &ret, sizeof (unsigned int));
- pos += ntohl (ret) + sizeof (unsigned int);
+ memcpy (&data[pos - sizes[i]],
+ &(bes[i])[1],
+ sizes[i]);
+ GNUNET_free (bes[i]);
}
GNUNET_free (sizes);
GNUNET_free (perm);
- GNUNET_free (ucs);
- GNUNET_assert (pos == size);
- return GNUNET_OK;
+ GNUNET_free (bes);
+ GNUNET_assert (pos == size);
+ GNUNET_CONTAINER_meta_data_destroy (bld->meta);
+ GNUNET_free (bld);
}
-#endif
-
/* end of fs_directory.c */
Modified: gnunet/src/fs/fs_file_information.c
===================================================================
--- gnunet/src/fs/fs_file_information.c 2009-08-26 20:47:59 UTC (rev 8862)
+++ gnunet/src/fs/fs_file_information.c 2009-08-27 11:14:19 UTC (rev 8863)
@@ -804,6 +804,7 @@
&fi->priority,
&fi->expirationTime,
&fi->client_info);
+ GNUNET_free_non_null (fi->data.dir.dir_data);
GNUNET_free (fi->data.dir.dirname);
}
else
@@ -821,7 +822,7 @@
&fi->expirationTime,
&fi->client_info);
}
-
+ GNUNET_free_non_null (fi->chk_tree);
/* clean up serialization */
if (0 != UNLINK (fi->serialization))
GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
Modified: gnunet/src/fs/fs_publish.c
===================================================================
--- gnunet/src/fs/fs_publish.c 2009-08-26 20:47:59 UTC (rev 8862)
+++ gnunet/src/fs/fs_publish.c 2009-08-27 11:14:19 UTC (rev 8863)
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet.
- (C) 2001, 2002, 2003, 2004, 2005, 2006, 2009 Christian Grothoff (and
other contributing authors)
+ (C) 2009 Christian Grothoff (and other contributing authors)
GNUnet is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published
@@ -24,17 +24,396 @@
* @see http://gnunet.org/encoding.php3
* @author Krista Bennett
* @author Christian Grothoff
+ *
+ * TODO:
+ * - directory creation
+ * - KBlocks
+ * - SBlocks
+ * - indexing support
+ * - calling of progress function
+ * - handling of IO errors (emsg)
+ * - code-sharing with unindex
+ * - datastore reservation support
+ * - persistence support
*/
#include "platform.h"
+#include "gnunet_constants.h"
#include "gnunet_util_lib.h"
#include "gnunet_fs_service.h"
#include "fs.h"
#define DEBUG_PUBLISH GNUNET_YES
+/**
+ * Main function that performs the upload.
+ * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
+ * @param tc task context
+ */
+static void
+do_upload (void *cls,
+ const struct GNUNET_SCHEDULER_TaskContext *tc);
+
/**
+ * Context for "ds_put_cont".
+ */
+struct PutContCtx
+{
+ /**
+ * Publishing context for which the datastore
+ * PUT request was executed.
+ */
+ struct GNUNET_FS_PublishContext *sc;
+
+ /**
+ * Specific file with the block.
+ */
+ struct GNUNET_FS_FileInformation *p;
+
+ /**
+ * Function to run next, if any (can be NULL).
+ */
+ GNUNET_SCHEDULER_Task cont;
+};
+
+/**
+ * Function called by the datastore API with
+ * the result from the PUT request.
+ *
+ * @param cls our closure
+ * @param success GNUNET_OK on success
+ * @param msg error message (or NULL)
+ */
+static void
+ds_put_cont (void *cls,
+ int success,
+ const char *msg)
+{
+ struct PutContCtx *pcc = cls;
+
+ if (GNUNET_OK != success)
+ {
+ // FIXME: call progress CB with error
+ // FIXME: update pcc->p to indicate abort
+ GNUNET_FS_file_information_sync (pcc->p);
+ return;
+ }
+ GNUNET_FS_file_information_sync (pcc->p);
+ if (NULL != pcc->cont)
+ pcc->sc->upload_task
+ = GNUNET_SCHEDULER_add_delayed (pcc->sc->h->sched,
+ GNUNET_NO,
+ GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
+ GNUNET_SCHEDULER_NO_TASK,
+ GNUNET_TIME_UNIT_ZERO,
+ pcc->cont,
+ pcc->sc);
+ GNUNET_free (pcc);
+}
+
+
+/**
+ * We need to publish a specific block. Do it. Then continue with
+ * the main task.
+ *
+ * @param sc overall upload data
+ * @param p file that the block belongs to (needed for options!)
+ * @param blk encoded block to publish
+ * @param blk_size size of the block
+ * @param blk_type type of the block
+ * @param cont function to run when done
+ */
+static void
+publish_block (struct GNUNET_FS_PublishContext *sc,
+ struct GNUNET_FS_FileInformation *p,
+ const void* blk,
+ uint16_t blk_size,
+ uint32_t blk_type,
+ GNUNET_SCHEDULER_Task cont)
+{
+ struct GNUNET_HashCode key;
+
+ // FIXME: GNUNET_FS_get_key (blk_type, blk, blk_size, &key);
+ // (or add "key" as argument to reduce hashing?)
+ dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx));
+ dpc_cls->cont = cont;
+ dpc_cls->sc = sc;
+ dpc_cls->p = p;
+ // FIXME: need to do something to "sc" to mark
+ // that "sc" can not be freed right now due to this
+ // pending, scheduled operation for which we don't have
+ // a task ID!
+ GNUNET_DATASTORE_put (sc->dsh,
+ sc->rid,
+ &key,
+ blk_size,
+ blk_type,
+ p->priority,
+ p->anonymity,
+ p->expirationTime,
+ GNUNET_CONSTANTS_SERVICE_TIMEOUT,
+ &ds_put_cont,
+ dpc_cls);
+}
+
+
+/**
+ * We are almost done publishing the structure,
+ * add SBlocks (if needed).
+ *
+ * @param sc overall upload data
+ */
+static void
+publish_sblock (struct GNUNET_FS_PublishContext *sc)
+{
+ struct GNUNET_FS_FileInformation *p;
+ p = sc->fi;
+
+ // FIXME: build sblock & call publish_block!
+
+ // FIXME: continuation should
+ // be releasing the datastore reserve
+ // (once implemented)
+}
+
+
+/**
+ * We have uploaded a file or directory; now publish
+ * the KBlocks in the global keyword space so that
+ * it can be found. Then continue with the
+ * main task.
+ *
+ * @param sc overall upload data
+ * @param p specific file or directory for which kblocks
+ * should be created
+ */
+static void
+publish_kblocks (struct GNUNET_FS_PublishContext *sc,
+ struct GNUNET_FS_FileInformation *p)
+{
+ // FIXME: build all kblocks
+ // call publish_kblock on each
+ // last continuation should then call the main continuation again
+}
+
+
+/**
+ * Compute the depth of the CHK tree.
+ *
+ * @param flen file length for which to compute the depth
+ * @return depth of the tree
+ */
+static unsigned int
+compute_depth (uint64_t flen)
+{
+ unsigned int treeDepth;
+ uint64_t fl;
+
+ treeDepth = 1;
+ fl = GNUNET_FS_DBLOCK_SIZE;
+ while (fl < flen)
+ {
+ treeDepth++;
+ if (fl * GNUNET_FS_CHK_PER_INODE < fl)
+ {
+ /* integer overflow, this is a HUGE file... */
+ return treeDepth;
+ }
+ fl = fl * GNUNET_FS_CHK_PER_INODE;
+ }
+ return treeDepth;
+}
+
+
+/**
+ * Compute the size of the current IBlock.
+ *
+ * @param height height of the IBlock in the tree (aka overall
+ * number of tree levels minus depth); 0 == DBlock
+ * @param offset current offset in the overall file
+ * @return size of the corresponding IBlock
+ */
+static uint16_t
+compute_iblock_size (unsigned int height,
+ uint64_t offset)
+{
+ unsigned int ret;
+ unsigned int i;
+ uint64_t mod;
+ uint64_t bds;
+
+ GNUNET_assert (height > 0);
+ bds = GNUNET_FS_DBLOCK_SIZE; /* number of bytes each CHK at level "i"
+ corresponds to */
+ for (i=0;i<height;i++)
+ bds *= GNUNET_FS_CHK_PER_INODE;
+ mod = offset % bds;
+ if (0 == mod)
+ {
+ /* we were triggered at the end of a full block */
+ ret = GNUNET_FS_CHK_PER_INODE;
+ }
+ else
+ {
+ /* we were triggered at the end of the file */
+ bds /= GNUNET_FS_CHK_PER_INODE;
+ ret = mod / bds;
+ if (0 != mod % bds)
+ ret++;
+ }
+ return (uint16_t) (ret * sizeof(struct ContentHashKey));
+}
+
+
+/**
+ * Compute the offset of the CHK for the
+ * current block in the IBlock above.
+ *
+ * @param height height of the IBlock in the tree (aka overall
+ * number of tree levels minus depth); 0 == DBlock
+ * @param offset current offset in the overall file
+ * @return (array of CHKs') offset in the above IBlock
+ */
+static unsigned int
+compute_chk_offset (unsigned int height,
+ uint64_t offset)
+{
+ uint64_t bds;
+ unsigned int ret;
+
+ bds = GNUNET_FS_DBLOCK_SIZE; /* number of bytes each CHK at level "i"
+ corresponds to */
+ for (i=0;i<height;i++)
+ bds *= GNUNET_FS_CHK_PER_INODE;
+ GNUNET_assert (0 == (offset % bds));
+ ret = offset / bds;
+ return ret % GNUNET_FS_CHK_PER_INODE;
+}
+
+
+/**
+ * We are uploading a file or directory; load (if necessary) the next
+ * block into memory, encrypt it and send it to the FS service. Then
+ * continue with the main task.
+ *
+ * @param sc overall upload data
+ * @param p specific file or directory for which kblocks
+ * should be created
+ */
+static void
+publish_content (struct GNUNET_FS_PublishContext *sc,
+ struct GNUNET_FS_FileInformation *p)
+{
+ struct ContentHashKey *chk;
+ const void *pt_block;
+ uint16_t pt_size;
+ char *emsg;
+ char iob[GNUNET_FS_DBLOCK_SIZE];
+ char enc[GNUNET_FS_DBLOCK_SIZE];
+ struct GNUNET_CRYPTO_AesSessionKey sk;
+ struct GNUNET_CRYPTO_AesInitializationVector iv;
+ uint64_t size;
+ unsigned int off;
+
+ // FIXME: figure out how to share this code
+ // with unindex!
+ size = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
+ if (NULL == p->chk_tree)
+ {
+ if (p->is_directory)
+ {
+ /* FIXME: create function to create directory
+ and use that API here! */
+ GNUNET_FS_directory_create (&p->data.dir.dir_size,
+ &p->data.dir.dir_data,
+ p->meta,
+ &directory_entry_lister,
+ p->data.dir.entries);
+ size = p->data.dir.data_size;
+ }
+ p->chk_tree_depth = compute_depth (size);
+ p->chk_tree = GNUNET_malloc (p->chk_tree_depth *
+ sizeof (struct ContentHashKey) *
+ GNUNET_FS_CHK_PER_INODE);
+ p->current_depth = p->chk_tree_depth;
+ }
+ if (p->current_depth == p->chk_tree_depth)
+ {
+ if (p->is_directory)
+ {
+ pt_size = GNUNET_MIN(GNUNET_FS_DBLOCK_SIZE,
+ p->data.dir.dir_size - p->publish_offset);
+ pt_block = &p->data.dir.dir_data[p->publish_offset];
+ }
+ else
+ {
+ pt_size = GNUNET_MIN(GNUNET_FS_DBLOCK_SIZE,
+ p->data.file.file_size - p->publish_offset);
+ p->data.file.reader (p->data.file.reader_cls,
+ p->publish_offset,
+ pt_size,
+ iob,
+ &emsg);
+ pt_block = iob;
+ }
+ }
+ else
+ {
+ pt_size = compute_iblock_size (p->chk_tree_depth - p->current_depth,
+ p->publish_offset);
+ pt_block = &p->chk_tree[p->current_depth *
+ GNUNET_FS_CHK_PER_INODE];
+ }
+ off = compute_chk_offset (p->chk_tree_depth - p->current_depth,
+ p->publish_offset);
+ chk = &p->chk_tree[(p->current_depth-1)*GNUNET_FS_CHK_PER_INODE+off];
+ GNUNET_CRYPTO_hash (pt_block, pt_size, &chk->key);
+ GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
+ GNUNET_CRYPTO_aes_encrypt (pt_block,
+ pt_size,
+ &sk,
+ &iv,
+ enc);
+ // NOTE: this call (and progress below) is all that really differs
+ // between publish/unindex! Parameterize & move this code!
+ // FIXME: something around here would need to change
+ // for indexing!
+ publish_block (sc, p, enc, pt_size,
+ (p->current_depth == p->chk_tree_depth)
+ ? GNUNET_DATASTORE_BLOCKTYPE_DBLOCK
+ : GNUNET_DATASTORE_BLOCKTYPE_IBLOCK,
+ &do_upload);
+ // FIXME: should call progress function somewhere here!
+ GNUNET_CRYPTO_hash (enc, pt_size, &chk->query);
+ if (p->current_depth == p->chk_tree_depth)
+ {
+ p->publish_offset += pt_size;
+ if ( (p->publish_offset == size) ||
+ (0 == p->publish_offset % (GNUNET_FS_CHK_PER_INODE *
GNUNET_FS_DBLOCK_SIZE) ) )
+ p->current_depth--;
+ }
+ else
+ {
+ if ( (off == GNUNET_FS_CHK_PER_INODE) ||
+ (p->publish_offset == size) )
+ p->current_depth--;
+ else
+ p->current_depth = p->chk_tree_depth;
+ }
+ if (0 == p->current_depth)
+ {
+ p->chk_uri = GNUNET_malloc (sizeof(struct GNUNET_FS_Uri));
+ p->chk_uri.type = chk;
+ p->chk_uri.data.chk.chk = p->chk_tree[0];
+ p->chk_uri.data.chk.file_length = size;
+ GNUNET_free (p->chk_tree);
+ p->chk_tree = NULL;
+ }
+}
+
+
+/**
* Main function that performs the upload.
* @param cls "struct GNUNET_FS_PublishContext" identifies the upload
* @param tc task context
@@ -44,11 +423,36 @@
const struct GNUNET_SCHEDULER_TaskContext *tc)
{
struct GNUNET_FS_PublishContext *sc = cls;
+ struct GNUNET_FS_FileInformation *p;
sc->upload_task = GNUNET_SCHEDULER_NO_TASK;
-
- // FIXME: find next block, process, schedule
- // transmission to FS service
+ p = sc->fi_pos;
+ if (NULL == p)
+ {
+ /* upload of entire hierarchy complete,
+ publish namespace entries */
+ publish_sblock (sc);
+ return;
+ }
+ if (NULL != p->chk_uri)
+ {
+ /* move on to next file */
+ if (NULL != p->next)
+ sc->fi_pos = p->next;
+ else
+ sc->fi_pos = p->dir;
+ /* upload of "p" complete, publish KBlocks! */
+ publish_kblocks (sc, p);
+ return;
+ }
+ if (p->do_index)
+ {
+ // FIXME: need to pre-compute hash over
+ // the entire file and ask FS to prepare
+ // for indexing!
+ return;
+ }
+ publish_content (sc, p);
}
@@ -75,8 +479,15 @@
const char *nuid)
{
struct GNUNET_FS_PublishContext *ret;
+ struct GNUNET_FS_FileInformation *p;
+ struct GNUNET_DATASTORE_Handle *dsh;
+ dsh = GNUNET_DATASTORE_connect (h->cfg,
+ h->sched);
+ if (NULL == dsh)
+ return NULL;
ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
+ ret->dsh = dsh;
ret->h = h;
ret->client_ctx = ctx;
ret->fi = fi;
@@ -90,6 +501,17 @@
ret->nuid = GNUNET_strdup (nuid);
}
// FIXME: make upload persistent!
+
+ /* find first leaf, DFS */
+ p = ret->fi;
+ while ( (p->is_directory) &&
+ (NULL != p->data.dir.entries) )
+ p = p->data.dir.entries;
+ ret->fi_pos = p;
+
+ // FIXME: calculate space needed for "fi"
+ // and reserve as first task (then trigger
+ // "do_upload" from that continuation)!
ret->upload_task
= GNUNET_SCHEDULER_add_delayed (h->sched,
GNUNET_NO,
@@ -120,429 +542,8 @@
GNUNET_FS_namespace_delete (sc->namespace, GNUNET_NO);
GNUNET_free_non_null (sc->nid);
GNUNET_free_non_null (sc->nuid);
+ GNUNET_DATASTORE_disconnect (sc->dsh);
GNUNET_free (sc);
}
-
-#if 0
-
-/**
- * Append the given key and query to the iblock[level]. If
- * iblock[level] is already full, compute its chk and push it to
- * level+1 and clear the level. iblocks is guaranteed to be big
- * enough.
- */
-static int
-pushBlock (struct GNUNET_ClientServerConnection *sock,
- const GNUNET_EC_ContentHashKey * chk,
- unsigned int level,
- GNUNET_DatastoreValue ** iblocks,
- unsigned int prio, GNUNET_CronTime expirationTime)
-{
- unsigned int size;
- unsigned int present;
- GNUNET_DatastoreValue *value;
- GNUNET_EC_DBlock *db;
- GNUNET_EC_ContentHashKey ichk;
-
- size = ntohl (iblocks[level]->size);
- GNUNET_GE_ASSERT (NULL, size > sizeof (GNUNET_DatastoreValue));
- size -= sizeof (GNUNET_DatastoreValue);
- GNUNET_GE_ASSERT (NULL,
- size - sizeof (GNUNET_EC_DBlock) <=
- GNUNET_ECRS_IBLOCK_SIZE);
- present =
- (size - sizeof (GNUNET_EC_DBlock)) / sizeof (GNUNET_EC_ContentHashKey);
- db = (GNUNET_EC_DBlock *) & iblocks[level][1];
- if (present == GNUNET_ECRS_CHK_PER_INODE)
- {
- GNUNET_EC_file_block_get_key (db, size, &ichk.key);
- GNUNET_EC_file_block_get_query (db, size, &ichk.query);
- if (GNUNET_OK != pushBlock (sock,
- &ichk, level + 1, iblocks, prio,
- expirationTime))
- return GNUNET_SYSERR;
- GNUNET_EC_file_block_encode (db, size, &ichk.query, &value);
- if (value == NULL)
- {
- GNUNET_GE_BREAK (NULL, 0);
- return GNUNET_SYSERR;
- }
- value->priority = htonl (prio);
- value->expiration_time = GNUNET_htonll (expirationTime);
- if (GNUNET_OK != GNUNET_FS_insert (sock, value))
- {
- GNUNET_free (value);
- return GNUNET_SYSERR;
- }
- GNUNET_free (value);
- size = sizeof (GNUNET_EC_DBlock); /* type */
- }
- /* append GNUNET_EC_ContentHashKey */
- memcpy (&((char *) db)[size], chk, sizeof (GNUNET_EC_ContentHashKey));
- size += sizeof (GNUNET_EC_ContentHashKey) + sizeof (GNUNET_DatastoreValue);
- GNUNET_GE_ASSERT (NULL, size < GNUNET_MAX_BUFFER_SIZE);
- iblocks[level]->size = htonl (size);
-
- return GNUNET_OK;
-}
-
-/**
- * Index or insert a file.
- *
- * @param priority what is the priority for OUR node to
- * keep this file available? Use 0 for maximum anonymity and
- * minimum reliability...
- * @param doIndex GNUNET_YES for index, GNUNET_NO for insertion,
- * GNUNET_SYSERR for simulation
- * @param uri set to the URI of the uploaded file
- * @return GNUNET_SYSERR if the upload failed (i.e. not enough space
- * or gnunetd not running)
- */
-int
-GNUNET_ECRS_file_upload (struct GNUNET_GE_Context *ectx,
- struct GNUNET_GC_Configuration *cfg,
- const char *filename,
- int doIndex,
- unsigned int anonymityLevel,
- unsigned int priority,
- GNUNET_CronTime expirationTime,
- GNUNET_ECRS_UploadProgressCallback upcb,
- void *upcbClosure,
- GNUNET_ECRS_TestTerminate tt,
- void *ttClosure, struct GNUNET_ECRS_URI **uri)
-{
- unsigned long long filesize;
- unsigned long long pos;
- unsigned int treedepth;
- int fd;
- int i;
- int ret;
- unsigned int size;
- GNUNET_DatastoreValue **iblocks;
- GNUNET_DatastoreValue *dblock;
- GNUNET_EC_DBlock *db;
- GNUNET_DatastoreValue *value;
- struct GNUNET_ClientServerConnection *sock;
- GNUNET_HashCode fileId;
- GNUNET_EC_ContentHashKey mchk;
- GNUNET_CronTime eta;
- GNUNET_CronTime start;
- GNUNET_CronTime now;
- GNUNET_EC_FileIdentifier fid;
-#if DEBUG_UPLOAD
- GNUNET_EncName enc;
-#endif
-
- GNUNET_GE_ASSERT (ectx, cfg != NULL);
- start = GNUNET_get_time ();
- memset (&mchk, 0, sizeof (GNUNET_EC_ContentHashKey));
- if (GNUNET_YES != GNUNET_disk_file_test (ectx, filename))
- {
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
- _("`%s' is not a file.\n"), filename);
- return GNUNET_SYSERR;
- }
- if (GNUNET_OK !=
- GNUNET_disk_file_size (ectx, filename, &filesize, GNUNET_YES))
- {
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
- _("Cannot get size of file `%s'"), filename);
-
- return GNUNET_SYSERR;
- }
- sock = GNUNET_client_connection_create (ectx, cfg);
- if (sock == NULL)
- {
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
- _("Failed to connect to gnunetd."));
- return GNUNET_SYSERR;
- }
- eta = 0;
- if (upcb != NULL)
- upcb (filesize, 0, eta, upcbClosure);
- if (doIndex == GNUNET_YES)
- {
- if (GNUNET_SYSERR == GNUNET_hash_file (ectx, filename, &fileId))
- {
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
- _("Cannot hash `%s'.\n"), filename);
-
- GNUNET_client_connection_destroy (sock);
- return GNUNET_SYSERR;
- }
- if (GNUNET_YES == GNUNET_FS_test_indexed (sock, &fileId))
- {
- /* file already indexed; simulate only to get the URI! */
- doIndex = GNUNET_SYSERR;
- }
- }
- if (doIndex == GNUNET_YES)
- {
- now = GNUNET_get_time ();
- eta = now + 2 * (now - start);
- /* very rough estimate: GNUNET_hash reads once through the file,
- we'll do that once more and write it. But of course
- the second read may be cached, and we have the encryption,
- so a factor of two is really, really just a rough estimate */
- start = now;
- /* reset the counter since the formula later does not
- take the time for GNUNET_hash_file into account */
-
- switch (GNUNET_FS_prepare_to_index (sock, &fileId, filename))
- {
- case GNUNET_SYSERR:
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
- _("Initialization for indexing file `%s' failed.\n"),
- filename);
- GNUNET_client_connection_destroy (sock);
- return GNUNET_SYSERR;
- case GNUNET_NO:
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
- _
- ("Indexing file `%s' failed. Suggestion: try to
insert the file.\n"),
- filename);
- GNUNET_client_connection_destroy (sock);
- return GNUNET_SYSERR;
- default:
- break;
- }
- }
- treedepth = GNUNET_ECRS_compute_depth (filesize);
- fd = GNUNET_disk_file_open (ectx, filename, O_RDONLY | O_LARGEFILE);
- if (fd == -1)
- {
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_USER,
- _("Cannot open file `%s': `%s'"), filename,
- STRERROR (errno));
-
- GNUNET_client_connection_destroy (sock);
- return GNUNET_SYSERR;
- }
-
- dblock =
- GNUNET_malloc (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE +
- sizeof (GNUNET_EC_DBlock));
- dblock->size =
- htonl (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE +
- sizeof (GNUNET_EC_DBlock));
- dblock->anonymity_level = htonl (anonymityLevel);
- dblock->priority = htonl (priority);
- dblock->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
- dblock->expiration_time = GNUNET_htonll (expirationTime);
- db = (GNUNET_EC_DBlock *) & dblock[1];
- db->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
- iblocks =
- GNUNET_malloc (sizeof (GNUNET_DatastoreValue *) * (treedepth + 1));
- for (i = 0; i <= treedepth; i++)
- {
- iblocks[i] =
- GNUNET_malloc (sizeof (GNUNET_DatastoreValue) +
- GNUNET_ECRS_IBLOCK_SIZE + sizeof (GNUNET_EC_DBlock));
- iblocks[i]->size =
- htonl (sizeof (GNUNET_DatastoreValue) + sizeof (GNUNET_EC_DBlock));
- iblocks[i]->anonymity_level = htonl (anonymityLevel);
- iblocks[i]->priority = htonl (priority);
- iblocks[i]->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
- iblocks[i]->expiration_time = GNUNET_htonll (expirationTime);
- ((GNUNET_EC_DBlock *) & iblocks[i][1])->type =
- htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
- }
-
- pos = 0;
- while (pos < filesize)
- {
- if (upcb != NULL)
- upcb (filesize, pos, eta, upcbClosure);
- if (tt != NULL)
- if (GNUNET_OK != tt (ttClosure))
- goto FAILURE;
- size = GNUNET_ECRS_DBLOCK_SIZE;
- if (size > filesize - pos)
- {
- size = filesize - pos;
- memset (&db[1], 0, GNUNET_ECRS_DBLOCK_SIZE);
- }
- GNUNET_GE_ASSERT (ectx,
- sizeof (GNUNET_DatastoreValue) + size +
- sizeof (GNUNET_EC_DBlock) < GNUNET_MAX_BUFFER_SIZE);
- dblock->size =
- htonl (sizeof (GNUNET_DatastoreValue) + size +
- sizeof (GNUNET_EC_DBlock));
- if (size != READ (fd, &db[1], size))
- {
- GNUNET_GE_LOG_STRERROR_FILE (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK |
- GNUNET_GE_ADMIN | GNUNET_GE_USER,
- "READ", filename);
- goto FAILURE;
- }
- if (tt != NULL)
- if (GNUNET_OK != tt (ttClosure))
- goto FAILURE;
- GNUNET_EC_file_block_get_key (db, size + sizeof (GNUNET_EC_DBlock),
- &mchk.key);
- GNUNET_EC_file_block_get_query (db, size + sizeof (GNUNET_EC_DBlock),
- &mchk.query);
-#if DEBUG_UPLOAD
- GNUNET_hash_to_enc (&mchk.query, &enc);
- fprintf (stderr,
- "Query for current block of size %u is `%s'\n", size,
- (const char *) &enc);
-#endif
- if (doIndex == GNUNET_YES)
- {
- if (GNUNET_SYSERR == GNUNET_FS_index (sock, &fileId, dblock, pos))
- {
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_ERROR | GNUNET_GE_BULK |
- GNUNET_GE_USER,
- _
- ("Indexing data of file `%s' failed at position
%llu.\n"),
- filename, pos);
- goto FAILURE;
- }
- }
- else
- {
- value = NULL;
- if (GNUNET_OK !=
- GNUNET_EC_file_block_encode (db,
- size + sizeof (GNUNET_EC_DBlock),
- &mchk.query, &value))
- {
- GNUNET_GE_BREAK (ectx, 0);
- goto FAILURE;
- }
- GNUNET_GE_ASSERT (ectx, value != NULL);
- *value = *dblock; /* copy options! */
- if ((doIndex == GNUNET_NO) &&
- (GNUNET_OK != (ret = GNUNET_FS_insert (sock, value))))
- {
- GNUNET_GE_BREAK (ectx, ret == GNUNET_NO);
- GNUNET_free (value);
- goto FAILURE;
- }
- GNUNET_free (value);
- }
- pos += size;
- now = GNUNET_get_time ();
- if (pos > 0)
- {
- eta = (GNUNET_CronTime) (start +
- (((double) (now - start) / (double) pos))
- * (double) filesize);
- }
- if (GNUNET_OK != pushBlock (sock, &mchk, 0, /* dblocks are on
level 0 */
- iblocks, priority, expirationTime))
- goto FAILURE;
- }
- if (tt != NULL)
- if (GNUNET_OK != tt (ttClosure))
- goto FAILURE;
-#if DEBUG_UPLOAD
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- "Tree depth is %u, walking up tree.\n", treedepth);
-#endif
- for (i = 0; i < treedepth; i++)
- {
- size = ntohl (iblocks[i]->size) - sizeof (GNUNET_DatastoreValue);
- GNUNET_GE_ASSERT (ectx, size < GNUNET_MAX_BUFFER_SIZE);
- if (size == sizeof (GNUNET_EC_DBlock))
- {
-#if DEBUG_UPLOAD
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- "Level %u is empty\n", i);
-#endif
- continue;
- }
- db = (GNUNET_EC_DBlock *) & iblocks[i][1];
- GNUNET_EC_file_block_get_key (db, size, &mchk.key);
-#if DEBUG_UPLOAD
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- "Computing query for %u bytes content.\n", size);
-#endif
- GNUNET_EC_file_block_get_query (db, size, &mchk.query);
-#if DEBUG_UPLOAD
- IF_GELOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- GNUNET_hash_to_enc (&mchk.query, &enc));
- GNUNET_GE_LOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- "Query for current block at level %u is `%s'.\n", i,
- &enc);
-#endif
- if (GNUNET_OK != pushBlock (sock,
- &mchk, i + 1, iblocks, priority,
- expirationTime))
- {
- GNUNET_GE_BREAK (ectx, 0);
- goto FAILURE;
- }
- GNUNET_EC_file_block_encode (db, size, &mchk.query, &value);
- if (value == NULL)
- {
- GNUNET_GE_BREAK (ectx, 0);
- goto FAILURE;
- }
- value->expiration_time = GNUNET_htonll (expirationTime);
- value->priority = htonl (priority);
- if ((doIndex != GNUNET_SYSERR) &&
- (GNUNET_SYSERR == GNUNET_FS_insert (sock, value)))
- {
- GNUNET_GE_BREAK (ectx, 0);
- GNUNET_free (value);
- goto FAILURE;
- }
- GNUNET_free (value);
- GNUNET_free (iblocks[i]);
- iblocks[i] = NULL;
- }
-#if DEBUG_UPLOAD
- IF_GELOG (ectx,
- GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- GNUNET_hash_to_enc (&mchk.query, &enc));
- GNUNET_GE_LOG (ectx, GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
- "Query for top block is %s\n", &enc);
-#endif
- /* build URI */
- fid.file_length = GNUNET_htonll (filesize);
- db = (GNUNET_EC_DBlock *) & iblocks[treedepth][1];
-
- fid.chk = *(GNUNET_EC_ContentHashKey *) & (db[1]);
- *uri = GNUNET_malloc (sizeof (URI));
- (*uri)->type = chk;
- (*uri)->data.fi = fid;
-
- /* free resources */
- GNUNET_free_non_null (iblocks[treedepth]);
- GNUNET_free (iblocks);
- GNUNET_free (dblock);
- if (upcb != NULL)
- upcb (filesize, filesize, eta, upcbClosure);
- CLOSE (fd);
- GNUNET_client_connection_destroy (sock);
- return GNUNET_OK;
-FAILURE:
- for (i = 0; i <= treedepth; i++)
- GNUNET_free_non_null (iblocks[i]);
- GNUNET_free (iblocks);
- GNUNET_free (dblock);
- CLOSE (fd);
- GNUNET_client_connection_destroy (sock);
- return GNUNET_SYSERR;
-}
-
-#endif
-
/* end of fs_publish.c */
Modified: gnunet/src/include/gnunet_constants.h
===================================================================
--- gnunet/src/include/gnunet_constants.h 2009-08-26 20:47:59 UTC (rev
8862)
+++ gnunet/src/include/gnunet_constants.h 2009-08-27 11:14:19 UTC (rev
8863)
@@ -55,7 +55,14 @@
*/
#define GNUNET_CONSTANTS_EXEC_WAIT GNUNET_TIME_relative_multiply
(GNUNET_TIME_UNIT_MILLISECONDS, 100)
+/**
+ * After how long do we consider a service irresponsive
+ * even if we assume that the service commonly does not
+ * respond instantly (DNS, Database, etc.).
+ */
+#define GNUNET_CONSTANTS_SERVICE_TIMEOUT GNUNET_TIME_relative_multiply
(GNUNET_TIME_UNIT_MINUTES, 10)
+
#if 0 /* keep Emacsens' auto-indent happy */
{
#endif
Modified: gnunet/src/include/gnunet_container_lib.h
===================================================================
--- gnunet/src/include/gnunet_container_lib.h 2009-08-26 20:47:59 UTC (rev
8862)
+++ gnunet/src/include/gnunet_container_lib.h 2009-08-27 11:14:19 UTC (rev
8863)
@@ -305,8 +305,8 @@
enum GNUNET_CONTAINER_MetaDataSerializationOptions
{
- GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL = GNUNET_NO,
- GNUNET_CONTAINER_META_DATA_SERIALIZE_PART = GNUNET_YES,
+ GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL = 0,
+ GNUNET_CONTAINER_META_DATA_SERIALIZE_PART = 1,
GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS = 2
};
@@ -323,10 +323,11 @@
* GNUNET_SYSERR on error (typically: not enough
* space)
*/
-int GNUNET_CONTAINER_meta_data_serialize (const struct
- GNUNET_CONTAINER_MetaData *md,
- char *target, unsigned int size,
- enum
+ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct
+ GNUNET_CONTAINER_MetaData *md,
+ char *target,
+ size_t size,
+ enum
GNUNET_CONTAINER_MetaDataSerializationOptions
opt);
@@ -337,12 +338,12 @@
* meta-data to match the size constraint,
* possibly discarding some data?
*/
-unsigned int GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
-
GNUNET_CONTAINER_MetaData
- *md,
- enum
-
GNUNET_CONTAINER_MetaDataSerializationOptions
- opt);
+ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
+
GNUNET_CONTAINER_MetaData
+ *md,
+ enum
+
GNUNET_CONTAINER_MetaDataSerializationOptions
+ opt);
/**
* Deserialize meta-data. Initializes md.
@@ -352,7 +353,7 @@
*/
struct GNUNET_CONTAINER_MetaData
*GNUNET_CONTAINER_meta_data_deserialize (const char *input,
- unsigned int size);
+ size_t size);
/**
* Does the meta-data claim that this is a directory?
Modified: gnunet/src/include/gnunet_crypto_lib.h
===================================================================
--- gnunet/src/include/gnunet_crypto_lib.h 2009-08-26 20:47:59 UTC (rev
8862)
+++ gnunet/src/include/gnunet_crypto_lib.h 2009-08-27 11:14:19 UTC (rev
8863)
@@ -394,7 +394,7 @@
/**
* Convert a hashcode into a key.
*/
-void GNUNET_CRYPTO_hash_to_AES_key (const GNUNET_HashCode * hc,
+void GNUNET_CRYPTO_hash_to_aes_key (const GNUNET_HashCode * hc,
struct GNUNET_CRYPTO_AesSessionKey *skey,
struct
GNUNET_CRYPTO_AesInitializationVector
Modified: gnunet/src/include/gnunet_datastore_service.h
===================================================================
--- gnunet/src/include/gnunet_datastore_service.h 2009-08-26 20:47:59 UTC
(rev 8862)
+++ gnunet/src/include/gnunet_datastore_service.h 2009-08-27 11:14:19 UTC
(rev 8863)
@@ -41,6 +41,12 @@
#endif
#endif
+#define GNUNET_DATASTORE_BLOCKTYPE_ANY 0
+#define GNUNET_DATASTORE_BLOCKTYPE_DBLOCK 1
+#define GNUNET_DATASTORE_BLOCKTYPE_IBLOCK 2
+#define GNUNET_DATASTORE_BLOCKTYPE_KBLOCK 3
+#define GNUNET_DATASTORE_BLOCKTYPE_SBLOCK 4
+#define GNUNET_DATASTORE_BLOCKTYPE_SKBLOCK 5
/**
* Handle to the datastore service.
Modified: gnunet/src/include/gnunet_fs_service.h
===================================================================
--- gnunet/src/include/gnunet_fs_service.h 2009-08-26 20:47:59 UTC (rev
8862)
+++ gnunet/src/include/gnunet_fs_service.h 2009-08-27 11:14:19 UTC (rev
8863)
@@ -2169,6 +2169,52 @@
void *dep_cls);
+/**
+ * Opaque handle to a directory builder.
+ */
+struct GNUNET_FS_DirectoryBuilder;
+
+/**
+ * Create a directory builder.
+ *
+ * @param mdir metadata for the directory
+ */
+struct GNUNET_FS_DirectoryBuilder *
+GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData
*mdir);
+
+
+/**
+ * Add an entry to a directory.
+ *
+ * @param bld directory to extend
+ * @param uri uri of the entry (must not be a KSK)
+ * @param md metadata of the entry
+ * @param data raw data of the entry, can be NULL, otherwise
+ * data must point to exactly the number of bytes specified
+ * by the uri
+ */
+void
+GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
+ const struct GNUNET_FS_Uri *uri,
+ const struct GNUNET_CONTAINER_MetaData *md,
+ const void *data);
+
+
+/**
+ * Finish building the directory. Frees the
+ * builder context and returns the directory
+ * in-memory.
+ *
+ * @param bld directory to finish
+ * @param size set to the number of bytes needed
+ * @param data set to the encoded directory
+ */
+void
+GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
+ size_t *size,
+ void **data);
+
+
#if 0 /* keep Emacsens' auto-indent happy */
{
#endif
Property changes on: gnunet/src/testing
___________________________________________________________________
Modified: svn:ignore
- Makefile.in
Makefile
.deps
+ test_testing.gcno
test_testing.gcda
test_testing
Makefile.in
Makefile
.deps
Property changes on: gnunet/src/transport
___________________________________________________________________
Modified: svn:ignore
- test_plugin_transport
transport_api.gcno
transport_api.gcda
plugin_transport_template.gcno
plugin_transport_tcp.gcno
plugin_transport_tcp.gcda
test_transport_api.gcda
gnunet-service-transport.gcda
test_transport_api.gcno
test_transport_api
Makefile.in
Makefile
gnunet-transport.gcno
gnunet-transport
gnunet-service-transport.gcno
gnunet-service-transport
core
.deps
+ test_plugin_transport.gcno
test_plugin_transport.gcda
test_plugin_transport
transport_api.gcno
transport_api.gcda
plugin_transport_template.gcno
plugin_transport_tcp.gcno
plugin_transport_tcp.gcda
test_transport_api.gcda
gnunet-service-transport.gcda
test_transport_api.gcno
test_transport_api
Makefile.in
Makefile
gnunet-transport.gcno
gnunet-transport
gnunet-service-transport.gcno
gnunet-service-transport
core
.deps
Property changes on: gnunet/src/upnp
___________________________________________________________________
Added: svn:ignore
+ Makefile.in
Modified: gnunet/src/util/container_meta_data.c
===================================================================
--- gnunet/src/util/container_meta_data.c 2009-08-26 20:47:59 UTC (rev
8862)
+++ gnunet/src/util/container_meta_data.c 2009-08-27 11:14:19 UTC (rev
8863)
@@ -442,9 +442,9 @@
* GNUNET_SYSERR on error (typically: not enough
* space)
*/
-int
+ssize_t
GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData
- *md, char *target, unsigned int max,
+ *md, char *target, size_t max,
enum
GNUNET_CONTAINER_MetaDataSerializationOptions
part)
@@ -463,7 +463,7 @@
while (1)
{
size = sizeof (struct MetaDataHeader);
- size += sizeof (unsigned int) * ic;
+ size += sizeof (uint32_t) * ic;
for (i = 0; i < ic; i++)
size += 1 + strlen (md->items[i].data);
while (size % 8 != 0)
@@ -472,8 +472,8 @@
hdr->version = htonl (md == NULL ? 1 : 0);
hdr->entries = htonl (ic);
for (i = 0; i < ic; i++)
- ((unsigned int *) &hdr[1])[i] =
- htonl ((unsigned int) md->items[i].type);
+ ((uint32_t *) &hdr[1])[i] =
+ htonl ((uint32_t) md->items[i].type);
pos = sizeof (struct MetaDataHeader);
pos += sizeof (unsigned int) * ic;
for (i = 0; i < ic; i++)
@@ -533,7 +533,7 @@
* serialized form. The estimate MAY be higher
* than what is strictly needed.
*/
-unsigned int
+ssize_t
GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
GNUNET_CONTAINER_MetaData *md,
enum
@@ -549,7 +549,7 @@
ic = md ? md->itemCount : 0;
size = sizeof (struct MetaDataHeader);
- size += sizeof (unsigned int) * ic;
+ size += sizeof (uint32_t) * ic;
for (i = 0; i < ic; i++)
size += 1 + strlen (md->items[i].data);
while (size % 8 != 0)
@@ -558,9 +558,9 @@
hdr->version = htonl (md == NULL ? 1 : 0);
hdr->entries = htonl (ic);
for (i = 0; i < ic; i++)
- ((unsigned int *) &hdr[1])[i] = htonl ((unsigned int) md->items[i].type);
+ ((uint32_t *) &hdr[1])[i] = htonl ((uint32_t) md->items[i].type);
pos = sizeof (struct MetaDataHeader);
- pos += sizeof (unsigned int) * ic;
+ pos += sizeof (uint32_t) * ic;
for (i = 0; i < ic; i++)
{
len = strlen (md->items[i].data) + 1;
@@ -590,7 +590,7 @@
* bad format)
*/
struct GNUNET_CONTAINER_MetaData *
-GNUNET_CONTAINER_meta_data_deserialize (const char *input, unsigned int size)
+GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size)
{
struct GNUNET_CONTAINER_MetaData *md;
const struct MetaDataHeader *hdr;
@@ -599,9 +599,9 @@
const char *cdata;
uint32_t dataSize;
int compressed;
- int i;
- unsigned int pos;
- int len;
+ uint32_t i;
+ size_t pos;
+ size_t len;
uint32_t version;
if (size < sizeof (struct MetaDataHeader))
@@ -651,7 +651,7 @@
}
}
- if ((sizeof (unsigned int) * ic + ic) > dataSize)
+ if ((sizeof (uint32_t) * ic + ic) > dataSize)
{
GNUNET_break (0);
goto FAILURE;
@@ -665,12 +665,12 @@
md = GNUNET_CONTAINER_meta_data_create ();
GNUNET_array_grow (md->items, md->itemCount, ic);
i = 0;
- pos = sizeof (unsigned int) * ic;
+ pos = sizeof (uint32_t) * ic;
while ((pos < dataSize) && (i < ic))
{
len = strlen (&cdata[pos]) + 1;
md->items[i].type = (EXTRACTOR_KeywordType)
- ntohl (MAKE_UNALIGNED (((const unsigned int *) cdata)[i]));
+ ntohl (MAKE_UNALIGNED (((const uint32_t *) cdata)[i]));
md->items[i].data = GNUNET_strdup (&cdata[pos]);
pos += len;
i++;
Modified: gnunet/src/util/crypto_hash.c
===================================================================
--- gnunet/src/util/crypto_hash.c 2009-08-26 20:47:59 UTC (rev 8862)
+++ gnunet/src/util/crypto_hash.c 2009-08-27 11:14:19 UTC (rev 8863)
@@ -709,7 +709,7 @@
* Convert a hashcode into a key.
*/
void
-GNUNET_CRYPTO_hash_to_AES_key (const GNUNET_HashCode * hc,
+GNUNET_CRYPTO_hash_to_aes_key (const GNUNET_HashCode * hc,
struct GNUNET_CRYPTO_AesSessionKey *skey,
struct GNUNET_CRYPTO_AesInitializationVector
*iv)
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r8863 - in gnunet: . doc/man src/datacache src/datastore src/fs src/include src/testing src/transport src/upnp src/util,
gnunet <=