[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r19157 - gnunet/src/fs
From: |
gnunet |
Subject: |
[GNUnet-SVN] r19157 - gnunet/src/fs |
Date: |
Sun, 15 Jan 2012 15:55:13 +0100 |
Author: grothoff
Date: 2012-01-15 15:55:13 +0100 (Sun, 15 Jan 2012)
New Revision: 19157
Modified:
gnunet/src/fs/fs_dirmetascan.c
gnunet/src/fs/gnunet-publish.c
Log:
-LRN: use filescanner in gnunet-publish
Modified: gnunet/src/fs/fs_dirmetascan.c
===================================================================
--- gnunet/src/fs/fs_dirmetascan.c 2012-01-15 12:16:42 UTC (rev 19156)
+++ gnunet/src/fs/fs_dirmetascan.c 2012-01-15 14:55:13 UTC (rev 19157)
@@ -568,6 +568,13 @@
EXTRACTOR_METATYPE_FILENAME,
EXTRACTOR_METAFORMAT_UTF8, "text/plain",
short_fn, strlen (short_fn) + 1);
+ if (ads->parent == NULL)
+ {
+ /* we're finished with the scan, make sure caller gets the top-level
+ * directory pointer
+ */
+ ads->adc->toplevel = item;
+ }
}
/**
@@ -869,7 +876,7 @@
* The function from which the scanner thread starts
*/
#if WINDOWS
-static DWORD
+DWORD
#else
static void *
#endif
@@ -1043,12 +1050,7 @@
if (0 != STAT (filename, &sbuf))
return NULL;
- /* TODO: consider generalizing this for files too! */
- if (!S_ISDIR (sbuf.st_mode))
- {
- GNUNET_break (0);
- return NULL;
- }
+
/* scan_directory() is guaranteed to be given expanded filenames,
* so expand we will!
*/
@@ -1211,30 +1213,35 @@
/* process a child entry (a file or a directory) and move to the next one*/
if (stack->item->is_directory)
stack->end_directory = GNUNET_NO;
- stack->dir_entry_count++;
- GNUNET_CONTAINER_meta_data_iterate (stack->item->meta,
&add_to_meta_counter, stack->metacounter);
+ if (stack->ctx->toplevel->is_directory)
+ {
+ stack->dir_entry_count++;
+ GNUNET_CONTAINER_meta_data_iterate (stack->item->meta,
&add_to_meta_counter, stack->metacounter);
- if (stack->item->is_directory)
- {
- char *user = getenv ("USER");
- if ((user == NULL) || (0 != strncasecmp (user,
stack->item->short_filename, strlen(user))))
+ if (stack->item->is_directory)
{
- /* only use filename if it doesn't match $USER */
- GNUNET_CONTAINER_meta_data_insert (stack->item->meta, "<libgnunetfs>",
- EXTRACTOR_METATYPE_FILENAME,
- EXTRACTOR_METAFORMAT_UTF8,
- "text/plain",
stack->item->short_filename,
- strlen (stack->item->short_filename)
+ 1);
- GNUNET_CONTAINER_meta_data_insert (stack->item->meta, "<libgnunetfs>",
-
EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
- EXTRACTOR_METAFORMAT_UTF8,
- "text/plain",
stack->item->short_filename,
- strlen (stack->item->short_filename)
+ 1);
+ char *user = getenv ("USER");
+ if ((user == NULL) || (0 != strncasecmp (user,
stack->item->short_filename, strlen(user))))
+ {
+ /* only use filename if it doesn't match $USER */
+ GNUNET_CONTAINER_meta_data_insert (stack->item->meta,
"<libgnunetfs>",
+ EXTRACTOR_METATYPE_FILENAME,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
stack->item->short_filename,
+ strlen
(stack->item->short_filename) + 1);
+ GNUNET_CONTAINER_meta_data_insert (stack->item->meta,
"<libgnunetfs>",
+
EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
stack->item->short_filename,
+ strlen
(stack->item->short_filename) + 1);
+ }
}
}
-
stack->item->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data
(stack->item->meta);
- GNUNET_FS_uri_ksk_get_keywords (stack->item->ksk_uri,
&add_to_keyword_counter, stack->keywordcounter);
+ if (stack->ctx->toplevel->is_directory)
+ {
+ GNUNET_FS_uri_ksk_get_keywords (stack->item->ksk_uri,
&add_to_keyword_counter, stack->keywordcounter);
+ }
stack->item = stack->item->next;
}
/* Call this task again later, if there are more entries to process */
@@ -1270,8 +1277,13 @@
ret->stack = GNUNET_malloc (sizeof (struct ProcessMetadataStackItem));
ret->stack->ctx = ret;
ret->stack->item = toplevel;
- ret->stack->keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024);
- ret->stack->metacounter = GNUNET_CONTAINER_multihashmap_create (1024);
+
+ if (ret->stack->ctx->toplevel->is_directory)
+ {
+ ret->stack->keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024);
+ ret->stack->metacounter = GNUNET_CONTAINER_multihashmap_create (1024);
+ }
+
ret->stack->dir_entry_count = 0;
ret->stack->end_directory = GNUNET_NO;
Modified: gnunet/src/fs/gnunet-publish.c
===================================================================
--- gnunet/src/fs/gnunet-publish.c 2012-01-15 12:16:42 UTC (rev 19156)
+++ gnunet/src/fs/gnunet-publish.c 2012-01-15 14:55:13 UTC (rev 19157)
@@ -66,7 +66,16 @@
static GNUNET_SCHEDULER_TaskIdentifier kill_task;
+static struct GNUNET_FS_DirScanner *ds;
+static struct GNUNET_FS_ShareTreeItem * directory_scan_intermediary_result;
+
+static struct GNUNET_FS_ShareTreeItem * directory_scan_result;
+
+static struct GNUNET_FS_ProcessMetadataContext *pmc;
+
+static struct GNUNET_FS_Namespace *namespace;
+
/**
* FIXME: docu
*/
@@ -340,7 +349,151 @@
ctx = NULL;
}
+static struct GNUNET_FS_FileInformation *
+get_file_information (struct GNUNET_FS_ShareTreeItem *item)
+{
+ struct GNUNET_FS_FileInformation *fi;
+ struct GNUNET_FS_FileInformation *fic;
+ struct GNUNET_FS_ShareTreeItem *child;
+ if (item->is_directory)
+ {
+ GNUNET_CONTAINER_meta_data_delete (item->meta,
+ EXTRACTOR_METATYPE_MIMETYPE, NULL, 0);
+ GNUNET_FS_meta_data_make_directory (item->meta);
+ GNUNET_FS_uri_ksk_add_keyword (item->ksk_uri, GNUNET_FS_DIRECTORY_MIME,
+ GNUNET_NO);
+ fi = GNUNET_FS_file_information_create_empty_directory (
+ ctx, NULL, item->ksk_uri,
+ item->meta, &bo, item->filename);
+ for (child = item->children_head; child; child = child->next)
+ {
+ fic = get_file_information (child);
+ GNUNET_break (GNUNET_OK == GNUNET_FS_file_information_add (fi, fic));
+ }
+ }
+ else
+ {
+ fi = GNUNET_FS_file_information_create_from_file (
+ ctx, NULL, item->filename,
+ item->ksk_uri, item->meta, !do_insert,
+ &bo);
+ }
+ GNUNET_CONTAINER_meta_data_destroy (item->meta);
+ GNUNET_FS_uri_destroy (item->ksk_uri);
+ GNUNET_free (item->short_filename);
+ GNUNET_free (item->filename);
+ GNUNET_free (item);
+ return fi;
+}
+
+static void
+directory_trim_complete (void *cls,
+ const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+ struct GNUNET_FS_FileInformation *fi;
+ directory_scan_result = directory_scan_intermediary_result;
+ fi = get_file_information (directory_scan_result);
+ directory_scan_result = NULL;
+ if (fi == NULL)
+ {
+ FPRINTF (stderr, "%s", _("Could not publish\n"));
+ if (namespace != NULL)
+ GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
+ GNUNET_FS_stop (ctx);
+ ret = 1;
+ return;
+ }
+ GNUNET_FS_file_information_inspect (fi, &publish_inspector, NULL);
+ if (extract_only)
+ {
+ if (namespace != NULL)
+ GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
+ GNUNET_FS_file_information_destroy (fi, NULL, NULL);
+ GNUNET_FS_stop (ctx);
+ return;
+ }
+ pc = GNUNET_FS_publish_start (ctx, fi, namespace, this_id, next_id,
+ (do_simulate) ?
+ GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY :
+ GNUNET_FS_PUBLISH_OPTION_NONE);
+ if (NULL == pc)
+ {
+ FPRINTF (stderr, "%s", _("Could not start publishing.\n"));
+ GNUNET_FS_stop (ctx);
+ ret = 1;
+ return;
+ }
+}
+
+static int
+directory_scan_cb (void *cls, struct GNUNET_FS_DirScanner *ds,
+ const char *filename, char is_directory,
+ enum GNUNET_FS_DirScannerProgressUpdateReason reason)
+{
+ switch (reason)
+ {
+ case GNUNET_DIR_SCANNER_NEW_FILE:
+ if (filename != NULL)
+ {
+ if (is_directory)
+ FPRINTF (stdout, _("Scanning directory `%s'.\n"), filename);
+ else
+ FPRINTF (stdout, _("Scanning file `%s'.\n"), filename);
+ }
+ break;
+ case GNUNET_DIR_SCANNER_DOES_NOT_EXIST:
+ if (filename != NULL)
+ {
+ FPRINTF (stdout,
+ _("Failed to scan `%s', because it does not exist.\n"),
+ filename);
+ }
+ break;
+ case GNUNET_DIR_SCANNER_ASKED_TO_STOP:
+ if (filename != NULL)
+ {
+ FPRINTF (stdout,
+ _("Scanner was about to scan `%s', but is now stopping.\n"),
+ filename);
+ }
+ else
+ FPRINTF (stdout, "%s", _("Scanner is stopping.\n"));
+ break;
+ case GNUNET_DIR_SCANNER_SHUTDOWN:
+ FPRINTF (stdout, "%s", _("Client is shutting down.\n"));
+ break;
+ case GNUNET_DIR_SCANNER_FINISHED:
+ FPRINTF (stdout, "%s", _("Scanner has finished.\n"));
+ break;
+ case GNUNET_DIR_SCANNER_PROTOCOL_ERROR:
+ FPRINTF (stdout, "%s",
+ _("There was a failure communicating with the scanner.\n"));
+ break;
+ default:
+ FPRINTF (stdout, _("Got unknown scanner update with filename `%s'.\n"),
+ filename);
+ break;
+ }
+ if ((filename == NULL && GNUNET_DIR_SCANNER_FINISHED)
+ || reason == GNUNET_DIR_SCANNER_PROTOCOL_ERROR
+ || reason == GNUNET_DIR_SCANNER_SHUTDOWN)
+ {
+ /* Any of this causes us to try to clean up the scanner */
+ directory_scan_intermediary_result = GNUNET_FS_directory_scan_cleanup (ds);
+ pmc = GNUNET_FS_trim_share_tree (directory_scan_intermediary_result,
+ &directory_trim_complete, NULL);
+
+ ds = NULL;
+ /* FIXME: change the tree processor to be able to free untrimmed trees
+ * right here instead of waiting for trimming to complete, if we need to
+ * cancel everything.
+ */
+ }
+ return 0;
+}
+
+
/**
* Main function that will be run by the scheduler.
*
@@ -353,11 +506,7 @@
run (void *cls, char *const *args, const char *cfgfile,
const struct GNUNET_CONFIGURATION_Handle *c)
{
- struct GNUNET_FS_FileInformation *fi;
- struct GNUNET_FS_Namespace *namespace;
struct EXTRACTOR_PluginList *plugins;
- struct GNUNET_FS_Uri *keywords;
- struct stat sbuf;
char *ex;
char *emsg;
@@ -451,6 +600,9 @@
GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
return;
}
+ ds = GNUNET_FS_directory_scan_start (args[0],
+ GNUNET_NO, NULL, directory_scan_cb, NULL);
+
plugins = NULL;
if (!disable_extractor)
{
@@ -465,63 +617,6 @@
GNUNET_free (ex);
}
}
- emsg = NULL;
- GNUNET_assert (NULL != args[0]);
- if (0 != STAT (args[0], &sbuf))
- {
- GNUNET_asprintf (&emsg, _("Could not access file: %s\n"), STRERROR
(errno));
- fi = NULL;
- }
- else if (S_ISDIR (sbuf.st_mode))
- {
- fi = GNUNET_FS_file_information_create_from_directory (ctx, NULL, args[0],
-
&GNUNET_FS_directory_scanner_default,
- plugins, !do_insert,
- &bo, &emsg);
- }
- else
- {
- if (meta == NULL)
- meta = GNUNET_CONTAINER_meta_data_create ();
- GNUNET_FS_meta_data_extract_from_file (meta, args[0], plugins);
- keywords = GNUNET_FS_uri_ksk_create_from_meta_data (meta);
- fi = GNUNET_FS_file_information_create_from_file (ctx, NULL, args[0],
- keywords, NULL,
- !do_insert, &bo);
- GNUNET_break (fi != NULL);
- GNUNET_FS_uri_destroy (keywords);
- }
- EXTRACTOR_plugin_remove_all (plugins);
- if (fi == NULL)
- {
- FPRINTF (stderr, _("Could not publish `%s': %s\n"), args[0], emsg);
- GNUNET_free (emsg);
- if (namespace != NULL)
- GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
- GNUNET_FS_stop (ctx);
- ret = 1;
- return;
- }
- GNUNET_FS_file_information_inspect (fi, &publish_inspector, NULL);
- if (extract_only)
- {
- if (namespace != NULL)
- GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
- GNUNET_FS_file_information_destroy (fi, NULL, NULL);
- GNUNET_FS_stop (ctx);
- return;
- }
- pc = GNUNET_FS_publish_start (ctx, fi, namespace, this_id, next_id,
- (do_simulate) ?
- GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY :
- GNUNET_FS_PUBLISH_OPTION_NONE);
- if (NULL == pc)
- {
- FPRINTF (stderr, "%s", _("Could not start publishing.\n"));
- GNUNET_FS_stop (ctx);
- ret = 1;
- return;
- }
kill_task =
GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
&do_stop_task,
NULL);
[Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r19157 - gnunet/src/fs,
gnunet <=