[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r2069 - Extractor Extractor/contrib Extractor/po Extractor/
|
From: |
grothoff |
|
Subject: |
[GNUnet-SVN] r2069 - Extractor Extractor/contrib Extractor/po Extractor/src/include Extractor/src/main Extractor/src/plugins Extractor/src/plugins/printable Extractor-docs/WWW GNUnet/contrib |
|
Date: |
Wed, 7 Sep 2005 21:47:32 -0700 (PDT) |
Author: grothoff
Date: 2005-09-07 21:46:11 -0700 (Wed, 07 Sep 2005)
New Revision: 2069
Added:
GNUnet/contrib/gprof-helper.c
Modified:
Extractor-docs/WWW/download.php3
Extractor-docs/WWW/index.php
Extractor/ChangeLog
Extractor/configure.ac
Extractor/contrib/doxygen
Extractor/po/de.po
Extractor/po/libextractor.pot
Extractor/po/ro.po
Extractor/po/rw.po
Extractor/src/include/extractor.h
Extractor/src/main/Makefile.am
Extractor/src/main/extractor.c
Extractor/src/plugins/Makefile.am
Extractor/src/plugins/manextractor.c
Extractor/src/plugins/printable/bloomfilter.c
Extractor/src/plugins/printable/dictionary-builder.c
Extractor/src/plugins/printable/printableextractor.c
Extractor/src/plugins/tarextractor.c
Log:
release
Modified: Extractor/ChangeLog
===================================================================
--- Extractor/ChangeLog 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/ChangeLog 2005-09-08 04:46:11 UTC (rev 2069)
@@ -1,3 +1,10 @@
+Wed Sep 7 21:41:35 PDT 2005
+ Added decompression of gz and bz2 streams to the LE core library
+ (avoids need to do this, possibly repeatedly, in plugins and makes
+ sure that all plugins work with compressed files). Eliminated gz
+ decompression from man and tar extractors.
+ Releasing libextractor 0.5.5.
+
Sun Sep 4 02:08:56 PDT 2005
Changed code to export fewer symbols (refactoring plus linker options,
goal is to address Mantis #925. Changed debian extractor to no longer
Modified: Extractor/configure.ac
===================================================================
--- Extractor/configure.ac 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/configure.ac 2005-09-08 04:46:11 UTC (rev 2069)
@@ -1,8 +1,8 @@
# Process this file with autoconf to produce a configure script.
AC_PREREQ(2.57)
-AC_INIT([libextractor], [0.5.4a], address@hidden)
+AC_INIT([libextractor], [0.5.5], address@hidden)
AC_REVISION($Revision: 1.67 $)
-AM_INIT_AUTOMAKE([libextractor], [0.5.4a])
+AM_INIT_AUTOMAKE([libextractor], [0.5.5])
AM_CONFIG_HEADER(src/include/config.h)
AH_TOP([#define _GNU_SOURCE 1])
@@ -132,6 +132,11 @@
AC_DEFINE(HAVE_ZLIB,1,[Have zlib])],
[AM_CONDITIONAL(HAVE_ZLIB, false)])
+AC_CHECK_LIB(bz2, BZ2_decompress,
+ [AM_CONDITIONAL(HAVE_BZ2, true)
+ AC_DEFINE(HAVE_LIBBZ2,1,[Have libbz2])],
+ [AM_CONDITIONAL(HAVE_BZ2, false)])
+
# restore LIBS
LIBS=$LIBSOLD
@@ -156,7 +161,7 @@
AC_HEADER_DIRENT
AC_HEADER_STDBOOL
AC_CHECK_HEADERS([fcntl.h netinet/in.h stdlib.h string.h unistd.h libintl.h
limits.h stddef.h zlib.h])
-AC_CHECK_HEADERS([ltdl.h iconv.h])
+AC_CHECK_HEADERS([ltdl.h iconv.h bzlib.h])
# fixme, we need to die here if a header is not found!
AC_CHECK_HEADERS([vorbis/vorbisfile.h])
Modified: Extractor/contrib/doxygen
===================================================================
--- Extractor/contrib/doxygen 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/contrib/doxygen 2005-09-08 04:46:11 UTC (rev 2069)
@@ -23,7 +23,7 @@
# This could be handy for archiving the generated documentation or
# if some version control system is used.
-PROJECT_NUMBER = 0.5.2
+PROJECT_NUMBER = 0.5.5
# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
# base path where the generated documentation will be put.
Modified: Extractor/po/de.po
===================================================================
--- Extractor/po/de.po 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/po/de.po 2005-09-08 04:46:11 UTC (rev 2069)
@@ -9,7 +9,7 @@
msgstr ""
"Project-Id-Version: libextractor 0.5.0\n"
"Report-Msgid-Bugs-To: address@hidden"
-"POT-Creation-Date: 2005-09-04 02:59-0700\n"
+"POT-Creation-Date: 2005-09-07 21:46-0700\n"
"PO-Revision-Date: 2005-06-22 15:05+0200\n"
"Last-Translator: Karl Eichwalder <address@hidden>\n"
"Language-Team: German <address@hidden>\n"
@@ -28,7 +28,7 @@
msgid "Binary RPM %d.%d"
msgstr "Binäres RPM %d.%d"
-#: src/plugins/printable/dictionary-builder.c:50
+#: src/plugins/printable/dictionary-builder.c:69
#, c-format
msgid ""
"Please provide the name of the language you are building\n"
@@ -37,12 +37,12 @@
"Bitte geben Sie den Namen der Sprache an, für die Sie ein Wörterbuch\n"
"erstellen. Zum Beispiel:\n"
-#: src/plugins/printable/dictionary-builder.c:63
+#: src/plugins/printable/dictionary-builder.c:82
#, c-format
msgid "Error opening file `%s': %s\n"
msgstr "Fehler beim Öffnen der Datei »%s«: %s\n"
-#: src/plugins/printable/dictionary-builder.c:71
+#: src/plugins/printable/dictionary-builder.c:90
#, c-format
msgid ""
"Error allocating: %s\n"
@@ -51,44 +51,44 @@
"Fehler beim Allokieren: %s\n"
"."
-#: src/plugins/printable/dictionary-builder.c:83
+#: src/plugins/printable/dictionary-builder.c:102
#, c-format
msgid "Increase ALLOCSIZE (in %s).\n"
msgstr "ALLOCSIZE vergrößern (in %s).\n"
-#: src/plugins/manextractor.c:128
+#: src/plugins/manextractor.c:140
msgid "Commands"
msgstr "Befehle"
-#: src/plugins/manextractor.c:133
+#: src/plugins/manextractor.c:145
msgid "System calls"
msgstr "Systemaufrufe"
-#: src/plugins/manextractor.c:138
+#: src/plugins/manextractor.c:150
msgid "Library calls"
msgstr "Bibliotheksaufrufe"
-#: src/plugins/manextractor.c:143
+#: src/plugins/manextractor.c:155
msgid "Special files"
msgstr "Spezialdateien"
-#: src/plugins/manextractor.c:148
+#: src/plugins/manextractor.c:160
msgid "File formats and conventions"
msgstr ""
-#: src/plugins/manextractor.c:153
+#: src/plugins/manextractor.c:165
msgid "Games"
msgstr "Spiele"
-#: src/plugins/manextractor.c:158
+#: src/plugins/manextractor.c:170
msgid "Conventions and miscellaneous"
msgstr ""
-#: src/plugins/manextractor.c:163
+#: src/plugins/manextractor.c:175
msgid "System management commands"
msgstr ""
-#: src/plugins/manextractor.c:168
+#: src/plugins/manextractor.c:180
msgid "Kernel routines"
msgstr "Kernelroutinen"
Modified: Extractor/po/libextractor.pot
===================================================================
--- Extractor/po/libextractor.pot 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/po/libextractor.pot 2005-09-08 04:46:11 UTC (rev 2069)
@@ -8,7 +8,7 @@
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: address@hidden"
-"POT-Creation-Date: 2005-09-04 02:59-0700\n"
+"POT-Creation-Date: 2005-09-07 21:46-0700\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <address@hidden>\n"
"Language-Team: LANGUAGE <address@hidden>\n"
@@ -26,63 +26,63 @@
msgid "Binary RPM %d.%d"
msgstr ""
-#: src/plugins/printable/dictionary-builder.c:50
+#: src/plugins/printable/dictionary-builder.c:69
#, c-format
msgid ""
"Please provide the name of the language you are building\n"
"a dictionary for. For example:\n"
msgstr ""
-#: src/plugins/printable/dictionary-builder.c:63
+#: src/plugins/printable/dictionary-builder.c:82
#, c-format
msgid "Error opening file `%s': %s\n"
msgstr ""
-#: src/plugins/printable/dictionary-builder.c:71
+#: src/plugins/printable/dictionary-builder.c:90
#, c-format
msgid ""
"Error allocating: %s\n"
"."
msgstr ""
-#: src/plugins/printable/dictionary-builder.c:83
+#: src/plugins/printable/dictionary-builder.c:102
#, c-format
msgid "Increase ALLOCSIZE (in %s).\n"
msgstr ""
-#: src/plugins/manextractor.c:128
+#: src/plugins/manextractor.c:140
msgid "Commands"
msgstr ""
-#: src/plugins/manextractor.c:133
+#: src/plugins/manextractor.c:145
msgid "System calls"
msgstr ""
-#: src/plugins/manextractor.c:138
+#: src/plugins/manextractor.c:150
msgid "Library calls"
msgstr ""
-#: src/plugins/manextractor.c:143
+#: src/plugins/manextractor.c:155
msgid "Special files"
msgstr ""
-#: src/plugins/manextractor.c:148
+#: src/plugins/manextractor.c:160
msgid "File formats and conventions"
msgstr ""
-#: src/plugins/manextractor.c:153
+#: src/plugins/manextractor.c:165
msgid "Games"
msgstr ""
-#: src/plugins/manextractor.c:158
+#: src/plugins/manextractor.c:170
msgid "Conventions and miscellaneous"
msgstr ""
-#: src/plugins/manextractor.c:163
+#: src/plugins/manextractor.c:175
msgid "System management commands"
msgstr ""
-#: src/plugins/manextractor.c:168
+#: src/plugins/manextractor.c:180
msgid "Kernel routines"
msgstr ""
Modified: Extractor/po/ro.po
===================================================================
--- Extractor/po/ro.po 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/po/ro.po 2005-09-08 04:46:11 UTC (rev 2069)
@@ -9,7 +9,7 @@
msgstr ""
"Project-Id-Version: libextractor 0.4.2\n"
"Report-Msgid-Bugs-To: address@hidden"
-"POT-Creation-Date: 2005-09-04 02:59-0700\n"
+"POT-Creation-Date: 2005-09-07 21:46-0700\n"
"PO-Revision-Date: 2005-02-25 12:00-0500\n"
"Last-Translator: Laurentiu Buzdugan <address@hidden>\n"
"Language-Team: Romanian <address@hidden>\n"
@@ -28,7 +28,7 @@
msgid "Binary RPM %d.%d"
msgstr "Binar RPM %d.%d"
-#: src/plugins/printable/dictionary-builder.c:50
+#: src/plugins/printable/dictionary-builder.c:69
#, c-format
msgid ""
"Please provide the name of the language you are building\n"
@@ -37,12 +37,12 @@
"V� rug�m furniza�i numele limbii pentru care contrui�i\n"
"un dic�ionar. De exemplu:\n"
-#: src/plugins/printable/dictionary-builder.c:63
+#: src/plugins/printable/dictionary-builder.c:82
#, c-format
msgid "Error opening file `%s': %s\n"
msgstr "Eroare deschidere fi�ier `%s': %s\n"
-#: src/plugins/printable/dictionary-builder.c:71
+#: src/plugins/printable/dictionary-builder.c:90
#, c-format
msgid ""
"Error allocating: %s\n"
@@ -51,44 +51,44 @@
"Eroare de alocare: %s\n"
"."
-#: src/plugins/printable/dictionary-builder.c:83
+#: src/plugins/printable/dictionary-builder.c:102
#, c-format
msgid "Increase ALLOCSIZE (in %s).\n"
msgstr "Cre�te�i ALLOCSIZE (�n %s).\n"
-#: src/plugins/manextractor.c:128
+#: src/plugins/manextractor.c:140
msgid "Commands"
msgstr "Comenzi"
-#: src/plugins/manextractor.c:133
+#: src/plugins/manextractor.c:145
msgid "System calls"
msgstr "Apeluri sistem"
-#: src/plugins/manextractor.c:138
+#: src/plugins/manextractor.c:150
msgid "Library calls"
msgstr "Apeluri de bibliotec�"
-#: src/plugins/manextractor.c:143
+#: src/plugins/manextractor.c:155
msgid "Special files"
msgstr "Fi�iere speciale"
-#: src/plugins/manextractor.c:148
+#: src/plugins/manextractor.c:160
msgid "File formats and conventions"
msgstr "Formate de fi�iere �i conven�ii"
-#: src/plugins/manextractor.c:153
+#: src/plugins/manextractor.c:165
msgid "Games"
msgstr "Jocuri"
-#: src/plugins/manextractor.c:158
+#: src/plugins/manextractor.c:170
msgid "Conventions and miscellaneous"
msgstr "Conven�ii �i diverse"
-#: src/plugins/manextractor.c:163
+#: src/plugins/manextractor.c:175
msgid "System management commands"
msgstr "Comenzi pentru managementul sistemului"
-#: src/plugins/manextractor.c:168
+#: src/plugins/manextractor.c:180
msgid "Kernel routines"
msgstr "Proceduri kernel"
Modified: Extractor/po/rw.po
===================================================================
--- Extractor/po/rw.po 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/po/rw.po 2005-09-08 04:46:11 UTC (rev 2069)
@@ -16,7 +16,7 @@
msgstr ""
"Project-Id-Version: libextractor 0.4.2\n"
"Report-Msgid-Bugs-To: address@hidden"
-"POT-Creation-Date: 2005-09-04 02:59-0700\n"
+"POT-Creation-Date: 2005-09-07 21:46-0700\n"
"PO-Revision-Date: 2005-04-04 10:55-0700\n"
"Last-Translator: Steven Michael Murphy <address@hidden>\n"
"Language-Team: Kinyarwanda <address@hidden>\n"
@@ -34,7 +34,7 @@
msgid "Binary RPM %d.%d"
msgstr ""
-#: src/plugins/printable/dictionary-builder.c:50
+#: src/plugins/printable/dictionary-builder.c:69
#, fuzzy, c-format
msgid ""
"Please provide the name of the language you are building\n"
@@ -42,62 +42,62 @@
msgstr "i Izina: Bya i Ururimi Inkoranyamagambo kugirango Urugero"
# basctl/source\basicide\basidesh.src:RID_STR_ERROROPENSTORAGE.text
-#: src/plugins/printable/dictionary-builder.c:63
+#: src/plugins/printable/dictionary-builder.c:82
#, fuzzy, c-format
msgid "Error opening file `%s': %s\n"
msgstr "Hari ikibazo mu gufungura dosiye"
-#: src/plugins/printable/dictionary-builder.c:71
+#: src/plugins/printable/dictionary-builder.c:90
#, c-format
msgid ""
"Error allocating: %s\n"
"."
msgstr ""
-#: src/plugins/printable/dictionary-builder.c:83
+#: src/plugins/printable/dictionary-builder.c:102
#, fuzzy, c-format
msgid "Increase ALLOCSIZE (in %s).\n"
msgstr "in"
-#: src/plugins/manextractor.c:128
+#: src/plugins/manextractor.c:140
msgid "Commands"
msgstr "amabwiriza"
-#: src/plugins/manextractor.c:133
+#: src/plugins/manextractor.c:145
#, fuzzy
msgid "System calls"
msgstr "Amahamagara:"
-#: src/plugins/manextractor.c:138
+#: src/plugins/manextractor.c:150
#, fuzzy
msgid "Library calls"
msgstr "Amahamagara:"
-#: src/plugins/manextractor.c:143
+#: src/plugins/manextractor.c:155
#, fuzzy
msgid "Special files"
msgstr "Idosiye"
-#: src/plugins/manextractor.c:148
+#: src/plugins/manextractor.c:160
#, fuzzy
msgid "File formats and conventions"
msgstr "Idosiye Imiterere Na"
-#: src/plugins/manextractor.c:153
+#: src/plugins/manextractor.c:165
msgid "Games"
msgstr ""
-#: src/plugins/manextractor.c:158
+#: src/plugins/manextractor.c:170
#, fuzzy
msgid "Conventions and miscellaneous"
msgstr "Na Binyuranye"
-#: src/plugins/manextractor.c:163
+#: src/plugins/manextractor.c:175
#, fuzzy
msgid "System management commands"
msgstr "Amabwiriza"
-#: src/plugins/manextractor.c:168
+#: src/plugins/manextractor.c:180
msgid "Kernel routines"
msgstr ""
Modified: Extractor/src/include/extractor.h
===================================================================
--- Extractor/src/include/extractor.h 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/src/include/extractor.h 2005-09-08 04:46:11 UTC (rev 2069)
@@ -29,7 +29,7 @@
* 0.2.6-1 => 0x00020601
* 4.5.2-0 => 0x04050200
*/
-#define EXTRACTOR_VERSION 0x00050401
+#define EXTRACTOR_VERSION 0x00050500
#include <stdio.h>
Modified: Extractor/src/main/Makefile.am
===================================================================
--- Extractor/src/main/Makefile.am 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/src/main/Makefile.am 2005-09-08 04:46:11 UTC (rev 2069)
@@ -23,10 +23,17 @@
dlflag=-ldl
endif
+#if HAVE_ZLIB
+ zlib =-lz
+#endif
+#if HAVE_BZ2
+ bz2lib = -lbz2
+#endif
+
libextractor_la_LDFLAGS = \
-export-dynamic -version-info 2:0:1 $(LIBICONV)
libextractor_la_LIBADD = \
- $(LIBLTDL) $(dlflag)
+ $(LIBLTDL) $(dlflag) $(zlib) $(bz2lib)
libextractor_la_DEPENDENCIES = \
$(LIBLTDL)
Modified: Extractor/src/main/extractor.c
===================================================================
--- Extractor/src/main/extractor.c 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/src/main/extractor.c 2005-09-08 04:46:11 UTC (rev 2069)
@@ -27,7 +27,14 @@
#include <../../libltdl/ltdl.h>
#endif
+#if HAVE_LIBBZ2
+#include <bzlib.h>
+#endif
+#if HAVE_ZLIB
+#include <zlib.h>
+#endif
+
#define DEBUG 1
/**
@@ -613,7 +620,180 @@
libraries = EXTRACTOR_removeLibrary (libraries, libraries->libname);
}
+
+
/**
+ * How many bytes do we actually try to scan? (from the beginning
+ * of the file). Limit to 1 GB.
+ */
+#define MAX_READ 1024 * 1024 * 1024
+
+/**
+ * How many bytes do we actually try to decompress? (from the beginning
+ * of the file). Limit to 16 MB.
+ */
+#define MAX_DECOMPRESS 16 * 1024 * 1024
+
+
+static EXTRACTOR_KeywordList *
+getKeywords (EXTRACTOR_ExtractorList * extractor,
+ const char * filename,
+ const unsigned char * data,
+ size_t size) {
+ EXTRACTOR_KeywordList *result;
+ char * buf;
+ size_t dsize;
+#if HAVE_ZLIB
+ z_stream strm;
+ int ret;
+ size_t pos;
+#endif
+#if HAVE_LIBBZ2
+ bz_stream bstrm;
+ int bret;
+ size_t bpos;
+#endif
+
+ buf = NULL;
+ dsize = 0;
+#if HAVE_ZLIB
+ /* try gzip decompression first */
+ if ( (data[0] == 0x1f) &&
+ (data[1] == 0x8b) &&
+ (data[2] == 0x08) ) {
+ memset(&strm,
+ 0,
+ sizeof(z_stream));
+ strm.next_in = (char*) data;
+ strm.avail_in = size;
+ strm.total_in = 0;
+ strm.zalloc = NULL;
+ strm.zfree = NULL;
+ strm.opaque = NULL;
+
+ if (Z_OK == inflateInit2(&strm,
+ 15 + 32)) {
+ dsize = 2 * size;
+ if (dsize > MAX_DECOMPRESS)
+ dsize = MAX_DECOMPRESS;
+ buf = malloc(dsize);
+ pos = 0;
+ if (buf == NULL) {
+ inflateEnd(&strm);
+ } else {
+ strm.next_out = buf;
+ strm.avail_out = dsize;
+ do {
+ ret = inflate(&strm,
+ Z_SYNC_FLUSH);
+ if (ret == Z_OK) {
+ if (dsize == MAX_DECOMPRESS)
+ break;
+ pos += strm.total_out;
+ dsize *= 2;
+ if (dsize > MAX_DECOMPRESS)
+ dsize = MAX_DECOMPRESS;
+ buf = realloc(buf, dsize);
+ strm.next_out = &buf[pos];
+ strm.avail_out = dsize - pos;
+ } else if (ret != Z_STREAM_END) {
+ /* error */
+ free(buf);
+ buf = NULL;
+ }
+ } while ( (buf != NULL) &&
+ (ret != Z_STREAM_END) );
+ dsize = pos + strm.total_out;
+ inflateEnd(&strm);
+ if (dsize == 0) {
+ free(buf);
+ buf = NULL;
+ }
+ }
+ }
+ }
+#endif
+
+#if HAVE_LIBBZ2
+ if ( (data[0] == 'B') &&
+ (data[1] == 'Z') &&
+ (data[2] == 'h') ) {
+ /* now try bz2 decompression */
+ memset(&bstrm,
+ 0,
+ sizeof(bz_stream));
+ bstrm.next_in = (char*) data;
+ bstrm.avail_in = size;
+ bstrm.total_in_lo32 = 0;
+ bstrm.total_in_hi32 = 0;
+ bstrm.bzalloc = NULL;
+ bstrm.bzfree = NULL;
+ bstrm.opaque = NULL;
+ if ( (buf == NULL) &&
+ (BZ_OK == BZ2_bzDecompressInit(&bstrm,
+ 0,
+ 0)) ) {
+ dsize = 2 * size;
+ if (dsize > MAX_DECOMPRESS)
+ dsize = MAX_DECOMPRESS;
+ buf = malloc(dsize);
+ bpos = 0;
+ if (buf == NULL) {
+ BZ2_bzDecompressEnd(&bstrm);
+ } else {
+ bstrm.next_out = buf;
+ bstrm.avail_out = dsize;
+ do {
+ bret = BZ2_bzDecompress(&bstrm);
+ if (bret == Z_OK) {
+ if (dsize == MAX_DECOMPRESS)
+ break;
+ bpos += bstrm.total_out_lo32;
+ dsize *= 2;
+ if (dsize > MAX_DECOMPRESS)
+ dsize = MAX_DECOMPRESS;
+ buf = realloc(buf, dsize);
+ bstrm.next_out = &buf[bpos];
+ bstrm.avail_out = dsize - pos;
+ } else if (bret != BZ_STREAM_END) {
+ /* error */
+ free(buf);
+ buf = NULL;
+ }
+ } while ( (buf != NULL) &&
+ (bret != BZ_STREAM_END) );
+ dsize = bpos + bstrm.total_out_lo32;
+ BZ2_bzDecompressEnd(&bstrm);
+ if (dsize == 0) {
+ free(buf);
+ buf = NULL;
+ }
+ }
+ }
+ }
+#endif
+
+
+ /* finally, call plugins */
+ if (buf != NULL) {
+ data = buf;
+ size = dsize;
+ }
+ result = NULL;
+ while (extractor != NULL) {
+ result = extractor->extractMethod(filename,
+ (char*) data,
+ size,
+ result,
+ extractor->options);
+ extractor = extractor->next;
+ }
+ if (buf != NULL)
+ free(buf);
+ return result;
+}
+
+/**
* Extract keywords from a file using the available extractors.
* @param extractor the list of extractor libraries
* @param filename the name of the file
@@ -646,21 +826,16 @@
return NULL;
}
- if (size > 1* 1024 * 1024 * 1024)
- size = 1 * 1024 * 1024 * 1024; /* do not mmap/read more than 1 GB! */
+ if (size > MAX_READ)
+ size = MAX_READ; /* do not mmap/read more than 1 GB! */
buffer = MMAP(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, file, 0);
close(file);
if ( (buffer == NULL) || (buffer == (void *) -1) )
return NULL;
- result = NULL;
- while (extractor != NULL) {
- result = extractor->extractMethod(filename,
- buffer,
- size,
- result,
- extractor->options);
- extractor = extractor->next;
- }
+ result = getKeywords(extractor,
+ filename,
+ buffer,
+ size);
if (size > 0)
MUNMAP (buffer, size);
else
@@ -684,20 +859,12 @@
EXTRACTOR_getKeywords2(EXTRACTOR_ExtractorList * extractor,
const char * data,
size_t size) {
- EXTRACTOR_KeywordList * result;
-
if (data == NULL)
return NULL;
- result = NULL;
- while (extractor != NULL) {
- result = extractor->extractMethod(NULL,
- (char*)data,
- size,
- result,
- extractor->options);
- extractor = extractor->next;
- }
- return result;
+ return getKeywords(extractor,
+ NULL,
+ data,
+ size);
}
static void
Modified: Extractor/src/plugins/Makefile.am
===================================================================
--- Extractor/src/plugins/Makefile.am 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor/src/plugins/Makefile.am 2005-09-08 04:46:11 UTC (rev 2069)
@@ -173,7 +173,6 @@
libextractor_tar_la_LDFLAGS = \
$(PLUGINFLAGS) -Wl,--retain-symbols-file -Wl,TAR_SYMBOLS
libextractor_tar_la_LIBADD = \
- -lz \
$(top_builddir)/src/main/libextractor.la
libextractor_lower_la_SOURCES = \
@@ -253,8 +252,6 @@
libextractor_man_la_LDFLAGS = \
$(top_builddir)/src/main/libextractor.la \
$(PLUGINFLAGS) -Wl,--retain-symbols-file -Wl,MAN_SYMBOLS
-libextractor_man_la_LIBADD = \
- -lz
libextractor_deb_la_SOURCES = \
debextractor.c
Modified: Extractor/src/plugins/manextractor.c
===================================================================
--- Extractor/src/plugins/manextractor.c 2005-09-08 04:44:19 UTC (rev
2068)
+++ Extractor/src/plugins/manextractor.c 2005-09-08 04:46:11 UTC (rev
2069)
@@ -78,13 +78,25 @@
(*end) = size+1;
}
-static struct EXTRACTOR_Keywords * tryParse(const char * buf,
- size_t size,
- struct EXTRACTOR_Keywords * prev) {
+/**
+ * How many bytes do we actually try to scan? (from the beginning
+ * of the file).
+ */
+#define MAX_READ (16 * 1024)
+
+
+
+struct EXTRACTOR_Keywords *
+libextractor_man_extract(const char * filename,
+ const char * buf,
+ size_t size,
+ struct EXTRACTOR_Keywords * prev) {
int pos;
size_t xsize;
const size_t xlen = strlen(".TH ");
+ if (size > MAX_READ)
+ size = MAX_READ;
pos = 0;
if (size < xlen)
return prev;
@@ -203,62 +215,4 @@
return prev;
}
-static voidpf Emalloc(voidpf opaque, uInt items, uInt size) {
- return malloc(size * items);
-}
-
-static void Efree(voidpf opaque, voidpf ptr) {
- free(ptr);
-}
-
-/**
- * How many bytes do we actually try to scan? (from the beginning
- * of the file).
- */
-#define MAX_READ 2048
-
-struct EXTRACTOR_Keywords *
-libextractor_man_extract(const char * filename,
- char * data,
- size_t size,
- struct EXTRACTOR_Keywords * prev) {
- z_stream strm;
- char * buf;
-
- memset(&strm,
- 0,
- sizeof(z_stream));
- strm.next_in = (char*) data;
- strm.avail_in = size;
- strm.total_in = 0;
- strm.zalloc = &Emalloc;
- strm.zfree = &Efree;
- strm.opaque = NULL;
- if (Z_OK == inflateInit2(&strm,
- 15 + 32)) {
- buf = malloc(MAX_READ);
- if (buf == NULL) {
- inflateEnd(&strm);
- return prev;
- }
- strm.next_out = buf;
- strm.avail_out = MAX_READ;
- inflate(&strm,
- Z_FINISH);
- if (strm.total_out > 0) {
- prev = tryParse(buf,
- strm.total_out,
- prev);
- inflateEnd(&strm);
- free(buf);
- return prev;
- }
- free(buf);
- inflateEnd(&strm);
- }
- return tryParse(data,
- size,
- prev);
-}
-
/* end of manextractor.c */
Modified: Extractor/src/plugins/printable/bloomfilter.c
===================================================================
--- Extractor/src/plugins/printable/bloomfilter.c 2005-09-08 04:44:19 UTC
(rev 2068)
+++ Extractor/src/plugins/printable/bloomfilter.c 2005-09-08 04:46:11 UTC
(rev 2069)
@@ -435,44 +435,4 @@
*arg = 0;
}
-/* *********************** INTERFACE **************** */
-
-/**
- * Test if an element is in the filter.
- *
- * @param e the element
- * @param bf the filter
- * @return 1 if the element is in the filter, 0 if not
- */
-static int testBloomfilter(Bloomfilter * bf,
- HashCode160 * e) {
- int res;
-
- if (NULL == bf)
- return 1;
- res = 1;
- iterateBits(bf,
- (BitIterator)&testBitCallback,
- &res,
- e);
- return res;
-}
-
-/**
- * Add an element to the filter
- *
- * @param bf the filter
- * @param e the element
- */
-static void addToBloomfilter(Bloomfilter * bf,
- HashCode160 * e) {
-
- if (NULL == bf)
- return;
- iterateBits(bf,
- &setBitCallback,
- NULL,
- e);
-}
-
/* ******************** end of bloomfilter.c *********** */
Modified: Extractor/src/plugins/printable/dictionary-builder.c
===================================================================
--- Extractor/src/plugins/printable/dictionary-builder.c 2005-09-08
04:44:19 UTC (rev 2068)
+++ Extractor/src/plugins/printable/dictionary-builder.c 2005-09-08
04:46:11 UTC (rev 2069)
@@ -30,6 +30,25 @@
#include "bloomfilter.h"
#include "bloomfilter.c"
+
+/**
+ * Add an element to the filter
+ *
+ * @param bf the filter
+ * @param e the element
+ */
+static void addToBloomfilter(Bloomfilter * bf,
+ HashCode160 * e) {
+
+ if (NULL == bf)
+ return;
+ iterateBits(bf,
+ &setBitCallback,
+ NULL,
+ e);
+}
+
+
#define ADDR_PER_ELEMENT 46
int main(int argc,
Modified: Extractor/src/plugins/printable/printableextractor.c
===================================================================
--- Extractor/src/plugins/printable/printableextractor.c 2005-09-08
04:44:19 UTC (rev 2068)
+++ Extractor/src/plugins/printable/printableextractor.c 2005-09-08
04:46:11 UTC (rev 2069)
@@ -32,6 +32,28 @@
#include "extractor.h"
#include "bloomfilter.c"
+/**
+ * Test if an element is in the filter.
+ *
+ * @param e the element
+ * @param bf the filter
+ * @return 1 if the element is in the filter, 0 if not
+ */
+static int testBloomfilter(Bloomfilter * bf,
+ HashCode160 * e) {
+ int res;
+
+ if (NULL == bf)
+ return 1;
+ res = 1;
+ iterateBits(bf,
+ (BitIterator)&testBitCallback,
+ &res,
+ e);
+ return res;
+}
+
+
extern Bloomfilter FILTER_NAME;
static char * xstrndup(const char * s, size_t n){
Modified: Extractor/src/plugins/tarextractor.c
===================================================================
--- Extractor/src/plugins/tarextractor.c 2005-09-08 04:44:19 UTC (rev
2068)
+++ Extractor/src/plugins/tarextractor.c 2005-09-08 04:46:11 UTC (rev
2069)
@@ -81,10 +81,11 @@
} USTarHeader;
-static struct EXTRACTOR_Keywords *
-tar_extract(const char * data,
- size_t size,
- struct EXTRACTOR_Keywords * prev) {
+struct EXTRACTOR_Keywords *
+libextractor_tar_extract(const char * filename,
+ const char * data,
+ size_t size,
+ struct EXTRACTOR_Keywords * prev) {
TarHeader * tar;
USTarHeader * ustar;
size_t pos;
@@ -135,87 +136,3 @@
}
return prev;
}
-
-static voidpf Emalloc(voidpf opaque, uInt items, uInt size) {
- return malloc(size * items);
-}
-
-static void Efree(voidpf opaque, voidpf ptr) {
- free(ptr);
-}
-
-/* do not decompress tar.gz files > 16 MB */
-#define MAX_TGZ_SIZE 16 * 1024 * 1024
-
-struct EXTRACTOR_Keywords *
-libextractor_tar_extract(const char * filename,
- const unsigned char * data,
- size_t size,
- struct EXTRACTOR_Keywords * prev) {
- if ( (data[0] == 0x1f) &&
- (data[1] == 0x8b) &&
- (data[2] == 0x08) ) {
- time_t ctime;
- char * buf;
- size_t bufSize;
- z_stream strm;
-
- /* Creation time */
- ctime = (((((( (unsigned int)data[7] << 8)
- | (unsigned int)data[6]) << 8)
- | (unsigned int)data[5]) << 8)
- | (unsigned int)data[4]);
- if (ctime) {
- struct tm ctm;
- char tmbuf[60];
-
- ctm = *gmtime(&ctime);
- if (strftime(tmbuf, sizeof(tmbuf),
- nl_langinfo(D_FMT),
- &ctm))
- prev = addKeyword(EXTRACTOR_CREATION_DATE, strdup(tmbuf), prev);
- }
-
- /* try for tar.gz */
- bufSize = data[size-4] + 256 * data[size-3] + 65536 * data[size-2] +
256*65536 * data[size-1];
- if (bufSize > MAX_TGZ_SIZE) {
- return prev;
- }
-
- memset(&strm,
- 0,
- sizeof(z_stream));
- strm.next_in = (char*) data;
- strm.avail_in = size;
- strm.total_in = 0;
- strm.zalloc = &Emalloc;
- strm.zfree = &Efree;
- strm.opaque = NULL;
- if (Z_OK != inflateInit2(&strm,
- 15 + 32))
- return prev;
- buf = malloc(bufSize);
- if (buf == NULL) {
- inflateEnd(&strm);
- return prev;
- }
- strm.next_out = buf;
- strm.avail_out = bufSize;
- inflate(&strm,
- Z_FINISH);
- if (strm.total_out == 0) {
- inflateEnd(&strm);
- free(buf);
- return prev;
- }
- bufSize = strm.total_out;
- inflateEnd(&strm);
- prev = tar_extract(buf, bufSize, prev);
- free(buf);
- return prev;
- } else {
- /* try for uncompressed tar */
- return tar_extract(data, size, prev);
- }
-}
-
Modified: Extractor-docs/WWW/download.php3
===================================================================
--- Extractor-docs/WWW/download.php3 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor-docs/WWW/download.php3 2005-09-08 04:46:11 UTC (rev 2069)
@@ -34,8 +34,8 @@
ANCHOR("tar");
H2("Tar Package");
W("Latest version is %s.",
- extlink_("download/libextractor-0.5.4.tar.gz",
- "libextractor-0.5.4.tar.gz"));
+ extlink_("download/libextractor-0.5.5.tar.gz",
+ "libextractor-0.5.5.tar.gz"));
BR();
W("Latest Java-binding is %s.",
extlink_("download/libextractor-java-0.5.1.tar.gz",
@@ -55,8 +55,8 @@
ANCHOR("win");
H2("Windows");
W("Latest Windows binary is %s.",
- extlink_("download/win/libextractor-0.5.3.zip",
- "libextractor-0.5.3.zip"));
+ extlink_("download/win/libextractor-0.5.4.zip",
+ "libextractor-0.5.4.zip"));
include("html_footer.php3");
?>
Modified: Extractor-docs/WWW/index.php
===================================================================
--- Extractor-docs/WWW/index.php 2005-09-08 04:44:19 UTC (rev 2068)
+++ Extractor-docs/WWW/index.php 2005-09-08 04:46:11 UTC (rev 2069)
@@ -37,6 +37,8 @@
H2("Recent News");
P();
echo "<dl>";
+DTDD("Wed Sep 7 21:41:35 PDT 2005 | libextractor v0.5.5 released.",
+ "This release fixes a problem with linkers that caused segmentation
faults for Debian unstable users. The deb extractor no longer uses pthreads.
Dead code was eliminated in the OLE2 and OO extractors. Minor bugfixes were
ported from libgsf to the OLE2 extractor. Mime-types are now detected for
various Microsoft Office formats. libextractor now automatically decompresses
GZ and BZ2 files before extracting keywords, adding support for compressed
files for all formats. Individual extractors do no longer perform full-file
decompression, avoiding some redundant computation.");
DTDD("Fri Aug 26 22:47:07 PDT 2005 | libextractor v0.5.4 released.",
"This release fixes a memory leak in the thumbnail extractor, character
set conversion in the OLE2 extractors and the build on OS X. Quotations now
follow GNU standards. A workaround for a bug in libstdc++ that could cause
segmentation fauls was added. A new version of the python binding has also
been released; this revision fixes various problems with the build process.");
DTDD("Sat Aug 13 19:08:46 PDT 2005 | libextractor v0.5.3 released.",
Added: GNUnet/contrib/gprof-helper.c
===================================================================
--- GNUnet/contrib/gprof-helper.c 2005-09-08 04:44:19 UTC (rev 2068)
+++ GNUnet/contrib/gprof-helper.c 2005-09-08 04:46:11 UTC (rev 2069)
@@ -0,0 +1,119 @@
+/* gprof-helper.c -- preload library to profile pthread-enabled programs
+ * *
+ * * Authors: Sam Hocevar <sam at zoy dot org>
+ * * Daniel J�nsson <danieljo at fagotten dot org>
+ * *
+ * * This program is free software; you can redistribute it and/or
+ * * modify it under the terms of the Do What The Fuck You Want To
+ * * Public License as published by Banlu Kemiyatorn. See
+ * * http://sam.zoy.org/projects/COPYING.WTFPL for more details.
+ * *
+ * * Compilation example:
+ * * gcc -shared -fPIC gprof-helper.c -o gprof-helper.so -lpthread
-ldl
+ * *
+ * * Usage example:
+ * * LD_PRELOAD=./gprof-helper.so your_program
+ * */
+
+#define _GNU_SOURCE
+#include <sys/time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <pthread.h>
+
+static void * wrapper_routine(void *);
+
+/* Original pthread function */
+static int (*pthread_create_orig)(pthread_t *__restrict,
+ __const pthread_attr_t
*__restrict,
+
void *(*)(void *),
+
void *__restrict) = NULL;
+
+/* Library initialization function */
+void wooinit(void) __attribute__((constructor));
+
+void wooinit(void)
+{
+ pthread_create_orig = dlsym(RTLD_NEXT, "pthread_create");
+ fprintf(stderr, "pthreads: using profiling hooks for gprof\n");
+ if(pthread_create_orig == NULL)
+ {
+ char *error = dlerror();
+ if(error == NULL)
+ {
+
error = "pthread_create is NULL";
+
}
+ fprintf(stderr,
"%s\n", error);
+
exit(EXIT_FAILURE);
+ }
+}
+
+/* Our data structure passed to the wrapper */
+typedef struct wrapper_s
+{
+ void * (*start_routine)(void *);
+ void * arg;
+
+ pthread_mutex_t lock;
+ pthread_cond_t wait;
+
+ struct itimerval itimer;
+
+} wrapper_t;
+
+/* The wrapper function in charge for setting the itimer value */
+static void * wrapper_routine(void * data)
+{
+ /* Put user data in thread-local variables */
+ void * (*start_routine)(void *) =
((wrapper_t*)data)->start_routine;
+ void * arg = ((wrapper_t*)data)->arg;
+
+ /* Set the profile timer value */
+ setitimer(ITIMER_PROF, &((wrapper_t*)data)->itimer, NULL);
+
+ /* Tell the calling thread that we don't need its data
anymore */
+ pthread_mutex_lock(&((wrapper_t*)data)->lock);
+ pthread_cond_signal(&((wrapper_t*)data)->wait);
+
pthread_mutex_unlock(&((wrapper_t*)data)->lock);
+
+ /* Call the real function */
+ return start_routine(arg);
+}
+
+/* Our wrapper function for the real pthread_create() */
+int pthread_create(pthread_t *__restrict thread,
+ __const pthread_attr_t *__restrict attr,
+ void *
(*start_routine)(void *),
+ void
*__restrict arg)
+{
+ wrapper_t wrapper_data;
+ int i_return;
+
+ /* Initialize the wrapper structure */
+ wrapper_data.start_routine = start_routine;
+ wrapper_data.arg = arg;
+ getitimer(ITIMER_PROF, &wrapper_data.itimer);
+ pthread_cond_init(&wrapper_data.wait, NULL);
+ pthread_mutex_init(&wrapper_data.lock,
NULL);
+ pthread_mutex_lock(&wrapper_data.lock);
+
+ /* The real pthread_create call */
+ i_return =
pthread_create_orig(thread,
+
attr,
+
&wrapper_routine,
+
&wrapper_data);
+
+ /* If the thread was
successfully spawned, wait for the data
+ * * to be released */
+ if(i_return == 0)
+ {
+
pthread_cond_wait(&wrapper_data.wait, &wrapper_data.lock);
+
}
+
+
pthread_mutex_unlock(&wrapper_data.lock);
+
pthread_mutex_destroy(&wrapper_data.lock);
+
pthread_cond_destroy(&wrapper_data.wait);
+
+ return
i_return;
+}
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r2069 - Extractor Extractor/contrib Extractor/po Extractor/src/include Extractor/src/main Extractor/src/plugins Extractor/src/plugins/printable Extractor-docs/WWW GNUnet/contrib,
grothoff <=