[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r6498 - GNUnet/src/applications/dstore_sqlite
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] r6498 - GNUnet/src/applications/dstore_sqlite |
|
Date: |
Sat, 1 Mar 2008 21:13:06 -0700 (MST) |
Author: grothoff
Date: 2008-03-01 21:13:05 -0700 (Sat, 01 Mar 2008)
New Revision: 6498
Modified:
GNUnet/src/applications/dstore_sqlite/dstore.c
GNUnet/src/applications/dstore_sqlite/dstore_quota_test.c
Log:
dstore fixes
Modified: GNUnet/src/applications/dstore_sqlite/dstore.c
===================================================================
--- GNUnet/src/applications/dstore_sqlite/dstore.c 2008-03-02 01:59:53 UTC
(rev 6497)
+++ GNUnet/src/applications/dstore_sqlite/dstore.c 2008-03-02 04:13:05 UTC
(rev 6498)
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet.
- (C) 2006, 2007 Christian Grothoff (and other contributing authors)
+ (C) 2006, 2007, 2008 Christian Grothoff (and other contributing authors)
GNUnet is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published
@@ -19,7 +19,7 @@
*/
/**
- * @file applications/dstore/dstore.c
+ * @file applications/dstore_sqlite/dstore.c
* @brief SQLite based implementation of the dstore service
* @author Christian Grothoff
* @todo Indexes, statistics
@@ -71,7 +71,7 @@
/**
* Estimate of the per-entry overhead (including indices).
*/
-#define OVERHEAD ((4+4+8+8*2+sizeof(GNUNET_HashCode)*2+32))
+#define OVERHEAD ((4*2+4*2+8*2+8*2+sizeof(GNUNET_HashCode)*5+32))
struct GNUNET_BloomFilter *bloom;
@@ -90,7 +90,7 @@
strlen (zSql), ppStmt, (const char **) &dummy);
}
-#define SQLITE3_EXEC(db, cmd) do { if (SQLITE_OK != sqlite3_exec(db, cmd,
NULL, NULL, &emsg)) { GNUNET_GE_LOG(coreAPI->ectx, GNUNET_GE_ERROR |
GNUNET_GE_ADMIN | GNUNET_GE_BULK, _("`%s' failed at %s:%d with error: %s\n"),
"sqlite3_exec", __FILE__, __LINE__, emsg); sqlite3_free(emsg); } } while(0)
+#define SQLITE3_EXEC(db, cmd) do { emsg = NULL; if (SQLITE_OK !=
sqlite3_exec(db, cmd, NULL, NULL, &emsg)) { GNUNET_GE_LOG(coreAPI->ectx,
GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, _("`%s' failed at %s:%d
with error: %s\n"), "sqlite3_exec", __FILE__, __LINE__, emsg);
sqlite3_free(emsg); } } while(0)
/**
* Log an error message at log-level 'level' that indicates
@@ -109,15 +109,17 @@
SQLITE3_EXEC (dbh, "PRAGMA count_changes=OFF");
SQLITE3_EXEC (dbh, "PRAGMA page_size=4092");
SQLITE3_EXEC (dbh,
- "CREATE TABLE ds071 ("
+ "CREATE TABLE ds080 ("
" size INTEGER NOT NULL DEFAULT 0,"
" type INTEGER NOT NULL DEFAULT 0,"
" puttime INTEGER NOT NULL DEFAULT 0,"
" expire INTEGER NOT NULL DEFAULT 0,"
- " key TEXT NOT NULL DEFAULT '',"
+ " key BLOB NOT NULL DEFAULT '',"
+ " vhash BLOB PRIMARY KEY,"
" value BLOB NOT NULL DEFAULT '')");
- SQLITE3_EXEC (dbh, "CREATE INDEX idx_key ON ds071 (key)");
- SQLITE3_EXEC (dbh, "CREATE INDEX idx_puttime ON ds071 (puttime)");
+ SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds080 (key,type,expire)");
+ SQLITE3_EXEC (dbh, "CREATE INDEX idx_allidx ON ds080 (key,vhash,type,size)");
+ SQLITE3_EXEC (dbh, "CREATE INDEX idx_puttime ON ds080 (puttime)");
}
static int
@@ -132,6 +134,7 @@
UNLINK (fn);
GNUNET_free (fn);
}
+ payload = 0;
tmpl = "/tmp/dstoreXXXXXX";
#ifdef MINGW
@@ -164,11 +167,9 @@
checkQuota (sqlite3 * dbh)
{
GNUNET_HashCode dkey;
+ GNUNET_HashCode vhash;
unsigned int dsize;
unsigned int dtype;
- GNUNET_CronTime dputtime;
- GNUNET_CronTime dexpire;
- char *dcontent;
sqlite3_stmt *stmt;
sqlite3_stmt *dstmt;
int err;
@@ -184,11 +185,11 @@
stmt = NULL;
dstmt = NULL;
if ((sq_prepare (dbh,
- "SELECT size, type, puttime, expire, key, value FROM ds071
ORDER BY puttime ASC",
+ "SELECT size, type, key, vhash FROM ds080 ORDER BY puttime
ASC LIMIT 1",
&stmt) != SQLITE_OK) ||
(sq_prepare (dbh,
- "DELETE FROM ds071 "
- "WHERE size = ? AND type = ? AND puttime = ? AND expire = ?
AND key = ? AND value = ?",
+ "DELETE FROM ds080 "
+ "WHERE key=? AND vhash=? AND type=? AND size=?",
&dstmt) != SQLITE_OK))
{
GNUNET_GE_LOG (coreAPI->ectx,
@@ -202,35 +203,29 @@
sqlite3_finalize (stmt);
return GNUNET_SYSERR;
}
- dcontent = GNUNET_malloc (MAX_CONTENT_SIZE);
err = SQLITE_DONE;
while ((payload * 10 > quota * 9) && /* we seem to be about 10% off */
((err = sqlite3_step (stmt)) == SQLITE_ROW))
{
dsize = sqlite3_column_int (stmt, 0);
dtype = sqlite3_column_int (stmt, 1);
- dputtime = sqlite3_column_int64 (stmt, 2);
- dexpire = sqlite3_column_int64 (stmt, 3);
GNUNET_GE_BREAK (NULL,
sqlite3_column_bytes (stmt,
- 4) == sizeof (GNUNET_HashCode));
- GNUNET_GE_BREAK (NULL, dsize == sqlite3_column_bytes (stmt, 5));
- memcpy (&dkey, sqlite3_column_blob (stmt, 4), sizeof (GNUNET_HashCode));
- if (dsize >= MAX_CONTENT_SIZE)
- {
- GNUNET_GE_BREAK (NULL, 0);
- dsize = MAX_CONTENT_SIZE;
- }
- memcpy (dcontent, sqlite3_column_blob (stmt, 5), dsize);
+ 2) == sizeof (GNUNET_HashCode));
+ GNUNET_GE_BREAK (NULL,
+ sqlite3_column_bytes (stmt,
+ 3) == sizeof (GNUNET_HashCode));
+ memcpy (&dkey, sqlite3_column_blob (stmt, 2), sizeof (GNUNET_HashCode));
+ memcpy (&vhash, sqlite3_column_blob (stmt, 3), sizeof (GNUNET_HashCode));
sqlite3_reset (stmt);
- sqlite3_bind_int (dstmt, 1, dsize);
- sqlite3_bind_int (dstmt, 2, dtype);
- sqlite3_bind_int64 (dstmt, 3, dputtime);
- sqlite3_bind_int64 (dstmt, 4, dexpire);
sqlite3_bind_blob (dstmt,
- 5, &dkey, sizeof (GNUNET_HashCode),
+ 1, &dkey, sizeof (GNUNET_HashCode),
SQLITE_TRANSIENT);
- sqlite3_bind_blob (dstmt, 6, dcontent, dsize, SQLITE_TRANSIENT);
+ sqlite3_bind_blob (dstmt,
+ 2, &vhash, sizeof (GNUNET_HashCode),
+ SQLITE_TRANSIENT);
+ sqlite3_bind_int (dstmt, 3, dtype);
+ sqlite3_bind_int (dstmt, 4, dsize);
if ((err = sqlite3_step (dstmt)) != SQLITE_DONE)
{
GNUNET_GE_LOG (coreAPI->ectx,
@@ -242,7 +237,12 @@
GNUNET_GE_BREAK (NULL, 0); /* should delete but cannot!? */
break;
}
- payload -= (dsize + OVERHEAD);
+ if (sqlite3_total_changes(dbh) > 0)
+ {
+ if (bloom != NULL)
+ GNUNET_bloomfilter_remove (bloom, &dkey);
+ payload -= (dsize + OVERHEAD);
+ }
#if DEBUG_DSTORE
GNUNET_GE_LOG (coreAPI->ectx,
GNUNET_GE_DEBUG | GNUNET_GE_REQUEST |
@@ -260,7 +260,6 @@
"sqlite3_step", __FILE__, __LINE__,
sqlite3_errmsg (dbh));
}
- GNUNET_free (dcontent);
sqlite3_finalize (dstmt);
sqlite3_finalize (stmt);
if (payload * 10 > quota * 9)
@@ -285,12 +284,14 @@
unsigned int type,
GNUNET_CronTime discard_time, unsigned int size, const char *data)
{
+ GNUNET_HashCode vhash;
sqlite3 *dbh;
sqlite3_stmt *stmt;
int ret;
if (size > MAX_CONTENT_SIZE)
return GNUNET_SYSERR;
+ GNUNET_hash(data, size, &vhash);
GNUNET_mutex_lock (lock);
if ((fn == NULL) || (SQLITE_OK != sqlite3_open (fn, &dbh)))
{
@@ -306,8 +307,8 @@
/* first try UPDATE */
if (sq_prepare (dbh,
- "UPDATE ds071 SET puttime=?, expire=? "
- "WHERE key=? AND type=? AND size=? AND value=?",
+ "UPDATE ds080 SET puttime=?, expire=? "
+ "WHERE key=? AND vhash=? AND type=? AND size=?",
&stmt) != SQLITE_OK)
{
GNUNET_GE_LOG (coreAPI->ectx,
@@ -318,15 +319,14 @@
GNUNET_mutex_unlock (lock);
return GNUNET_SYSERR;
}
- if ((SQLITE_OK != sqlite3_bind_int64 (stmt, 1, GNUNET_get_time ())) ||
- (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, discard_time)) ||
+ if ((SQLITE_OK !=
+ sqlite3_bind_blob (stmt, 1, key, sizeof (GNUNET_HashCode),
+ SQLITE_TRANSIENT)) ||
(SQLITE_OK !=
- sqlite3_bind_blob (stmt, 3, key, sizeof (GNUNET_HashCode),
- SQLITE_TRANSIENT))
- || (SQLITE_OK != sqlite3_bind_int (stmt, 4, type))
- || (SQLITE_OK != sqlite3_bind_int (stmt, 5, size))
- || (SQLITE_OK !=
- sqlite3_bind_blob (stmt, 6, data, size, SQLITE_TRANSIENT)))
+ sqlite3_bind_blob (stmt, 2, &vhash, sizeof (GNUNET_HashCode),
+ SQLITE_TRANSIENT)) ||
+ (SQLITE_OK != sqlite3_bind_int (stmt, 3, type)) ||
+ (SQLITE_OK != sqlite3_bind_int (stmt, 4, size)))
{
GNUNET_GE_LOG (coreAPI->ectx,
GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
@@ -358,9 +358,6 @@
GNUNET_mutex_unlock (lock);
return GNUNET_OK;
}
- if (bloom != NULL)
- GNUNET_bloomfilter_add (bloom, key);
-
if (GNUNET_OK != checkQuota (dbh))
{
sqlite3_close (dbh);
@@ -368,9 +365,9 @@
return GNUNET_SYSERR;
}
if (sq_prepare (dbh,
- "INSERT INTO ds071 "
- "(size, type, puttime, expire, key, value) "
- "VALUES (?, ?, ?, ?, ?, ?)", &stmt) != SQLITE_OK)
+ "INSERT INTO ds080 "
+ "(size, type, puttime, expire, key, vhash, value) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?)", &stmt) != SQLITE_OK)
{
GNUNET_GE_LOG (coreAPI->ectx,
GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
@@ -386,16 +383,25 @@
(SQLITE_OK == sqlite3_bind_int64 (stmt, 4, discard_time)) &&
(SQLITE_OK ==
sqlite3_bind_blob (stmt, 5, key, sizeof (GNUNET_HashCode),
+ SQLITE_TRANSIENT)) &&
+ (SQLITE_OK ==
+ sqlite3_bind_blob (stmt, 6, &vhash, sizeof (GNUNET_HashCode),
SQLITE_TRANSIENT))
&& (SQLITE_OK ==
- sqlite3_bind_blob (stmt, 6, data, size, SQLITE_TRANSIENT)))
+ sqlite3_bind_blob (stmt, 7, data, size, SQLITE_TRANSIENT)))
{
if (SQLITE_DONE != sqlite3_step (stmt))
- LOG_SQLITE (dbh,
- GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN |
- GNUNET_GE_BULK, "sqlite3_step");
+ {
+ LOG_SQLITE (dbh,
+ GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN |
+ GNUNET_GE_BULK, "sqlite3_step");
+ }
else
- payload += size + OVERHEAD;
+ {
+ payload += size + OVERHEAD;
+ if (bloom != NULL)
+ GNUNET_bloomfilter_add (bloom, key);
+ }
if (SQLITE_OK != sqlite3_finalize (stmt))
LOG_SQLITE (dbh,
GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN |
@@ -441,6 +447,9 @@
unsigned int size;
const char *dat;
unsigned int cnt;
+ unsigned int off;
+ unsigned int total;
+ char scratch[256];
GNUNET_mutex_lock (lock);
if ((bloom != NULL) && (GNUNET_NO == GNUNET_bloomfilter_test (bloom, key)))
@@ -461,7 +470,7 @@
#endif
now = GNUNET_get_time ();
if (sq_prepare (dbh,
- "SELECT size, value FROM ds071 WHERE key=? AND type=? AND
expire >= ?",
+ "SELECT count(*) FROM ds080 WHERE key=? AND type=? AND
expire >= ?",
&stmt) != SQLITE_OK)
{
GNUNET_GE_LOG (coreAPI->ectx,
@@ -469,6 +478,7 @@
_("`%s' failed at %s:%d with error: %s\n"),
"sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh));
sqlite3_close (dbh);
+ db_reset (dbh);
GNUNET_mutex_unlock (lock);
return GNUNET_SYSERR;
}
@@ -476,22 +486,70 @@
SQLITE_TRANSIENT);
sqlite3_bind_int (stmt, 2, type);
sqlite3_bind_int (stmt, 3, now);
- cnt = 0;
- while (sqlite3_step (stmt) == SQLITE_ROW)
+ if (SQLITE_ROW != sqlite3_step(stmt))
{
- size = sqlite3_column_int (stmt, 0);
- if (size != sqlite3_column_bytes (stmt, 1))
- {
- GNUNET_GE_BREAK (NULL, 0);
- continue;
- }
- dat = sqlite3_column_blob (stmt, 1);
- cnt++;
- if ( (handler != NULL) &&
- (GNUNET_OK != handler (key, type, size, dat, closure)) )
- break;
+ LOG_SQLITE (dbh,
+ GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER |
+ GNUNET_GE_BULK, "sqlite_step");
+ sqlite3_reset (stmt);
+ sqlite3_finalize (stmt);
+ db_reset (dbh);
+ GNUNET_mutex_unlock (lock);
+ return GNUNET_SYSERR;
}
+ total = sqlite3_column_int (stmt, 0);
+ sqlite3_reset (stmt);
sqlite3_finalize (stmt);
+ if ( (total == 0) ||
+ (handler == NULL) )
+ {
+ sqlite3_close (dbh);
+ GNUNET_mutex_unlock (lock);
+ return total;
+ }
+
+ cnt = 0;
+ off = GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, total);
+ while (cnt < total) {
+ off = (off + 1) % total;
+ GNUNET_snprintf(scratch, 256,
+ "SELECT size, value FROM ds080 WHERE key=? AND type=? AND
expire >= ? LIMIT 1 OFFSET %u",
+ off);
+ if (sq_prepare (dbh,
+ scratch,
+ &stmt) != SQLITE_OK)
+ {
+ GNUNET_GE_LOG (coreAPI->ectx,
+ GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
+ _("`%s' failed at %s:%d with error: %s\n"),
+ "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh));
+ sqlite3_close (dbh);
+ GNUNET_mutex_unlock (lock);
+ return GNUNET_SYSERR;
+ }
+ sqlite3_bind_blob (stmt, 1, key, sizeof (GNUNET_HashCode),
+ SQLITE_TRANSIENT);
+ sqlite3_bind_int (stmt, 2, type);
+ sqlite3_bind_int (stmt, 3, now);
+ if (sqlite3_step (stmt) != SQLITE_ROW)
+ break;
+ size = sqlite3_column_int (stmt, 0);
+ if (size != sqlite3_column_bytes (stmt, 1))
+ {
+ GNUNET_GE_BREAK (NULL, 0);
+ sqlite3_finalize (stmt);
+ continue;
+ }
+ dat = sqlite3_column_blob (stmt, 1);
+ cnt++;
+ if ( (handler != NULL) &&
+ (GNUNET_OK != handler (key, type, size, dat, closure)) )
+ {
+ sqlite3_finalize (stmt);
+ break;
+ }
+ sqlite3_finalize (stmt);
+ }
sqlite3_close (dbh);
GNUNET_mutex_unlock (lock);
return cnt;
@@ -508,6 +566,7 @@
GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
"SQLite Dstore: initializing database\n");
#endif
+ coreAPI = capi;
if (GNUNET_OK != db_reset ())
{
GNUNET_GE_BREAK (capi->ectx, 0);
@@ -516,7 +575,6 @@
lock = GNUNET_mutex_create (GNUNET_NO);
- coreAPI = capi;
api.get = &d_get;
api.put = &d_put;
GNUNET_GC_get_configuration_value_number (coreAPI->cfg,
Modified: GNUnet/src/applications/dstore_sqlite/dstore_quota_test.c
===================================================================
--- GNUnet/src/applications/dstore_sqlite/dstore_quota_test.c 2008-03-02
01:59:53 UTC (rev 6497)
+++ GNUnet/src/applications/dstore_sqlite/dstore_quota_test.c 2008-03-02
04:13:05 UTC (rev 6498)
@@ -70,7 +70,7 @@
{
fprintf(stderr, ".");
GNUNET_hash (&k, sizeof (GNUNET_HashCode), &n);
- if (i < 5)
+ if (i < 2)
ASSERT (0 == api->get (&k, i, NULL, NULL));
if (i == 9)
ASSERT (0 != api->get (&k, i, NULL, NULL));
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r6498 - GNUnet/src/applications/dstore_sqlite,
gnunet <=