[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[taler-wallet-core] 02/02: fix DB indexing issues
|
From: |
gnunet |
|
Subject: |
[taler-wallet-core] 02/02: fix DB indexing issues |
|
Date: |
Tue, 11 Jan 2022 22:16:02 +0100 |
This is an automated email from the git hooks/post-receive script.
dold pushed a commit to branch master
in repository wallet-core.
commit a74cdf05295764258fe9e7f66f73a442a9b46697
Author: Florian Dold <florian@dold.me>
AuthorDate: Tue Jan 11 21:00:12 2022 +0100
fix DB indexing issues
---
packages/idb-bridge/src/MemoryBackend.ts | 216 ++++++++++++++-------
packages/idb-bridge/src/bridge-idb.ts | 8 +-
.../idb-wpt-ported/idbcursor-update-index.test.ts | 1 -
packages/idb-bridge/src/index.ts | 1 +
packages/idb-bridge/src/util/structuredClone.ts | 76 ++++++++
packages/taler-wallet-cli/src/bench1.ts | 12 +-
packages/taler-wallet-core/src/db.ts | 44 ++++-
packages/taler-wallet-core/src/headless/helpers.ts | 23 ++-
packages/taler-wallet-core/src/index.node.ts | 3 +
.../taler-wallet-core/src/operations/README.md | 2 +-
.../src/operations/backup/import.ts | 25 ++-
.../taler-wallet-core/src/operations/balance.ts | 7 +
.../taler-wallet-core/src/operations/deposits.ts | 32 +--
.../taler-wallet-core/src/operations/pending.ts | 167 ++++++++--------
.../taler-wallet-core/src/operations/refresh.ts | 3 +
.../taler-wallet-core/src/operations/reserves.ts | 9 +
.../taler-wallet-core/src/operations/withdraw.ts | 4 +-
17 files changed, 455 insertions(+), 178 deletions(-)
diff --git a/packages/idb-bridge/src/MemoryBackend.ts
b/packages/idb-bridge/src/MemoryBackend.ts
index c1a1c12e..b37dd376 100644
--- a/packages/idb-bridge/src/MemoryBackend.ts
+++ b/packages/idb-bridge/src/MemoryBackend.ts
@@ -229,6 +229,16 @@ function furthestKey(
}
}
+export interface AccessStats {
+ writeTransactions: number;
+ readTransactions: number;
+ writesPerStore: Record<string, number>;
+ readsPerStore: Record<string, number>;
+ readsPerIndex: Record<string, number>;
+ readItemsPerIndex: Record<string, number>;
+ readItemsPerStore: Record<string, number>;
+}
+
/**
* Primitive in-memory backend.
*
@@ -266,6 +276,18 @@ export class MemoryBackend implements Backend {
enableTracing: boolean = false;
+ trackStats: boolean = true;
+
+ accessStats: AccessStats = {
+ readTransactions: 0,
+ writeTransactions: 0,
+ readsPerStore: {},
+ readsPerIndex: {},
+ readItemsPerIndex: {},
+ readItemsPerStore: {},
+ writesPerStore: {},
+ };
+
/**
* Load the data in this IndexedDB backend from a dump in JSON format.
*
@@ -512,6 +534,14 @@ export class MemoryBackend implements Backend {
throw Error("unsupported transaction mode");
}
+ if (this.trackStats) {
+ if (mode === "readonly") {
+ this.accessStats.readTransactions++;
+ } else if (mode === "readwrite") {
+ this.accessStats.writeTransactions++;
+ }
+ }
+
myDb.txRestrictObjectStores = [...objectStores];
this.connectionsByTransaction[transactionCookie] = myConn;
@@ -1153,6 +1183,13 @@ export class MemoryBackend implements Backend {
lastIndexPosition: req.lastIndexPosition,
lastObjectStorePosition: req.lastObjectStorePosition,
});
+ if (this.trackStats) {
+ const k = `${req.objectStoreName}.${req.indexName}`;
+ this.accessStats.readsPerIndex[k] =
+ (this.accessStats.readsPerIndex[k] ?? 0) + 1;
+ this.accessStats.readItemsPerIndex[k] =
+ (this.accessStats.readItemsPerIndex[k] ?? 0) + resp.count;
+ }
} else {
if (req.advanceIndexKey !== undefined) {
throw Error("unsupported request");
@@ -1167,6 +1204,13 @@ export class MemoryBackend implements Backend {
lastIndexPosition: req.lastIndexPosition,
lastObjectStorePosition: req.lastObjectStorePosition,
});
+ if (this.trackStats) {
+ const k = `${req.objectStoreName}`;
+ this.accessStats.readsPerStore[k] =
+ (this.accessStats.readsPerStore[k] ?? 0) + 1;
+ this.accessStats.readItemsPerStore[k] =
+ (this.accessStats.readItemsPerStore[k] ?? 0) + resp.count;
+ }
}
if (this.enableTracing) {
console.log(`TRACING: getRecords got ${resp.count} results`);
@@ -1180,6 +1224,11 @@ export class MemoryBackend implements Backend {
): Promise<RecordStoreResponse> {
if (this.enableTracing) {
console.log(`TRACING: storeRecord`);
+ console.log(
+ `key ${storeReq.key}, record ${JSON.stringify(
+ structuredEncapsulate(storeReq.value),
+ )}`,
+ );
}
const myConn = this.requireConnectionFromTransaction(btx);
const db = this.databases[myConn.dbName];
@@ -1199,6 +1248,12 @@ export class MemoryBackend implements Backend {
}', transaction is over ${JSON.stringify(db.txRestrictObjectStores)}`,
);
}
+
+ if (this.trackStats) {
+ this.accessStats.writesPerStore[storeReq.objectStoreName] =
+ (this.accessStats.writesPerStore[storeReq.objectStoreName] ?? 0) + 1;
+ }
+
const schema = myConn.modifiedSchema;
const objectStoreMapEntry =
myConn.objectStoreMap[storeReq.objectStoreName];
@@ -1275,7 +1330,9 @@ export class MemoryBackend implements Backend {
}
}
- const objectStoreRecord: ObjectStoreRecord = {
+ const oldStoreRecord = modifiedData.get(key);
+
+ const newObjectStoreRecord: ObjectStoreRecord = {
// FIXME: We should serialize the key here, not just clone it.
primaryKey: structuredClone(key),
value: structuredClone(value),
@@ -1283,7 +1340,7 @@ export class MemoryBackend implements Backend {
objectStoreMapEntry.store.modifiedData = modifiedData.with(
key,
- objectStoreRecord,
+ newObjectStoreRecord,
true,
);
@@ -1297,6 +1354,11 @@ export class MemoryBackend implements Backend {
}
const indexProperties =
schema.objectStores[storeReq.objectStoreName].indexes[indexName];
+
+ // Remove old index entry first!
+ if (oldStoreRecord) {
+ this.deleteFromIndex(index, key, oldStoreRecord.value,
indexProperties);
+ }
try {
this.insertIntoIndex(index, key, value, indexProperties);
} catch (e) {
@@ -1482,31 +1544,28 @@ function getIndexRecords(req: {
const primaryKeys: Key[] = [];
const values: Value[] = [];
const { unique, range, forward, indexData } = req;
- let indexPos = req.lastIndexPosition;
- let objectStorePos: IDBValidKey | undefined = undefined;
- let indexEntry: IndexRecord | undefined = undefined;
- const rangeStart = forward ? range.lower : range.upper;
- const dataStart = forward ? indexData.minKey() : indexData.maxKey();
- indexPos = furthestKey(forward, indexPos, rangeStart);
- indexPos = furthestKey(forward, indexPos, dataStart);
- function nextIndexEntry(): IndexRecord | undefined {
- assertInvariant(indexPos != null);
+ function nextIndexEntry(prevPos: IDBValidKey): IndexRecord | undefined {
const res: [IDBValidKey, IndexRecord] | undefined = forward
- ? indexData.nextHigherPair(indexPos)
- : indexData.nextLowerPair(indexPos);
- if (res) {
- indexEntry = res[1];
- indexPos = indexEntry.indexKey;
- return indexEntry;
- } else {
- indexEntry = undefined;
- indexPos = undefined;
- return undefined;
- }
+ ? indexData.nextHigherPair(prevPos)
+ : indexData.nextLowerPair(prevPos);
+ return res ? res[1] : undefined;
}
function packResult(): RecordGetResponse {
+ // Collect the values based on the primary keys,
+ // if requested.
+ if (req.resultLevel === ResultLevel.Full) {
+ for (let i = 0; i < numResults; i++) {
+ const result = req.storeData.get(primaryKeys[i]);
+ if (!result) {
+ console.error("invariant violated during read");
+ console.error("request was", req);
+ throw Error("invariant violated during read");
+ }
+ values.push(structuredClone(result.value));
+ }
+ }
return {
count: numResults,
indexKeys:
@@ -1517,18 +1576,39 @@ function getIndexRecords(req: {
};
}
- if (indexPos == null) {
+ let firstIndexPos = req.lastIndexPosition;
+ {
+ const rangeStart = forward ? range.lower : range.upper;
+ const dataStart = forward ? indexData.minKey() : indexData.maxKey();
+ firstIndexPos = furthestKey(forward, firstIndexPos, rangeStart);
+ firstIndexPos = furthestKey(forward, firstIndexPos, dataStart);
+ }
+
+ if (firstIndexPos == null) {
return packResult();
}
+ let objectStorePos: IDBValidKey | undefined = undefined;
+ let indexEntry: IndexRecord | undefined = undefined;
+
// Now we align at indexPos and after objectStorePos
- indexEntry = indexData.get(indexPos);
+ indexEntry = indexData.get(firstIndexPos);
if (!indexEntry) {
// We're not aligned to an index key, go to next index entry
- nextIndexEntry();
- }
- if (indexEntry) {
+ indexEntry = nextIndexEntry(firstIndexPos);
+ if (!indexEntry) {
+ return packResult();
+ }
+ objectStorePos = nextKey(true, indexEntry.primaryKeys, undefined);
+ } else if (
+ req.lastIndexPosition != null &&
+ compareKeys(req.lastIndexPosition, indexEntry.indexKey) !== 0
+ ) {
+ // We're already past the desired lastIndexPosition, don't use
+ // lastObjectStorePosition.
+ objectStorePos = nextKey(true, indexEntry.primaryKeys, undefined);
+ } else {
objectStorePos = nextKey(
true,
indexEntry.primaryKeys,
@@ -1536,43 +1616,56 @@ function getIndexRecords(req: {
);
}
+ // Now skip lower/upper bound of open ranges
+
if (
forward &&
range.lowerOpen &&
range.lower != null &&
- compareKeys(range.lower, indexPos) === 0
+ compareKeys(range.lower, indexEntry.indexKey) === 0
) {
- const e = nextIndexEntry();
- objectStorePos = e?.primaryKeys.minKey();
+ indexEntry = nextIndexEntry(indexEntry.indexKey);
+ if (!indexEntry) {
+ return packResult();
+ }
+ objectStorePos = indexEntry.primaryKeys.minKey();
}
if (
!forward &&
range.upperOpen &&
range.upper != null &&
- compareKeys(range.upper, indexPos) === 0
+ compareKeys(range.upper, indexEntry.indexKey) === 0
) {
- const e = nextIndexEntry();
- objectStorePos = e?.primaryKeys.minKey();
+ indexEntry = nextIndexEntry(indexEntry.indexKey);
+ if (!indexEntry) {
+ return packResult();
+ }
+ objectStorePos = indexEntry.primaryKeys.minKey();
}
+ // If requested, return only unique results
+
if (
unique &&
- indexPos != null &&
req.lastIndexPosition != null &&
- compareKeys(indexPos, req.lastIndexPosition) === 0
+ compareKeys(indexEntry.indexKey, req.lastIndexPosition) === 0
) {
- const e = nextIndexEntry();
- objectStorePos = e?.primaryKeys.minKey();
+ indexEntry = nextIndexEntry(indexEntry.indexKey);
+ if (!indexEntry) {
+ return packResult();
+ }
+ objectStorePos = indexEntry.primaryKeys.minKey();
}
- if (req.advancePrimaryKey) {
- indexPos = furthestKey(forward, indexPos, req.advanceIndexKey);
- if (indexPos) {
- indexEntry = indexData.get(indexPos);
- if (!indexEntry) {
- nextIndexEntry();
- }
+ if (req.advanceIndexKey != null) {
+ const ik = furthestKey(forward, indexEntry.indexKey, req.advanceIndexKey)!;
+ indexEntry = indexData.get(ik);
+ if (!indexEntry) {
+ indexEntry = nextIndexEntry(ik);
+ }
+ if (!indexEntry) {
+ return packResult();
}
}
@@ -1580,9 +1673,7 @@ function getIndexRecords(req: {
if (
req.advanceIndexKey != null &&
req.advancePrimaryKey &&
- indexPos != null &&
- indexEntry &&
- compareKeys(indexPos, req.advanceIndexKey) == 0
+ compareKeys(indexEntry.indexKey, req.advanceIndexKey) == 0
) {
if (
objectStorePos == null ||
@@ -1597,13 +1688,10 @@ function getIndexRecords(req: {
}
while (1) {
- if (indexPos === undefined) {
- break;
- }
if (req.limit != 0 && numResults == req.limit) {
break;
}
- if (!range.includes(indexPos)) {
+ if (!range.includes(indexEntry.indexKey)) {
break;
}
if (indexEntry === undefined) {
@@ -1611,14 +1699,16 @@ function getIndexRecords(req: {
}
if (objectStorePos == null) {
// We don't have any more records with the current index key.
- nextIndexEntry();
- if (indexEntry) {
- objectStorePos = indexEntry.primaryKeys.minKey();
+ indexEntry = nextIndexEntry(indexEntry.indexKey);
+ if (!indexEntry) {
+ return packResult();
}
+ objectStorePos = indexEntry.primaryKeys.minKey();
continue;
}
- indexKeys.push(indexEntry.indexKey);
- primaryKeys.push(objectStorePos);
+
+ indexKeys.push(structuredClone(indexEntry.indexKey));
+ primaryKeys.push(structuredClone(objectStorePos));
numResults++;
if (unique) {
objectStorePos = undefined;
@@ -1627,20 +1717,6 @@ function getIndexRecords(req: {
}
}
- // Now we can collect the values based on the primary keys,
- // if requested.
- if (req.resultLevel === ResultLevel.Full) {
- for (let i = 0; i < numResults; i++) {
- const result = req.storeData.get(primaryKeys[i]);
- if (!result) {
- console.error("invariant violated during read");
- console.error("request was", req);
- throw Error("invariant violated during read");
- }
- values.push(result.value);
- }
- }
-
return packResult();
}
diff --git a/packages/idb-bridge/src/bridge-idb.ts
b/packages/idb-bridge/src/bridge-idb.ts
index 5d5f531b..8264b43e 100644
--- a/packages/idb-bridge/src/bridge-idb.ts
+++ b/packages/idb-bridge/src/bridge-idb.ts
@@ -64,7 +64,10 @@ import { makeStoreKeyValue } from "./util/makeStoreKeyValue";
import { normalizeKeyPath } from "./util/normalizeKeyPath";
import { openPromise } from "./util/openPromise";
import queueTask from "./util/queueTask";
-import { structuredClone } from "./util/structuredClone";
+import {
+ checkStructuredCloneOrThrow,
+ structuredClone,
+} from "./util/structuredClone";
import { validateKeyPath } from "./util/validateKeyPath";
import { valueToKey } from "./util/valueToKey";
@@ -303,7 +306,7 @@ export class BridgeIDBCursor implements IDBCursor {
try {
// Only called for the side effect of throwing an exception
- structuredClone(value);
+ checkStructuredCloneOrThrow(value);
} catch (e) {
throw new DataCloneError();
}
@@ -327,6 +330,7 @@ export class BridgeIDBCursor implements IDBCursor {
}
const { btx } = this.source._confirmStartedBackendTransaction();
await this._backend.storeRecord(btx, storeReq);
+ // FIXME: update the index position here!
};
return transaction._execRequestAsync({
operation,
diff --git
a/packages/idb-bridge/src/idb-wpt-ported/idbcursor-update-index.test.ts
b/packages/idb-bridge/src/idb-wpt-ported/idbcursor-update-index.test.ts
index 53866545..dcbee2b1 100644
--- a/packages/idb-bridge/src/idb-wpt-ported/idbcursor-update-index.test.ts
+++ b/packages/idb-bridge/src/idb-wpt-ported/idbcursor-update-index.test.ts
@@ -10,7 +10,6 @@ import {
// IDBCursor.update() - index - modify a record in the object store
test.cb("WPT test idbcursor_update_index.htm", (t) => {
var db: any,
- count = 0,
records = [
{ pKey: "primaryKey_0", iKey: "indexKey_0" },
{ pKey: "primaryKey_1", iKey: "indexKey_1" },
diff --git a/packages/idb-bridge/src/index.ts b/packages/idb-bridge/src/index.ts
index 0abbf105..c4dbb828 100644
--- a/packages/idb-bridge/src/index.ts
+++ b/packages/idb-bridge/src/index.ts
@@ -72,6 +72,7 @@ export type {
};
export { MemoryBackend } from "./MemoryBackend";
+export type { AccessStats } from "./MemoryBackend";
// globalThis polyfill, see https://mathiasbynens.be/notes/globalthis
(function () {
diff --git a/packages/idb-bridge/src/util/structuredClone.ts
b/packages/idb-bridge/src/util/structuredClone.ts
index 51e4483e..c33dc5e3 100644
--- a/packages/idb-bridge/src/util/structuredClone.ts
+++ b/packages/idb-bridge/src/util/structuredClone.ts
@@ -171,6 +171,75 @@ export function mkDeepClone() {
}
}
+/**
+ * Check if an object is deeply cloneable.
+ * Only called for the side-effect of throwing an exception.
+ */
+export function mkDeepCloneCheckOnly() {
+ const refs = [] as any;
+
+ return clone;
+
+ function cloneArray(a: any) {
+ var keys = Object.keys(a);
+ refs.push(a);
+ for (var i = 0; i < keys.length; i++) {
+ var k = keys[i] as any;
+ var cur = a[k];
+ checkCloneableOrThrow(cur);
+ if (typeof cur !== "object" || cur === null) {
+ // do nothing
+ } else if (cur instanceof Date) {
+ // do nothing
+ } else if (ArrayBuffer.isView(cur)) {
+ // do nothing
+ } else {
+ var index = refs.indexOf(cur);
+ if (index !== -1) {
+ // do nothing
+ } else {
+ clone(cur);
+ }
+ }
+ }
+ refs.pop();
+ }
+
+ function clone(o: any) {
+ checkCloneableOrThrow(o);
+ if (typeof o !== "object" || o === null) return o;
+ if (o instanceof Date) return;
+ if (Array.isArray(o)) return cloneArray(o);
+ if (o instanceof Map) return cloneArray(Array.from(o));
+ if (o instanceof Set) return cloneArray(Array.from(o));
+ refs.push(o);
+ for (var k in o) {
+ if (Object.hasOwnProperty.call(o, k) === false) continue;
+ var cur = o[k] as any;
+ checkCloneableOrThrow(cur);
+ if (typeof cur !== "object" || cur === null) {
+ // do nothing
+ } else if (cur instanceof Date) {
+ // do nothing
+ } else if (cur instanceof Map) {
+ cloneArray(Array.from(cur));
+ } else if (cur instanceof Set) {
+ cloneArray(Array.from(cur));
+ } else if (ArrayBuffer.isView(cur)) {
+ // do nothing
+ } else {
+ var i = refs.indexOf(cur);
+ if (i !== -1) {
+ // do nothing
+ } else {
+ clone(cur);
+ }
+ }
+ }
+ refs.pop();
+ }
+}
+
function internalEncapsulate(
val: any,
outRoot: any,
@@ -358,3 +427,10 @@ export function structuredRevive(val: any): any {
export function structuredClone(val: any): any {
return mkDeepClone()(val);
}
+
+/**
+ * Structured clone for IndexedDB.
+ */
+export function checkStructuredCloneOrThrow(val: any): void {
+ return mkDeepCloneCheckOnly()(val);
+}
diff --git a/packages/taler-wallet-cli/src/bench1.ts
b/packages/taler-wallet-cli/src/bench1.ts
index 1a6a26b6..c7e570b4 100644
--- a/packages/taler-wallet-cli/src/bench1.ts
+++ b/packages/taler-wallet-cli/src/bench1.ts
@@ -22,13 +22,15 @@ import {
codecForNumber,
codecForString,
codecOptional,
+ j2s,
Logger,
} from "@gnu-taler/taler-util";
import {
- getDefaultNodeWallet,
+ getDefaultNodeWallet2,
NodeHttpLib,
WalletApiOperation,
Wallet,
+ AccessStats,
} from "@gnu-taler/taler-wallet-core";
/**
@@ -64,6 +66,7 @@ export async function runBench1(configJson: any):
Promise<void> {
}
let wallet = {} as Wallet;
+ let getDbStats: () => AccessStats;
for (let i = 0; i < numIter; i++) {
// Create a new wallet in each iteration
@@ -72,12 +75,16 @@ export async function runBench1(configJson: any):
Promise<void> {
if (i % restartWallet == 0) {
if (Object.keys(wallet).length !== 0) {
wallet.stop();
+ console.log("wallet DB stats", j2s(getDbStats!()));
}
- wallet = await getDefaultNodeWallet({
+
+ const res = await getDefaultNodeWallet2({
// No persistent DB storage.
persistentStoragePath: undefined,
httpLib: myHttpLib,
});
+ wallet = res.wallet;
+ getDbStats = res.getDbStats;
if (trustExchange) {
wallet.setInsecureTrustExchange();
}
@@ -119,6 +126,7 @@ export async function runBench1(configJson: any):
Promise<void> {
}
wallet.stop();
+ console.log("wallet DB stats", j2s(getDbStats!()));
}
/**
diff --git a/packages/taler-wallet-core/src/db.ts
b/packages/taler-wallet-core/src/db.ts
index 07e0f4b0..772061fb 100644
--- a/packages/taler-wallet-core/src/db.ts
+++ b/packages/taler-wallet-core/src/db.ts
@@ -42,6 +42,7 @@ import {
import { RetryInfo } from "./util/retries.js";
import { PayCoinSelection } from "./util/coinSelection.js";
import { Event, IDBDatabase } from "@gnu-taler/idb-bridge";
+import { PendingTaskInfo } from "./pending-types.js";
/**
* Name of the Taler database. This is effectively the major
@@ -153,6 +154,8 @@ export interface ReserveRecord {
*/
timestampCreated: Timestamp;
+ operationStatus: OperationStatus;
+
/**
* Time when the information about this reserve was posted to the bank.
*
@@ -914,10 +917,19 @@ export enum RefreshCoinStatus {
Frozen = "frozen",
}
+export enum OperationStatus {
+ Finished = "finished",
+ Pending = "pending",
+}
+
export interface RefreshGroupRecord {
+ operationStatus: OperationStatus;
+
/**
* Retry info, even present when the operation isn't active to allow indexing
* on the next retry timestamp.
+ *
+ * FIXME: No, this can be optional, indexing is still possible
*/
retryInfo: RetryInfo;
@@ -1350,6 +1362,8 @@ export interface WithdrawalGroupRecord {
*/
timestampFinish?: Timestamp;
+ operationStatus: OperationStatus;
+
/**
* Amount including fees (i.e. the amount subtracted from the
* reserve to withdraw all coins in this withdrawal session).
@@ -1561,6 +1575,8 @@ export interface DepositGroupRecord {
timestampFinished: Timestamp | undefined;
+ operationStatus: OperationStatus;
+
lastError: TalerErrorDetails | undefined;
/**
@@ -1601,6 +1617,18 @@ export interface TombstoneRecord {
id: string;
}
+export interface BalancePerCurrencyRecord {
+ currency: string;
+
+ availableNow: AmountString;
+
+ availableExpected: AmountString;
+
+ pendingIncoming: AmountString;
+
+ pendingOutgoing: AmountString;
+}
+
export const WalletStoresV1 = {
coins: describeStore(
describeContents<CoinRecord>("coins", {
@@ -1671,7 +1699,9 @@ export const WalletStoresV1 = {
describeContents<RefreshGroupRecord>("refreshGroups", {
keyPath: "refreshGroupId",
}),
- {},
+ {
+ byStatus: describeIndex("byStatus", "operationStatus"),
+ },
),
recoupGroups: describeStore(
describeContents<RecoupGroupRecord>("recoupGroups", {
@@ -1686,6 +1716,7 @@ export const WalletStoresV1 = {
"byInitialWithdrawalGroupId",
"initialWithdrawalGroupId",
),
+ byStatus: describeIndex("byStatus", "operationStatus"),
},
),
purchases: describeStore(
@@ -1716,6 +1747,7 @@ export const WalletStoresV1 = {
}),
{
byReservePub: describeIndex("byReservePub", "reservePub"),
+ byStatus: describeIndex("byStatus", "operationStatus"),
},
),
planchets: describeStore(
@@ -1753,7 +1785,9 @@ export const WalletStoresV1 = {
describeContents<DepositGroupRecord>("depositGroups", {
keyPath: "depositGroupId",
}),
- {},
+ {
+ byStatus: describeIndex("byStatus", "operationStatus"),
+ },
),
tombstones: describeStore(
describeContents<TombstoneRecord>("tombstones", { keyPath: "id" }),
@@ -1765,6 +1799,12 @@ export const WalletStoresV1 = {
}),
{},
),
+ balancesPerCurrency: describeStore(
+ describeContents<BalancePerCurrencyRecord>("balancesPerCurrency", {
+ keyPath: "currency",
+ }),
+ {},
+ ),
};
export interface MetaConfigRecord {
diff --git a/packages/taler-wallet-core/src/headless/helpers.ts
b/packages/taler-wallet-core/src/headless/helpers.ts
index 191c4844..d8616f71 100644
--- a/packages/taler-wallet-core/src/headless/helpers.ts
+++ b/packages/taler-wallet-core/src/headless/helpers.ts
@@ -37,6 +37,7 @@ import type { IDBFactory } from "@gnu-taler/idb-bridge";
import { WalletNotification } from "@gnu-taler/taler-util";
import { Wallet } from "../wallet.js";
import * as fs from "fs";
+import { AccessStats } from "@gnu-taler/idb-bridge/src/MemoryBackend";
const logger = new Logger("headless/helpers.ts");
@@ -80,6 +81,21 @@ function makeId(length: number): string {
export async function getDefaultNodeWallet(
args: DefaultNodeWalletArgs = {},
): Promise<Wallet> {
+ const res = await getDefaultNodeWallet2(args);
+ return res.wallet;
+}
+
+/**
+ * Get a wallet instance with default settings for node.
+ *
+ * Extended version that allows getting DB stats.
+ */
+export async function getDefaultNodeWallet2(
+ args: DefaultNodeWalletArgs = {},
+): Promise<{
+ wallet: Wallet;
+ getDbStats: () => AccessStats;
+}> {
BridgeIDBFactory.enableTracing = false;
const myBackend = new MemoryBackend();
myBackend.enableTracing = false;
@@ -121,7 +137,7 @@ export async function getDefaultNodeWallet(
BridgeIDBFactory.enableTracing = false;
const myBridgeIdbFactory = new BridgeIDBFactory(myBackend);
- const myIdbFactory: IDBFactory = (myBridgeIdbFactory as any) as IDBFactory;
+ const myIdbFactory: IDBFactory = myBridgeIdbFactory as any as IDBFactory;
let myHttpLib;
if (args.httpLib) {
@@ -164,5 +180,8 @@ export async function getDefaultNodeWallet(
if (args.notifyHandler) {
w.addNotificationListener(args.notifyHandler);
}
- return w;
+ return {
+ wallet: w,
+ getDbStats: () => myBackend.accessStats,
+ };
}
diff --git a/packages/taler-wallet-core/src/index.node.ts
b/packages/taler-wallet-core/src/index.node.ts
index 0860ccc2..7a6ad6a7 100644
--- a/packages/taler-wallet-core/src/index.node.ts
+++ b/packages/taler-wallet-core/src/index.node.ts
@@ -20,6 +20,9 @@ export * from "./index.js";
export { NodeHttpLib } from "./headless/NodeHttpLib.js";
export {
getDefaultNodeWallet,
+ getDefaultNodeWallet2,
DefaultNodeWalletArgs,
} from "./headless/helpers.js";
export * from "./crypto/workers/nodeThreadWorker.js";
+
+export type { AccessStats } from "@gnu-taler/idb-bridge";
diff --git a/packages/taler-wallet-core/src/operations/README.md
b/packages/taler-wallet-core/src/operations/README.md
index 32e2fbfc..ca7140d6 100644
--- a/packages/taler-wallet-core/src/operations/README.md
+++ b/packages/taler-wallet-core/src/operations/README.md
@@ -4,4 +4,4 @@ This folder contains the implementations for all wallet
operations that operate
To avoid cyclic dependencies, these files must **not** reference each other.
Instead, other operations should only be accessed via injected dependencies.
-Avoiding cyclic dependencies is important for module bundlers.
\ No newline at end of file
+Avoiding cyclic dependencies is important for module bundlers.
diff --git a/packages/taler-wallet-core/src/operations/backup/import.ts
b/packages/taler-wallet-core/src/operations/backup/import.ts
index 564d3979..5ca1ebb9 100644
--- a/packages/taler-wallet-core/src/operations/backup/import.ts
+++ b/packages/taler-wallet-core/src/operations/backup/import.ts
@@ -47,6 +47,7 @@ import {
WireInfo,
WalletStoresV1,
RefreshCoinStatus,
+ OperationStatus,
} from "../../db.js";
import { PayCoinSelection } from "../../util/coinSelection.js";
import { j2s } from "@gnu-taler/taler-util";
@@ -180,8 +181,11 @@ async function getDenomSelStateFromBackup(
const d = await tx.denominations.get([exchangeBaseUrl, s.denom_pub_hash]);
checkBackupInvariant(!!d);
totalCoinValue = Amounts.add(totalCoinValue, d.value).amount;
- totalWithdrawCost = Amounts.add(totalWithdrawCost, d.value, d.feeWithdraw)
- .amount;
+ totalWithdrawCost = Amounts.add(
+ totalWithdrawCost,
+ d.value,
+ d.feeWithdraw,
+ ).amount;
}
return {
selectedDenoms,
@@ -475,6 +479,8 @@ export async function importBackup(
backupExchangeDetails.base_url,
backupReserve.initial_selected_denoms,
),
+ // FIXME!
+ operationStatus: OperationStatus.Pending,
});
}
for (const backupWg of backupReserve.withdrawal_groups) {
@@ -507,6 +513,9 @@ export async function importBackup(
timestampFinish: backupWg.timestamp_finish,
withdrawalGroupId: backupWg.withdrawal_group_id,
denomSelUid: backupWg.selected_denoms_id,
+ operationStatus: backupWg.timestamp_finish
+ ? OperationStatus.Finished
+ : OperationStatus.Pending,
});
}
}
@@ -758,7 +767,8 @@ export async function importBackup(
// FIXME!
payRetryInfo: initRetryInfo(),
download,
- paymentSubmitPending:
!backupPurchase.timestamp_first_successful_pay,
+ paymentSubmitPending:
+ !backupPurchase.timestamp_first_successful_pay,
refundQueryRequested: false,
payCoinSelection: await recoverPayCoinSelection(
tx,
@@ -809,10 +819,8 @@ export async function importBackup(
reason = RefreshReason.Scheduled;
break;
}
- const refreshSessionPerCoin: (
- | RefreshSessionRecord
- | undefined
- )[] = [];
+ const refreshSessionPerCoin: (RefreshSessionRecord | undefined)[] =
+ [];
for (const oldCoin of backupRefreshGroup.old_coins) {
const c = await tx.coins.get(oldCoin.coin_pub);
checkBackupInvariant(!!c);
@@ -848,6 +856,9 @@ export async function importBackup(
? RefreshCoinStatus.Finished
: RefreshCoinStatus.Pending,
),
+ operationStatus: backupRefreshGroup.timestamp_finish
+ ? OperationStatus.Finished
+ : OperationStatus.Pending,
inputPerCoin: backupRefreshGroup.old_coins.map((x) =>
Amounts.parseOrThrow(x.input_amount),
),
diff --git a/packages/taler-wallet-core/src/operations/balance.ts
b/packages/taler-wallet-core/src/operations/balance.ts
index 29889392..61bae828 100644
--- a/packages/taler-wallet-core/src/operations/balance.ts
+++ b/packages/taler-wallet-core/src/operations/balance.ts
@@ -47,6 +47,10 @@ export async function getBalancesInsideTransaction(
withdrawalGroups: typeof WalletStoresV1.withdrawalGroups;
}>,
): Promise<BalancesResponse> {
+ return {
+ balances: [],
+ };
+
const balanceStore: Record<string, WalletBalance> = {};
/**
@@ -148,6 +152,9 @@ export async function getBalancesInsideTransaction(
export async function getBalances(
ws: InternalWalletState,
): Promise<BalancesResponse> {
+ return {
+ balances: [],
+ };
logger.trace("starting to compute balance");
const wbal = await ws.db
diff --git a/packages/taler-wallet-core/src/operations/deposits.ts
b/packages/taler-wallet-core/src/operations/deposits.ts
index 0a90e021..afe8e6f3 100644
--- a/packages/taler-wallet-core/src/operations/deposits.ts
+++ b/packages/taler-wallet-core/src/operations/deposits.ts
@@ -50,7 +50,7 @@ import {
getRandomBytes,
stringToBytes,
} from "@gnu-taler/taler-util";
-import { DepositGroupRecord } from "../db.js";
+import { DepositGroupRecord, OperationStatus } from "../db.js";
import { guardOperationException } from "../errors.js";
import { PayCoinSelection, selectPayCoins } from "../util/coinSelection.js";
import { readSuccessResponseJsonOrThrow } from "../util/http.js";
@@ -281,6 +281,7 @@ async function processDepositGroupImpl(
}
if (allDeposited) {
dg.timestampFinished = getTimestampNow();
+ dg.operationStatus = OperationStatus.Finished;
delete dg.lastError;
delete dg.retryInfo;
await tx.depositGroups.put(dg);
@@ -409,11 +410,7 @@ export async function getFeeForDeposit(
refund_deadline: { t_ms: 0 },
};
- const contractData = extractContractData(
- contractTerms,
- "",
- "",
- );
+ const contractData = extractContractData(contractTerms, "", "");
const candidates = await getCandidatePayCoins(ws, contractData);
@@ -436,7 +433,6 @@ export async function getFeeForDeposit(
amount,
payCoinSel,
);
-
}
export async function createDepositGroup(
@@ -570,6 +566,7 @@ export async function createDepositGroup(
salt: wireSalt,
},
retryInfo: initRetryInfo(),
+ operationStatus: OperationStatus.Pending,
lastError: undefined,
};
@@ -708,8 +705,10 @@ export async function getTotalFeeForDepositAmount(
.filter((x) =>
Amounts.isSameCurrency(x.value, pcs.coinContributions[i]),
);
- const amountLeft = Amounts.sub(denom.value, pcs.coinContributions[i])
- .amount;
+ const amountLeft = Amounts.sub(
+ denom.value,
+ pcs.coinContributions[i],
+ ).amount;
const refreshCost = getTotalRefreshCost(allDenoms, denom, amountLeft);
refreshFee.push(refreshCost);
}
@@ -736,8 +735,17 @@ export async function getTotalFeeForDepositAmount(
});
return {
- coin: coinFee.length === 0 ? Amounts.getZero(total.currency) :
Amounts.sum(coinFee).amount,
- wire: wireFee.length === 0 ? Amounts.getZero(total.currency) :
Amounts.sum(wireFee).amount,
- refresh: refreshFee.length === 0 ? Amounts.getZero(total.currency) :
Amounts.sum(refreshFee).amount
+ coin:
+ coinFee.length === 0
+ ? Amounts.getZero(total.currency)
+ : Amounts.sum(coinFee).amount,
+ wire:
+ wireFee.length === 0
+ ? Amounts.getZero(total.currency)
+ : Amounts.sum(wireFee).amount,
+ refresh:
+ refreshFee.length === 0
+ ? Amounts.getZero(total.currency)
+ : Amounts.sum(refreshFee).amount,
};
}
diff --git a/packages/taler-wallet-core/src/operations/pending.ts
b/packages/taler-wallet-core/src/operations/pending.ts
index e3d22bfe..07c29e87 100644
--- a/packages/taler-wallet-core/src/operations/pending.ts
+++ b/packages/taler-wallet-core/src/operations/pending.ts
@@ -28,6 +28,7 @@ import {
WalletStoresV1,
BackupProviderStateTag,
RefreshCoinStatus,
+ OperationStatus,
} from "../db.js";
import {
PendingOperationsResponse,
@@ -37,6 +38,8 @@ import {
import {
getTimestampNow,
isTimestampExpired,
+ j2s,
+ Logger,
Timestamp,
} from "@gnu-taler/taler-util";
import { InternalWalletState } from "../common.js";
@@ -82,33 +85,35 @@ async function gatherReservePending(
now: Timestamp,
resp: PendingOperationsResponse,
): Promise<void> {
- await tx.reserves.iter().forEach((reserve) => {
- const reserveType = reserve.bankInfo
- ? ReserveType.TalerBankWithdraw
- : ReserveType.Manual;
- switch (reserve.reserveStatus) {
- case ReserveRecordStatus.DORMANT:
- // nothing to report as pending
- break;
- case ReserveRecordStatus.WAIT_CONFIRM_BANK:
- case ReserveRecordStatus.QUERYING_STATUS:
- case ReserveRecordStatus.REGISTERING_BANK:
- resp.pendingOperations.push({
- type: PendingTaskType.Reserve,
- givesLifeness: true,
- timestampDue: reserve.retryInfo.nextRetry,
- stage: reserve.reserveStatus,
- timestampCreated: reserve.timestampCreated,
- reserveType,
- reservePub: reserve.reservePub,
- retryInfo: reserve.retryInfo,
- });
- break;
- default:
- // FIXME: report problem!
- break;
- }
- });
+ await tx.reserves.indexes.byStatus
+ .iter(OperationStatus.Pending)
+ .forEach((reserve) => {
+ const reserveType = reserve.bankInfo
+ ? ReserveType.TalerBankWithdraw
+ : ReserveType.Manual;
+ switch (reserve.reserveStatus) {
+ case ReserveRecordStatus.DORMANT:
+ // nothing to report as pending
+ break;
+ case ReserveRecordStatus.WAIT_CONFIRM_BANK:
+ case ReserveRecordStatus.QUERYING_STATUS:
+ case ReserveRecordStatus.REGISTERING_BANK:
+ resp.pendingOperations.push({
+ type: PendingTaskType.Reserve,
+ givesLifeness: true,
+ timestampDue: reserve.retryInfo.nextRetry,
+ stage: reserve.reserveStatus,
+ timestampCreated: reserve.timestampCreated,
+ reserveType,
+ reservePub: reserve.reservePub,
+ retryInfo: reserve.retryInfo,
+ });
+ break;
+ default:
+ // FIXME: report problem!
+ break;
+ }
+ });
}
async function gatherRefreshPending(
@@ -116,24 +121,26 @@ async function gatherRefreshPending(
now: Timestamp,
resp: PendingOperationsResponse,
): Promise<void> {
- await tx.refreshGroups.iter().forEach((r) => {
- if (r.timestampFinished) {
- return;
- }
- if (r.frozen) {
- return;
- }
- resp.pendingOperations.push({
- type: PendingTaskType.Refresh,
- givesLifeness: true,
- timestampDue: r.retryInfo.nextRetry,
- refreshGroupId: r.refreshGroupId,
- finishedPerCoin: r.statusPerCoin.map(
- (x) => x === RefreshCoinStatus.Finished,
- ),
- retryInfo: r.retryInfo,
+ await tx.refreshGroups.indexes.byStatus
+ .iter(OperationStatus.Pending)
+ .forEach((r) => {
+ if (r.timestampFinished) {
+ return;
+ }
+ if (r.frozen) {
+ return;
+ }
+ resp.pendingOperations.push({
+ type: PendingTaskType.Refresh,
+ givesLifeness: true,
+ timestampDue: r.retryInfo.nextRetry,
+ refreshGroupId: r.refreshGroupId,
+ finishedPerCoin: r.statusPerCoin.map(
+ (x) => x === RefreshCoinStatus.Finished,
+ ),
+ retryInfo: r.retryInfo,
+ });
});
- });
}
async function gatherWithdrawalPending(
@@ -144,29 +151,31 @@ async function gatherWithdrawalPending(
now: Timestamp,
resp: PendingOperationsResponse,
): Promise<void> {
- await tx.withdrawalGroups.iter().forEachAsync(async (wsr) => {
- if (wsr.timestampFinish) {
- return;
- }
- let numCoinsWithdrawn = 0;
- let numCoinsTotal = 0;
- await tx.planchets.indexes.byGroup
- .iter(wsr.withdrawalGroupId)
- .forEach((x) => {
- numCoinsTotal++;
- if (x.withdrawalDone) {
- numCoinsWithdrawn++;
- }
+ await tx.withdrawalGroups.indexes.byStatus
+ .iter(OperationStatus.Pending)
+ .forEachAsync(async (wsr) => {
+ if (wsr.timestampFinish) {
+ return;
+ }
+ let numCoinsWithdrawn = 0;
+ let numCoinsTotal = 0;
+ await tx.planchets.indexes.byGroup
+ .iter(wsr.withdrawalGroupId)
+ .forEach((x) => {
+ numCoinsTotal++;
+ if (x.withdrawalDone) {
+ numCoinsWithdrawn++;
+ }
+ });
+ resp.pendingOperations.push({
+ type: PendingTaskType.Withdraw,
+ givesLifeness: true,
+ timestampDue: wsr.retryInfo.nextRetry,
+ withdrawalGroupId: wsr.withdrawalGroupId,
+ lastError: wsr.lastError,
+ retryInfo: wsr.retryInfo,
});
- resp.pendingOperations.push({
- type: PendingTaskType.Withdraw,
- givesLifeness: true,
- timestampDue: wsr.retryInfo.nextRetry,
- withdrawalGroupId: wsr.withdrawalGroupId,
- lastError: wsr.lastError,
- retryInfo: wsr.retryInfo,
});
- });
}
async function gatherProposalPending(
@@ -199,20 +208,22 @@ async function gatherDepositPending(
now: Timestamp,
resp: PendingOperationsResponse,
): Promise<void> {
- await tx.depositGroups.iter().forEach((dg) => {
- if (dg.timestampFinished) {
- return;
- }
- const timestampDue = dg.retryInfo?.nextRetry ?? getTimestampNow();
- resp.pendingOperations.push({
- type: PendingTaskType.Deposit,
- givesLifeness: true,
- timestampDue,
- depositGroupId: dg.depositGroupId,
- lastError: dg.lastError,
- retryInfo: dg.retryInfo,
+ await tx.depositGroups.indexes.byStatus
+ .iter(OperationStatus.Pending)
+ .forEach((dg) => {
+ if (dg.timestampFinished) {
+ return;
+ }
+ const timestampDue = dg.retryInfo?.nextRetry ?? getTimestampNow();
+ resp.pendingOperations.push({
+ type: PendingTaskType.Deposit,
+ givesLifeness: true,
+ timestampDue,
+ depositGroupId: dg.depositGroupId,
+ lastError: dg.lastError,
+ retryInfo: dg.retryInfo,
+ });
});
- });
}
async function gatherTipPending(
diff --git a/packages/taler-wallet-core/src/operations/refresh.ts
b/packages/taler-wallet-core/src/operations/refresh.ts
index 00eaa0ea..5b589f1f 100644
--- a/packages/taler-wallet-core/src/operations/refresh.ts
+++ b/packages/taler-wallet-core/src/operations/refresh.ts
@@ -26,6 +26,7 @@ import {
CoinSourceType,
CoinStatus,
DenominationRecord,
+ OperationStatus,
RefreshCoinStatus,
RefreshGroupRecord,
WalletStoresV1,
@@ -127,6 +128,7 @@ function updateGroupStatus(rg: RefreshGroupRecord): void {
rg.retryInfo = initRetryInfo();
} else {
rg.timestampFinished = getTimestampNow();
+ rg.operationStatus = OperationStatus.Finished;
rg.retryInfo = initRetryInfo();
}
}
@@ -929,6 +931,7 @@ export async function createRefreshGroup(
}
const refreshGroup: RefreshGroupRecord = {
+ operationStatus: OperationStatus.Pending,
timestampFinished: undefined,
statusPerCoin: oldCoinPubs.map(() => RefreshCoinStatus.Pending),
lastError: undefined,
diff --git a/packages/taler-wallet-core/src/operations/reserves.ts
b/packages/taler-wallet-core/src/operations/reserves.ts
index 75d517d6..1550d946 100644
--- a/packages/taler-wallet-core/src/operations/reserves.ts
+++ b/packages/taler-wallet-core/src/operations/reserves.ts
@@ -41,6 +41,7 @@ import {
} from "@gnu-taler/taler-util";
import { InternalWalletState } from "../common.js";
import {
+ OperationStatus,
ReserveBankInfo,
ReserveRecord,
ReserveRecordStatus,
@@ -155,6 +156,7 @@ export async function createReserve(
lastError: undefined,
currency: req.amount.currency,
requestedQuery: false,
+ operationStatus: OperationStatus.Pending,
};
const exchangeInfo = await updateExchangeFromUrl(ws, req.exchange);
@@ -250,6 +252,7 @@ export async function forceQueryReserve(
switch (reserve.reserveStatus) {
case ReserveRecordStatus.DORMANT:
reserve.reserveStatus = ReserveRecordStatus.QUERYING_STATUS;
+ reserve.operationStatus = OperationStatus.Pending;
break;
default:
reserve.requestedQuery = true;
@@ -338,6 +341,7 @@ async function registerReserveWithBank(
}
r.timestampReserveInfoPosted = getTimestampNow();
r.reserveStatus = ReserveRecordStatus.WAIT_CONFIRM_BANK;
+ r.operationStatus = OperationStatus.Pending;
if (!r.bankInfo) {
throw Error("invariant failed");
}
@@ -419,6 +423,7 @@ async function processReserveBankStatusImpl(
const now = getTimestampNow();
r.timestampBankConfirmed = now;
r.reserveStatus = ReserveRecordStatus.BANK_ABORTED;
+ r.operationStatus = OperationStatus.Finished;
r.retryInfo = initRetryInfo();
await tx.reserves.put(r);
});
@@ -455,6 +460,7 @@ async function processReserveBankStatusImpl(
const now = getTimestampNow();
r.timestampBankConfirmed = now;
r.reserveStatus = ReserveRecordStatus.QUERYING_STATUS;
+ r.operationStatus = OperationStatus.Pending;
r.retryInfo = initRetryInfo();
} else {
switch (r.reserveStatus) {
@@ -658,6 +664,7 @@ async function updateReserve(
if (denomSelInfo.selectedDenoms.length === 0) {
newReserve.reserveStatus = ReserveRecordStatus.DORMANT;
+ newReserve.operationStatus = OperationStatus.Finished;
newReserve.lastError = undefined;
newReserve.retryInfo = initRetryInfo();
await tx.reserves.put(newReserve);
@@ -684,11 +691,13 @@ async function updateReserve(
denomsSel: denomSelectionInfoToState(denomSelInfo),
secretSeed: encodeCrock(getRandomBytes(64)),
denomSelUid: encodeCrock(getRandomBytes(32)),
+ operationStatus: OperationStatus.Pending,
};
newReserve.lastError = undefined;
newReserve.retryInfo = initRetryInfo();
newReserve.reserveStatus = ReserveRecordStatus.DORMANT;
+ newReserve.operationStatus = OperationStatus.Finished;
await tx.reserves.put(newReserve);
await tx.withdrawalGroups.put(withdrawalRecord);
diff --git a/packages/taler-wallet-core/src/operations/withdraw.ts
b/packages/taler-wallet-core/src/operations/withdraw.ts
index 8b72c40e..c44435e8 100644
--- a/packages/taler-wallet-core/src/operations/withdraw.ts
+++ b/packages/taler-wallet-core/src/operations/withdraw.ts
@@ -53,6 +53,7 @@ import {
DenomSelectionState,
ExchangeDetailsRecord,
ExchangeRecord,
+ OperationStatus,
PlanchetRecord,
} from "../db.js";
import { walletCoreDebugFlags } from "../util/debugFlags.js";
@@ -968,7 +969,8 @@ async function processWithdrawGroupImpl(
if (wg.timestampFinish === undefined && numFinished === numTotalCoins) {
finishedForFirstTime = true;
wg.timestampFinish = getTimestampNow();
- wg.lastError = undefined;
+ wg.operationStatus = OperationStatus.Finished;
+ delete wg.lastError;
wg.retryInfo = initRetryInfo();
}
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.