[libeufin] branch master updated (75bd4dcf -> a1820a97)

[gnunet]

This is an automated email from the git hooks/post-receive script.

ms pushed a change to branch master
in repository libeufin.

    from 75bd4dcf addressing #6988
     new 88338078 Logging.
     new 8a74c824 EBICS checks.
     new a1820a97 addressing #6988

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../tech/libeufin/nexus/BankConnectionProtocol.kt  | 14 +++-
 nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt |  1 -
 .../tech/libeufin/nexus/bankaccount/BankAccount.kt | 13 +++-
 .../tech/libeufin/nexus/ebics/EbicsClient.kt       | 82 ++++++++++++++--------
 .../kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt | 14 +++-
 .../tech/libeufin/nexus/server/NexusServer.kt      | 20 +++++-
 6 files changed, 107 insertions(+), 37 deletions(-)

diff --git 
a/nexus/src/main/kotlin/tech/libeufin/nexus/BankConnectionProtocol.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/BankConnectionProtocol.kt
index c56524ff..deaa007a 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/BankConnectionProtocol.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/BankConnectionProtocol.kt
@@ -58,14 +58,22 @@ interface BankConnectionProtocol {
     // Send to the bank a previously prepared payment instruction.
     suspend fun submitPaymentInitiation(httpClient: HttpClient, 
paymentInitiationId: Long)
 
-    // Downloads transactions from the bank, according to the specification
-    // given in the arguments.
+    /**
+     * Downloads transactions from the bank, according to the specification
+     * given in the arguments.
+     *
+     * This function returns a possibly empty list of exceptions.
+     * That helps not to stop fetching if ONE operation fails.  Notably,
+     * C52 and C53 may be asked along one invocation of this function,
+     * therefore storing the exception on C52 allows the C53 to still
+     * take place.  The caller then decides how to handle the exceptions.
+     */
     suspend fun fetchTransactions(
         fetchSpec: FetchSpecJson,
         client: HttpClient,
         bankConnectionId: String,
         accountId: String
-    )
+    ): List<Exception>?
 }
 
 fun getConnectionPlugin(connId: String): BankConnectionProtocol {
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt
index 957373d8..52dca293 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt
@@ -502,7 +502,6 @@ private suspend fun historyIncoming(call: ApplicationCall) {
  * In the future, a dedicate "add-incoming" facade should
  * be provided, offering the mean to store the credentials
  * at configuration time.
- *
  */
 private suspend fun addIncoming(call: ApplicationCall) {
     val facadeId = ensureNonNull(call.parameters["fcid"])
diff --git 
a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
index 7dab22ff..b32b814a 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -153,7 +153,15 @@ data class CamtTransactionsCount(
      * Total number of transactions that were included in a report
      * or a statement.
      */
-    val downloadedTransactions: Int
+    val downloadedTransactions: Int,
+    /**
+     * Exceptions occurred while fetching transactions.  Fetching
+     * transactions can be done via multiple EBICS messages, therefore
+     * a failing one should not prevent other messages to be sent.
+     * This list collects all the exceptions that happened during the
+     * execution of a batch of messages.
+     */
+    var errors: List<Exception>? = null
 )
 
 /**
@@ -426,7 +434,7 @@ suspend fun fetchBankAccountTransactions(
      * document into the database.  This function tries to download
      * both reports AND statements even if the first one fails.
      */
-    getConnectionPlugin(res.connectionType).fetchTransactions(
+    val errors: List<Exception>? = 
getConnectionPlugin(res.connectionType).fetchTransactions(
         fetchSpec,
         client,
         res.connectionName,
@@ -437,6 +445,7 @@ suspend fun fetchBankAccountTransactions(
     ingestFacadeTransactions(accountId, "taler-wire-gateway", ::talerFilter, 
::maybeTalerRefunds)
     ingestFacadeTransactions(accountId, "anastasis", ::anastasisFilter, null)
 
+    ingestionResult.errors = errors
     return ingestionResult
 }
 
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
index ffaefd4f..33f2f720 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
@@ -89,8 +89,8 @@ class EbicsDownloadBankErrorResult(
 ) : EbicsDownloadResult()
 
 /**
- * Do an EBICS download transaction.  This includes the initialization phase, 
transaction phase
- * and receipt phase.
+ * Do an EBICS download transaction.  This includes
+ * the initialization phase, transaction phase and receipt phase.
  */
 suspend fun doEbicsDownloadTransaction(
     client: HttpClient,
@@ -105,55 +105,73 @@ suspend fun doEbicsDownloadTransaction(
     val initResponseStr = client.postToBank(subscriberDetails.ebicsUrl, 
initDownloadRequestStr)
     val initResponse = parseAndValidateEbicsResponse(subscriberDetails, 
initResponseStr)
 
+    val transactionID: String? = initResponse.transactionID
+    // Checking for EBICS communication problems.
     when (initResponse.technicalReturnCode) {
         EbicsReturnCode.EBICS_OK -> {
-            // Success, nothing to do!
+            /**
+             * The EBICS communication succeeded, but business problems
+             * may be reported along the 'bank technical' code; this check
+             * takes place later.
+             */
         }
         else -> {
-            /**
-             * The bank gave a valid response that contains however
-             * an error.  Hence Nexus sent not processable instructions. */
-
+            // The bank gave a valid XML response but EBICS had problems.
             throw EbicsProtocolError(
                 HttpStatusCode.UnprocessableEntity,
-                "unexpected return code ${initResponse.technicalReturnCode}",
+                "Unexpected return code ${initResponse.technicalReturnCode}," +
+                        " for order type $orderType and transaction ID: 
$transactionID," +
+                        " at init phase.",
                 initResponse.technicalReturnCode
             )
         }
     }
-
+    /**
+     * At this point, the EBICS init phase went through,
+     * therefore the message should carry a transaction ID!
+     */
+    if (transactionID == null) throw NexusError(
+        HttpStatusCode.BadGateway,
+        "EBICS-correct init response should contain" +
+                " a transaction ID, $orderType did not!"
+    )
+    // Checking the 'bank technical' code.
     when (initResponse.bankReturnCode) {
         EbicsReturnCode.EBICS_OK -> {
             // Success, nothing to do!
         }
         else -> {
-            logger.warn("Bank return code was: ${initResponse.bankReturnCode}")
+            logger.error(
+                "Bank return code at init phase was: 
${initResponse.bankReturnCode}, for" +
+                        " order type $orderType and transaction ID 
$transactionID."
+            )
             return EbicsDownloadBankErrorResult(initResponse.bankReturnCode)
         }
     }
-
-    val transactionID =
-        initResponse.transactionID ?: throw NexusError(
-            HttpStatusCode.BadGateway,
-            "Initial response must contain transaction ID"
-        )
-    logger.debug("Bank acknowledges EBICS download initialization.  
Transaction ID: $transactionID.")
+    logger.debug("Bank acknowledges EBICS download initialization." +
+            "  Transaction ID: $transactionID.")
     val encryptionInfo = initResponse.dataEncryptionInfo
         ?: throw NexusError(
             HttpStatusCode.BadGateway,
-            "initial response did not contain encryption info"
+            "initial response did not contain encryption info.  " +
+                    "Order type $orderType, transaction ID $transactionID"
         )
 
     val initOrderDataEncChunk = initResponse.orderDataEncChunk
         ?: throw NexusError(
             HttpStatusCode.BadGateway,
-            "initial response for download transaction does not contain data 
transfer"
+            "initial response for download transaction does not " +
+                    "contain data transfer.  Order type $orderType, 
transaction ID $transactionID."
         )
 
     payloadChunks.add(initOrderDataEncChunk)
 
     val numSegments = initResponse.numSegments
-        ?: throw NexusError(HttpStatusCode.FailedDependency, "missing segment 
number in EBICS download init response")
+        ?: throw NexusError(
+            HttpStatusCode.FailedDependency,
+            "missing segment number in EBICS download init response." +
+                    "  Order type $orderType, transaction ID $transactionID"
+        )
 
     // Transfer phase
     for (x in 2 .. numSegments) {
@@ -169,7 +187,9 @@ suspend fun doEbicsDownloadTransaction(
             else -> {
                 throw NexusError(
                     HttpStatusCode.FailedDependency,
-                    "unexpected technical return code 
${transferResponse.technicalReturnCode}"
+                    "unexpected EBICS technical return code at transfer phase: 
" +
+                            "${transferResponse.technicalReturnCode}." +
+                            "  Order type $orderType, transaction ID 
$transactionID"
                 )
             }
         }
@@ -178,14 +198,17 @@ suspend fun doEbicsDownloadTransaction(
                 // Success, nothing to do!
             }
             else -> {
-                logger.warn("Bank return code was: 
${transferResponse.bankReturnCode}")
+                logger.error("Bank return code " +
+                        "at transfer phase was: 
${transferResponse.bankReturnCode}." +
+                        "  Order type $orderType, transaction ID 
$transactionID")
                 return 
EbicsDownloadBankErrorResult(transferResponse.bankReturnCode)
             }
         }
         val transferOrderDataEncChunk = transferResponse.orderDataEncChunk
             ?: throw NexusError(
                 HttpStatusCode.BadGateway,
-                "transfer response for download transaction does not contain 
data transfer"
+                "transfer response for download transaction " +
+                        "does not contain data transfer.  Order type 
$orderType, transaction ID $transactionID"
             )
         payloadChunks.add(transferOrderDataEncChunk)
         logger.debug("Download transfer phase of ${transactionID}: bank 
acknowledges $x")
@@ -194,7 +217,6 @@ suspend fun doEbicsDownloadTransaction(
     val respPayload = decryptAndDecompressResponse(subscriberDetails, 
encryptionInfo, payloadChunks)
 
     // Acknowledgement phase
-
     val ackRequest = createEbicsRequestForDownloadReceipt(subscriberDetails, 
transactionID)
     val ackResponseStr = client.postToBank(
         subscriberDetails.ebicsUrl,
@@ -207,7 +229,9 @@ suspend fun doEbicsDownloadTransaction(
         else -> {
             throw NexusError(
                 HttpStatusCode.InternalServerError,
-                "unexpected return code: 
${ackResponse.technicalReturnCode.name}"
+                "Unexpected EBICS return code" +
+                        " at acknowledgement phase: 
${ackResponse.technicalReturnCode.name}." +
+                        "  Order type $orderType, transaction ID 
$transactionID"
             )
         }
     }
@@ -215,9 +239,7 @@ suspend fun doEbicsDownloadTransaction(
     return EbicsDownloadSuccessResult(respPayload)
 }
 
-/**
- * Currently only 1-segment requests.
- */
+// Currently only 1-segment requests.
 suspend fun doEbicsUploadTransaction(
     client: HttpClient,
     subscriberDetails: EbicsClientSubscriberDetails,
@@ -226,7 +248,9 @@ suspend fun doEbicsUploadTransaction(
     orderParams: EbicsOrderParams
 ) {
     if (subscriberDetails.bankEncPub == null) {
-        throw NexusError(HttpStatusCode.BadRequest, "bank encryption key 
unknown, request HPB first")
+        throw NexusError(HttpStatusCode.BadRequest,
+            "bank encryption key unknown, request HPB first"
+        )
     }
     val preparedUploadData = prepareUploadPayload(subscriberDetails, payload)
     val req = createEbicsRequestForUploadInitialization(subscriberDetails, 
orderType, orderParams, preparedUploadData)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
index b02e8193..209c9384 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -402,13 +402,20 @@ fun formatHex(ba: ByteArray): String {
     return out
 }
 
+/**
+ * This function returns a possibly empty list of Exception.
+ * That helps not to stop fetching if ONE operation fails.  Notably,
+ * C52 and C53 may be asked along one invocation of this function,
+ * therefore storing the exception on C52 allows the C53 to still
+ * take place.  The caller then decides how to handle the exceptions.
+ */
 class EbicsBankConnectionProtocol: BankConnectionProtocol {
     override suspend fun fetchTransactions(
         fetchSpec: FetchSpecJson,
         client: HttpClient,
         bankConnectionId: String,
         accountId: String
-    ) {
+    ): List<Exception>? {
         val subscriberDetails = transaction { 
getEbicsSubscriberDetails(bankConnectionId) }
         val lastTimes = transaction {
             val acct = NexusBankAccountEntity.findByName(accountId)
@@ -507,6 +514,7 @@ class EbicsBankConnectionProtocol: BankConnectionProtocol {
             }
         }
         // Downloads and stores the bank message into the database.  No 
ingestion.
+        val errors = mutableListOf<Exception>()
         for (spec in specs) {
             try {
                 fetchEbicsC5x(
@@ -518,8 +526,12 @@ class EbicsBankConnectionProtocol: BankConnectionProtocol {
                 )
             } catch (e: Exception) {
                 logger.warn("Fetching transactions (${spec.orderType}) 
excepted: ${e.message}.")
+                errors.add(e)
             }
         }
+        if (errors.size > 0)
+            return errors
+        return null
     }
     /**
      * Submit one Pain.001 for one payment initiations.
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
index b600fbcd..79ab9adf 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
@@ -729,7 +729,25 @@ val nexusApp: Application.() -> Unit = {
                 )
             }
             val ingestionResult = fetchBankAccountTransactions(client, 
fetchSpec, accountid)
-            call.respond(ingestionResult)
+            var statusCode = HttpStatusCode.OK
+            /**
+             * Client errors are unlikely here, because authentication
+             * and JSON validity fail earlier.  Hence either Nexus or the
+             * bank had a problem.  NOTE: because this handler triggers 
multiple
+             * fetches, it is ALSO possible that although one error is 
reported,
+             * SOME transactions made it to the database!
+             */
+            if (ingestionResult.errors != null)
+            /**
+             * 500 is intentionally generic, because multiple errors
+             * may suggest different statuses.  The response body however
+             * informs the client about what failed.
+             */
+            statusCode = HttpStatusCode.InternalServerError
+            call.respond(
+                status = statusCode,
+                ingestionResult
+            )
             return@post
         }
 

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.