[taler-grid5k] 36/73: Update image build, add install script to install taler binaries in running experiment

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit 600c67872d2ff51290209c0f4d5190f2b254c714
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Wed Dec 1 11:47:00 2021 +0100

    Update image build, add install script to install taler binaries in running 
experiment
---
 configs/etc/taler/conf.d/exchange-business.conf    |  2 +
 .../systemd/system/taler-exchange-httpd.service    |  3 +-
 .../systemd/system/taler-exchange-httpd@.service   |  3 +-
 docker/Dockerfile                                  |  6 ++
 docker/entrypoint.sh                               | 85 +++++++++++++---------
 experiment/README.md                               | 19 +++++
 experiment/env                                     |  2 +
 experiment/scripts/database.sh                     |  4 +-
 experiment/scripts/exchange.sh                     |  2 +-
 experiment/scripts/helpers.sh                      |  2 +-
 experiment/scripts/install.sh                      | 70 ++++++++++++++++++
 experiment/scripts/setup.sh                        |  8 +-
 image/README.md                                    |  2 +
 image/taler-debian11.yaml                          |  4 +
 14 files changed, 170 insertions(+), 42 deletions(-)

diff --git a/configs/etc/taler/conf.d/exchange-business.conf 
b/configs/etc/taler/conf.d/exchange-business.conf
index 0e6dd1e..7a8b299 100644
--- a/configs/etc/taler/conf.d/exchange-business.conf
+++ b/configs/etc/taler/conf.d/exchange-business.conf
@@ -12,6 +12,8 @@ MASTER_PUBLIC_KEY = <MASTER_KEY_HERE>
 # BASE_URL = https://example.com/
 BASE_URL = <BASE_URL_HERE>
 
+MAX_REQUESTS = 1024
+
 # For your terms of service and privacy policy, you should specify
 # an Etag that must be updated whenever there are significant
 # changes to either document.  The format is up to you, what matters
diff --git a/configs/usr/lib/systemd/system/taler-exchange-httpd.service 
b/configs/usr/lib/systemd/system/taler-exchange-httpd.service
index 51e62d1..2df2697 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-httpd.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-httpd.service
@@ -8,7 +8,8 @@ PartOf=taler-exchange.target
 [Service]
 User=taler-exchange-httpd
 Type=simple
-Restart=on-failure
+Restart=always
+RestartSec=1ms
 EnvironmentFile=/etc/environment
 ExecStart=<CMD_PREFIX_HERE>/usr/bin/taler-exchange-httpd -c 
/etc/taler/taler.conf $TALER_ARGS $EXCHANGE_ARGS
 StandardOutput=journal
diff --git a/configs/usr/lib/systemd/system/taler-exchange-httpd@.service 
b/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
index e0930f0..3f51bcc 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
@@ -8,7 +8,8 @@ PartOf=taler-exchange.target
 [Service]
 User=taler-exchange-httpd
 Type=simple
-Restart=on-failure
+Restart=always
+RestartSec=1ms
 EnvironmentFile=/etc/environment
 ExecStart=<CMD_PREFIX_HERE>/usr/bin/taler-exchange-httpd -c 
/etc/taler/taler.conf $TALER_ARGS $EXCHANGE_ARGS
 StandardOutput=journal
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 8805d3e..f735e49 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,5 +1,9 @@
 FROM debian:11
 
+LABEL author="Boss Marco" \
+      description="Build the Taler Grid5000 environment image" \
+      company="Taler"
+
 ENV DEBIAN_FRONTEND=noninteractive
 
 ENV GRID5K_DEST=lyon,lille
@@ -9,6 +13,8 @@ ENV MERCHANT_COMMIT_SHA=master
 ENV WALLET_COMMIT_SHA=master
 ENV GRID5K_COMMIT_SHA=master
 
+ENV TALER_HOME=/taler
+
 RUN apt update && \
     apt upgrade -y && \
     apt install -y \
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 2862722..ea91a20 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -1,4 +1,8 @@
 #!/bin/bash
+# Docker Entrypoint script
+# Automatically builds the Taler performance 
+# environment image for Grid5000
+
 set -e
 
 while [[ $# -gt 0 ]]; do
@@ -30,71 +34,79 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-TALER_HOME=/taler
+# Run a command and fail when there was an error
+# $1: Command to run
+# $2: Log file to write to (default /dev/null)
+function run_command() {
+  if ! $(eval ${1} > "${2:-/dev/null}" 2>&1) ; then
+    echo "ERROR cmd ${1} failed"
+    cat "${2}" || true
+    exit 1
+  fi
+}
 
+# Prepare a git repo 
+# $1: Git repo name
+# $2: Git commit to checkout to
 function prepare() {
   cd "${TALER_HOME}/$1"
   git checkout master > /dev/null && (git pull > /dev/null 2>&1 || true)
   git checkout "$2" > /dev/null && (git pull > /dev/null 2>&1 || true)
 }
 
-function _make() {
+# Build from a repo
+function build() {
   echo "INFO running bootstrap and configure"
-  if ! ./bootstrap > bootstrap.log 2>&1; then
-    echo "ERROR bootstrap failed"
-    cat bootstrap.log
-    exit 1
-  fi
+  run_command "./bootstrap" "bootstrap.log"
   if [ -f contrib/gana.sh ]; then
-    ./contrib/gana.sh > gana.log 2>&1
-  fi
-  if ! (./configure --enable-logging=verbose --prefix=/usr > configure.log 
2>&1 || \
-       ./configure > configure.log 2>&1); then
-    echo "ERROR configure failed"
-    cat configure.log
-    exit 1
+    run_command "./contrib/gana.sh" "gana.log"
   fi
+  run_command \
+      "(./configure --enable-logging=verbose --prefix=/usr || ./configure)" \
+      "configure.log"
+  run_command "make" "make.log"
   if [ "$VERBOSE" = true ]; then
     cat bootstrap.log || true
     cat gana.log || true
     cat configure.log || true
+    cat make.log || true
   fi
 }
 
-function build() {
+# Build a dist version (build is run first)
+# $1: Git repo to build
+# $2: Git commit to checkout to
+# $3: Custom command to run before running make dist
+function dist() {
   prepare "$1" "$2"
   if [ "$CLEAN" = true ]; then
     echo "INFO cleaning previous build"
     make clean > /dev/null 2>&1 || true
   fi
   if ! make > /dev/null 2>&1; then
-    _make
+    build
   fi
   if [ ! -z "$3" ]; then
     echo "INFO running custom command '$3'"
     /bin/bash -c "$3"
   fi
   echo "INFO running 'make dist'"
-  if ! make dist > dist.log 2>&1;  then
-    echo "ERROR make dist failed"
-    cat dist.log
-    exit 1
-  fi
+  run_command "make dist" "dist.log"
   if [ "$VERBOSE" = true ]; then
     cat dist.log || true
   fi
 }
 
+# Install from a git repo (dist is run first)
+# $1: Git repo to build
+# $2: Git commit to checkout to
+# $3: Custom command to run before running make dist
 function install() {
-  build "$1" "$2" "$3"
+  dist "$1" "$2" "$3"
   echo "INFO installing"
   if [[ $(ldconfig -p | grep "$1") == "" ]] || \
      [ "$CLEAN" = true ]; then
-    if ! make install > install.log 2>&1; then
-      echo "ERROR install failed"
-      cat install.log
-      exit 1
-    fi
+    run_command "make install" "install.log"
     ldconfig
   fi
   if [ "$VERBOSE" = true ]; then
@@ -104,6 +116,8 @@ function install() {
 
 if [ "$COPY" != false ];
 then
+  # Add the certificate to the agent 
+  # no more passwords are needed when it succeeds
   eval $(ssh-agent)
   
   if [[ -f "/root/cert.pem" ]]; then
@@ -140,15 +154,16 @@ fi
 echo "INFO preparing Grid5000 repository"
 prepare grid5k \
       "${GRID5K_COMMIT_SHA}" 
+
+# Test if the image specification is correct
+# To do this the files copied into it must be present
 touch 
${TALER_HOME}/grid5k/image/grid5000/steps/data/{gnunet,wallet,exchange,merchant}.tar.gz
-if ! kameleon dryrun ${TALER_HOME}/grid5k/image/taler-debian11.yaml > 
dryrun.log 2>&1; then
-  echo "ERROR Grid5000 image configuration contains errors"
-  cat dryrun.log
-  exit 1
-else
-  rm ${TALER_HOME}/grid5k/image/grid5000/steps/data/*.tar.gz
-fi
+run_command \
+    "kameleon dryrun ${TALER_HOME}/grid5k/image/taler-debian11.yaml" \
+    "dryrun.log"
+rm ${TALER_HOME}/grid5k/image/grid5000/steps/data/*.tar.gz
 
+# Prepare and package all applications
 echo "INFO preparing GNUnet"
 install gnunet \
       "${GNUNET_COMMIT_SHA}"
@@ -163,7 +178,7 @@ install merchant \
       "${MERCHANT_COMMIT_SHA}"
 
 echo "INFO preparing Taler Wallet"
-build wallet-core \
+dist wallet-core \
       "${WALLET_COMMIT_SHA}"
 
 cd ${TALER_HOME}/grid5k/image
diff --git a/experiment/README.md b/experiment/README.md
index 4c6de59..c34bb9a 100644
--- a/experiment/README.md
+++ b/experiment/README.md
@@ -35,6 +35,25 @@ TODO - DOES NOT WORK YET - DNS host needs to be known ..
 * Click (Re)Run ESpec for the wallet job (type directory)
 * Run `talet-perf update prometheus` on any deployed node to make prometheus 
aware of the freshly added wallets
 
+## Rebuild Taler Binaries
+
+To quickly test fixes of new commits in gnunet,exchange,merchant and wallet, 
there is a script `scripts/instal.sh`
+which can be run inside a running experiment rather than rebuilding the whole 
image.
+
+To do so copy the following snippet into the `Multi Command` window in jFed:
+(Please adjust commits as needed)
+
+```bash
+#!/bin/bash
+
+export GNUNET_COMMIT_SHA=master
+export EXCHANGE_COMMIT_SHA=master
+export MERCHANT_COMMIT_SHA=master
+export WALLET_COMMIT_SHA=master
+
+./scripts/install.sh
+```
+
 ## Actions in running Experiment
 
 To add more exchange processes run `taler-perf stop ecxchange <NUM>` on any 
node
diff --git a/experiment/env b/experiment/env
index 9fa06af..2f5875a 100644
--- a/experiment/env
+++ b/experiment/env
@@ -54,6 +54,8 @@ DB_NAME=taler-exchange
 DB_USER=taler
 # Database password
 DB_PASSWORD=taler
+# If exchanges should connect to pgBouncer rather than to the db directly
+USE_PGBOUNCER=false
 
 # Initial number of wallets to start in parallel per host.
 # This is a bulk size with default = 10
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 17276f3..072c605 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -41,7 +41,7 @@ function setup_config() {
   # Enable password for taler since this is commonly the case
   # For the postgres user do not enable authentication (used in metrics)
   echo "
-  # host all ${DB_USER} 172.16.0.0/12 md5 
+  host all ${DB_USER} 172.16.0.0/12 md5
   host all postgres 172.16.0.0/12 trust
   " >> /etc/postgresql/13/main/pg_hba.conf
 
@@ -60,7 +60,7 @@ fi
 rm -f /var/run/postgresql/pgbouncer.pid
 rm -f /var/run/postgresql/.s.PGSQL.6432
   
-systemctl restart postgresql pgbouncer
+systemctl restart postgresql # pgbouncer
 
 su postgres << EOF
 psql postgres -tAc "SELECT 1 FROM pg_roles WHERE 
rolname='taler-exchange-httpd'" | \
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index d6ed1ed..e5f3b65 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -2,7 +2,7 @@
 set -eux
 
 function setup_config() {
-  sed -i 
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}:6432/${DB_NAME}\g"
 \
+  sed -i 
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}:${DB_PORT}/${DB_NAME}\g"
 \
        /etc/taler/secrets/exchange-db.secret.conf
   
   sed -i "s/<BANK_HOST_HERE>/${BANK_DOMAIN}/g" \
diff --git a/experiment/scripts/helpers.sh b/experiment/scripts/helpers.sh
index 00ecac9..3c64be4 100755
--- a/experiment/scripts/helpers.sh
+++ b/experiment/scripts/helpers.sh
@@ -43,7 +43,7 @@ function wait_for_db() {
         -h "${DATABASE_DOMAIN}" \
         -U "${DB_USER}" \
         -d "${DB_NAME}" \
-       -p "6432" \
+       -p "${DB_PORT}" \
         -c '\q';
   do
     echo "Database not ready yet"
diff --git a/experiment/scripts/install.sh b/experiment/scripts/install.sh
new file mode 100644
index 0000000..63ee692
--- /dev/null
+++ b/experiment/scripts/install.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+TALER_HOME=~/taler
+
+# Run a command and fail when there was an error
+# $1: Command to run
+# $2: Log file to write to (default /dev/null)
+function run_command() {
+  if ! $(eval ${1} > "${2:-/dev/null}" 2>&1) ; then
+    echo "ERROR cmd ${1} failed"
+    cat "${2}" || true
+    exit 1 
+  fi
+}
+
+# Prepare the repository
+# $1: Git repo to clone
+# $2: Commit to checkout to
+function prepare() {
+  DIR="${TALER_HOME}/$(basename ${1%.*})"
+  test -d "${DIR}" || git clone "${1}" "${DIR}"
+  cd "${DIR}"
+  git checkout master > /dev/null && \
+    (git pull > /dev/null 2>&1 || true)
+  git checkout "$2" > /dev/null && \
+    (git pull > /dev/null 2>&1 || true)
+}
+
+function build() {
+  echo "INFO running bootstrap and configure"
+  run_command "./bootstrap" "bootstrap.log" 
+  if [ -f contrib/gana.sh ]; then
+    run_command "./contrib/gana.sh" "gana.log"
+  fi
+  run_command \
+    "(./configure --enable-logging=verbose --prefix=/usr || ./configure)" \
+    "configure.log"
+  run_command "make" "make.log"
+}
+
+# Install from a git repo
+# $1: Git repo to clone
+# $2: Commit to checkout to
+function install() {
+  prepare "$1" "$2" 
+  build 
+  echo "INFO installing"
+  run_command "make install" "install.log"
+  ldconfig
+}
+
+if [ ! -d "${TALER_HOME}" ]; then
+  mkdir "${TALER_HOME}"
+fi
+
+echo "INFO preparing GNUnet"
+install "https://git.gnunet.org/gnunet.git"; \
+        "${GNUNET_COMMIT_SHA:-master}"
+
+echo "INFO preparing Taler Exchange"
+install "https://git.taler.net/exchange.git"; \
+        "${EXCHANGE_COMMIT_SHA:-master}"
+
+echo "INFO preparing Taler Merchant"
+install "https://git.taler.net/merchant.git"; \
+        "${MERCHANT_COMMIT_SHA:-master}"
+
+echo "INFO preparing Taler Wallet"
+install "https://git.taler.net/wallet-core.git"; \
+        "${WALLET_COMMIT_SHA:-master}"
diff --git a/experiment/scripts/setup.sh b/experiment/scripts/setup.sh
index d4f2c04..9274dc4 100644
--- a/experiment/scripts/setup.sh
+++ b/experiment/scripts/setup.sh
@@ -44,13 +44,19 @@ function setup_log() {
 function setup_environment() {
   set +x
   
+  if [ "$USE_PGBOUNCER" = "true" ]; then
+    echo "DB_PORT=6432" > ~/.env
+  else
+    echo "DB_PORT=5432" > ~/.env
+  fi
+ 
   # Needed for envsubst to work 
   export DNS_ZONE=${DNS_ZONE}
   # Set the hostnames completely with substituting ${DNS_ZONE}
   cat ~/.env | envsubst > .env && mv .env ~/.env
   # Add the environment config for following shells
   cat ~/.env | grep -v API_KEY | tee /etc/environment
-  
+
   # Reload the env since HOST_* and *_DOMAIN was added
   source ~/.env
   
diff --git a/image/README.md b/image/README.md
index 866af19..16e0e7c 100644
--- a/image/README.md
+++ b/image/README.md
@@ -18,6 +18,8 @@ To build the image, the following repositories need to be 
added as dist archives
 * [taler-merchant](https://git.taler.net/merchant.git): merchant.tar.gz
 * [taler-wallet-core](https://git.taler.net/wallet-core.git): wallet.tar.gz
 
+This is necessary since bootstrap seems not to work when creating the image.
+
 Replace `<G5K_USER> `and `<G5K_HOST>` in `taler-debian11.yaml` in the 
following line:
 `g5k_tar_path: 
"http://public.<G5K_HOST>.grid5000.fr/~<G5K_USER>/taler-debian11.tar.zst"`
 
diff --git a/image/taler-debian11.yaml b/image/taler-debian11.yaml
index ce70f84..50cac31 100644
--- a/image/taler-debian11.yaml
+++ b/image/taler-debian11.yaml
@@ -101,6 +101,10 @@ setup:
       - exec_in: |
          cd $${taler_path} 
 
+         git clone https://git.gnunet.org/gnunet.git
+         cd gnunet
+         ./bootstrap 
+
          tar -xvf $${gnunet_tar}
          cd gnunet-* 
          CFLAGS="-O0 -g" ./configure --enable-logging=verbose --prefix=/usr # 
--enable-sanitizer

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.