[taler-grid5k] 09/73: add config options for all exchange processes

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit fe1c69356205431071ac0b740b2168ce04375f30
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Thu Nov 25 14:54:21 2021 +0100

    add config options for all exchange processes
---
 README.md                                          |  5 ++-
 .../system/taler-exchange-aggregator.service       |  2 +-
 .../systemd/system/taler-exchange-closer.service   |  2 +-
 .../systemd/system/taler-exchange-httpd.service    |  2 +-
 .../systemd/system/taler-exchange-httpd@.service   |  2 +-
 .../system/taler-exchange-secmod-eddsa.service     |  2 +-
 .../system/taler-exchange-secmod-rsa.service       |  2 +-
 .../systemd/system/taler-exchange-transfer.service |  2 +-
 .../system/taler-exchange-wirewatch.service        |  2 +-
 .../system/taler-exchange-wirewatch@.service       |  2 +-
 experiment/README.md                               | 31 +++++++---------
 experiment/env                                     | 17 +++++++--
 experiment/scripts/monitor.sh                      | 43 +++++++++++++---------
 experiment/scripts/setup.sh                        |  2 +-
 14 files changed, 65 insertions(+), 51 deletions(-)

diff --git a/README.md b/README.md
index bf0d198..ed22ec8 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ which describes how the parts in there can be used.
 
 Contains all configurations and scripts needed
 for the experiment to run in the grid. They are setup to 
-work with (jFed)[https://jfed.ilabt.imec.be/].
+work with [jFed](https://jfed.ilabt.imec.be/).
 
 ### Image
 
@@ -31,5 +31,6 @@ data of experiments.
 ### Configs
 
 Contains the configurations for the applications in the environment.
-They will be copied and adjusted once an experiment is started.
+They will be adjusted copied to '/' (some make sure to add the correct 
directory strucutre) 
+once an experiment is started.
 
diff --git a/configs/usr/lib/systemd/system/taler-exchange-aggregator.service 
b/configs/usr/lib/systemd/system/taler-exchange-aggregator.service
index 453b516..9aba5a4 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-aggregator.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-aggregator.service
@@ -7,7 +7,7 @@ User=taler-exchange-aggregator
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=/usr/bin/taler-exchange-aggregator -c /etc/taler/taler.conf 
$EXCHANGE_ARGS
+ExecStart=/usr/bin/taler-exchange-aggregator -c /etc/taler/taler.conf 
$TALER_ARGS $AGGREGATOR_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=yes
diff --git a/configs/usr/lib/systemd/system/taler-exchange-closer.service 
b/configs/usr/lib/systemd/system/taler-exchange-closer.service
index 5d060f2..0728ce2 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-closer.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-closer.service
@@ -7,7 +7,7 @@ User=taler-exchange-closer
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=/usr/bin/taler-exchange-closer -c /etc/taler/taler.conf 
$EXCHANGE_ARGS
+ExecStart=/usr/bin/taler-exchange-closer -c /etc/taler/taler.conf $TALER_ARGS 
$CLOSER_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=yes
diff --git a/configs/usr/lib/systemd/system/taler-exchange-httpd.service 
b/configs/usr/lib/systemd/system/taler-exchange-httpd.service
index 52b60ad..2f4ec18 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-httpd.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-httpd.service
@@ -10,7 +10,7 @@ User=taler-exchange-httpd
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=<CMD_PREFIX_HERE> /usr/bin/taler-exchange-httpd -c 
/etc/taler/taler.conf $EXCHANGE_ARGS
+ExecStart=<CMD_PREFIX_HERE> /usr/bin/taler-exchange-httpd -c 
/etc/taler/taler.conf $TALER_ARGS $EXCHANGE_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=no
diff --git a/configs/usr/lib/systemd/system/taler-exchange-httpd@.service 
b/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
index 4c04144..37d82f5 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
@@ -10,7 +10,7 @@ User=taler-exchange-httpd
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=<CMD_PREFIX_HERE> /usr/bin/taler-exchange-httpd -c 
/etc/taler/taler.conf $EXCHANGE_ARGS
+ExecStart=<CMD_PREFIX_HERE> /usr/bin/taler-exchange-httpd -c 
/etc/taler/taler.conf $TALER_ARGS $EXCHANGE_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=no
diff --git a/configs/usr/lib/systemd/system/taler-exchange-secmod-eddsa.service 
b/configs/usr/lib/systemd/system/taler-exchange-secmod-eddsa.service
index eba283e..f96bee3 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-secmod-eddsa.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-secmod-eddsa.service
@@ -8,7 +8,7 @@ User=taler-exchange-secmod-eddsa
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=/usr/bin/taler-exchange-secmod-eddsa -c /etc/taler/taler.conf 
$EXCHANGE_ARGS
+ExecStart=/usr/bin/taler-exchange-secmod-eddsa -c /etc/taler/taler.conf 
$TALER_ARGS $EDDSA_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=no
diff --git a/configs/usr/lib/systemd/system/taler-exchange-secmod-rsa.service 
b/configs/usr/lib/systemd/system/taler-exchange-secmod-rsa.service
index 0265055..433fd5a 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-secmod-rsa.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-secmod-rsa.service
@@ -8,7 +8,7 @@ User=taler-exchange-secmod-rsa
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=/usr/bin/taler-exchange-secmod-rsa -c /etc/taler/taler.conf 
$EXCHANGE_ARGS
+ExecStart=/usr/bin/taler-exchange-secmod-rsa -c /etc/taler/taler.conf 
$TALER_ARGS $RSA_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=no
diff --git a/configs/usr/lib/systemd/system/taler-exchange-transfer.service 
b/configs/usr/lib/systemd/system/taler-exchange-transfer.service
index 5b6ee2f..325a37a 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-transfer.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-transfer.service
@@ -8,7 +8,7 @@ User=taler-exchange-wire
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=/usr/bin/taler-exchange-transfer -c /etc/taler/taler.conf 
$EXCHANGE_ARGS
+ExecStart=/usr/bin/taler-exchange-transfer -c /etc/taler/taler.conf 
$TALER_ARGS $TRANSFER_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=yes
diff --git a/configs/usr/lib/systemd/system/taler-exchange-wirewatch.service 
b/configs/usr/lib/systemd/system/taler-exchange-wirewatch.service
index 9e948c4..7efe7de 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-wirewatch.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-wirewatch.service
@@ -8,7 +8,7 @@ User=taler-exchange-wire
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=/usr/bin/taler-exchange-wirewatch -c /etc/taler/taler.conf 
$EXCHANGE_ARGS
+ExecStart=/usr/bin/taler-exchange-wirewatch -c /etc/taler/taler.conf 
$TALER_ARGS $WIREWATCH_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=yes
diff --git a/configs/usr/lib/systemd/system/taler-exchange-wirewatch@.service 
b/configs/usr/lib/systemd/system/taler-exchange-wirewatch@.service
index 9e948c4..7efe7de 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-wirewatch@.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-wirewatch@.service
@@ -8,7 +8,7 @@ User=taler-exchange-wire
 Type=simple
 Restart=on-failure
 EnvironmentFile=/etc/environment
-ExecStart=/usr/bin/taler-exchange-wirewatch -c /etc/taler/taler.conf 
$EXCHANGE_ARGS
+ExecStart=/usr/bin/taler-exchange-wirewatch -c /etc/taler/taler.conf 
$TALER_ARGS $WIREWATCH_ARGS
 StandardOutput=journal
 StandardError=journal
 PrivateTmp=yes
diff --git a/experiment/README.md b/experiment/README.md
index 599beb3..bc01f41 100644
--- a/experiment/README.md
+++ b/experiment/README.md
@@ -3,8 +3,9 @@
 ## Files
 
 * experiment-specitication.yml: [ESpec](https://jfed.ilabt.imec.be/espec/) for 
jFed 
-* infra.rspec: Contains all nodes for the Taler infrastructure
-* wallets.rspec: Contains a set of nodes for wallets
+* taler.rspec: Complete set of nodes to run an experiment
+* infra.rspec: Contains all nodes for the Taler infrastructure only
+* wallets.rspec: Contains an additional set of nodes for wallets
 * env: template file to add enviroment variables needed for the experiment
 * scripts: Bash scripts which will be run in the experiment
   
@@ -20,21 +21,19 @@ To successfully run an experiment the following steps must 
be made:
 * Copy the environment default configuration `env` to `.env`
 * Read through `.env` and define the missing variables
 * Start jFed Experimenter GUI 
-* Load infra.rspec and click Run
+* Load taler.rspec and click Run
 * Specify the experiment name and time
-* Wait until infra.rspec is allocated successfully
-* Configure `.env` and set the hosts - the allocated nodes can be found in the 
tab `RSpec Viewer`
-  (this can be fixed once we can use one rspec only)
-* Wait until the nodes are ready
-* Click (Re)Run ESpec for the infra job (use the type Directory and select 
this directory (experiment))
+* Wait until taler.rspec is allocated successfully and nodes are ready
+* Click (Re)Run ESpec for the job (use the type Directory and select this 
directory (experiment))
+
+### Add addtional Wallet Hosts
+
+TODO - DOES NOT WORK YET - DNS host needs to be known ..
+
 * Load wallets.rspec into jFed and click Run
-* Click (Re)Run ESpec for the wallets job (type directory)
+* Click (Re)Run ESpec for the wallet job (type directory)
 * Run `talet-perf update prometheus` on any deployed node to make prometheus 
aware of the freshly added wallets
 
-**NOTE** If for some reason the espec needs to be rerun, all uploaded files 
need to be deleted currently.
-         This can be achieved on all nodes by pressing `Multi Command` and 
specifying the command:
-         `rm -rf scripts .env`
-
 ## Actions in running Experiment
 
 To add more exchange processes run `taler-perf stop ecxchange <NUM>` on any 
node
@@ -54,8 +53,4 @@ plots based on a configuration. Please refer to the README 
located in the specif
 The logs of the taler applications will be moved periodically to 
`/home/<G5K_USER>/taler-logs`. 
 They can then be retrieved using scp to the respective site they where running 
on
 (e.g. nancy/taler-logs for grvingt and others).
-Make sure to take them away before restarting an experiment, since they will 
be deleted in a new experiment.
-
-## Notes
-
-Hopefully the bind dns gets useless once the experiment can be run in one 
espec.
+Make sure to back them up before restarting an experiment, since they will be 
deleted in a new one.
diff --git a/experiment/env b/experiment/env
index 04fae1a..02b83b0 100644
--- a/experiment/env
+++ b/experiment/env
@@ -23,10 +23,19 @@ MERCHANT_DOMAIN=merchant.${DNS_ZONE}
    PROXY_DOMAIN=proxy.${DNS_ZONE}
   WALLET_DOMAIN=wallet.*.${DNS_ZONE}
 
-# Arguments to pass to taler-exchange-* binaries at startup.
-EXCHANGE_ARGS="-L INFO"
+# Arguments to pass to all taler-exchange-* binaries at startup.
+# E.g. -L INFO
+     TALER_ARGS="-L INFO"
+# Service specific arguments to pass at startup
+AGGREGATOR_ARGS=""
+    CLOSER_ARGS=""
+  EXCHANGE_ARGS=""
+  TRANSFER_ARGS=""
+ WIREWATCH_ARGS=""
+     EDDSA_ARGS=""
+       RSA_ARGS=""
 # Additional arguments to prepend to taler-exchange-httpd.service
-# files ExecStart command, e.g. valgrind --log-file=/tmp/exch%p.log
+# files ExecStart command, e.g. /usr/bin/valgrind 
--log-file=/tmp/exchange-%N.log
 EXCHANGE_CMD_PREFIX=""
 
 # Datasource names registered in the grafana instance
@@ -66,6 +75,8 @@ NUM_WIREWATTCH_PROCESSES=1
 ENABLE_EXPORTERS=true
 
 # Host where grafana is reachable including scheme and port.
+# If this one or the key is left empty, no grafana instance
+# is assumed, Loki and Prometheus will run anyway since they are used by 
rsyslog.
 GRAFANA_HOST=https://147.87.255.221:3000
 # API key for grafana
 # Needs admin level since datasources need to be updated.
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index c861bce..cab6045 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -2,10 +2,24 @@
 set -ex
 
 source ~/scripts/helpers.sh
-source ~/.env
 
-if [[ "$1" == "init" ]];
-then
+function update_datasource() {
+  ID=$(jq --arg name "$1" '.[] | select(.name == $name) | .id' ds.json)
+
+  jq --arg url "https://$(hostname | cut -d "." -f 1,2 
-).$2.proxy.grid5000.fr" \
+     --arg name "$1" \
+     '.[] | select(.name == $name) | .url = $url' \
+     ds.json | tee /dev/tty | curl -X PUT -k -f -d @- \
+       -H "${AUTH_HEADER}" \
+       -H "Content-Type: application/json" \
+       -H "Accept: application/json" \
+        "${GRAFANA_API}/datasources/${ID}"
+}
+
+function update_grafana() {
+  if [ -z "${GRAFANA_HOST}" || -z ${GRAFANA_API_KEY} ]; then
+    return
+  fi
   AUTH_HEADER="Authorization: Bearer ${GRAFANA_API_KEY}"
   GRAFANA_API="${GRAFANA_HOST}/api"
   
@@ -17,21 +31,14 @@ then
     exit $?
   fi
   
-  function update_datasource() {
-    ID=$(jq --arg name "$1" '.[] | select(.name == $name) | .id' ds.json)
-  
-    jq --arg url "https://$(hostname | cut -d "." -f 1,2 
-).$2.proxy.grid5000.fr" \
-       --arg name "$1" \
-       '.[] | select(.name == $name) | .url = $url' \
-       ds.json | tee /dev/tty | curl -X PUT -k -f -d @- \
-         -H "${AUTH_HEADER}" \
-         -H "Content-Type: application/json" \
-         -H "Accept: application/json" \
-          "${GRAFANA_API}/datasources/${ID}"
-  }
-  
   update_datasource "${PROMETHEUS_DATASOURCE_NAME}" 
"${PROMETHEUS_G5K_PROXY_PORT}"
   update_datasource "${LOKI_DATASOURCE_NAME}" "${LOKI_G5K_PROXY_PORT}"
+}
+  
+if [[ "$1" == "init" ]];
+then
+
+  update_grafana
   
   systemctl restart loki \
                  promtail
@@ -53,7 +60,7 @@ then
     cat /etc/monitor/node-exporters.yaml.tpl | envsubst >> 
/etc/monitor/prometheus.yaml
     for WALLET in $(get_wallet_domains);
     do
-      sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ - 
'wallet.${WALLET}.${DNS_ZONE}:9100'" \
+      sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ - 
'${WALLET_DOMAIN//\*/${WALLET}}:9100'" \
            /etc/monitor/prometheus.yaml
     done
   fi
@@ -69,7 +76,7 @@ else
   then
     for WALLET in $(get_wallet_domains);
     do
-      if ! grep -q "wallet.${WALLET}.${DNS_ZONE}:9100" 
/etc/monitor/prometheus.yaml; 
+      if ! grep -q "${WALLET_DOMAIN//\*/${WALLET}}:9100" 
/etc/monitor/prometheus.yaml; 
       then
         sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ - 
'wallet.${WALLET}.${DNS_ZONE}:9100'" \
                 /etc/monitor/prometheus.yaml
diff --git a/experiment/scripts/setup.sh b/experiment/scripts/setup.sh
index 739efef..6971c93 100644
--- a/experiment/scripts/setup.sh
+++ b/experiment/scripts/setup.sh
@@ -125,4 +125,4 @@ setup_environment
 setup_config
 setup_dns
 
-exit 0
+exec ~/scripts/createusers.sh

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.