[taler-grid5k] 58/73: sanitize scripts

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit 93009e27e8d6b0df70e5409f2be096baa013a33a
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Thu Dec 9 20:36:01 2021 +0100

    sanitize scripts
---
 README.md                        | 10 +++++++
 experiment/scripts/database.sh   |  2 +-
 experiment/scripts/exchange.sh   | 59 +++++++++++++++++++++++++---------------
 experiment/scripts/helpers.sh    | 22 +++++++++++++++
 experiment/scripts/monitor.sh    | 41 +++++++++++++++++++++-------
 experiment/scripts/proxy.sh      | 56 +++++++++++++++++++++++++++++---------
 experiment/scripts/taler-perf.sh | 58 +++++++++++++++++++++++++++++++++++++--
 experiment/scripts/wallet.sh     | 23 ++++------------
 8 files changed, 204 insertions(+), 67 deletions(-)

diff --git a/README.md b/README.md
index 7a86c9b..3f170a0 100644
--- a/README.md
+++ b/README.md
@@ -35,3 +35,13 @@ Contains the configurations for the applications in the 
environment.
 They will be adjusted copied to '/' (some make sure to add the correct 
directory strucutre) 
 once an experiment is started.
 
+## Quick Start
+
+To run an experiment, you must
+
+* (optionally) have a grafana instance
+* Make sure the environment exists in the public direcory which is configured 
in 
+  `experiment/taler.rspec`. If its not in the grid, use `image/README.md` or 
`docker/README.md`
+  to see how to build such an environment.
+* Read `experiment/README.md` for instructions on how to run an experiment 
inside the grid 
+  
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index c07517f..3f0631e 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -109,7 +109,7 @@ function setup_config() {
 
 if [[ "$1" == "init" ]]; then
   setup_config
-  setup_disks
+#  setup_disks
 fi
 
 systemctl restart postgresql
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index 4d747c0..6300b45 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eux
 
+source ~/scripts/helpers.sh
+
 function setup_config() {
   sed -i 
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}:${DB_PORT}/${DB_NAME}\g"
 \
        /etc/taler/secrets/exchange-db.secret.conf
@@ -33,9 +35,7 @@ function setup_exchange() {
   taler-exchange-offline upload < fee-res.json
 }
 
-if [[ "$1" == "init" ]];
-then   
-  source ~/scripts/helpers.sh
+function init_exchanges() {
   restart_rsyslog
 
   setup_config
@@ -44,29 +44,44 @@ then
   # Wait another second to make sure user has permissions
   sleep 5
 
-  NUM_PROCESSES=$((${NUM_EXCHANGE_PROCESSES:-10}-1))
-
   setup_exchange
+}
+
+function start_exchanges() {
+  RUNNING=$(ps -aux | grep "[taler]-exchange-httpd" | wc -l)
+
+  for i in $(seq ${1}); do
+    let "i+=${RUNNING}-1" || true
+    let "i+=10000"
+    systemctl restart taler-exchange-httpd@"${i}".socket \
+                      taler-exchange-httpd@"${i}".service
+    sleep 0.05
+  done
+}
 
-  # || true is needed since when 1-1 is run let returns an error
-  let "NUM_WIREWATCH_PROCESSES-=1" || true
-  for i in $(seq ${NUM_WIREWATCH_PROCESSES}); 
-  do
-    systemctl restart taler-exchange-wirewatch@${i}.service
+function start_wirewatchers() {
+  for i in $(seq ${1}); do
+    systemctl restart taler-exchange-wirewatch@"${i}".service
   done
+}
 
-else
-  NUM_PROCESSES=$1
-fi
-  
-RUNNING=$(ps -aux | grep "[taler]-exchange-httpd" | wc -l)
+function stop_exchanges() {
+  stop_numbered_services "taler-exchange-httpd" $1
+}
 
-for i in $(seq ${NUM_PROCESSES});
-do
-  let "i+=${RUNNING}-1" || true
-  let "i+=10000"
-  systemctl restart taler-exchange-httpd@"${i}".socket 
taler-exchange-httpd@"${i}".service
-  sleep 0.05
-done
+case $1 in
+  init)
+    init_exchanges
+    # TODO adjust when started manually and not with .target
+    start_exchanges "$((${NUM_EXCHANGE_PROCESSES}-1))"
+    start_wirewatchers "$((${NUM_WIREWATCH_PROCESSES}-1))"
+    ;;
+  start)
+    start_exchanges $2
+    ;;
+  stop)
+    stop_exchanges $2
+    ;;
+esac
 
 exit 0
diff --git a/experiment/scripts/helpers.sh b/experiment/scripts/helpers.sh
index d6a05c0..5557a5c 100755
--- a/experiment/scripts/helpers.sh
+++ b/experiment/scripts/helpers.sh
@@ -82,6 +82,28 @@ function restart_rsyslog() {
   systemctl restart rsyslog
 }
 
+# Stop services which have the form name@number.service
+# will stop the services starting from the biggest numbers
+# Expects the following arguments:
+#   $1: service base name, e.g. taler-exchange-httpd
+#   $2: amount of services to stop
+function stop_numbered_services() {
+  # Get all running processes of the service $1,
+  # extract their number, limit by number of 
+  # to stop and stop those
+  N=$(\
+    systemctl status "${1}"@*.service | \
+    grep -E "${1}@[0-9]+.service -" | \
+    awk '{print $2}' | \
+    sed 's/[^0-9]*//g' | \
+    sort -n -r | \
+    head -n $2 \
+  )
+  for i in ${N}; do
+    systemctl stop ${1}@${i}.service
+  done
+}
+
 function get_wallet_domains() {
   IFS=$'\n' read -r -d '' -a WALLETS < <(\
     dig -t AXFR "${DNS_ZONE}" "@${DNS_HOSTS}" \
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index 8d9dd56..9992827 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -89,7 +89,21 @@ function add_exchanges_to_prometheus() {
   fi
 }
 
-function init() {
+function remove_exchanges_from_prometheus() {
+  TO_STOP=$(\
+    get_exchanges | \
+    tr " " "\n" | \
+    sort -t ":" -k 2 -n -r | \
+    head -n ${1}\
+  ) 
+
+  for EXCH in ${TO_STOP}; do
+    sed -i "/${EXCH}/d" \
+            /etc/monitor/prometheus.yaml
+  done
+}
+
+function init_monitor() {
 
   update_grafana
   
@@ -109,15 +123,22 @@ function init() {
   add_exchanges_to_prometheus "init"
 
   systemctl restart prometheus-nginx-exporter \
-                    prometheus-postgres-exporter 
+                    prometheus-postgres-exporter \
+                    prometheus
 
 }
   
-if [[ "$1" == "init" ]]; then
-  init
-else 
-  add_wallet_nodes_to_prometheus ""
-  add_exchanges_to_prometheus ""
-fi
-
-systemctl restart prometheus
+case $1 in
+  init)
+    init_monitor
+    ;;
+  start)
+    add_wallet_nodes_to_prometheus ""
+    add_exchanges_to_prometheus ""
+    ;;
+  stop-exchanges)
+    remove_exchanges_from_prometheus $2
+    ;;
+esac
+
+systemctl reload prometheus
diff --git a/experiment/scripts/proxy.sh b/experiment/scripts/proxy.sh
index 5e04919..f6543c1 100755
--- a/experiment/scripts/proxy.sh
+++ b/experiment/scripts/proxy.sh
@@ -1,28 +1,26 @@
 #!/bin/bash
 set -eux
 
-if [[ "$1" == "init" ]];
-then
   source ~/scripts/helpers.sh
   
-  sed -e "/<SERVERS_HERE>/a \ \ server ${EXCHANGE_DOMAIN}:80;" \
-      -e "s/<MONITOR_HOST_HERE>/${MONITOR_DOMAIN}/g" \
-          /etc/nginx/sites-available/proxy > /etc/nginx/sites-enabled/proxy
+function setup_config() {
+  sed "/<SERVERS_HERE>/a \ \ server ${EXCHANGE_DOMAIN}:80;" \
+       /etc/nginx/sites-available/proxy > /etc/nginx/sites-enabled/proxy
   
   # We want n processes, one is already enabled on port 80
-  for (( i=1; i < ${NUM_EXCHANGE_PROCESSES}; i++ ))
-  do
+  for i in $(seq $((${NUM_EXCHANGE_PROCESSES:-10}-1)) ); do
     let "PORT=i+10000"
     sed -i "/<SERVERS_HERE>/a \ \ server ${EXCHANGE_DOMAIN}:${PORT};" \
           /etc/nginx/sites-enabled/proxy
   done
   
+  # Nginx will log to our rsyslog directly an then from there it will be 
+  # redirected to promtail - there was an issue doing it directly that's
+  # why it is done this way
   sed -i -e '/module(load="imudp")/s/^#//g' \
          -e '/input(type="imudp" port="514")/s/^#//g' \
           /etc/rsyslog.conf
   
-  systemctl restart rsyslog
-  
   echo "
   fs.file-max=50000
   " >> /etc/sysctl.conf
@@ -33,23 +31,55 @@ then
   " >> /etc/security/limits.conf
   
   sysctl -p
+}
+
+function init_proxy() {
+  setup_config
+
+  restart_rsyslog
   
   # Nginx does not start until the destination server is reachable - wait here
   # nginx: [emerg] host not found in upstream "exch.perf.taler" ...
   wait_for_keys "${EXCHANGE_DOMAIN}"
   
   systemctl restart nginx
-else
+}
 
+function add_exchanges() {
   ADDED=$(grep -r "  server ${EXCHANGE_DOMAIN}:" 
/etc/nginx/sites-enabled/proxy | wc -l)
   
-  for i in $(seq $1); do
+  for i in $(seq ${1}); do
     let "i+=${ADDED}-1"
     let "i+=10000"
     sed -i "/<SERVERS_HERE>/a \ \ server ${EXCHANGE_DOMAIN}:${i};" \
-          /etc/nginx/sites-enabled/proxy
+            /etc/nginx/sites-enabled/proxy
+  done
+}
+
+function remove_exchanges() {
+  TO_STOP=$(\
+    get_exchanges | \
+    tr " " "\n" | \
+    sort -t ":" -k 2 -n -r | \
+    head -n ${1}\
+  ) 
+
+  for EXCH in ${TO_STOP}; do
+    sed -i "/${EXCH};/d" \
+            /etc/nginx/sites-enabled/proxy
   done
+}
 
-fi
+case $1 in
+  init)
+    init_proxy
+    ;;
+  start)
+    add_exchanges $2
+    ;;
+  stop)
+    remove_exchanges $2
+    ;;
+esac
 
 systemctl reload nginx
diff --git a/experiment/scripts/taler-perf.sh b/experiment/scripts/taler-perf.sh
index 253038a..8171e0b 100644
--- a/experiment/scripts/taler-perf.sh
+++ b/experiment/scripts/taler-perf.sh
@@ -6,7 +6,8 @@ source ~/scripts/helpers.sh
 function update_processes() {
   case "$1" in 
     prometheus)
-      ssh -A -o StrictHostKeyChecking=no ${MONITOR_DOMAIN} "/bin/bash -c 
/root/scripts/monitor.sh"
+      ssh -A -o StrictHostKeyChecking=no ${MONITOR_DOMAIN} \
+                "/bin/bash /root/scripts/monitor.sh start"
       ;;
     *)
       echo "Unknown argument '$1' for function ${FUNCNAME[0]}"
@@ -33,12 +34,24 @@ function stop_wallets() {
 
 function start_exchanges() {
   ssh -o StrictHostKeyChecking=no ${EXCHANGE_DOMAIN} \
-         "/bin/bash /root/scripts/exchange.sh ${1:-${NUM_EXCHANGE_PROCESSES}}" 
+         "/bin/bash /root/scripts/exchange.sh start 
${1:-${NUM_EXCHANGE_PROCESSES}}" 
   ssh -o StrictHostKeyChecking=no ${PROXY_DOMAIN} \
-         "/bin/bash /root/scripts/proxy.sh ${1:-${NUM_EXCHANGE_PROCESSES}}"
+         "/bin/bash /root/scripts/proxy.sh start 
${1:-${NUM_EXCHANGE_PROCESSES}}"
   update_processes "prometheus"
 }
 
+function stop_exchanges() {
+  # must remove the exchange form the monitor host before the one from the 
nginx
+  # since helpers.sh gets all exchanges from the nginx config
+  ssh -A -o StrictHostKeyChecking=no ${MONITOR_DOMAIN} \
+         "/bin/bash /root/scripts/monitor.sh stop-exchanges 
${1:-${NUM_EXCHANGE_PROCESSES}}"
+  ssh -A -o StrictHostKeyChecking=no ${PROXY_DOMAIN} \
+         "/bin/bash /root/scripts/proxy.sh stop 
${1:-${NUM_EXCHANGE_PROCESSES}}"
+  sleep 5
+  ssh -o StrictHostKeyChecking=no ${EXCHANGE_DOMAIN} \
+         "/bin/bash /root/scripts/exchange.sh stop 
${1:-${NUM_EXCHANGE_PROCESSES}}" 
+}
+
 function start_processes() {
   case "$1" in
     wallet)
@@ -59,6 +72,9 @@ function stop_processes() {
     wallet)
       stop_wallets $2
       ;;
+    exchange)
+      stop_exchanges $2
+      ;;
     *)
       echo "Unknown argument '$1' for function ${FUNCNAME[0]}"
       echo "Usage: stop [wallet] NUM"
@@ -66,6 +82,39 @@ function stop_processes() {
    esac
 }
 
+function rebuild() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --exchange|-e)
+        EXCHANGE_COMMIT_SHA=$2
+       shift 2
+       ;;
+      --gnunet|-g)
+        GNUNET_COMMIT_SHA=$2
+       shift 2
+       ;;
+      --wallet|-w)
+        WALLET_COMMIT_SHA=$2
+       shift 2
+       ;;
+      --merchant|-m)
+        MERCHANT_COMMIT_SHA=$2
+       shift 2
+       ;;
+      *)
+        echo "Unkown argument $1"
+       echo "Usage rebuilt [target]"
+       echo "Targets: "
+       echo "-e|--exchange <commit-sha>"
+        echo "-g|--gnunet <commit-sha>"
+       echo "-w|--wallet <commit-sha>"
+       echo "-m|--merchant <commit-sha>"
+       exit 1
+    esac
+  done
+  # TODO
+}
+
 case "$1" in
   start)
     shift
@@ -78,6 +127,9 @@ case "$1" in
   update)
     shift
     update_processes $@
+  rebuild)
+    shift
+    rebuild
     ;;
 esac
 
diff --git a/experiment/scripts/wallet.sh b/experiment/scripts/wallet.sh
index aa38e98..29df965 100755
--- a/experiment/scripts/wallet.sh
+++ b/experiment/scripts/wallet.sh
@@ -1,10 +1,9 @@
 #!/bin/bash
 set -eux
 
-NUM_PROCESSES=${NUM_WALLET_PROCESSES:-10}
+source ~/scripts/helpers.sh
 
 function init_wallets() {
-  source ~/scripts/helpers.sh
   restart_rsyslog
   wait_for_keys "${PROXY_DOMAIN}"
   sleep 5
@@ -29,33 +28,21 @@ function start_wallets() {
   done
 }
 
-function stop_wallet() {
+function stop_wallets() {
   if [[ "$1" == "all" ]]; then
     systemctl stop taler-wallet@*.service
   else
-    # Get all running wallet processes,
-    # extract their number, limit by number of 
-    # wallets to stop and stop those
-    N=$(\
-      systemctl status taler-wallet@*.service | \
-      grep -E "taler-wallet@[0-9]+.service -" | \
-      sed 's/[^0-9]*//g' | \
-      sort -n -r | \
-      head -n $1 \
-    )
-    for i in ${N}; do
-      systemctl stop taler-wallet@${i}.service
-    done
+    stop_numbered_services "taler-wallet" $1
   fi
 }
 
 case "$1" in
   init)
     init_wallets
-    start_wallets $NUM_PROCESSES
+    start_wallets ${NUM_WALLET_PROCESSES:-10}
     ;;
   stop)
-    stop_wallet $2
+    stop_wallets $2
     ;;
   start)
     start_wallets $2

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.