[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[taler-grid5k] 64/73: comment scripts and add help
|
From: |
gnunet |
|
Subject: |
[taler-grid5k] 64/73: comment scripts and add help |
|
Date: |
Tue, 14 Dec 2021 15:10:46 +0100 |
This is an automated email from the git hooks/post-receive script.
marco-boss pushed a commit to branch master
in repository grid5k.
commit 2f71d0e70ac0db749c866e1fccd69b1e98532dbd
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Sat Dec 11 15:54:46 2021 +0100
comment scripts and add help
---
...r-logbackup.service => taler-logrotate.service} | 2 +-
...taler-logbackup.timer => taler-logrotate.timer} | 4 +-
experiment/scripts/bank.sh | 26 +++++++-
experiment/scripts/benchmark.sh | 61 ++++++++++++-----
experiment/scripts/createusers.sh | 8 ++-
experiment/scripts/database.sh | 46 +++++++++----
experiment/scripts/exchange.sh | 42 +++++++++++-
experiment/scripts/helpers.sh | 74 ++++++++++++++++-----
experiment/scripts/install.sh | 8 +++
experiment/scripts/merchant.sh | 76 ++++++++++++++--------
experiment/scripts/monitor.sh | 48 ++++++++++++--
experiment/scripts/ping.sh | 24 ++++++-
experiment/scripts/proxy.sh | 30 ++++++++-
experiment/scripts/run.sh | 15 +++--
experiment/scripts/setup.sh | 34 +++++++++-
experiment/scripts/taler-perf.sh | 4 +-
experiment/scripts/wallet.sh | 30 +++++++--
17 files changed, 422 insertions(+), 110 deletions(-)
diff --git a/configs/usr/lib/systemd/system/taler-logbackup.service
b/configs/usr/lib/systemd/system/taler-logrotate.service
similarity index 86%
rename from configs/usr/lib/systemd/system/taler-logbackup.service
rename to configs/usr/lib/systemd/system/taler-logrotate.service
index 45001c2..0dd22af 100644
--- a/configs/usr/lib/systemd/system/taler-logbackup.service
+++ b/configs/usr/lib/systemd/system/taler-logrotate.service
@@ -1,6 +1,6 @@
[Unit]
Description=Compresses and moves taler logs to the grid5000 nfs
-Wants=taler-logbackup.timer
+Wants=taler-logrotate.timer
[Service]
Type=oneshot
diff --git a/configs/usr/lib/systemd/system/taler-logbackup.timer
b/configs/usr/lib/systemd/system/taler-logrotate.timer
similarity index 68%
rename from configs/usr/lib/systemd/system/taler-logbackup.timer
rename to configs/usr/lib/systemd/system/taler-logrotate.timer
index fe33a6a..7cb2ae3 100644
--- a/configs/usr/lib/systemd/system/taler-logbackup.timer
+++ b/configs/usr/lib/systemd/system/taler-logrotate.timer
@@ -1,9 +1,9 @@
[Unit]
Description=Compresses and moves taler logs to the grid5000 nfs
-Requires=taler-logbackup.service
+Requires=taler-logrotate.service
[Timer]
-Unit=taler-logbackup.service
+Unit=taler-logrotate.service
OnCalendar=*:0/15
[Install]
diff --git a/experiment/scripts/bank.sh b/experiment/scripts/bank.sh
index 4723a8c..bfdc483 100755
--- a/experiment/scripts/bank.sh
+++ b/experiment/scripts/bank.sh
@@ -1,10 +1,30 @@
#!/bin/bash
set -eux
-
source ~/scripts/helpers.sh
-restart_rsyslog
+INFO_MSG="
+Setup the bank node for the experiments
+(Start the taler-fakebank)
+"
+OPT_MSG="
+init:
+ Configure and start the taler-fakebank
+"
+
+# Start the taler-fakebank
+function init_bank() {
+ restart_rsyslog
+
+ systemctl restart taler-fakebank.service
+}
-systemctl restart taler-fakebank.service
+case $1 in
+ init)
+ init_bank
+ ;;
+ *)
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+ ;;
+esac
exit 0
diff --git a/experiment/scripts/benchmark.sh b/experiment/scripts/benchmark.sh
index 5733eac..be5bb16 100755
--- a/experiment/scripts/benchmark.sh
+++ b/experiment/scripts/benchmark.sh
@@ -1,22 +1,47 @@
#!/bin/bash
+set -eu
+source ~/scripts/helpers.sh
-LOG_LEVEL=ERROR
+INFO_MSG="
+Start a wallet benchmark loop with 100'000 iterations
+"
+OPT_MSG="
+<N>:
+ Any number
+ If it is dividable by 10 then the wallet will log in INFO level
+"
-# Every thenth wallet should log messages
-if ! (($1 % 10)) || [ $1 == "1" ]; then
- LOG_LEVEL=INFO
-fi
+# Start a wallet benchmark loop
+function start_wallet_bench() {
+ LOG_LEVEL=ERROR
+
+ # Every thenth wallet should log messages
+ if ! (($1 % 10)) || [ $1 == "1" ]; then
+ LOG_LEVEL=INFO
+ fi
+
+ taler-wallet-cli \
+ -L ${LOG_LEVEL} \
+ advanced bench1 \
+ --config-json "
+ {
+ \"exchange\": \"http://${PROXY_DOMAIN}/\";,
+ \"bank\": \"http://${BANK_DOMAIN}/\";,
+ \"currency\": \"KUDOS\",
+ \"payto\": \"payto://x-taler-bank/${BANK_DOMAIN}/foo\",
+ \"iterations\": 100000,
+ \"deposits\": 10,
+ \"restartAfter\": 2
+ }"
+}
-taler-wallet-cli \
- -L ${LOG_LEVEL} \
- advanced bench1 \
- --config-json "
-{
- \"exchange\": \"http://${PROXY_DOMAIN}/\";,
- \"bank\": \"http://${BANK_DOMAIN}/\";,
- \"currency\": \"KUDOS\",
- \"payto\": \"payto://x-taler-bank/${BANK_DOMAIN}/foo\",
- \"iterations\": 100000,
- \"deposits\": 10,
- \"restartAfter\": 2
-}"
+case $1 in
+ [0-9]*)
+ start_wallet_bench $1
+ ;;
+ *)
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+ ;;
+esac
+
+exit 0
diff --git a/experiment/scripts/createusers.sh
b/experiment/scripts/createusers.sh
index a7de2a7..a5b3705 100755
--- a/experiment/scripts/createusers.sh
+++ b/experiment/scripts/createusers.sh
@@ -1,9 +1,15 @@
#!/bin/bash
+# Create all necessary users and groups
+# for the taler applications.
+# (normaly done automatically when installing from packages)
+#
+# Usage: ./createusers.sh
set -e
-. /usr/share/debconf/confmodule
+source /usr/share/debconf/confmodule
TALER_HOME="/var/lib/taler"
+
GROUPNAME=taler-exchange-secmod
DBGROUPNAME=taler-exchange-db
EUSERNAME=taler-exchange-httpd
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 706a0a7..68ccaa2 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -1,9 +1,16 @@
#!/bin/bash
set -eux
-
source ~/scripts/helpers.sh
-# move to tmp to prevent change dir errors
+INFO_MSG="
+Setup the database node (start postgresql)
+"
+OPT_MSG="
+init:
+ Initialize and start the taler database
+"
+
+# move to tmp to prevent change directory errors
cd /tmp
function setup_disks() {
@@ -19,11 +26,12 @@ function setup_disks() {
fi
}
+# Setup the postgresql configuration
function setup_config() {
sed -i "s\<DB_URL_HERE>\postgresql:///${DB_NAME}\g" \
/etc/taler/secrets/exchange-db.secret.conf
- # Enable password for taler since this is commonly the case
+ # Enable password for taler since this is the case in real world deployments
# For the postgres user do not enable authentication (used in metrics)
if ! grep -q "host all ${DB_USER} 127.16.0.0/12 md5" \
/etc/postgresql/13/main/pg_hba.conf; then
@@ -33,22 +41,25 @@ function setup_config() {
" >> /etc/postgresql/13/main/pg_hba.conf
fi
+ # Get hardware info to tune in postgresql.conf
SHARED_MEM=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) / 3 ))
CACHE_SIZE=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) * 3/4))
NUM_CPU=$(lscpu | grep "CPU(s)" | head -n 1 | awk '{print $2}')
+ # Enable huge pages
# Size for huge_pages =~ shared_buffers * 1.25 so that there is enough
VM_PEAK=$((${SHARED_MEM} * 10/8))
HUGE_PAGES_SIZE=$(grep ^Hugepagesize /proc/meminfo | awk '{print $2}')
NUM_PAGES=$((${VM_PEAK} / ${HUGE_PAGES_SIZE}))
-
if ! grep -q "vm.nr_hugepages'" /etc/sysctl.conf; then
echo "vm.nr_hugepages=${NUM_PAGES}" >> /etc/sysctl.conf
sysctl -p
fi
+ # Configure postgres with an additional file and include this
+ # in the main configuration
echo "
listen_addresses='*'
log_destination=syslog
@@ -99,16 +110,16 @@ function setup_config() {
echo "include = 'exchange.conf'" >> \
/etc/postgresql/13/main/postgresql.conf
fi
-
- sed -i -e "s/<DB_USER_HERE>/${DB_USER}/g" \
- -e "s/<DB_PASSWORD_HERE>/${DB_PASSWORD}/g" \
- /etc/pgbouncer/userlist.txt
- sed -i -e "s/<DB_NAME_HERE>/${DB_NAME}/g" \
- /etc/pgbouncer/pgbouncer.ini
}
+# Configure and start pgBouncer if $USE_PGBOUNCER is true
function setup_pgbouncer() {
if [ "${USE_PGBOUNCER}" = "true" ]; then
+ sed -i -e "s/<DB_USER_HERE>/${DB_USER}/g" \
+ -e "s/<DB_PASSWORD_HERE>/${DB_PASSWORD}/g" \
+ /etc/pgbouncer/userlist.txt
+ sed -i -e "s/<DB_NAME_HERE>/${DB_NAME}/g" \
+ /etc/pgbouncer/pgbouncer.ini
# pgbouncer does not cleanup those sometimes
rm -f /var/run/postgresql/pgbouncer.pid
rm -f /var/run/postgresql/.s.PGSQL.6432
@@ -116,9 +127,11 @@ function setup_pgbouncer() {
fi
}
+# Initialize the database for taler exchange
function init_db() {
systemctl restart postgresql
+ # Create the role taler-exchange-httpd and the database
su postgres << EOF
psql postgres -tAc "SELECT 1 FROM pg_roles WHERE
rolname='taler-exchange-httpd'" | \
grep -q 1 || \
@@ -132,15 +145,17 @@ EOF
# sudo -u taler-exchange-httpd taler-exchange-dbinit -s || true
sudo -u taler-exchange-httpd taler-exchange-dbinit
+ # Create the remote user "$DB_USER" and load pg_stat_statements for metrics
su postgres << EOF
psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='${DB_USER}'" | \
- grep -q 1 \
- || psql << END
- CREATE USER "${DB_USER}" with encrypted password '${DB_PASSWORD}';
- CREATE EXTENSION pg_stat_statements;
+ grep -q 1 || \
+ psql << END
+ CREATE USER "${DB_USER}" with encrypted password '${DB_PASSWORD}';
+ CREATE EXTENSION pg_stat_statements;
END
EOF
+ # Grant access to the databse to the remote user
su taler-exchange-httpd -s /bin/bash << EOF
psql -d "${DB_NAME}"
GRANT SELECT,INSERT,UPDATE ON ALL TABLES IN SCHEMA public TO "${DB_USER}";
@@ -156,6 +171,9 @@ case ${1} in
setup_pgbouncer
restart_rsyslog
;;
+ *)
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+ ;;
esac
exit 0
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index d896d62..2d4f5bc 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -1,8 +1,31 @@
#!/bin/bash
set -eux
-
source ~/scripts/helpers.sh
+INFO_MSG="
+Setup the Exchange node
+Start taler-exchnage-*
+-----
+ could change later to single nodes
+ for aggregator, wirewatch etc..
+-----
+Each exchange-http daemon, will get its own port starting from 10001,
+unless the first, this one will get port 80
+"
+
+OPT_MSG="
+init:
+ Initialize the applications and start them
+ uses NUM_EXCHANGE_PROCESSES and NUM_WIREWATCH_PROCESSES
+
+start NUM:
+ Start another NUM exchange-http daemons
+
+stop NUM:
+ Stop NUM exchange-httpd daemons
+"
+
+# Setup the configuration in /etc/taler
function setup_config() {
sed -i
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}:${DB_PORT}/${DB_NAME}\g"
\
/etc/taler/secrets/exchange-db.secret.conf
@@ -19,6 +42,7 @@ function setup_config() {
/etc/taler/conf.d/exchange-business.conf
}
+# Setup the exchange with the taler-exchange-offline signing procedure
function setup_exchange() {
systemctl restart taler-exchange.target
@@ -35,6 +59,8 @@ function setup_exchange() {
taler-exchange-offline upload < fee-res.json
}
+# Initialize all stuff needed
+# logs, configs, exchanges
function init_exchanges() {
restart_rsyslog
@@ -47,26 +73,37 @@ function init_exchanges() {
setup_exchange
}
+# Start N new exchange-http daemons
+# $1: N - number of new exchanges to start
function start_exchanges() {
+ # Get all currently running ones so the next free
+ # port can be calculated - 10000 + RUNNING
RUNNING=$(ps -aux | grep "[taler]-exchange-httpd" | wc -l)
# We cant do seq 0 n, if n=0 it would yield 0, thus do seq n and decrement
# by one, then seq 0 yields nothing
for i in $(seq ${1}); do
+ # seq starts at 1 - substract it
let "i+=${RUNNING}-1" || true
let "i+=10000"
systemctl restart taler-exchange-httpd@"${i}".socket \
taler-exchange-httpd@"${i}".service
+ # Wait so they have some small delay in between their routines from the
start
sleep 0.05
done
}
+# Start N new exchange-wirewatch processes
+# $1: N - number of new wirewatchers to start
+# NOTE: only for init purposes currently
function start_wirewatchers() {
for i in $(seq ${1}); do
systemctl restart taler-exchange-wirewatch@"${i}".service
done
}
+# Stop N exchange daemons
+# $1: N -- number of exchanges to stop
function stop_exchanges() {
stop_numbered_services "taler-exchange-httpd" $1
}
@@ -84,6 +121,9 @@ case $1 in
stop)
stop_exchanges $2
;;
+ *)
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+ ;;
esac
exit 0
diff --git a/experiment/scripts/helpers.sh b/experiment/scripts/helpers.sh
index 5162b36..413043e 100755
--- a/experiment/scripts/helpers.sh
+++ b/experiment/scripts/helpers.sh
@@ -1,17 +1,24 @@
#/!bin/bash
+# Useful functions which are neeeded across
+# multiple scripts
+#
+# Usage: source ./helpers.sh
+# Disable verbose output when loading .env so secret
+# Variables are not in logs unecessarily
set +x
source ~/.env
set -x
# Set a dynamic domain name in our own dns
-# argument: the domain to be added
+# IP address will be resolved on the node this function
+# is executed on
+# $1: The domain to be added
function set_ddn() {
# There are still issues sometimes
- # hostname -I ... failure syntax error
- # Even when looping with while ! hostname -I the next one
- # can fail.
- # Try to fix it with eval
+ # hostname -i ... failure syntax error
+ # Even when looping with while ! hostname -i the next one
+ # possibly may fail. Thus we try to fix it with eval
while ! eval $(echo IP=$(hostname -i)) ; do
sleep 5;
systemctl restart dnsmasq
@@ -24,12 +31,20 @@ send
EOF
}
+# Set the host (role) which the node executing this
+# function is assigned to in the experiments
+# This can later be used to setup log directories per role or in
+# log entries - such as net-delay (ping.sh) for example
function set_host() {
+ # Add it to all sources and export it to the current shell
echo "TALER_HOST=${1}" >> /etc/environment
echo "TALER_HOST=${1}" >> /root/.env
export "TALER_HOST=${1}"
}
+# Enable the service which logs round trip times between nodes
+# $1: Destination to ping
+# NOTE: currenly only one destination is supported
function enable_netdelay() {
sed -i "s/<PING_DESTINATION>/${1}/g" \
/usr/lib/systemd/system/taler-netdelay.service
@@ -37,22 +52,27 @@ function enable_netdelay() {
systemctl restart taler-netdelay.timer
}
+# Setup the log directiories in the Grid5000 shared home directory (NFS)
function setup_log() {
HOST_LOG_DIR=${LOG_DIR}/${TALER_HOST}
- test -d ${HOST_LOG_DIR} || mkdir ${HOST_LOG_DIR} || true #needed for wallets
+ # || true is sill needed when e.g. wallets want to create the same
+ # directory at the same time
+ test -d ${HOST_LOG_DIR} || mkdir ${HOST_LOG_DIR} || true
# Send all logs about taler to promtail on the monitoring node
sed -i -e "s/<MONITOR_DOMAIN_HERE>/${MONITOR_DOMAIN}/g" \
-e "s|<LOG_DIR_HERE>|${HOST_LOG_DIR}|g" \
/etc/rsyslog.d/taler.conf
-
+ # Enable log rotating for all logs in the host log dir on NFS
sed -i "s|<LOG_DIR_HERE>|${HOST_LOG_DIR}|g" \
/etc/logrotate.d/taler
}
-function enable_logbackup() {
- systemctl restart taler-logbackup.timer
+# Enable the logbackup (NFS) - logrotation service
+function enable_logrotate() {
+ systemctl restart taler-logrotate.timer
}
+# Wait for the database to be acessible by $DB_USER from remote
function wait_for_db() {
until PGPASSWORD="${DB_PASSWORD}" psql \
-h "${DATABASE_DOMAIN}" \
@@ -66,8 +86,8 @@ function wait_for_db() {
done
}
-# Arguments
-# 1: Domain to request /keys from
+# Wait until the exchange is ready to present key materials
+# $1: Domain to request /keys from (exch.perf.taler</management>)
function wait_for_keys() {
until wget http://${1}/keys \
--spider \
@@ -80,6 +100,8 @@ function wait_for_keys() {
done
}
+# Restart (enable) the rsyslog to send the logs to the monitoring
+# Node and to the shared log dir (NFS)
function restart_rsyslog() {
# rsyslg fails to apply the taler rule if remote is not reachable
while ! nc -z ${MONITOR_DOMAIN} 1514;
@@ -96,12 +118,11 @@ function restart_rsyslog() {
# Stop services which have the form name@number.service
# will stop the services starting from the biggest numbers
-# Expects the following arguments:
-# $1: service base name, e.g. taler-exchange-httpd
-# $2: amount of services to stop
+# $1: service base name, e.g. taler-exchange-httpd
+# $2: amount of services to stop
function stop_numbered_services() {
# Get all running processes of the service $1,
- # extract their number, limit by number of
+ # extract their number, limit by number of
# to stop and stop those
N=$(\
systemctl status "${1}"@*.service | \
@@ -116,7 +137,9 @@ function stop_numbered_services() {
done
}
-function get_wallet_domains() {
+# Get all Grid5000 hosts which host wallets
+# Returns only the Grid5000 node - e.g. graoully-1
+function get_wallet_hosts() {
IFS=$'\n' read -r -d '' -a WALLETS < <(\
dig -t AXFR "${DNS_ZONE}" "@${DNS_HOSTS}" \
| grep ${WALLET_DOMAIN} | awk '{print $1}' | cut -d '.' -f 2 \
@@ -124,6 +147,8 @@ function get_wallet_domains() {
echo ${WALLETS[@]}
}
+# Get all exchanges which are registered in the nginx proxy config
+# Returns the whole domain - e.g. exchange.perf.taler:80
function get_exchanges() {
IFS=$'\n' read -r -d '' -a EXCHANGES < <(\
ssh -o StrictHostKeyChecking=no ${PROXY_DOMAIN} \
@@ -132,3 +157,20 @@ function get_exchanges() {
)
echo ${EXCHANGES[@]}
}
+
+# Display a help message and exit
+# $1: script name to display help for
+# $2: info message about the script to display
+# $3: option description to display
+function taler_perf_help() {
+ set +x
+ echo "=================== Taler Performance Help ===================="
+ echo "$2"
+ echo "Usage: $1 OPTIONS"
+ echo ""
+ echo "OPTIONS"
+ echo ""
+ echo "$3"
+ echo "==============================================================="
+ exit 2
+}
diff --git a/experiment/scripts/install.sh b/experiment/scripts/install.sh
index 090dfbe..9e3703f 100755
--- a/experiment/scripts/install.sh
+++ b/experiment/scripts/install.sh
@@ -1,4 +1,8 @@
#!/bin/bash
+# Rebuild the taler binaries from source
+# Requires the following optional variables to be set,
+# if not set the corresponding repo will not be rebuilt.
+# <GUNET|EXCHANGE|MERCHANT|WALLET>_COMMIT_SHA
TALER_HOME=~/taler
@@ -15,6 +19,8 @@ function prepare() {
(git pull > /dev/null 2>&1 || true)
}
+# Build the binaries in the current directory with make
+# (runs ./bootstrap & ./configure)
function build() {
echo "INFO running bootstrap and configure"
./bootstrap
@@ -63,3 +69,5 @@ if [ ! -z "${WALLET_COMMIT_SHA}" ]; then
install "https://git.taler.net/wallet-core.git"; \
"${WALLET_COMMIT_SHA:-master}"
fi
+
+exit 0
diff --git a/experiment/scripts/merchant.sh b/experiment/scripts/merchant.sh
index 8ad1a4c..62d1ba3 100755
--- a/experiment/scripts/merchant.sh
+++ b/experiment/scripts/merchant.sh
@@ -1,14 +1,21 @@
#!/bin/bash
set -eux
-
source ~/scripts/helpers.sh
-cd /tmp
+INFO_MSG="
+Setup the merchant node
+(Start taler-merchant-httpd)
+"
+OPT_MSG="
+init:
+ Configure and start the merchant together with its database
+"
-restart_rsyslog
+# Prevent change directory errors
+cd /tmp
-if [[ "$1" == "init" ]];
-then
+# Configurre the merchants files in /etc/taler
+function configure_merchant() {
wait_for_keys "${PROXY_DOMAIN}"
MASTER_KEY=$(
@@ -20,39 +27,56 @@ then
sed -i -e "s\<EXCHANGE_URL_HERE>\http://${EXCHANGE_DOMAIN}/\g"; \
-e "s/<EXCHANGE_MASTER_KEY_HERE>/${MASTER_KEY}/g" \
/etc/taler/conf.d/merchant.conf
-
- sed -i "s/local\s* all\s* postgres\s* peer/local all postgres trust/g" \
- /etc/postgresql/13/main/pg_hba.conf
+}
+# Setup the merchants db on the same host
+function configure_db() {
+ # Allow the remote hosts (monitor) access with the postgres user
echo "
host all postgres 172.16.0.0/12 trust
" >> /etc/postgresql/13/main/pg_hba.conf
+ # Listen on all interfaces so the monitors db exporter can reach the db
echo "
listen_addresses='*'
shared_preload_libraries='pg_stat_statements,auto_explain'
" >> /etc/postgresql/13/main/postgresql.conf
-fi
+}
+
+# Start postgres an initialize the merchant's database
+function init_db() {
+ systemctl restart postgresql
-systemctl restart postgresql
-
-su postgres << EOF
-psql postgres -tAc "SELECT 1 FROM pg_roles WHERE
rolname='taler-merchant-httpd'" | \
- grep -q 1 || \
- createuser taler-merchant-httpd
-psql -tAc "SELECT 1 FROM pg_database WHERE datname='taler-merchant'" | \
- grep -q 1 || \
- createdb -O taler-merchant-httpd taler-merchant
-psql
-CREATE EXTENSION pg_stat_statements;
+ su postgres << EOF
+ psql postgres -tAc "SELECT 1 FROM pg_roles WHERE
rolname='taler-merchant-httpd'" | \
+ grep -q 1 || \
+ createuser taler-merchant-httpd
+ psql -tAc "SELECT 1 FROM pg_database WHERE datname='taler-merchant'" | \
+ grep -q 1 || \
+ createdb -O taler-merchant-httpd taler-merchant
+ psql
+ CREATE EXTENSION pg_stat_statements;
EOF
-if [[ "$1" == "init" ]];
-then
sudo -u taler-merchant-httpd taler-merchant-dbinit
-fi
+}
+
+# Initialize the merchant
+function init_merchant() {
+ restart_rsyslog
+ configure_merchant
+ configure_db
+ init_db
+ systemctl restart taler-merchant-httpd
+}
-sed -i "s\<DB_URL_HERE>\user=postgres host=/run/postgresql
dbname=taler-merchant\g" \
- /etc/default/prometheus-postgres-exporter
+case $1 in
+ init)
+ init_merchant
+ ;;
+ *)
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+ ;;
+esac
-systemctl restart taler-merchant-httpd
+exit 0
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index 129adbd..51de000 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -1,8 +1,27 @@
#!/bin/bash
set -ex
-
source ~/scripts/helpers.sh
+INFO_MSG="
+Setup and start the experiment monitoring infrastructure such as
+Loki, Promtail, Prometheus and various Prometheus exporters
+"
+OPT_MSG="
+init
+ Initialize and start all services
+
+start:
+ Automatically update wallet / exchange nodes which are monitored
+ (does add but not remove)
+
+stop-exchanges NUM:
+ Stop NUM exchanges from being monitored
+"
+
+# Update a data source on the external grafana instance
+# $1: Data-source name (configured in .env)
+# $2: Grid5000 external url for the node where the data source is hosted
+# - https://www.grid5000.fr/w/HTTP/HTTPs_access
function update_datasource() {
ID=$(jq --arg name "$1" '.[] | select(.name == $name) | .id' ds.json)
@@ -16,6 +35,10 @@ function update_datasource() {
"${GRAFANA_API}/datasources/${ID}"
}
+# Update the external grafana instance and tell it
+# about the nodes which should be queried
+# If GRAFANA_HOST or GRAFANA_API_KEY are empty this
+# step is skipped - requires admin level api key to update data sources
function update_grafana() {
if [ -z "${GRAFANA_HOST}" ] || [ -z ${GRAFANA_API_KEY} ]; then
return
@@ -35,6 +58,7 @@ function update_grafana() {
update_datasource "${LOKI_DATASOURCE_NAME}" "${LOKI_G5K_PROXY_PORT}"
}
+# Configure all exporters which run on this host
function configure_prometheus_and_exporters() {
sed -i "s/<MERCHANT_HOST_HERE>/${MERCHANT_DOMAIN}/g" \
/etc/monitor/prometheus.yaml
@@ -47,9 +71,10 @@ function configure_prometheus_and_exporters() {
/etc/default/prometheus-nginx-exporter
}
-
+# Add wallet nodes to be monitored
+# Requires no argument since wallet nodes are retrieved from the DNS
function add_wallet_nodes_to_prometheus() {
- for WALLET in $(get_wallet_domains); do
+ for WALLET in $(get_wallet_hosts); do
if ! grep -q "${WALLET_DOMAIN//\*/${WALLET}}:9100"
/etc/monitor/prometheus.yaml;
then
sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ -
'${WALLET_DOMAIN//\*/${WALLET}}:9100'" \
@@ -58,6 +83,8 @@ function add_wallet_nodes_to_prometheus() {
done
}
+# Add new exchanges to be monitored
+# Requires no argument since exchanges are retrieved from the proxy
function add_exchanges_to_prometheus() {
if ! grep -q "${EXCHANGE_DOMAIN}:80" /etc/monitor/prometheus.yaml; then
# unfortunately we do not have access to the other hosts when running from
ESpec
@@ -81,6 +108,9 @@ function add_exchanges_to_prometheus() {
done
}
+# Remove N exchanges which have been stopped
+# from the config so they will not be scraped anymore
+# $1: N - number of exchanges to remove
function remove_exchanges_from_prometheus() {
TO_STOP=$(\
get_exchanges | \
@@ -95,6 +125,8 @@ function remove_exchanges_from_prometheus() {
done
}
+# Initialize all applications needed for the
+# observation of the experiments
function init_monitor() {
update_grafana
@@ -104,14 +136,12 @@ function init_monitor() {
configure_prometheus_and_exporters
- wait_for_db
-
- # Proxy takes longer to start
wait_for_keys "${PROXY_DOMAIN}"
if [[ "${ENABLE_EXPORTERS}" == "true" ]]; then
if ! grep -q "job_name: 'nodes'" /etc/monitor/prometheus.yaml; then
- cat /etc/monitor/node-exporters.yaml.tpl | envsubst >>
/etc/monitor/prometheus.yaml
+ cat /etc/monitor/node-exporters.yaml.tpl | \
+ envsubst >> /etc/monitor/prometheus.yaml
add_wallet_nodes_to_prometheus
fi
fi
@@ -139,6 +169,10 @@ case $1 in
stop-exchanges)
remove_exchanges_from_prometheus $2
;;
+ *)
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+ ;;
esac
systemctl reload prometheus
+exit 0
diff --git a/experiment/scripts/ping.sh b/experiment/scripts/ping.sh
index 9f82ca0..ff314a1 100755
--- a/experiment/scripts/ping.sh
+++ b/experiment/scripts/ping.sh
@@ -1,13 +1,31 @@
#!/bin/bash
+source ~/scripts/helpers.sh
-DELAY=$(ping -c 2 $1 | sed -n "3p" | awk '{print $(NF-1)$NF}')
+INFO_MSG="
+Measure and log round-trip-times between experiment nodes
+Logs to promtail directly
+"
+OPT_MSG="
+<DEST>:
+ Destination host to ping
+"
-if [[ "$DELAY" == "time="* ]];
+if [ -z "${1}" ]; then
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+fi
+
+RTT=$(\
+ ping -c 2 ${1} | \
+ sed -n "3p" | \
+ awk '{print $(NF-1)$NF}' \
+)
+
+if [[ "${RTT}" == "time="* ]];
then
logger -s --tcp \
--port 1514 \
--server ${MONITOR_DOMAIN} \
--tag taler-network \
- "src=${TALER_HOST} dst=${1} ${DELAY}" \
+ "src=${TALER_HOST} dst=${1} ${RTT}" \
|| true # Ignore errors (mostly because NXDOMAIN)
fi
diff --git a/experiment/scripts/proxy.sh b/experiment/scripts/proxy.sh
index 75c38cb..11a566e 100755
--- a/experiment/scripts/proxy.sh
+++ b/experiment/scripts/proxy.sh
@@ -1,9 +1,26 @@
#!/bin/bash
set -eux
-
source ~/scripts/helpers.sh
+INFO_MSG="
+Confiure and start the nginx proxy server
+"
+OPT_MSG="
+init:
+ Initialize and start the proxy with NUM_EXCHANGE_PROCESSES upstreams
+
+start NUM:
+ Add another NUM exchanges to the list of upstreams
+
+stop NUM:
+ Remove NUM exchanges from the list of upstreams
+"
+
+# Add N exchanges to the upstream servers in the proxy configuration
+# $1: Number of exchanges to add
function add_exchanges() {
+ # Determine the number of exchanges which are already added
+ # and calculate new ports based on that info
ADDED=$(\
grep -r " server ${EXCHANGE_DOMAIN}:" /etc/nginx/sites-enabled/proxy | \
wc -l
@@ -14,6 +31,8 @@ function add_exchanges() {
# The first exchange to add is the default one from the target on port 80
i="80"
else
+ # seq starts at 1 so remove it again
+ # (0 N cannot be used since seq 0 0 would yield 0 while seq N=0 yields
nothing)
let "i+=${ADDED}-1"
let "i+=10000"
fi
@@ -22,6 +41,7 @@ function add_exchanges() {
done
}
+# Setup the node and proxy configuration
function setup_config() {
add_exchanges ${NUM_EXCHANGE_PROCESSES}
@@ -32,6 +52,7 @@ function setup_config() {
-e '/input(type="imudp" port="514")/s/^#//g' \
/etc/rsyslog.conf
+ # Allow enough files to be opened by nginx (www-data)
echo "
fs.file-max=50000
" >> /etc/sysctl.conf
@@ -44,6 +65,7 @@ function setup_config() {
sysctl -p
}
+# Initialize and start the proxy
function init_proxy() {
setup_config
@@ -56,7 +78,8 @@ function init_proxy() {
systemctl restart nginx
}
-
+# Remove N exchanges from the upstream list
+# $1: N - number of exchanges to remove
function remove_exchanges() {
TO_STOP=$(\
get_exchanges | \
@@ -81,6 +104,9 @@ case $1 in
stop)
remove_exchanges $2
;;
+ *)
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+ ;;
esac
systemctl reload nginx
diff --git a/experiment/scripts/run.sh b/experiment/scripts/run.sh
index 48e2a11..69a451e 100644
--- a/experiment/scripts/run.sh
+++ b/experiment/scripts/run.sh
@@ -1,7 +1,8 @@
#!/bin/bash
# Run the experiment
+# Will determine the role of the node
+# based on its hostname and start role specific scripts
set -euax
-
source ~/scripts/helpers.sh
if [[ "${ENABLE_EXPORTERS}" == "true" ]];
@@ -14,21 +15,21 @@ case "${HOSTNAME}" in
set_host bank
set_ddn ${BANK_DOMAIN}
setup_log
- enable_logbackup
- exec ~/scripts/bank.sh
+ enable_logrotate
+ exec ~/scripts/bank.sh init
;;
${DB_HOSTS})
set_host database
set_ddn ${DATABASE_DOMAIN}
setup_log
- enable_logbackup
+ enable_logrotate
exec ~/scripts/database.sh init
;;
${EXCHANGE_HOSTS})
set_host exchange
set_ddn ${EXCHANGE_DOMAIN}
setup_log
- enable_logbackup
+ enable_logrotate
enable_netdelay ${DATABASE_DOMAIN}
exec ~/scripts/exchange.sh init
;;
@@ -36,7 +37,7 @@ case "${HOSTNAME}" in
set_host merchant
set_ddn ${MERCHANT_DOMAIN}
setup_log
- enable_logbackup
+ enable_logrotate
exec ~/scripts/merchant.sh init
;;
${MONITOR_HOSTS})
@@ -49,7 +50,7 @@ case "${HOSTNAME}" in
set_ddn ${PROXY_DOMAIN}
enable_netdelay ${EXCHANGE_DOMAIN}
setup_log
- enable_logbackup
+ enable_logrotate
exec ~/scripts/proxy.sh init
;;
${DNS_HOSTS})
diff --git a/experiment/scripts/setup.sh b/experiment/scripts/setup.sh
index da8fafa..802a735 100644
--- a/experiment/scripts/setup.sh
+++ b/experiment/scripts/setup.sh
@@ -5,10 +5,13 @@
echo "G5K_USER=$(cat ~/experiment-info.json | jq -r '.user.name')" >> ~/.env
source ~/.env
-
set -euax
+# Parse and export the experiment nodes specified in the meta file
+# experiment-info.json
function parse_experiment_nodes() {
+ # Get the nodes and add them to nodes.json in the form:
+ # {node: jFed node-name, host: Grid5000 node}
if [ ! -f ~/nodes.json ]; then
cat ~/experiment-info.json | \
jq '.nodes | to_entries | .[] | {node: .key, host:
.value.ssh_login[1].hostname }' | \
@@ -16,6 +19,14 @@ function parse_experiment_nodes() {
> ~/nodes.json
fi
+ # Read the jFed node-names which are defined in the NODES env
+ # and export their Grid5000 hostname to .env
+ # Assume the jFed nodes Exchange-1 Exchange-2, then Exchange must be added
in NODES
+ # The env will then be the following:
+ # EXCHANGE_HOSTS=node-1.site-1.grid5000.fr|node-2.site-2.grid5000.fr
+ # When jFed has only Exchange, the env will be just
+ # EXCHANGE_HOSTS=node-1.site-1.grid5000.fr
+ # This will be used in run.sh to determine which role script to execute
for NODE in ${NODES}; do
echo "${NODE^^}_HOSTS=\"$(\
cat ~/nodes.json | \
@@ -27,13 +38,17 @@ function parse_experiment_nodes() {
echo "WALLET_HOSTS=*" >> ~/.env
}
+# Determine and create the base log directory
+# If NFS exists, it will be created in the users home dir on the NFS
function setup_log_dir() {
LOG_DIR=/home/${G5K_USER}/taler-logs
if [ -d ${LOG_DIR} ]; then
- rm -rf ${LOG_DIR}/*
+ # If multiple nodes want to delete the same dir we run into errors
+ # Let it fail safely with || true
+ rm -rf ${LOG_DIR}/* || true
elif [ -d /home/${G5K_USER} ]; then
- mkdir ${LOG_DIR}
+ mkdir ${LOG_DIR} | true
else
LOG_DIR=/tmp/taler && mkdir ${LOG_DIR}
fi
@@ -41,9 +56,11 @@ function setup_log_dir() {
echo "LOG_DIR=${LOG_DIR}" >> ~/.env
}
+# Setup the environment configuration
function setup_environment() {
set +x
+ # Determine the port the taler-exchange-* should use
if [ "$USE_PGBOUNCER" = "true" ]; then
echo "DB_PORT=6432" >> ~/.env
else
@@ -53,6 +70,8 @@ function setup_environment() {
# Needed for envsubst to work
export DNS_ZONE=${DNS_ZONE}
# Set the hostnames completely with substituting ${DNS_ZONE}
+ # Important: dont use cat env | envsubst > env - this will truncate env
before its read
+ # and thus it will be empty in the end
cat ~/.env | envsubst > /tmp/.env && mv /tmp/.env ~/.env
# Add the environment config for following shells
cat ~/.env | grep -v API_KEY | tee /etc/environment
@@ -63,6 +82,7 @@ function setup_environment() {
set -x
}
+# Setup shared configurations such as the ones from configs/*
function setup_config() {
# Temporarily checkout to the feature branch
cd "${G5K_HOME}" && git checkout node-setup && git pull && cd
@@ -81,18 +101,23 @@ function setup_config() {
fi
}
+# Configure the experiments DNS
function setup_dns() {
NS_IP=$(host ${DNS_HOSTS} | sed -n 1p | awk '{print $4}')
+ # Set our DNS to be the only DNS to query by the stub resolver
if ! grep -Fxq "server=${NS_IP}" /etc/dnsmasq.conf ; then
echo "server=${NS_IP}" >> /etc/dnsmasq.conf
fi
+ # Set dnsmasq as our only resolver (stub)
if ! grep -Fxq "nameserver 127.0.0.1" /etc/resolv.conf ; then
mv /etc/resolv.conf /etc/resolv.conf.bak
echo "nameserver 127.0.0.1" > /etc/resolv.conf
fi
+ # Add the Grid5000 DNS servers as forwarders to our DNS so grid5000 stuff
+ # gets resolved correctly
BIND_SERVERS=$(grep nameserver /etc/resolv.conf.bak | awk '{print $2}'
ORS='; ')
sed -i "s/<DNS_ZONE_HERE>/${DNS_ZONE}/g" \
/etc/bind/named.conf.local
@@ -102,6 +127,8 @@ function setup_dns() {
-e "s/<NS_IP_HERE>/${NS_IP}/g" \
/var/lib/bind/perf.taler
+ # Remove potentionally expired zone journals so bind will not complain
+ # (this can happen when the Espec - or just this script are executed
multiple times)
if [ -f /var/lib/bind/perf.taler.jnl ]; then
rm -rf /var/lib/bind/perf.taler.jnl
fi
@@ -116,6 +143,7 @@ function setup_dns() {
# hostname -I couldn't get address for 'x': failure syntax error
sleep 5
else
+ # Start the DNS when we are the DNS host
systemctl restart named
fi
}
diff --git a/experiment/scripts/taler-perf.sh b/experiment/scripts/taler-perf.sh
index 9414d4b..e1ff428 100644
--- a/experiment/scripts/taler-perf.sh
+++ b/experiment/scripts/taler-perf.sh
@@ -17,7 +17,7 @@ function update_processes() {
}
function start_wallets() {
- for WALLET in $(get_wallet_domains); do
+ for WALLET in $(get_wallet_hosts); do
ssh -o StrictHostKeyChecking=no ${WALLET_DOMAIN//\*/${WALLET}} \
"/bin/bash /root/scripts/wallet.sh start
${1:-${NUM_WALLET_PROCESSES}}" &
done
@@ -25,7 +25,7 @@ function start_wallets() {
}
function stop_wallets() {
- for WALLET in $(get_wallet_domains); do
+ for WALLET in $(get_wallet_hosts); do
ssh -o StrictHostKeyChecking=no ${WALLET_DOMAIN//\*/${WALLET}} \
"/bin/bash /root/scripts/wallet.sh stop
${1:-${NUM_WALLET_PROCESSES}}" &
done
diff --git a/experiment/scripts/wallet.sh b/experiment/scripts/wallet.sh
index 29df965..86bd7f4 100755
--- a/experiment/scripts/wallet.sh
+++ b/experiment/scripts/wallet.sh
@@ -1,26 +1,43 @@
#!/bin/bash
set -eux
-
source ~/scripts/helpers.sh
+INFO_MSG="
+Configure and start taler-wallets
+"
+OPT_MSG="
+init:
+ Initialize the wallets and start an inital NUM_WALLET_PROCESSES
+
+start NUM:
+ Start NUM new wallet benchmark processes
+
+stop NUM:
+ Stop NUM wallet benchmark processes
+"
+
+# Enable rsyslog and wait until the exchange is ready
function init_wallets() {
restart_rsyslog
wait_for_keys "${PROXY_DOMAIN}"
sleep 5
}
+# Start N wallets
+# If CPU load is over 75%, no wallets are started here
+# $1 Number of wallets to start
function start_wallets() {
CPU_USAGE=$[100-$(vmstat 1 2 | tail -1 | awk '{print $15}')]
echo "CPU Usage: ${CPU_USAGE}%"
- # count the running wallets
- RUNNING=$(ps -aux | grep "[wallet]-cli" | wc -l)
-
if [ "${CPU_USAGE}" -gt "75" ]; then
echo "Not starting any more wallets"
exit 1
fi
+ # count the running wallets so that numbers can be increased
+ RUNNING=$(ps -aux | grep "[wallet]-cli" | wc -l)
+
for i in $(seq ${1}); do
let "i+=${RUNNING}"
systemctl restart taler-wallet@${i}.service
@@ -28,6 +45,8 @@ function start_wallets() {
done
}
+# Stop N wallets
+# $1: N - number of wallets to stop or "all"
function stop_wallets() {
if [[ "$1" == "all" ]]; then
systemctl stop taler-wallet@*.service
@@ -47,6 +66,9 @@ case "$1" in
start)
start_wallets $2
;;
+ *)
+ taler_perf_help $0 "$INFO_MSG" "$OPT_MSG"
+ ;;
esac
exit 0
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.
- [taler-grid5k] 40/73: document variables, (continued)
- [taler-grid5k] 40/73: document variables, gnunet, 2021/12/14
- [taler-grid5k] 70/73: update plotter README and dashboards, gnunet, 2021/12/14
- [taler-grid5k] 65/73: filter wallet getRecord messages - they spam the logs, gnunet, 2021/12/14
- [taler-grid5k] 51/73: add auto explain - not tested yet, gnunet, 2021/12/14
- [taler-grid5k] 67/73: update dashboards, gnunet, 2021/12/14
- [taler-grid5k] 53/73: add more description about huge pages, gnunet, 2021/12/14
- [taler-grid5k] 71/73: remove unused rpecs and handled notes, gnunet, 2021/12/14
- [taler-grid5k] 69/73: update plotter script, gnunet, 2021/12/14
- [taler-grid5k] 72/73: add possibility to define grid5k repo branch for experiments, gnunet, 2021/12/14
- [taler-grid5k] 66/73: update dashboard exports, gnunet, 2021/12/14
- [taler-grid5k] 64/73: comment scripts and add help,
gnunet <=
- [taler-grid5k] 73/73: scripts documented and commented, gnunet, 2021/12/14
- [taler-grid5k] 68/73: update grafana dashboards, add option for ram mount in db, gnunet, 2021/12/14