[taler-grid5k] 54/73: enable huge_pages, implement exchange scrape logic

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit 760069f8647b895c91c42c40d5c67e1355a97dbb
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Wed Dec 8 10:57:46 2021 +0100

    enable huge_pages, implement exchange scrape logic
---
 configs/etc/monitor/exchange-exporters.yaml.tpl |  6 +++
 experiment/scripts/database.sh                  | 51 ++++++++++++++++++-------
 experiment/scripts/helpers.sh                   |  8 ++++
 experiment/scripts/monitor.sh                   | 14 +++++++
 experiment/scripts/taler-perf.sh                | 29 +++++++-------
 5 files changed, 80 insertions(+), 28 deletions(-)

diff --git a/configs/etc/monitor/exchange-exporters.yaml.tpl 
b/configs/etc/monitor/exchange-exporters.yaml.tpl
new file mode 100644
index 0000000..74a4992
--- /dev/null
+++ b/configs/etc/monitor/exchange-exporters.yaml.tpl
@@ -0,0 +1,6 @@
+  - job_name: 'taler'
+    static_configs:
+    - labels:
+        component: 'exchange'
+      targets:
+      # <EXCHANGES_HERE>
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 2569991..8f83a8f 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -12,10 +12,30 @@ function setup_config() {
   
   # Enable password for taler since this is commonly the case
   # For the postgres user do not enable authentication (used in metrics)
-  echo "
-  host all ${DB_USER} 172.16.0.0/12 md5
-  host all postgres 172.16.0.0/12 trust
-  " >> /etc/postgresql/13/main/pg_hba.conf
+  if ! grep -q "host all ${DB_USER} 127.16.0.0/12 md5" \
+    /etc/postgresql/13/main/pg_hba.conf; then
+    echo "
+    host all ${DB_USER} 172.16.0.0/12 md5
+    host all postgres 172.16.0.0/12 trust
+    " >> /etc/postgresql/13/main/pg_hba.conf
+  fi
+
+  SHARED_MEM=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) / 3 ))
+  CACHE_SIZE=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) * 3/4))
+  NUM_CPU=$(lscpu | grep "CPU(s)" | head -n 1 | awk '{print $2}')
+  NUM_PARALLEL_WORKERS=$((${NUM_CPU} / 2))
+
+  # Size for huge_pages =~ shared_buffers * 1.25 so that there is enough
+  VM_PEAK=$((${SHARED_MEM} * 10/8))
+
+  HUGE_PAGES_SIZE=$(grep ^Hugepagesize /proc/meminfo | awk '{print $2}')
+  NUM_PAGES=$((${VM_PEAK} / ${HUGE_PAGES_SIZE}))
+
+
+  if ! grep -q "vm.nr_hugepages'" /etc/sysctl.conf; then
+    echo "vm.nr_hugepages=${NUM_PAGES}" >> /etc/sysctl.conf
+    sysctl -p
+  fi
 
   echo "
   listen_addresses='*'
@@ -30,18 +50,18 @@ function setup_config() {
   
   # use 25% of the available memory 
   # (https://www.postgresql.org/docs/13/runtime-config-resource.html)
-  shared_buffers=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) / 3 ))kB
-  effective_cache_size=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) * 
3/4))kB
+  shared_buffers=${SHARED_MEM}kB
+  effective_cache_size=${CACHE_SIZE}kB
 
-  # huge_pages=on
+  huge_pages=on
   
   # 
(https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-MAX-WAL-SIZE)
   min_wal_size=4GB
   max_wal_size=8GB
   wal_buffers=16MB
   
-  max_worker_processes=$(lscpu | grep "CPU(s)" | head -n 1 | awk '{print $2}')
-  max_parallel_workers=$(($(lscpu | grep "CPU(s)" | head -n 1 | awk '{print 
$2}') * 1/2))
+  max_worker_processes=${NUM_CPU}
+  max_parallel_workers=${NUM_PARALLEL_WORKERS}
   max_connections=500
 
   max_parallel_maintenance_workers=4
@@ -72,11 +92,14 @@ if [[ "$1" == "init" ]]; then
   setup_config
 fi
 
-# pgbouncer does not cleanup those sometimes
-rm -f /var/run/postgresql/pgbouncer.pid
-rm -f /var/run/postgresql/.s.PGSQL.6432
-  
-systemctl restart postgresql # pgbouncer
+systemctl restart postgresql
+
+if [ "${USE_PGBOUNCER}" = "true" ]; then
+  # pgbouncer does not cleanup those sometimes
+  rm -f /var/run/postgresql/pgbouncer.pid
+  rm -f /var/run/postgresql/.s.PGSQL.6432
+  systemctl restart pgbouncer
+fi
 
 su postgres << EOF
 psql postgres -tAc "SELECT 1 FROM pg_roles WHERE 
rolname='taler-exchange-httpd'" | \
diff --git a/experiment/scripts/helpers.sh b/experiment/scripts/helpers.sh
index e497444..d6a05c0 100755
--- a/experiment/scripts/helpers.sh
+++ b/experiment/scripts/helpers.sh
@@ -90,3 +90,11 @@ function get_wallet_domains() {
   echo ${WALLETS[@]}
 }
 
+function get_exchanges() {
+  IFS=$'\n' read -r -d '' -a EXCHANGES < <(\
+    ssh -o StrictHostKeyChecking=no ${PROXY_DOMAIN} \
+      grep -E "^[[:space:]]*server[[:space:]]exch" 
/etc/nginx/sites-enabled/proxy \
+      | cut -d ";" -f 1 | cut -d " " -f 4 \
+  )
+  echo ${EXCHANGES[@]}
+}
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index 3811818..7e779c9 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -68,6 +68,20 @@ function add_wallet_nodes_to_prometheus() {
   fi
 }
 
+function add_exchange_nodes_to_prometheus() {
+  if [[ "$1" == "init" ]]; then 
+    cat /etc/monitor/exchange-exporters.yaml.tpl >> 
/etc/monitor/prometheus.yaml
+  fi
+
+  for EXCH in $(get_exchanges); do 
+    if ! grep -q "${EXCH}" /etc/monitor/prometheus.yaml;
+    then
+      sed -i "/<EXCHANGES_HERE>/a  \ \ \ \ \ \ - '${EXCH}'" \
+              /etc/monitor/prometheus.yaml
+    fi
+  done
+}
+
 function init() {
 
   update_grafana
diff --git a/experiment/scripts/taler-perf.sh b/experiment/scripts/taler-perf.sh
index 2511e41..8a63cc1 100644
--- a/experiment/scripts/taler-perf.sh
+++ b/experiment/scripts/taler-perf.sh
@@ -3,9 +3,21 @@ set -e
 
 source ~/scripts/helpers.sh
 
+function update_processes() {
+  case "$1" in 
+    prometheus)
+      ssh -o StrictHostKeyChecking=no ${MONITOR_DOMAIN} "/bin/bash -c 
/root/scripts/monitor.sh"
+      ;;
+    *)
+      echo "Unknown argument '$1' for function ${FUNCNAME[0]}"
+      echo "Usage: update [prometheus]"
+      ;;
+  esac
+}
+
 function start_wallets() {
   for WALLET in $(get_wallet_domains); do
-    ssh -o StrictHostKeyChecking=no wallet.${WALLET}.${DNS_ZONE} \
+    ssh -o StrictHostKeyChecking=no ${WALLET_DOMAIN//\*/${WALLET}} \
            "/bin/bash /root/scripts/wallet.sh start 
${1:-${NUM_WALLET_PROCESSES}}" &
   done
   wait
@@ -13,7 +25,7 @@ function start_wallets() {
 
 function stop_wallets() {
  for WALLET in $(get_wallet_domains); do
-   ssh -o StrictHostKeyChecking=no wallet.${WALLET}.${DNS_ZONE} \
+   ssh -o StrictHostKeyChecking=no ${WALLET_DOMAIN//\*/${WALLET}} \
           "/bin/bash /root/scripts/wallet.sh stop 
${1:-${NUM_WALLET_PROCESSES}}" &
   done
   wait
@@ -24,6 +36,7 @@ function start_exchanges() {
          "/bin/bash /root/scripts/exchange.sh ${1:-${NUM_EXCHANGE_PROCESSES}}" 
   ssh -o StrictHostKeyChecking=no ${PROXY_DOMAIN} \
          "/bin/bash /root/scripts/proxy.sh ${1:-${NUM_EXCHANGE_PROCESSES}}"
+  update_processes "prometheus"
 }
 
 function start_processes() {
@@ -53,18 +66,6 @@ function stop_processes() {
    esac
 }
 
-function update_processes() {
-  case "$1" in 
-    prometheus)
-      ssh -o StrictHostKeyChecking=no ${MONITOR_DOMAIN} "/bin/bash -c 
/root/scripts/monitor.sh"
-      ;;
-    *)
-      echo "Unknown argument '$1' for function ${FUNCNAME[0]}"
-      echo "Usage: update [prometheus]"
-      ;;
-  esac
-}
-
 case "$1" in
   start)
     shift

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.