[taler-grid5k] 24/73: cleanup with functions - try work_mem and idle_in_transaction

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit 01793c1134d093c4475a086de12911131a470951
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Tue Nov 30 10:16:25 2021 +0100

    cleanup with functions - try work_mem and idle_in_transaction
---
 experiment/scripts/database.sh | 16 +++++++--
 experiment/scripts/exchange.sh | 30 +++++++++++------
 experiment/scripts/monitor.sh  | 76 +++++++++++++++++++++++-------------------
 3 files changed, 74 insertions(+), 48 deletions(-)

diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 69503c0..fd20388 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -6,8 +6,7 @@ source ~/scripts/helpers.sh
 # move to tmp to prevent change dir errors
 cd /tmp 
 
-if [[ "$1" == "init" ]];
-then
+function setup_config() {
   sed -i "s\<DB_URL_HERE>\postgresql:///${DB_NAME}\g" \
        /etc/taler/secrets/exchange-db.secret.conf
   
@@ -22,14 +21,21 @@ then
   # (https://www.postgresql.org/docs/13/runtime-config-resource.html)
   shared_buffers=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) / 4 ))kB
   effective_cache_size=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) * 
3/4))kB
+
   # 
(https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-MAX-WAL-SIZE)
   max_wal_size=2GB 
   wal_buffers=16MB
+
   max_worker_processes=$(lscpu | grep "CPU(s)" | head -n 1 | awk '{print $2}')
   max_connections=200
+
   # out of shared memory
   max_locks_per_transaction=85
-  # idle_in_transaction_session_timeout=5000
+
+  # Increase work mem to lower I/O utilization (max used =~ work_mem * 
max_connections)
+  # NOTE: This formula is not completely correct 
+  work_mem=128MB
+  idle_in_transaction_session_timeout=3000
   " >> /etc/postgresql/13/main/postgresql.conf
   
   # Enable password for taler since this is commonly the case
@@ -38,6 +44,10 @@ then
   host all ${DB_USER} 172.16.0.0/12 md5 
   host all postgres 172.16.0.0/12 trust
   " >> /etc/postgresql/13/main/pg_hba.conf
+}
+
+if [[ "$1" == "init" ]]; then
+  setup_config
 fi
   
 systemctl restart postgresql 
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index 0094231..bb37d69 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -1,11 +1,7 @@
 #!/bin/bash
 set -eux
 
-if [[ "$1" == "init" ]];
-then   
-  source ~/scripts/helpers.sh
-  restart_rsyslog
-  
+function setup_config() {
   sed -i 
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}/${DB_NAME}\g"
 \
        /etc/taler/secrets/exchange-db.secret.conf
   
@@ -18,13 +14,9 @@ then
          -e "s\<BASE_URL_HERE>\http://${EXCHANGE_DOMAIN}/\g"; \
          -e "s/<MASTER_KEY_HERE>/${MASTER_KEY}/g" \
        /etc/taler/conf.d/exchange-business.conf
-  
-  wait_for_db
-  # Wait another second to make sure user has permissions
-  sleep 5
-
-  NUM_PROCESSES=$((${NUM_EXCHANGE_PROCESSES:-10}-1))
+}
 
+function setup_exchange() {
   systemctl restart taler-exchange.target
 
   wait_for_keys "${EXCHANGE_DOMAIN}/management"
@@ -38,6 +30,22 @@ then
   taler-exchange-offline upload < sig-res.json
   taler-exchange-offline upload < acct-res.json
   taler-exchange-offline upload < fee-res.json
+}
+
+if [[ "$1" == "init" ]];
+then   
+  source ~/scripts/helpers.sh
+  restart_rsyslog
+
+  setup_config
+  
+  wait_for_db
+  # Wait another second to make sure user has permissions
+  sleep 5
+
+  NUM_PROCESSES=$((${NUM_EXCHANGE_PROCESSES:-10}-1))
+
+  setup_exchange
 
   # || true is needed since when 1-1 is run let returns an error
   let "NUM_WIREWATCH_PROCESSES-=1" || true
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index cab6045..034f78e 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -17,7 +17,7 @@ function update_datasource() {
 }
 
 function update_grafana() {
-  if [ -z "${GRAFANA_HOST}" || -z ${GRAFANA_API_KEY} ]; then
+  if [ -z "${GRAFANA_HOST}" ] || [ -z ${GRAFANA_API_KEY} ]; then
     return
   fi
   AUTH_HEADER="Authorization: Bearer ${GRAFANA_API_KEY}"
@@ -34,15 +34,8 @@ function update_grafana() {
   update_datasource "${PROMETHEUS_DATASOURCE_NAME}" 
"${PROMETHEUS_G5K_PROXY_PORT}"
   update_datasource "${LOKI_DATASOURCE_NAME}" "${LOKI_G5K_PROXY_PORT}"
 }
-  
-if [[ "$1" == "init" ]];
-then
 
-  update_grafana
-  
-  systemctl restart loki \
-                 promtail
-  
+function configure_prometheus_and_exporters() {
   sed -i "s/<MERCHANT_HOST_HERE>/${MERCHANT_DOMAIN}/g" \
        /etc/monitor/prometheus.yaml
   
@@ -51,39 +44,54 @@ then
   
   sed -i "s\<PROXY_URL_HERE>\http://${PROXY_DOMAIN}/stub_status\g"; \
        /etc/default/prometheus-nginx-exporter
-  
-  wait_for_db
-  
-  # Initialize prometheus after the db is ready, then all dns records have 
been set for sure
-  if [[ "${ENABLE_EXPORTERS}" == "true" ]];
-  then
-    cat /etc/monitor/node-exporters.yaml.tpl | envsubst >> 
/etc/monitor/prometheus.yaml
-    for WALLET in $(get_wallet_domains);
-    do
-      sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ - 
'${WALLET_DOMAIN//\*/${WALLET}}:9100'" \
-           /etc/monitor/prometheus.yaml
-    done
-  fi
-  
-  # Proxy takes longer to start
-  wait_for_keys "${PROXY_DOMAIN}"
-  
-  systemctl restart prometheus-postgres-exporter \
-                    prometheus-nginx-exporter
-else 
+}
+
+
+function add_wallet_nodes_to_prometheus() {
+  if [[ "${ENABLE_EXPORTERS}" == "true" ]]; then
+
+    if [[ "$1" == "init" ]]; then
+      cat /etc/monitor/node-exporters.yaml.tpl | envsubst >> 
/etc/monitor/prometheus.yaml
+    fi
+
+    for WALLET in $(get_wallet_domains); do
 
-  if [[ "${ENABLE_EXPORTERS}" == "true" ]];
-  then
-    for WALLET in $(get_wallet_domains);
-    do
       if ! grep -q "${WALLET_DOMAIN//\*/${WALLET}}:9100" 
/etc/monitor/prometheus.yaml; 
       then
-        sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ - 
'wallet.${WALLET}.${DNS_ZONE}:9100'" \
+        sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ - 
'${WALLET_DOMAIN//\*/${WALLET}}:9100'" \
                 /etc/monitor/prometheus.yaml
       fi
+
     done
+
   fi
+}
 
+function init() {
+
+  update_grafana
+  
+  systemctl restart loki \
+                   promtail
+  
+  configure_prometheus_and_exporters
+  
+  wait_for_db
+  
+  add_wallet_nodes_to_prometheus "init"
+  
+  # Proxy takes longer to start
+  wait_for_keys "${PROXY_DOMAIN}"
+  
+  systemctl restart prometheus-nginx-exporter 
+                   # prometheus-postgres-exporter 
+
+}
+  
+if [[ "$1" == "init" ]]; then
+  init
+else 
+  add_wallet_nodes_to_prometheus ""
 fi
 
 systemctl restart prometheus

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.