[taler-grid5k] 39/141: first monitoring setup

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit 74c413e677b7259f4605461817cbec6b2246cbaf
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Tue Oct 12 00:12:14 2021 +0200

    first monitoring setup
---
 configs/etc/monitor/loki.yaml                      | 69 ++++++++++++++++++++++
 configs/etc/monitor/prometheus.yaml                |  0
 configs/etc/monitor/promtail.yaml                  | 18 ++++++
 configs/usr/lib/systemd/system/loki.service        |  9 +++
 configs/usr/lib/systemd/system/promtail.service    |  9 +++
 .../usr/lib/systemd/system}/taler-fakebank.service |  0
 experiment/cleardns.sh                             |  2 +-
 experiment/env                                     |  7 +++
 experiment/run.sh                                  | 19 ++++++
 experiment/scripts/database.sh                     |  2 +-
 experiment/scripts/exchange.sh                     |  4 +-
 experiment/scripts/monitor.sh                      |  3 +
 experiment/scripts/proxy.sh                        |  2 +-
 13 files changed, 140 insertions(+), 4 deletions(-)

diff --git a/configs/etc/monitor/loki.yaml b/configs/etc/monitor/loki.yaml
new file mode 100644
index 0000000..3ba496a
--- /dev/null
+++ b/configs/etc/monitor/loki.yaml
@@ -0,0 +1,69 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 8080
+  grpc_listen_port: 0
+
+ingester:
+  wal:
+    enabled: true
+    dir: /tmp/wal
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time 
will be flushed
+  max_chunk_age: 1h           # All chunks will be flushed when they hit this 
age, default is 1h
+  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, 
flushing first if chunk_idle_period or max_chunk_age is reached first
+  chunk_retain_period: 30s    # Must be greater than index read cache TTL if 
using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /tmp/loki/boltdb-shipper-active
+    cache_location: /tmp/loki/boltdb-shipper-cache
+    cache_ttl: 24h         # Can be increased for faster performance over 
longer query periods, uses more disk space
+    shared_store: filesystem
+  filesystem:
+    directory: /tmp/loki/chunks
+
+compactor:
+  working_directory: /tmp/loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  unordered_writes: true
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+    #ruler:
+    #  storage:
+    #    type: local
+    #    local:
+    #      directory: /tmp/loki/rules
+    #  rule_path: /tmp/loki/rules-temp
+    #  alertmanager_url: http://localhost:9093
+    #  ring:
+    #    kvstore:
+    #      store: inmemory
+    #  enable_api: true
diff --git a/configs/etc/monitor/prometheus.yaml 
b/configs/etc/monitor/prometheus.yaml
new file mode 100644
index 0000000..e69de29
diff --git a/configs/etc/monitor/promtail.yaml 
b/configs/etc/monitor/promtail.yaml
new file mode 100644
index 0000000..a63a6c2
--- /dev/null
+++ b/configs/etc/monitor/promtail.yaml
@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://monitor.perf.taler:8080/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+    - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log
diff --git a/configs/usr/lib/systemd/system/loki.service 
b/configs/usr/lib/systemd/system/loki.service
new file mode 100644
index 0000000..8e5acdb
--- /dev/null
+++ b/configs/usr/lib/systemd/system/loki.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Grafana Loki instance
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/loki --config.file=/etc/monitor/loki.yaml
+
+[Install]
+WantedBy=multi-user.target
diff --git a/configs/usr/lib/systemd/system/promtail.service 
b/configs/usr/lib/systemd/system/promtail.service
new file mode 100644
index 0000000..a0f971d
--- /dev/null
+++ b/configs/usr/lib/systemd/system/promtail.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Grafana Promtail instance
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/promtail --config.file=/etc/monitor/promtail.yaml
+
+[Install]
+WantedBy=multi-user.target
diff --git a/image/grid5000/steps/data/taler-fakebank.service 
b/configs/usr/lib/systemd/system/taler-fakebank.service
similarity index 100%
rename from image/grid5000/steps/data/taler-fakebank.service
rename to configs/usr/lib/systemd/system/taler-fakebank.service
diff --git a/experiment/cleardns.sh b/experiment/cleardns.sh
index 0e899ad..f71c17b 100644
--- a/experiment/cleardns.sh
+++ b/experiment/cleardns.sh
@@ -6,7 +6,7 @@ source .env
 
 # Delete all old dns entries of the previous experitment
 while IFS='=' read -r name value; do
-  if [[ $name == *"_DOMAIN" && $value == *".${DNS_ZONE}" ]]; then 
+  if [[ $name == *"_DOMAIN" || $name == *"_CNAME" ]] && [[ $value == 
*".${DNS_ZONE}" ]]; then 
     nsupdate -k /root/ddns.key -v << EOF
 server ${DNS_HOST}
 zone ${DNS_ZONE}
diff --git a/experiment/env b/experiment/env
index 6c894e4..fe2465a 100755
--- a/experiment/env
+++ b/experiment/env
@@ -3,6 +3,7 @@ EXCHANGE_HOSTS=
 PROXY_HOSTS=
 BANK_HOSTS=
 WALLET_HOSTS=
+MONITOR_HOSTS=
 
 DNS_HOST=taler.net
 DNS_ZONE=perf.taler
@@ -12,6 +13,12 @@ EXCHANGE_DOMAIN=exch.perf.taler
 DATABASE_DOMAIN=db.perf.taler
 BANK_DOMAIN=bank.perf.taler
 
+PROMETHEUS_CNAME=prometheus.perf.taler
+LOKI_CNAME=loki.perf.taler
+
+PROMETHEUS_PORT=http
+LOKI_PORT=http8080
+
 DB_NAME=taler-exchange
 DB_USER=taler
 DB_PASSWORD=
diff --git a/experiment/run.sh b/experiment/run.sh
index a80f184..1ba6a5c 100644
--- a/experiment/run.sh
+++ b/experiment/run.sh
@@ -16,6 +16,18 @@ send
 EOF
 }
 
+# Set a cname record to the external grid5000 proxy
+# argument 1: cname to be added
+# argument 2: grid5000 port, one of http, http8080
+function set_cname() {
+  nsupdate -k /root/ddns.key -v << EOF
+server ${DNS_HOST}
+zone ${DNS_ZONE}
+update add $1 IN CNAME $(hostname | cut -d "." -f 1,2 -).$2.proxy.grid5000.fr
+send
+EOF
+}
+
 case "${HOSTNAME}" in 
   ${BANK_HOSTS}) 
     echo "BANK"
@@ -41,6 +53,13 @@ case "${HOSTNAME}" in
     set_ddn ${EXCHANGE_DOMAIN}
     exec ~/scripts/exchange.sh 
     ;;
+  ${MONITOR_HOSTS})
+    echo "MONITOR"
+    set_ddn ${MONITOR_DOMAIN}
+    set_cname ${PROMETHEUS_CNAME} ${PROMETHEUS_PORT}
+    set_cname ${LOKI_CNAME} ${LOKI_PORT}
+    exec ~/scripts/monitor.sh
+    ;;
 esac
 
 exit 0
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 490a686..bf351de 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -10,7 +10,7 @@ sed -i "s\<DB_URL_HERE>\postgresql:///${DB_NAME}\g" \
 echo "listen_addresses='*'" >> /etc/postgresql/13/main/postgresql.conf
 echo "host all all 172.16.0.0/12 md5" >> /etc/postgresql/13/main/pg_hba.conf
 
-systemctl start postgresql
+systemctl restart postgresql prometheus-postgres-exporter
 
 su postgres << EOF
 createuser taler-exchange-httpd
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index a832429..aa15bfc 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eux
 
+systemctl restart promtail
+
 sed -i 
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}/${DB_NAME}\g"
 \
        /etc/taler/secrets/exchange-db.secret.conf
 
@@ -30,7 +32,7 @@ done
 
 sleep 1
 
-systemctl start taler-exchange.target
+systemctl restart taler-exchange.target
 
 until wget http://${EXCHANGE_DOMAIN}/management/keys \
       --spider \
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
new file mode 100755
index 0000000..33ce041
--- /dev/null
+++ b/experiment/scripts/monitor.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+systemctl enable prometheus loki 
diff --git a/experiment/scripts/proxy.sh b/experiment/scripts/proxy.sh
index 3b65787..05b75b4 100755
--- a/experiment/scripts/proxy.sh
+++ b/experiment/scripts/proxy.sh
@@ -14,4 +14,4 @@ do
   sleep 1
 done
 
-systemctl restart nginx
+systemctl restart nginx prometheus-nginx-exporter

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.