[taler-grid5k] 38/73: speedup image build

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit 29059d67979c35bf8717527fb5f917def76f887b
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Wed Dec 1 14:17:11 2021 +0100

    speedup image build
---
 docker/Dockerfile         |  22 +++++----
 docker/README.md          |  21 +++++++--
 docker/entrypoint.sh      | 114 +++++-----------------------------------------
 image/README.md           |  34 ++++----------
 image/taler-debian11.yaml |  17 +++----
 5 files changed, 57 insertions(+), 151 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index f735e49..5c29514 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,19 +1,21 @@
 FROM debian:11
 
 LABEL author="Boss Marco" \
-      description="Build the Taler Grid5000 environment image" \
-      company="Taler"
+      company="Taler" \
+      description="Image to automate Grid5000 image build for the  \
+Taler performance experiments. \
+This image contains all dev libraries used to also \
+build the binaries directly"
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-ENV GRID5K_DEST=lyon,lille
-ENV GNUNET_COMMIT_SHA=master
-ENV EXCHANGE_COMMIT_SHA=master
-ENV MERCHANT_COMMIT_SHA=master
-ENV WALLET_COMMIT_SHA=master
-ENV GRID5K_COMMIT_SHA=master
-
-ENV TALER_HOME=/taler
+ENV GRID5K_DEST=lyon,lille \
+    GNUNET_COMMIT_SHA=master \
+    EXCHANGE_COMMIT_SHA=master \
+    MERCHANT_COMMIT_SHA=master \
+    WALLET_COMMIT_SHA=master \
+    GRID5K_COMMIT_SHA=master \
+    TALER_HOME=/taler
 
 RUN apt update && \
     apt upgrade -y && \
diff --git a/docker/README.md b/docker/README.md
index 0e7b579..270db82 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -33,6 +33,15 @@ docker run -it --rm \
 **NOTE** about the port 5900, this one can be used for vncviewer to see whats 
happening inside the image which 
 will be created. Run `vncviewver :0`.
 
+#### Manual Build
+
+To get an interactive shell into the image override the entrypoint by adding 
the following argument
+before the last line in the command above:
+
+```bash
+--entrypoint=/bin/bash
+```
+
 ### docker-compose
 
 Assuming an env file `.env` with the following contents:
@@ -50,6 +59,14 @@ ARGUMENTS=<ARGUMENTS> GRID5K_CERT_PASSWD=<cert_passwd> 
docker-compose up --build
 
 Or place the password (also arguments) in .env too but make sure its protected.
 
+#### Manual Build
+
+You can also use the docker compose to get an interactive shell into the image:
+
+```bash
+docker-compose run --entrypoint /bin/bash taler-build
+```
+
 ### Notes
 
 #### Environment Variables
@@ -59,10 +76,8 @@ Or place the password (also arguments) in .env too but make 
sure its protected.
 **GRID5K_CERT_PASSWD**: the password to decrypt `GRID5K_CERT`
 **GRID5K_DEST**: comma separated list of where to copy the image to in the 
grid (default: lille,lyon)
 **ARGUMENTS**: args to pass to entrypoint, one of 
-  -c|--clean (run make clean) 
-  -r|--rebuild (rebuild the image - automatically set if -c is used) 
+  -r|--rebuild (rebuild the image)
   -n|--no-copy (do not copy the generated image to Grid5000 - make sure output 
volume is mounted)
-  -v|--verbose (enable verbose logging)
  As per default, running the docker command again will not clean or rebuild 
the image
 
 ##### Additional
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index ea91a20..ccda5d7 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -7,12 +7,6 @@ set -e
 
 while [[ $# -gt 0 ]]; do
   case "$1" in 
-    -c|--clean)
-      CLEAN=true
-      REBUILD=true
-      echo "INFO will clean previous builds and image"
-      shift
-      ;;
     -r|--rebuild)
       REBUILD=true
       echo "INFO will rebuild image"
@@ -23,11 +17,6 @@ while [[ $# -gt 0 ]]; do
       echo "INFO will not copy image to Grid5000"
       shift
       ;;
-    -v|--verbose)
-      VERBOSE=true
-      echo "INFO verbose output enabled"
-      shift
-      ;;
     *)
       shift
       ;;
@@ -54,66 +43,6 @@ function prepare() {
   git checkout "$2" > /dev/null && (git pull > /dev/null 2>&1 || true)
 }
 
-# Build from a repo
-function build() {
-  echo "INFO running bootstrap and configure"
-  run_command "./bootstrap" "bootstrap.log"
-  if [ -f contrib/gana.sh ]; then
-    run_command "./contrib/gana.sh" "gana.log"
-  fi
-  run_command \
-      "(./configure --enable-logging=verbose --prefix=/usr || ./configure)" \
-      "configure.log"
-  run_command "make" "make.log"
-  if [ "$VERBOSE" = true ]; then
-    cat bootstrap.log || true
-    cat gana.log || true
-    cat configure.log || true
-    cat make.log || true
-  fi
-}
-
-# Build a dist version (build is run first)
-# $1: Git repo to build
-# $2: Git commit to checkout to
-# $3: Custom command to run before running make dist
-function dist() {
-  prepare "$1" "$2"
-  if [ "$CLEAN" = true ]; then
-    echo "INFO cleaning previous build"
-    make clean > /dev/null 2>&1 || true
-  fi
-  if ! make > /dev/null 2>&1; then
-    build
-  fi
-  if [ ! -z "$3" ]; then
-    echo "INFO running custom command '$3'"
-    /bin/bash -c "$3"
-  fi
-  echo "INFO running 'make dist'"
-  run_command "make dist" "dist.log"
-  if [ "$VERBOSE" = true ]; then
-    cat dist.log || true
-  fi
-}
-
-# Install from a git repo (dist is run first)
-# $1: Git repo to build
-# $2: Git commit to checkout to
-# $3: Custom command to run before running make dist
-function install() {
-  dist "$1" "$2" "$3"
-  echo "INFO installing"
-  if [[ $(ldconfig -p | grep "$1") == "" ]] || \
-     [ "$CLEAN" = true ]; then
-    run_command "make install" "install.log"
-    ldconfig
-  fi
-  if [ "$VERBOSE" = true ]; then
-    cat install.log || true
-  fi
-}
-
 if [ "$COPY" != false ];
 then
   # Add the certificate to the agent 
@@ -152,48 +81,29 @@ then
 fi
 
 echo "INFO preparing Grid5000 repository"
-prepare grid5k \
-      "${GRID5K_COMMIT_SHA}" 
+prepare grid5k "${GRID5K_COMMIT_SHA}"
 
 # Test if the image specification is correct
-# To do this the files copied into it must be present
-touch 
${TALER_HOME}/grid5k/image/grid5000/steps/data/{gnunet,wallet,exchange,merchant}.tar.gz
 run_command \
-    "kameleon dryrun ${TALER_HOME}/grid5k/image/taler-debian11.yaml" \
+    "kameleon dryrun -g g5k_user:${G5K_USER} 
${TALER_HOME}/grid5k/image/taler-debian11.yaml" \
     "dryrun.log"
-rm ${TALER_HOME}/grid5k/image/grid5000/steps/data/*.tar.gz
-
-# Prepare and package all applications
-echo "INFO preparing GNUnet"
-install gnunet \
-      "${GNUNET_COMMIT_SHA}"
-
-echo "INFO preparing Taler Exchange"
-install exchange \
-      "${EXCHANGE_COMMIT_SHA}" \
-      'find . -name "*Makefile*" -exec sed -i "/x-taler-bank.fee/d" {} \;'
-
-echo "INFO preparing Taler Merchant"
-install merchant \
-      "${MERCHANT_COMMIT_SHA}"
-
-echo "INFO preparing Taler Wallet"
-dist wallet-core \
-      "${WALLET_COMMIT_SHA}"
 
 cd ${TALER_HOME}/grid5k/image
 
 if [ ! -f "build/taler-debian11/taler-debian11.tar.zst" ] || \
    [ "$REBUILD" = true ];
 then
-  mv "${TALER_HOME}"/gnunet/gnunet*.tar.gz grid5000/steps/data/gnunet.tar.gz
-  mv "${TALER_HOME}"/exchange/taler-exchange*.tar.gz 
grid5000/steps/data/exchange.tar.gz
-  mv "${TALER_HOME}"/merchant/taler-merchant*.tar.gz 
grid5000/steps/data/merchant.tar.gz
-  mv "${TALER_HOME}"/wallet-core/taler-wallet*.tar.gz 
grid5000/steps/data/wallet.tar.gz
-
   rm -rf /tmp/taler-debian11 || true
   echo "INFO building image"
-  yes r | kameleon build -b /tmp taler-debian11.yaml | tee build.log 
+  yes r | kameleon build \
+          -b /tmp taler-debian11.yaml \
+          -g gnunet_commit_sha:${GNUNET_COMMIT_SHA} \
+             exchange_commit_sha:${EXCHANGE_COMMIT_SHA} \
+             merchant_commit_sha:${MERCHANT_COMMIT_SHA} \
+             wallet_commit_sha:${WALLET_COMMIT_SHA} \
+            grid5k_commit_sha:${GRID5K_COMMIT_SHA} \
+             g5k_user:${G5K_USER} \
+       | tee build.log
   mv /tmp/taler-debian11 ${TALER_HOME}
 fi
 
@@ -208,7 +118,7 @@ if [ -f "/root/cert.pem" ] && [ "$COPY" != false ]; then
   for G5K_HOST in "${G5K_HOSTS[@]}"; do
     echo "Copying image to ${G5K_HOST}"
 
-    sed "s/<G5K_HOST>/${G5K_HOST}/g; s/<G5K_USER>/${GRID5K_USER}/g" \
+    sed "/g5k_tar_path/s/lyon/${G5K_HOST}/g" \
         taler-debian11.dsc.bak > taler-debian11.dsc
 
     scp -o StrictHostKeyChecking=no taler-debian11.tar.zst taler-debian11.dsc \
diff --git a/image/README.md b/image/README.md
index 16e0e7c..4d42c12 100644
--- a/image/README.md
+++ b/image/README.md
@@ -8,42 +8,24 @@ Official documentation can be found on these links:
 
 ## Manual Build
 
-### Prerequisites
+Replace `<G5K_USER>` with your Grid5000 username.
+This variable is required, if not specified the build will fail.
 
-To build the image, the following repositories need to be added as dist 
archives in 
-`grid5000/steps/data`
+`kameleon build -g g5k_user:<G5K_USER> taler-debian11`
 
-* [gnunet](https://git.gnunet.org/gnunet.git): gnunet.tar.gz
-* [taler-exchange](https://git.taler.net/exchange.git): exchange.tar.gz
-* [taler-merchant](https://git.taler.net/merchant.git): merchant.tar.gz
-* [taler-wallet-core](https://git.taler.net/wallet-core.git): wallet.tar.gz
-
-This is necessary since bootstrap seems not to work when creating the image.
-
-Replace `<G5K_USER> `and `<G5K_HOST>` in `taler-debian11.yaml` in the 
following line:
-`g5k_tar_path: 
"http://public.<G5K_HOST>.grid5000.fr/~<G5K_USER>/taler-debian11.tar.zst"`
-
-**G5K_USER**: Your Grid5000 username.
-**G5K_HOST**: The host where the image will be copied to in the last step.
-
-Alternatively they can be replaced after the build in 
`build/taler-debian11/taler-debian11.dsc`
-
-### Build
-
-`kameleon build taler-debian11`
-
-**NOTE** Make sure that all dependencies listed in Grid5000 Environments are 
installed
+**NOTE** Make sure that all dependencies listed in `Grid5000 Environment` are 
installed
 
 ### Deploy
 
-Copy the image to a Grid5000 node:
+Copy the image to a Grid5000 site:
 
 ```bash
 cd build/taler-debian11
-scp taler-debian11.* <G5K_USER>@access.grid5000.fr:<G5K_NODE>/public/
+scp taler-debian11.* <G5K_USER>@access.grid5000.fr:<G5K_SITE>/public/
 ```
 
-**NOTE** G5K_USER and G5K_HOST should match the ones in taler-debian11.yaml
+**NOTE** G5K_USER and G5K_SITE should match the ones in taler-debian11.dsc
+G5K_SITE defaults to `lyon`.
 
 ## Usage
 
diff --git a/image/taler-debian11.yaml b/image/taler-debian11.yaml
index 4aa4403..323eea1 100644
--- a/image/taler-debian11.yaml
+++ b/image/taler-debian11.yaml
@@ -27,7 +27,7 @@ global:
   g5k_version: 2
   ## Environment image path and compression
   # Note: setup for the docker build image - please replace G5K_HOST and 
G5K_USER for manual builds
-  g5k_tar_path: 
"http://public.<G5K_HOST>.grid5000.fr/~<G5K_USER>/taler-debian11.tar.zst"
+  g5k_tar_path: 
"http://public.lyon.grid5000.fr/~$${g5k_user}/taler-debian11.tar.zst";
   # g5k_tar_compression: "zstd"
   ## Environment postinstall path, compression, and script command
   # g5k_postinst_path: server:///grid5000/postinstalls/g5k-postinstall.tgz
@@ -49,6 +49,7 @@ global:
   exchange_commit_sha: master
   merchant_commit_sha: master
   wallet_commit_sha: master
+  grid5k_commit_sha: master
   ## Other parameters can be changed, see kameleon info debian10-taler.yaml
 
 bootstrap:
@@ -75,6 +76,8 @@ setup:
          apt install -y $${taler_packages} $${taler_build_packages}
          apt install -y --no-install-recommends 
$${taler_packages_no_recommends}
 
+         mkdir -p $${taler_path}
+
     - gnunet: 
       - exec_in: |
          cd $${taler_path} 
@@ -90,8 +93,6 @@ setup:
          make install
          ldconfig
 
-         cd .. && rm -rf gnunet-*
-
     - exchange:
       - exec_in: |
          cd $${taler_path}
@@ -109,8 +110,6 @@ setup:
          make install
          ldconfig
 
-         cd .. && rm -rf taler-exchange-*
-
     - merchant:
       - exec_in: |
          cd $${taler_path}
@@ -127,8 +126,6 @@ setup:
          make install
          ldconfig
 
-         cd .. && rm -rf taler-merchant-*
-
     - wallet: 
       - exec_in: |
          cd $${taler_path}
@@ -145,15 +142,15 @@ setup:
          make || true
          make install
 
-         cd .. && rm -rf taler-wallet-*
-
     - grid5k:
       - exec_in: |
          cd $${taler_path}
 
          git clone git://git.taler.net/grid5k.git 
+         cd grid5k
+         git checkout $${grid5k_commit_sha} || true
 
-         ./grid5k/experiment/scripts/createusers.sh
+         ./experiment/scripts/createusers.sh
 
     - loki-promtail:
       - exec_in: |

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.