[lmi-commits] [lmi] odd/multiarch e06e543: Build for multiple '--host' architectures

[Greg Chicares]

branch: odd/multiarch
commit e06e543560fc95beedfc5f993272224326b2628b
Author: Gregory W. Chicares <address@hidden>
Commit: Gregory W. Chicares <address@hidden>

    Build for multiple '--host' architectures
---
 GNUmakefile                  | 16 +++++++---
 gui_test.sh                  |  3 ++
 install_libxml2_libxslt.make | 15 ++++++----
 install_msw.sh               | 71 ++++++++++++++++++++++++++++++++++++++++----
 install_wx.sh                |  9 ++++--
 install_wxpdfdoc.sh          |  5 ++--
 nychthemeral_test.sh         | 58 ++++++++++++++++++++++++++++--------
 7 files changed, 145 insertions(+), 32 deletions(-)

diff --git a/GNUmakefile b/GNUmakefile
index 4924b35..cf5a0e7 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -86,7 +86,8 @@ MAKEFLAGS := \
 
 prefix          := /opt/lmi
 # parent directory for machine-specific binaries
-exec_prefix     := $(prefix)
+# assume no harm from trailing slash if $LMI_HOST is undefined
+exec_prefix     := $(prefix)/$(LMI_HOST)
 # binaries that users can run
 bindir          := $(exec_prefix)/bin
 # binaries that administrators can run
@@ -121,8 +122,11 @@ srcdir          := $(CURDIR)
 localbindir     := $(exec_prefix)/local/bin
 locallibdir     := $(exec_prefix)/local/lib
 localincludedir := $(exec_prefix)/local/include
-test_dir        := $(exec_prefix)/test
-touchstone_dir  := $(exec_prefix)/touchstone
+test_dir        := $(prefix)/test
+touchstone_dir  := $(prefix)/touchstone
+# deferred evaluation to reflect variables ($(build_type), e.g.) defined later
+# Eventually prepend "$(toolset)-" to $(exec_prefix) .
+build_directory  = $(exec_prefix)/build/$(build_type)
 
 
################################################################################
 
@@ -173,8 +177,12 @@ $(srcdir)/local_options.make:: ;
 
 build_type ?= ship
 toolset ?= gcc
-build_directory := \
+XXbuild_directory := \
   $(srcdir)/../build/$(notdir $(srcdir))/$(uname)/$(toolset)/$(build_type)
+# Eventually prepend "$(toolset)-" to $(exec_prefix) .
+# Is it better to define "build_directory =" above (deferred), with other
+# directories...or here, "build_directory :=", away from other directories?
+# build_directory := $(exec_prefix)/build/$(build_type)
 
 gpl_files := \
   COPYING \
diff --git a/gui_test.sh b/gui_test.sh
index b03c9e9..ea086eb 100755
--- a/gui_test.sh
+++ b/gui_test.sh
@@ -141,4 +141,7 @@ mkdir --parents /tmp/lmi/logs
 
 cd /opt/lmi/src/lmi
 
+# NOT! "$PERFORM" /opt/lmi/"$LMI_HOST"/bin/wx_test --ash_nazg 
--data_path=/opt/lmi/data 2>&1 | tee /tmp/lmi/logs/gui-test | sed -e 
"$gui_test_clutter"
+# Specifying the "true" path to a binary too often requires setting
+# $PATH and $WINEPATH, which is complex and fragile; symlinks are much better.
 "$PERFORM" /opt/lmi/bin/wx_test --ash_nazg --data_path=/opt/lmi/data 2>&1 | 
tee /tmp/lmi/logs/gui-test | sed -e "$gui_test_clutter"
diff --git a/install_libxml2_libxslt.make b/install_libxml2_libxslt.make
index 4e2e79e..7533fe9 100644
--- a/install_libxml2_libxslt.make
+++ b/install_libxml2_libxslt.make
@@ -48,24 +48,29 @@ $(xz_version).tar.gz: host_path := projects/lzmautils/files
 
 mingw_dir     := /MinGW_
 
-prefix        := /opt/lmi/local
+# Move $host_type here to make it configurable.
+# Move $build_type here to keep it with $host_type.
+# This build_type is overridden below for cygwin; is there no good
+# way to get it dynamically, OAOO, except 'config.guess', which may
+# not be globally available?
+build_type    := x86_64-unknown-linux-gnu
+host_type     := $(if $(LMI_HOST),$(LMI_HOST),i686-w64-mingw32)
+
+prefix        := /opt/lmi/$(host_type)/local
 exec_prefix   := $(prefix)
 
 cache_dir     := /cache_for_lmi/downloads
 
-xml_dir       := /opt/lmi/xml-scratch
+xml_dir       := /opt/lmi/$(host_type)/xml-scratch
 
 # Variables that normally should be left alone 
#################################
 
 mingw_bin_dir :=
-build_type    := x86_64-unknown-linux-gnu
-host_type     := i686-w64-mingw32
 
 uname := $(shell uname 2>/dev/null)
 ifeq (CYGWIN,$(findstring CYGWIN,$(uname)))
   mingw_bin_dir := $(mingw_dir)/bin/
   build_type    := i686-pc-cygwin
-  host_type     := i686-w64-mingw32
 endif
 
 xz_cflags := \
diff --git a/install_msw.sh b/install_msw.sh
index c0322b0..d1cd5c1 100755
--- a/install_msw.sh
+++ b/install_msw.sh
@@ -38,17 +38,31 @@ echo "Started: $stamp0"
 #   (but don't force it to 4 if it's already set).
 # '--output-sync=recurse' is also used, passim, to facilitate log
 #   comparison.
+#
+# Here and elsewhere: assign variable on one line, export it on another?
+# Is that just an affectation that's no longer relevant?
+# POSIX requires that 'export foo=bar' work as intended:
+#   https://pubs.opengroup.org/onlinepubs/009604499/utilities/export.html
+# shellcheck doesn't complain if RHS of assignment is a literal:
+#   https://github.com/koalaman/shellcheck/wiki/SC2155
 if [ -z "$coefficiency" ]
 then
-    export coefficiency='--jobs=4'
+    coefficiency='--jobs=4'
+    export coefficiency
 fi
 
-export platform
+# Original version had 'export' here, before 'case'. Apparently the order
+# doesn't matter--POSIX is not entirely clear:
+#   https://pubs.opengroup.org/onlinepubs/009604499/utilities/export.html
+# although zsh is (but this is a 'sh' script):
+#   http://zsh.sourceforge.net/Doc/Release/Shell-Builtin-Commands.html
+#   "If a parameter specified does not already exist, it is created in the 
global scope."
 case $(uname) in
     CYGWIN*)
         platform=CYGWIN
         ;;
 esac
+export platform
 
 if [ "CYGWIN" = "$platform" ]
 then
@@ -115,6 +129,11 @@ then
     git config --global core.fileMode false
 fi
 
+# Temporary expedient: to test changes to the build system,
+# don't reinstall anything that isn't compiled (to save time),
+# and don't reinstall lmi (so that local modifications to
+# makefiles and scripts aren't overwritten).
+cat >/dev/null <<SUPPRESSED
 java -version
 
 mkdir --parents /opt/lmi/src
@@ -169,19 +188,34 @@ mount
 
 md5sum "$0"
 find /cache_for_lmi/downloads -type f | xargs md5sum
+SUPPRESSED
 
 rm --force --recursive scratch
 
+# Eventually, this should be used for cygwin as well, to support
+# building with different '--host' settings (but for now $LMI_HOST
+# is also used to determine whether 'wine' is used, which is wrong
+# because that should properly depend on the '--build' system).
+LMI_HOST=${LMI_HOST:-"i686-w64-mingw32"}
+export LMI_HOST
+
 if [ "CYGWIN" = "$platform" ]
 then
     # For Cygwin, install and use this msw-native compiler.
     rm --force --recursive /MinGW_
     make $coefficiency --output-sync=recurse -f install_mingw.make
-else
-    # For real *nix, set LMI_HOST to specify a cross compiler.
-    export LMI_HOST=i686-w64-mingw32
 fi
 
+# It seems like a good idea to build both 32- and 64-bit msw binaries,
+# at least until we abandon 32-bit builds altogether.
+for LMI_HOST in i686-w64-mingw32 x86_64-w64-mingw32
+do
+
+# Symlinks make multi-arch work fairly simple. Commands like these can
+# be run manually (at the command line) to switch between architectures.
+ln --symbolic --force --no-dereference /opt/lmi/"$LMI_HOST"/local 
/opt/lmi/local
+ln --symbolic --force --no-dereference /opt/lmi/"$LMI_HOST"/bin /opt/lmi/bin
+
 make $coefficiency --output-sync=recurse -f install_miscellanea.make clobber
 make $coefficiency --output-sync=recurse -f install_miscellanea.make
 
@@ -196,11 +230,38 @@ find /cache_for_lmi/downloads -type f | xargs md5sum
 export         PATH=/opt/lmi/local/bin:/opt/lmi/local/lib:$PATH
 export 
minimal_path=/opt/lmi/local/bin:/opt/lmi/local/lib:/usr/bin:/bin:/usr/sbin:/sbin
 
+# USE SYMLINKS INSTEAD
+#exec_prefix=/opt/lmi/"$LMI_HOST"
+#localbindir="$exec_prefix"/local/bin
+#locallibdir="$exec_prefix"/local/lib
+#
+#        PATH="$localbindir":"$locallibdir":"$PATH"
+#minimal_path="$localbindir":"$locallibdir":/usr/bin:/bin:/usr/sbin:/sbin
+#export         PATH
+#export minimal_path
+
+# USE SYMLINKS INSTEAD
+# condition should be cross compilation, not !cygwin
+# double quotes inside double quotes--wrong?
+# 'winepath' doesn't DTRT with '/first/path:/second/path'--need to
+#   run it against each component, and separate results with ';', not ':'
+#if [ "CYGWIN" != "$platform" ]
+#then
+#   extrabindir=/opt/lmi/third_party/bin
+#   w0="$(winepath -w "$localbindir" | sed -e's/\\/\\\\/g')"
+#   w1="$(winepath -w "$locallibdir" | sed -e's/\\/\\\\/g')"
+#   w2="$(winepath -w "$extrabindir" | sed -e's/\\/\\\\/g')"
+#   WINEPATH="$w0;$w1:$w2"
+#   export WINEPATH
+#fi
+
 make $coefficiency --output-sync=recurse PATH=$minimal_path wx_config_check
 make $coefficiency --output-sync=recurse PATH=$minimal_path show_flags
 make $coefficiency --output-sync=recurse PATH=$minimal_path clean
 make $coefficiency --output-sync=recurse PATH=$minimal_path install
 
+done
+
 if [ "CYGWIN" = "$platform" ]
 then
     # No lmi binary should depend on any Cygwin library.
diff --git a/install_wx.sh b/install_wx.sh
index 64d64c4..11f13ca 100755
--- a/install_wx.sh
+++ b/install_wx.sh
@@ -41,11 +41,13 @@ coefficiency=${coefficiency:-"--jobs=4"}
 
 MAKE=${MAKE:-"make $coefficiency"}
 
+host_type=${LMI_HOST:-"i686-w64-mingw32"}
+
 # Variables that normally should be left alone 
#################################
 
 mingw_dir=/MinGW_
 
-prefix=/opt/lmi/local
+prefix=/opt/lmi/"$host_type"/local
 exec_prefix="$prefix"
 
 repo_name="wxWidgets"
@@ -78,7 +80,6 @@ git checkout "$wx_commit_sha"
 git submodule update "$coefficiency" --recursive --init
 
 build_type=$("$proxy_wx_dir"/config.guess)
-host_type=i686-w64-mingw32
 
 case $(uname) in
     CYGWIN*)
@@ -88,8 +89,10 @@ esac
 
 # Construct a vendor string for this build using the compiler name and version
 # and the unique commit SHA-1.
+#
+# Does $vendor have two separate "-gcc" literals where one would suffice?
 gcc_version=$(${mingw_bin_dir}${host_type}-gcc -dumpversion|tr -d '\r')
-vendor=gcc-$gcc_version-$wx_commit_sha
+vendor="$host_type"-gcc-$gcc_version-$wx_commit_sha
 
 # Configuration reference:
 #   http://lists.nongnu.org/archive/html/lmi/2007-11/msg00001.html
diff --git a/install_wxpdfdoc.sh b/install_wxpdfdoc.sh
index eda698c..cc150d9 100755
--- a/install_wxpdfdoc.sh
+++ b/install_wxpdfdoc.sh
@@ -41,11 +41,13 @@ coefficiency=${coefficiency:-"--jobs=4"}
 
 MAKE=${MAKE:-"make $coefficiency"}
 
+host_type=${LMI_HOST:-"i686-w64-mingw32"}
+
 # Variables that normally should be left alone 
#################################
 
 mingw_dir=/MinGW_
 
-prefix=/opt/lmi/local
+prefix=/opt/lmi/"$host_type"/local
 exec_prefix="$prefix"
 
 repo_name="wxpdfdoc"
@@ -75,7 +77,6 @@ fi
 git checkout "$wxpdfdoc_commit_sha"
 
 build_type=$("$proxy_wxpdfdoc_dir"/admin/build-aux/config.guess)
-host_type=i686-w64-mingw32
 
 case $(uname) in
     CYGWIN*)
diff --git a/nychthemeral_test.sh b/nychthemeral_test.sh
index b0d7c00..0090239 100755
--- a/nychthemeral_test.sh
+++ b/nychthemeral_test.sh
@@ -22,7 +22,11 @@
 # snail: Chicares, 186 Belle Woods Drive, Glastonbury CT 06033, USA
 
 # Suggested use:
-#   $make clobber; ./nychthemeral_test.sh 2>&1 | tee /tmp/lmi/logs/log | sed 
-f errors.sed
+# [It seems moderately distasteful to insert "$(LMI_HOST)/" here,
+# and maybe some bad thing would happen if $LMI_HOST is not defined;
+# but is there any cleaner way to do this? Would it be worthwhile
+# to set up a symlink for /tmp/lmi/$(LMI_HOST) ?
+#   $make clobber; ./nychthemeral_test.sh 2>&1 | tee 
/tmp/lmi/$(LMI_HOST)/logs/log | sed -f errors.sed
 # Omitting the 'clobber' step when it's known to be unnecessary makes
 # that command take two minutes instead of five on a dual E5-2630 v3
 # machine. What's difficult is knowing when it's truly unnecessary.
@@ -36,6 +40,33 @@ set -e
 # provides no convenient alternative):
 setopt PIPE_FAIL
 
+# Don't manipulate paths--use symlinks instead.
+cat >/dev/null <<SUPPRESSED
+# --- DISTRESSINGLY COPIED BLOCK
+# INSTEAD TRY SYMLINKS?
+exec_prefix=/opt/lmi/"$LMI_HOST"
+localbindir="$exec_prefix"/local/bin
+locallibdir="$exec_prefix"/local/lib
+
+        PATH="$localbindir":"$locallibdir":"$PATH"
+minimal_path="$localbindir":"$locallibdir":/usr/bin:/bin:/usr/sbin:/sbin
+export         PATH
+export minimal_path
+
+# condition should be cross compilation, not !cygwin
+# double quotes inside double quotes--wrong?
+if [ "CYGWIN" != "$platform" ]
+then
+    extrabindir=/opt/lmi/third_party/bin
+    w0="$(winepath -w "$localbindir" | sed -e's/\\/\\\\/g')"
+    w1="$(winepath -w "$locallibdir" | sed -e's/\\/\\\\/g')"
+    w2="$(winepath -w "$extrabindir" | sed -e's/\\/\\\\/g')"
+    WINEPATH="$w0;$w1:$w2"
+    export WINEPATH
+fi
+# --- DISTRESSINGLY COPIED BLOCK
+SUPPRESSED
+
 case "$LMI_HOST" in
     ("i686-w64-mingw32" | "x86_64-w64-mingw32")
         PERFORM=wine
@@ -102,7 +133,8 @@ schemata_clutter='
 '
 
 # Directory for test logs.
-mkdir --parents /tmp/lmi/logs
+logdir=/tmp/lmi/"$LMI_HOST"/logs
+mkdir --parents "$logdir"
 
 cd /opt/lmi/src/lmi
 
@@ -110,30 +142,30 @@ printf '\n# test concinnity\n\n'
 make "$coefficiency" check_concinnity 2>&1 | sed -e "$build_clutter" -e 
"$concinnity_clutter"
 
 printf '# install; check physical closure\n\n'
-make "$coefficiency" install check_physical_closure 2>&1 | tee 
/tmp/lmi/logs/install | sed -e "$build_clutter" -e "$install_clutter"
+make "$coefficiency" install check_physical_closure 2>&1 | tee 
"$logdir"/install | sed -e "$build_clutter" -e "$install_clutter"
 
 printf 'Production system built--ready to start GUI test in another 
session.\n' > /dev/tty
 
 printf '\n# cgi and cli tests\n\n'
-make "$coefficiency" --output-sync=recurse cgi_tests cli_tests 2>&1 | tee 
/tmp/lmi/logs/cgi-cli | sed -e "$build_clutter" -e "$cli_cgi_clutter"
+make "$coefficiency" --output-sync=recurse cgi_tests cli_tests 2>&1 | tee 
"$logdir"/cgi-cli | sed -e "$build_clutter" -e "$cli_cgi_clutter"
 
 printf '\n# system test\n\n'
-make "$coefficiency" system_test 2>&1 | tee /tmp/lmi/logs/system-test | sed -e 
"$build_clutter" -e "$install_clutter"
+make "$coefficiency" system_test 2>&1 | tee "$logdir"/system-test | sed -e 
"$build_clutter" -e "$install_clutter"
 
 printf '\n# unit tests\n\n'
-make "$coefficiency" unit_tests 2>&1 | tee >(grep '\*\*\*') >(grep \?\?\?\?) 
>(grep '!!!!' --count | xargs printf '%d tests succeeded\n') 
>/tmp/lmi/logs/unit-tests
+make "$coefficiency" unit_tests 2>&1 | tee >(grep '\*\*\*') >(grep \?\?\?\?) 
>(grep '!!!!' --count | xargs printf '%d tests succeeded\n') 
>"$logdir"/unit-tests
 
 printf '\n# build with shared-object attributes\n\n'
-make "$coefficiency" all build_type=so_test USE_SO_ATTRIBUTES=1 2>&1 | tee 
/tmp/lmi/logs/so_test | sed -e "$build_clutter"
+make "$coefficiency" all build_type=so_test USE_SO_ATTRIBUTES=1 2>&1 | tee 
"$logdir"/so_test | sed -e "$build_clutter"
 
 printf '\n# cgi and cli tests in libstdc++ debug mode\n\n'
-make "$coefficiency" --output-sync=recurse cgi_tests cli_tests 
build_type=safestdlib 2>&1 | tee /tmp/lmi/logs/cgi-cli-safestdlib | sed -e 
"$build_clutter" -e "$cli_cgi_clutter"
+make "$coefficiency" --output-sync=recurse cgi_tests cli_tests 
build_type=safestdlib 2>&1 | tee "$logdir"/cgi-cli-safestdlib | sed -e 
"$build_clutter" -e "$cli_cgi_clutter"
 
 printf '\n# unit tests in libstdc++ debug mode\n\n'
-make "$coefficiency" unit_tests build_type=safestdlib 2>&1 | tee >(grep 
'\*\*\*') >(grep \?\?\?\?) >(grep '!!!!' --count | xargs printf '%d tests 
succeeded\n') >/tmp/lmi/logs/unit-tests-safestdlib
+make "$coefficiency" unit_tests build_type=safestdlib 2>&1 | tee >(grep 
'\*\*\*') >(grep \?\?\?\?) >(grep '!!!!' --count | xargs printf '%d tests 
succeeded\n') >"$logdir"/unit-tests-safestdlib
 
 printf '\n# xrc tests\n\n'
-java -jar /opt/lmi/third_party/rng/jing.jar -c xrc.rnc *.xrc 2>&1 | tee 
/tmp/lmi/logs/xrc
+java -jar /opt/lmi/third_party/rng/jing.jar -c xrc.rnc *.xrc 2>&1 | tee 
"$logdir"/xrc
 
 # Run the following tests in a throwaway directory so that the files
 # they create can be cleaned up easily.
@@ -148,12 +180,12 @@ cp /opt/lmi/src/lmi/sample.cns .
 
 printf '\n# test all valid emission types\n\n'
 
-"$PERFORM" /opt/lmi/bin/lmi_cli_shared --file=/tmp/lmi/tmp/sample.ill --accept 
--ash_nazg --data_path=/opt/lmi/data 
--emit=emit_test_data,emit_spreadsheet,emit_text_stream,emit_custom_0,emit_custom_1
 >/dev/null
+"$PERFORM" /opt/lmi/"$LMI_HOST"/bin/lmi_cli_shared 
--file=/tmp/lmi/tmp/sample.ill --accept --ash_nazg --data_path=/opt/lmi/data 
--emit=emit_test_data,emit_spreadsheet,emit_text_stream,emit_custom_0,emit_custom_1
 >/dev/null
 
-"$PERFORM" /opt/lmi/bin/lmi_cli_shared --file=/tmp/lmi/tmp/sample.cns --accept 
--ash_nazg --data_path=/opt/lmi/data 
--emit=emit_test_data,emit_spreadsheet,emit_group_roster,emit_text_stream,emit_custom_0,emit_custom_1
 >/dev/null
+"$PERFORM" /opt/lmi/"$LMI_HOST"/bin/lmi_cli_shared 
--file=/tmp/lmi/tmp/sample.cns --accept --ash_nazg --data_path=/opt/lmi/data 
--emit=emit_test_data,emit_spreadsheet,emit_group_roster,emit_text_stream,emit_custom_0,emit_custom_1
 >/dev/null
 
 printf '\n# schema tests\n\n'
-/opt/lmi/src/lmi/test_schemata.sh 2>&1 | tee /tmp/lmi/logs/schemata | sed -e 
"$schemata_clutter"
+/opt/lmi/src/lmi/test_schemata.sh 2>&1 | tee "$logdir"/schemata | sed -e 
"$schemata_clutter"
 
 # Clean up stray output. (The zsh '(N)' glob qualifier turns on
 # null_glob for a single expansion.)