[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
02/03: build-system/go: Build with a filesystem union of Go dependencies
|
From: |
guix-commits |
|
Subject: |
02/03: build-system/go: Build with a filesystem union of Go dependencies. |
|
Date: |
Tue, 12 Mar 2019 16:22:17 -0400 (EDT) |
lfam pushed a commit to branch wip-go-build-system
in repository guix.
commit ce41a7729f75fdee243ae12000513a5dbbcc67d1
Author: Leo Famulari <address@hidden>
Date: Sat Jan 6 15:47:47 2018 -0500
build-system/go: Build with a filesystem union of Go dependencies.
This basically changes (guix build-system go) so that instead of looking
up its dependencies in a list of directories in $GOPATH, all the
Go dependencies are symlinked into a single directory.
Fixes <https://bugs.gnu.org/33620>.
* guix/build/go-build-system.scm (setup-go-environment): New variable.
(setup-environment, install-source): Remove variables.
(unpack): Unpack the source relative to $GOPATH.
(install): Do not install the compiled objects in the 'pkg' directory.
Install the source code in this phase, and only install the source of
the package named by IMPORT-PATH.
* doc/guix.texi (Build Systems): Adjust accordingly.
* gnu/packages/docker.scm (docker): Import (guix build union) on the build
side
and adjust to build phase name changes in (guix build-system go).
* gnu/packages/shellutils.scm (direnv): Likewise.
* gnu/packages/databases.scm (mongo-tools)[arguments]:
Set '#:install-source #f'.
* gnu/packages/music.scm (demlo)[arguments]: Move the 'install-scripts'
phase after the 'install' phase.
---
doc/guix.texi | 4 +-
gnu/packages/databases.scm | 3 +-
gnu/packages/docker.scm | 6 +-
gnu/packages/music.scm | 2 +-
gnu/packages/shellutils.scm | 6 +-
guix/build-system/go.scm | 2 +
guix/build/go-build-system.scm | 139 ++++++++++++++++++++++-------------------
7 files changed, 90 insertions(+), 72 deletions(-)
diff --git a/doc/guix.texi b/doc/guix.texi
index 82cf2ba..0ef4814 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -5726,8 +5726,8 @@ some cases, you will need to unpack the package's source
code to a
different directory structure than the one indicated by the import path,
and @code{#:unpack-path} should be used in such cases.
-Packages that provide Go libraries should be installed along with their
-source code. The key @code{#:install-source?}, which defaults to
+Packages that provide Go libraries should install their source code into
+the built output. The key @code{#:install-source?}, which defaults to
@code{#t}, controls whether or not the source code is installed. It can
be set to @code{#f} for packages that only provide executable files.
@end defvr
diff --git a/gnu/packages/databases.scm b/gnu/packages/databases.scm
index be02b79..921304e 100644
--- a/gnu/packages/databases.scm
+++ b/gnu/packages/databases.scm
@@ -2610,6 +2610,7 @@ transforms idiomatic python function calls to well-formed
SQL queries.")
#:modules ((srfi srfi-1)
(guix build go-build-system)
(guix build utils))
+ #:install-source? #f
#:phases
(let ((all-tools
'("bsondump" "mongodump" "mongoexport" "mongofiles"
@@ -2629,8 +2630,6 @@ transforms idiomatic python function calls to well-formed
SQL queries.")
(("skipping restore of system.profile collection\", db)")
"skipping restore of system.profile collection\")"))
#t))
- ;; We don't need to install the source code for end-user
applications
- (delete 'install-source)
(replace 'build
(lambda _
(for-each (lambda (tool)
diff --git a/gnu/packages/docker.scm b/gnu/packages/docker.scm
index 7cb8f96..4b6d04f 100644
--- a/gnu/packages/docker.scm
+++ b/gnu/packages/docker.scm
@@ -249,9 +249,11 @@ network attachments.")
`(#:modules
((guix build gnu-build-system)
((guix build go-build-system) #:prefix go:)
+ (guix build union)
(guix build utils))
#:imported-modules
(,@%gnu-build-system-modules
+ (guix build union)
(guix build go-build-system))
#:phases
(modify-phases %standard-phases
@@ -412,8 +414,8 @@ network attachments.")
;; Make build faster
(setenv "GOCACHE" "/tmp")
#t))
- (add-before 'build 'setup-environment
- (assoc-ref go:%standard-phases 'setup-environment))
+ (add-before 'build 'setup-go-environment
+ (assoc-ref go:%standard-phases 'setup-go-environment))
(replace 'build
(lambda _
;; Our LD doesn't like the statically linked relocatable things
diff --git a/gnu/packages/music.scm b/gnu/packages/music.scm
index 7d30204..52631bb 100644
--- a/gnu/packages/music.scm
+++ b/gnu/packages/music.scm
@@ -4361,7 +4361,7 @@ console music players.")
dir "/sbin"))
(list ffmpeg chromaprint))))
#t)))
- (add-after 'install-source 'install-scripts
+ (add-after 'install 'install-scripts
(lambda* (#:key outputs #:allow-other-keys)
(let* ((out (assoc-ref outputs "out"))
(root (string-append out
"/src/gitlab.com/ambrevar/demlo"))
diff --git a/gnu/packages/shellutils.scm b/gnu/packages/shellutils.scm
index a672aa0..88c0f32 100644
--- a/gnu/packages/shellutils.scm
+++ b/gnu/packages/shellutils.scm
@@ -118,15 +118,17 @@ are already there.")
#:make-flags (list (string-append "DESTDIR=" (assoc-ref %outputs
"out")))
#:modules ((guix build gnu-build-system)
((guix build go-build-system) #:prefix go:)
+ (guix build union)
(guix build utils))
#:imported-modules (,@%gnu-build-system-modules
+ (guix build union)
(guix build go-build-system))
#:phases
(modify-phases %standard-phases
(delete 'configure)
;; Help the build scripts find the Go language dependencies.
- (add-after 'unpack 'setup-go-environment
- (assoc-ref go:%standard-phases 'setup-environment)))))
+ (add-before 'unpack 'setup-go-environment
+ (assoc-ref go:%standard-phases 'setup-go-environment)))))
(inputs
`(("go" ,go)
("go-github-com-burntsushi-toml" ,go-github-com-burntsushi-toml)
diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm
index cf91163..1b916af 100644
--- a/guix/build-system/go.scm
+++ b/guix/build-system/go.scm
@@ -39,6 +39,7 @@
(define %go-build-system-modules
;; Build-side modules imported and used by default.
`((guix build go-build-system)
+ (guix build union)
,@%gnu-build-system-modules))
(define (default-go)
@@ -87,6 +88,7 @@
(guile #f)
(imported-modules %go-build-system-modules)
(modules '((guix build go-build-system)
+ (guix build union)
(guix build utils))))
(define builder
`(begin
diff --git a/guix/build/go-build-system.scm b/guix/build/go-build-system.scm
index 022d4fe..1a716ce 100644
--- a/guix/build/go-build-system.scm
+++ b/guix/build/go-build-system.scm
@@ -1,6 +1,6 @@
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2016 Petter <address@hidden>
-;;; Copyright © 2017 Leo Famulari <address@hidden>
+;;; Copyright © 2017, 2019 Leo Famulari <address@hidden>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -19,6 +19,7 @@
(define-module (guix build go-build-system)
#:use-module ((guix build gnu-build-system) #:prefix gnu:)
+ #:use-module (guix build union)
#:use-module (guix build utils)
#:use-module (ice-9 match)
#:use-module (srfi srfi-1)
@@ -38,24 +39,26 @@
;; results. [0]
;; Go software is developed and built within a particular file system hierarchy
-;; structure called a 'workspace' [1]. This workspace is found by Go
-;; via the GOPATH environment variable. Typically, all Go source code
-;; and compiled objects are kept in a single workspace, but it is
-;; possible for GOPATH to contain a list of directories, and that is
-;; what we do in this go-build-system. [2]
+;; structure called a 'workspace' [1]. This workspace can be found by Go via
+;; the GOPATH environment variable. Typically, all Go source code and compiled
+;; objects are kept in a single workspace, but GOPATH may be a list of
+;; directories [2]. In this go-build-system we create a filesystem union of
+;; the Go-language dependencies. Previously, we made GOPATH a list of store
+;; directories, but stopped because Go programs started keeping references to
+;; these directories in Go 1.11:
+;; <https://bugs.gnu.org/33620>.
;;
-;; Go software, whether a package or a command, is uniquely named using
-;; an 'import path'. The import path is based on the URL of the
-;; software's source. Since most source code is provided over the
-;; internet, the import path is typically a combination of the remote
-;; URL and the source repository's file system structure. For example,
-;; the Go port of the common `du` command is hosted on github.com, at
-;; <https://github.com/calmh/du>. Thus, the import path is
-;; <github.com/calmh/du>. [3]
+;; Go software, whether a package or a command, is uniquely named using an
+;; 'import path'. The import path is based on the URL of the software's
source.
+;; Because most source code is provided over the internet, the import path is
+;; typically a combination of the remote URL and the source repository's file
+;; system structure. For example, the Go port of the common `du` command is
+;; hosted on github.com, at <https://github.com/calmh/du>. Thus, the import
+;; path is <github.com/calmh/du>. [3]
;;
-;; It may be possible to programatically guess a package's import path
-;; based on the source URL, but we don't try that in this revision of
-;; the go-build-system.
+;; It may be possible to automatically guess a package's import path based on
+;; the source URL, but we don't try that in this revision of the
+;; go-build-system.
;;
;; Modules of modular Go libraries are named uniquely with their
;; file system paths. For example, the supplemental but "standardized"
@@ -75,6 +78,17 @@
;; file system union of the required modules of such libraries. I think
;; this could be improved in future revisions of the go-build-system.
;;
+;; TODO:
+;; * Avoid copying dependencies into the build environment and / or avoid using
+;; a tmpdir when creating the inputs union.
+;; * Use Go modules [4]
+;; * Re-use compiled packages [5]
+;; * Avoid the go-inputs hack
+;; * Stop needing remove-go-references (-trimpath ? )
+;; * Remove module packages, only offering the full Git repos? This is
+;; more idiomatic, I think, because Go downloads Git repos, not modules.
+;; What are the trade-offs?
+;;
;; [0] `go build`:
;; https://golang.org/cmd/go/#hdr-Compile_packages_and_dependencies
;; `go install`:
@@ -107,18 +121,44 @@
;;
;; [2] https://golang.org/doc/code.html#GOPATH
;; [3] https://golang.org/doc/code.html#ImportPaths
+;; [4] https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more
+;; [5] https://bugs.gnu.org/32919
;;
;; Code:
+(define* (setup-go-environment #:key inputs outputs #:allow-other-keys)
+ "Prepare a Go build environment for INPUTS and OUTPUTS. Build a filesystem
+union of INPUTS. Export GOPATH, which helps the compiler find the source code
+of the package being built and its dependencies, and GOBIN, which determines
+where executables (\"commands\") are installed to. This phase is sometimes
used
+by packages that use (guix build-system gnu) but have a handful of Go
+dependencies, so it should be self-contained."
+ ;; Using the current working directory as GOPATH makes it easier for
packagers
+ ;; who need to manipulate the unpacked source code.
+ (setenv "GOPATH" (getcwd))
+ (setenv "GOBIN" (string-append (assoc-ref outputs "out") "/bin"))
+ (let ((tmpdir (tmpnam)))
+ (match (go-inputs inputs)
+ (((names . directories) ...)
+ (union-build tmpdir (filter directory-exists? directories)
+ #:create-all-directories? #t
+ #:log-port (%make-void-port "w"))))
+ ;; XXX A little dance because (guix build union) doesn't use mkdir-p.
+ (copy-recursively tmpdir
+ (string-append (getenv "GOPATH"))
+ #:keep-mtime? #t)
+ (delete-file-recursively tmpdir))
+ #t)
+
(define* (unpack #:key source import-path unpack-path #:allow-other-keys)
- "Unpack SOURCE in the UNPACK-PATH, or the IMPORT-PATH is the UNPACK-PATH is
-unset. When SOURCE is a directory, copy it instead of unpacking."
+ "Relative to $GOPATH, unpack SOURCE in the UNPACK-PATH, or the IMPORT-PATH is
+the UNPACK-PATH is unset. When SOURCE is a directory, copy it instead of
+unpacking."
(if (string-null? import-path)
((display "WARNING: The Go import path is unset.\n")))
(if (string-null? unpack-path)
(set! unpack-path import-path))
- (mkdir "src")
- (let ((dest (string-append "src/" unpack-path)))
+ (let ((dest (string-append (getenv "GOPATH") "/src/" unpack-path)))
(mkdir-p dest)
(if (file-is-directory? source)
(begin
@@ -128,15 +168,6 @@ unset. When SOURCE is a directory, copy it instead of
unpacking."
(invoke "unzip" "-d" dest source)
(invoke "tar" "-C" dest "-xvf" source)))))
-(define* (install-source #:key install-source? outputs #:allow-other-keys)
- "Install the source code to the output directory."
- (let* ((out (assoc-ref outputs "out"))
- (source "src")
- (dest (string-append out "/" source)))
- (when install-source?
- (copy-recursively source dest #:keep-mtime? #t))
- #t))
-
(define (go-package? name)
(string-prefix? "go-" name))
@@ -155,27 +186,6 @@ unset. When SOURCE is a directory, copy it instead of
unpacking."
(_ #f))
inputs))))
-(define* (setup-environment #:key inputs outputs #:allow-other-keys)
- "Export the variables GOPATH and GOBIN, which are based on INPUTS and
OUTPUTS,
-respectively."
- (let ((out (assoc-ref outputs "out")))
- ;; GOPATH is where Go looks for the source code of the build's
dependencies.
- (set-path-environment-variable "GOPATH"
- ;; XXX Matching "." hints that we could do
- ;; something simpler here...
- (list ".")
- (match (go-inputs inputs)
- (((_ . dir) ...)
- dir)))
-
- ;; Add the source code of the package being built to GOPATH.
- (if (getenv "GOPATH")
- (setenv "GOPATH" (string-append (getcwd) ":" (getenv "GOPATH")))
- (setenv "GOPATH" (getcwd)))
- ;; Where to install compiled executable files ('commands' in Go parlance').
- (setenv "GOBIN" (string-append out "/bin"))
- #t))
-
(define* (build #:key import-path #:allow-other-keys)
"Build the package named by IMPORT-PATH."
(with-throw-handler
@@ -193,22 +203,26 @@ respectively."
"Here are the results of `go env`:\n"))
(invoke "go" "env"))))
+;; Can this also install commands???
(define* (check #:key tests? import-path #:allow-other-keys)
"Run the tests for the package named by IMPORT-PATH."
(when tests?
(invoke "go" "test" import-path))
#t)
-(define* (install #:key outputs #:allow-other-keys)
- "Install the compiled libraries. `go install` installs these files to
-$GOPATH/pkg, so we have to copy them into the output directory manually.
-Compiled executable files should have already been installed to the store based
-on $GOBIN in the build phase."
- ;; TODO: From go-1.10 onward, the pkg folder should not be needed (see
- ;; https://lists.gnu.org/archive/html/guix-devel/2018-11/msg00208.html).
- ;; Remove it?
- (when (file-exists? "pkg")
- (copy-recursively "pkg" (string-append (assoc-ref outputs "out") "/pkg")))
+(define* (install #:key install-source? outputs import-path unpack-path
#:allow-other-keys)
+ "Install the source code of IMPORT-PATH to the primary output directory.
+Compiled executable files (Go \"commands\") should have already been installed
+to the store based on $GOBIN in the build phase.
+XXX We can't make us of compiled libraries (Go \"packages\")."
+ (when install-source?
+ (if (string-null? import-path)
+ ((display "WARNING: The Go import path is unset.\n")))
+ (let* ((out (assoc-ref outputs "out"))
+ (source (string-append (getenv "GOPATH") "/src/" import-path))
+ (dest (string-append out "/src/" import-path)))
+ (mkdir-p dest)
+ (copy-recursively source dest #:keep-mtime? #t)))
#t)
(define* (remove-store-reference file file-name
@@ -269,9 +283,8 @@ files in OUTPUTS."
(delete 'bootstrap)
(delete 'configure)
(delete 'patch-generated-file-shebangs)
+ (add-before 'unpack 'setup-go-environment setup-go-environment)
(replace 'unpack unpack)
- (add-after 'unpack 'install-source install-source)
- (add-before 'build 'setup-environment setup-environment)
(replace 'build build)
(replace 'check check)
(replace 'install install)