From 21cf092c67e10e60682f3c14d6b438ce7d905eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Fri, 28 Dec 2018 18:27:59 +0100 Subject: [PATCH 4/5] publish: Add IPFS support. * guix/scripts/publish.scm (show-help, %options): Add '--ipfs'. (narinfo-string): Add IPFS parameter and honor it. (render-narinfo/cached): Add #:ipfs? and honor it. (bake-narinfo+nar, make-request-handler, run-publish-server): Likewise. (guix-publish): Honor '--ipfs' and parameterize %IPFS-BASE-URL. --- doc/guix.texi | 34 +++++++++++++++++++ guix/scripts/publish.scm | 73 +++++++++++++++++++++++++++------------- 2 files changed, 83 insertions(+), 24 deletions(-) diff --git a/doc/guix.texi b/doc/guix.texi index 1f33fd3b76..e52083fc5d 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -12267,6 +12267,16 @@ http://example.org/file/hello-2.10.tar.gz/sha256/0ssi1@dots{}ndq1i Obviously, these URLs only work for files that are in the store; in other cases, they return 404 (``Not Found''). +@cindex peer-to-peer, substitute distribution +@cindex distributed storage, of substitutes +@cindex IPFS, for substitutes + +It is also possible to publish substitutes over @uref{https://ipfs.io, IFPS}, +a distributed, peer-to-peer storage mechanism. To enable it, pass the +@option{--ipfs} option alongside @option{--cache}, and make sure you're +running @command{ipfs daemon}. Capable clients will then be able to choose +whether to fetch substitutes over HTTP or over IPFS. + @cindex build logs, publication Build logs are available from @code{/log} URLs like: @@ -12363,6 +12373,30 @@ thread per CPU core is created, but this can be customized. See When @option{--ttl} is used, cached entries are automatically deleted when they have expired. +@item --ifps[=@var{gateway}] +When used in conjunction with @option{--cache}, instruct @command{guix +publish} to publish substitutes over the @uref{https://ipfs.io, IPFS +distributed data store} in addition to HTTP. + +@quotation Note +As of version @value{VERSION}, IPFS support is experimental. You're welcome +to share your experience with the developers by emailing +@email{guix-devel@@gnu.org}! +@end quotation + +The IPFS HTTP interface must be reachable at @var{gateway}, by default +@code{localhost:5001}. To get it up and running, it is usually enough to +install IPFS and start the IPFS daemon: + +@example +$ guix package -i go-ipfs +$ ipfs init +$ ipfs daemon +@end example + +For more information on how to get started with IPFS, please refer to the +@uref{https://docs.ipfs.io/introduction/usage/, IPFS documentation}. + @item --workers=@var{N} When @option{--cache} is used, request the allocation of @var{N} worker threads to ``bake'' archives. diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index c31cef3181..998dfa560d 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -64,8 +64,8 @@ #:use-module ((guix build utils) #:select (dump-port mkdir-p find-files)) #:use-module ((guix build syscalls) #:select (set-thread-name)) + #:use-module ((guix ipfs) #:prefix ipfs:) #:export (%default-gzip-compression - %public-key %private-key signed-string @@ -94,6 +94,8 @@ Publish ~a over HTTP.\n") %store-directory) (display (G_ " --cache-bypass-threshold=SIZE serve store items below SIZE even when not cached")) + (display (G_ " + --ipfs[=GATEWAY] publish items over IPFS via GATEWAY")) (display (G_ " --workers=N use N workers to bake items")) (display (G_ " @@ -210,6 +212,10 @@ usage." (lambda (opt name arg result) (alist-cons 'cache-bypass-threshold (size->number arg) result))) + (option '("ipfs") #f #t + (lambda (opt name arg result) + (alist-cons 'ipfs (or arg (ipfs:%ipfs-base-url)) + result))) (option '("workers") #t #f (lambda (opt name arg result) (alist-cons 'workers (string->number* arg) @@ -308,14 +314,16 @@ with COMPRESSION, starting at NAR-PATH." (define* (narinfo-string store store-path key #:key (compressions (list %no-compression)) - (nar-path "nar") (file-sizes '())) + (nar-path "nar") (file-sizes '()) ipfs) "Generate a narinfo key/value string for STORE-PATH; an exception is raised if STORE-PATH is invalid. Produce a URL that corresponds to COMPRESSION. The narinfo is signed with KEY. NAR-PATH specifies the prefix for nar URLs. Optionally, FILE-SIZES is a list of compression/integer pairs, where the integer is size in bytes of the compressed NAR; it informs the client of how -much needs to be downloaded." +much needs to be downloaded. + +When IPFS is true, it is the IPFS object identifier for STORE-PATH." (let* ((path-info (query-path-info store store-path)) (compressions (actual-compressions store-path compressions)) (hash (bytevector->nix-base32-string @@ -363,7 +371,12 @@ References: ~a~%" (apply throw args)))))) (signature (base64-encode-string (canonical-sexp->string (signed-string info))))) - (format #f "~aSignature: 1;~a;~a~%" info (gethostname) signature))) + (format #f "~aSignature: 1;~a;~a~%~a" info (gethostname) signature + + ;; Append IPFS info below the signed part. + (if ipfs + (string-append "IPFS: " ipfs "\n") + "")))) (define* (not-found request #:key (phrase "Resource not found") @@ -510,10 +523,12 @@ interpreted as the basename of a store item." (define* (render-narinfo/cached store request hash #:key ttl (compressions (list %no-compression)) (nar-path "nar") - cache pool) + cache pool ipfs?) "Respond to the narinfo request for REQUEST. If the narinfo is available in CACHE, then send it; otherwise, return 404 and \"bake\" that nar and narinfo -requested using POOL." +requested using POOL. + +When IPFS? is true, additionally publish binaries over IPFS." (define (delete-entry narinfo) ;; Delete NARINFO and the corresponding nar from CACHE. (let* ((nar (string-append (string-drop-right narinfo @@ -556,7 +571,8 @@ requested using POOL." (bake-narinfo+nar cache item #:ttl ttl #:compressions compressions - #:nar-path nar-path))) + #:nar-path nar-path + #:ipfs? ipfs?))) (when ttl (single-baker 'cache-cleanup @@ -617,7 +633,7 @@ requested using POOL." (define* (bake-narinfo+nar cache item #:key ttl (compressions (list %no-compression)) - (nar-path "/nar")) + (nar-path "/nar") ipfs?) "Write the narinfo and nar for ITEM to CACHE." (define (compressed-nar-size compression) (let* ((nar (nar-cache-file cache item #:compression compression)) @@ -644,7 +660,11 @@ requested using POOL." (%private-key) #:nar-path nar-path #:compressions compressions - #:file-sizes sizes) + #:file-sizes sizes + #:ipfs + (and ipfs? + (ipfs:content-name + (ipfs:add-file-tree item)))) port))) ;; Make the cached narinfo world-readable, contrary to what @@ -996,7 +1016,8 @@ methods, return the applicable compression." cache pool narinfo-ttl (nar-path "nar") - (compressions (list %no-compression))) + (compressions (list %no-compression)) + ipfs?) (define compression-type? string->compression-type) @@ -1027,7 +1048,9 @@ methods, return the applicable compression." #:pool pool #:ttl narinfo-ttl #:nar-path nar-path - #:compressions compressions) + #:compressions compressions + #:compressions compressions + #:ipfs? ipfs?) (render-narinfo store request hash #:ttl narinfo-ttl #:nar-path nar-path @@ -1089,7 +1112,7 @@ methods, return the applicable compression." advertise? port (compressions (list %no-compression)) (nar-path "nar") narinfo-ttl - cache pool) + cache pool ipfs?) (when advertise? (let ((name (service-name))) ;; XXX: Use a callback from Guile-Avahi here, as Avahi can pick a @@ -1098,13 +1121,13 @@ methods, return the applicable compression." (avahi-publish-service-thread name #:type publish-service-type #:port port))) - (run-server (make-request-handler store #:cache cache #:pool pool #:nar-path nar-path #:narinfo-ttl narinfo-ttl - #:compressions compressions) + #:compressions compressions + #:ipfs? ipfs?) concurrent-http-server `(#:socket ,socket))) @@ -1166,6 +1189,7 @@ methods, return the applicable compression." (repl-port (assoc-ref opts 'repl)) (cache (assoc-ref opts 'cache)) (workers (assoc-ref opts 'workers)) + (ipfs (assoc-ref opts 'ipfs)) ;; Read the key right away so that (1) we fail early on if we can't ;; access them, and (2) we can then drop privileges. @@ -1204,16 +1228,17 @@ consider using the '--user' option!~%"))) (set-thread-name "guix publish") (with-store store - (run-publish-server socket store - #:advertise? advertise? - #:port port - #:cache cache - #:pool (and cache (make-pool workers - #:thread-name - "publish worker")) - #:nar-path nar-path - #:compressions compressions - #:narinfo-ttl ttl)))))) + (parameterize ((ipfs:%ipfs-base-url ipfs)) + (run-publish-server socket store + #:advertise? advertise? + #:port port + #:cache cache + #:pool (and cache (make-pool workers + #:thread-name + "publish worker")) + #:nar-path nar-path + #:compressions compressions + #:narinfo-ttl ttl))))))) ;;; Local Variables: ;;; eval: (put 'single-baker 'scheme-indent-function 1) -- 2.29.2