From 5f79b0502f62bd1dacc8ea143c1dbd9ef7cfc29d Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:00 -0400 Subject: [PATCH 2/9] import: pypi: Do not parse optional requirements from source. * guix/import/pypi.scm: Export PARSE-REQUIRES.TXT. (guess-requirements): Move the READ-REQUIREMENTS procedure to the top level, and rename it to PARSE-REQUIRES.TXT. Move the CLEAN-REQUIREMENT and COMMENT? functions inside the READ-REQUIREMENTS procedure. (parse-requires.txt): Add a SECTION-HEADER? predicate, and use it to prevent parsing optional requirements. * tests/pypi.scm (test-requires-with-sections): New variable. ("parse-requires.txt, with sections"): New test. ("pypi->guix-package"): Mute tar output to stdout. --- guix/import/pypi.scm | 76 +++++++++++++++++++++++++++----------------- tests/pypi.scm | 21 ++++++++++-- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 8269aa61d7..91e987e9f1 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -47,7 +47,8 @@ #:use-module (guix upstream) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) - #:export (guix-package->pypi-name + #:export (parse-requires.txt + guix-package->pypi-name pypi-recursive-import pypi->guix-package %pypi-updater)) @@ -117,6 +118,49 @@ package definition." ((package-inputs ...) `((propagated-inputs (,'quasiquote ,package-inputs)))))) +(define (clean-requirement s) + ;; Given a requirement LINE, as can be found in a setuptools requires.txt + ;; file, remove everything other than the actual name of the required + ;; package, and return it. + (string-take s (or (string-index s (lambda (chr) + (member chr '(#\space #\> #\= #\<)))) + (string-length s)))) + +(define (parse-requires.txt requires.txt) + "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of +requirement names." + ;; This is a very incomplete parser, which job is to select the non-optional + ;; dependencies and strip them out of any version information. + ;; Alternatively, we could implement a PEG parser with the (ice-9 peg) + ;; library and the requirements grammar defined by PEP-0508 + ;; (https://www.python.org/dev/peps/pep-0508/). + + (define (comment? line) + ;; Return #t if the given LINE is a comment, #f otherwise. + (eq? (string-ref (string-trim line) 0) #\#)) + + (define (section-header? line) + ;; Return #t if the given LINE is a section header, #f otherwise. + (let ((trimmed-line (string-trim line))) + (and (not (string-null? trimmed-line)) + (eq? (string-ref trimmed-line 0) #\[)))) + + (call-with-input-file requires.txt + (lambda (port) + (let loop ((result '())) + (let ((line (read-line port))) + ;; Stop when a section is encountered, as sections contains optional + ;; (extra) requirements. Non-optional requirements must appear + ;; before any section is defined. + (if (or (eof-object? line) (section-header? line)) + (reverse result) + (cond + ((or (string-null? line) (comment? line)) + (loop result)) + (else + (loop (cons (clean-requirement line) + result)))))))))) + (define (guess-requirements source-url wheel-url tarball) "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of the required packages specified in the requirements.txt file. TARBALL will @@ -139,34 +183,6 @@ be extracted in a temporary directory." cannot determine package dependencies")) #f))))) - (define (clean-requirement s) - ;; Given a requirement LINE, as can be found in a Python requirements.txt - ;; file, remove everything other than the actual name of the required - ;; package, and return it. - (string-take s - (or (string-index s (lambda (chr) (member chr '(#\space #\> #\= #\<)))) - (string-length s)))) - - (define (comment? line) - ;; Return #t if the given LINE is a comment, #f otherwise. - (eq? (string-ref (string-trim line) 0) #\#)) - - (define (read-requirements requirements-file) - ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list - ;; of name/variable pairs describing the requirements. - (call-with-input-file requirements-file - (lambda (port) - (let loop ((result '())) - (let ((line (read-line port))) - (if (eof-object? line) - result - (cond - ((or (string-null? line) (comment? line)) - (loop result)) - (else - (loop (cons (clean-requirement line) - result)))))))))) - (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's ;; requirements. @@ -212,7 +228,7 @@ cannot determine package dependencies")) (current-output-port (%make-void-port "rw+"))) (system* "tar" "xf" tarball "-C" dir requires.txt)))) (if (zero? exit-code) - (read-requirements (string-append dir "/" requires.txt)) + (parse-requires.txt (string-append dir "/" requires.txt)) (begin (warning (G_ "Failed to extract file: ~a from source.~%") diff --git a/tests/pypi.scm b/tests/pypi.scm index 335be42644..e4b7142311 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -60,6 +60,15 @@ bar baz > 13.37 ") +(define test-requires-with-sections "\ +# A comment +foo ~= 3 +bar != 2 + +[test] +pytest (>=2.5.0) +") + (define test-metadata "{ \"run_requires\": [ @@ -99,6 +108,12 @@ baz > 13.37 (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz" (pypi-uri "cram" "0.7")))))))) +(test-equal "parse-requires.txt, with sections" + '("foo" "bar") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-requires.txt test-requires-with-sections))) + (test-assert "pypi->guix-package" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch @@ -110,7 +125,8 @@ baz > 13.37 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () (display test-requires.txt))) - (system* "tar" "czvf" file-name "foo-1.0.0/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") (set! test-source-hash (call-with-input-file file-name port-sha256)))) @@ -161,7 +177,8 @@ baz > 13.37 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () (display test-requires.txt))) - (system* "tar" "czvf" file-name "foo-1.0.0/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") (set! test-source-hash (call-with-input-file file-name port-sha256)))) -- 2.21.0