[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
06/09: gnu: Add python-pytorch.
|
From: |
guix-commits |
|
Subject: |
06/09: gnu: Add python-pytorch. |
|
Date: |
Wed, 4 Aug 2021 11:23:16 -0400 (EDT) |
civodul pushed a commit to branch master
in repository guix.
commit dd55ad7ff896a0a099c669ef0b420ae2bcdb4498
Author: Ludovic Courtès <ludovic.courtes@inria.fr>
AuthorDate: Wed Aug 4 16:15:17 2021 +0200
gnu: Add python-pytorch.
* gnu/packages/machine-learning.scm (python-pytorch): New variable.
* gnu/packages/patches/python-pytorch-runpath.patch,
gnu/packages/patches/python-pytorch-system-libraries.patch: New files.
* gnu/local.mk (dist_patch_DATA): Add them.
---
gnu/local.mk | 2 +
gnu/packages/machine-learning.scm | 143 ++++++++++++++++++++-
gnu/packages/patches/python-pytorch-runpath.patch | 25 ++++
.../patches/python-pytorch-system-libraries.patch | 131 +++++++++++++++++++
4 files changed, 300 insertions(+), 1 deletion(-)
diff --git a/gnu/local.mk b/gnu/local.mk
index 82e5fbc..51a76e3 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -1649,6 +1649,8 @@ dist_patch_DATA =
\
%D%/packages/patches/python-pydot-regression-test.patch \
%D%/packages/patches/python2-pygobject-2-deprecation.patch \
%D%/packages/patches/python-pygpgme-fix-pinentry-tests.patch \
+ %D%/packages/patches/python-pytorch-runpath.patch \
+ %D%/packages/patches/python-pytorch-system-libraries.patch \
%D%/packages/patches/python-robotframework-source-date-epoch.patch \
%D%/packages/patches/python-seaborn-kde-test.patch \
%D%/packages/patches/python2-subprocess32-disable-input-test.patch \
diff --git a/gnu/packages/machine-learning.scm
b/gnu/packages/machine-learning.scm
index c3480d6..4149def 100644
--- a/gnu/packages/machine-learning.scm
+++ b/gnu/packages/machine-learning.scm
@@ -65,10 +65,12 @@
#:use-module (gnu packages graphviz)
#:use-module (gnu packages gstreamer)
#:use-module (gnu packages image)
+ #:use-module (gnu packages libffi)
#:use-module (gnu packages linux)
#:use-module (gnu packages llvm)
#:use-module (gnu packages maths)
#:use-module (gnu packages mpi)
+ #:use-module (gnu packages ninja)
#:use-module (gnu packages ocaml)
#:use-module (gnu packages onc-rpc)
#:use-module (gnu packages parallel)
@@ -2546,7 +2548,7 @@ general non-linear dimension reduction.")
(package
(name "xnnpack")
(version (git-version version revision commit))
- (home-page "https://github.com/google/XNNPACK";)
+ (home-page "https://github.com/google/XNNPACK";) ;fork of QNNPACK
(source (origin
(method git-fetch)
(uri (git-reference (url home-page) (commit commit)))
@@ -2586,3 +2588,142 @@ high-level machine learning frameworks, such as
TensorFlow Lite,
TensorFlow.js, PyTorch, and MediaPipe.")
(license license:bsd-3))))
+(define-public python-pytorch
+ (package
+ (name "python-pytorch")
+ (version "1.9.0")
+ (source (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/pytorch/pytorch";)
+ (commit (string-append "v" version))
+ (recursive? #t)))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ "0cznsh68hwk5761gv7iijb4g6jgjpvs3bbixwpzzmkbkbn2q96c1"))
+ (patches (search-patches "python-pytorch-system-libraries.patch"
+ "python-pytorch-runpath.patch"))
+ (modules '((guix build utils)))
+ (snippet
+ '(begin
+ ;; XXX: Let's be clear: this package is a bundling fest. We
+ ;; delete as much as we can, but there's still a lot left.
+ (for-each (lambda (directory)
+ (delete-file-recursively
+ (string-append "third_party/" directory)))
+ '("benchmark" "cpuinfo" "eigen"
+
+ ;; FIXME: QNNPACK (of which XNNPACK is a fork)
+ ;; needs these.
+ ;; "FP16" "FXdiv" "gemmlowp" "psimd"
+
+ "gloo" "googletest" "ios-cmake"
+ "onnx" "protobuf" "pthreadpool"
+ "pybind11" "python-enum" "python-peachpy"
+ "python-six" "tbb" "XNNPACK" "zstd"))
+
+ ;; Adjust references to the onnx-optimizer headers.
+ (substitute* "caffe2/onnx/backend.cc"
+ (("onnx/optimizer/")
+ "onnxoptimizer/"))))))
+ (build-system python-build-system)
+ (arguments
+ '(#:phases (modify-phases %standard-phases
+ (add-before 'build 'use-system-libraries
+ (lambda* (#:key outputs #:allow-other-keys)
+ ;; Tell 'setup.py' to let 'CMakeLists.txt' know that we
+ ;; want to use "system libraries" instead of the bundled
+ ;; ones.
+ (setenv "USE_SYSTEM_LIBS" "1")
+
+ ;; XXX: Disable that for simplicity for now.
+ (setenv "USE_FBGEMM" "0")
+
+ ;; Set 'OUT/lib' rather than '$ORIGIN' as the RUNPATH on
+ ;; binaries that go to 'lib/python3.8/torch/bin' & co.
+ #;(substitute* "cmake/Dependencies.cmake"
+ (("^set\\(CMAKE_INSTALL_RPATH .*")
+ (string-append "set(CMAKE_INSTALL_RPATH \""
+ (assoc-ref outputs "out")
+ "/lib\")\n")))))
+ (add-before 'build 'make-things-writable
+ (lambda _
+ ;; The 'build_caffe2' function in
+ ;; 'tools/build_pytorch_libs.py', called from the
+ ;; top-level 'setup.py', needs write access to this
+ ;; directory.
+ (for-each make-file-writable
+ (find-files "caffe2/proto" "."
+ #:directories? #t))))
+ (replace 'check
+ (lambda* (#:key inputs outputs tests? #:allow-other-keys)
+ ;; Run the test suite following the instructions in
+ ;; 'CONTRIBUTING.md'. XXX: Unfortunately this doesn't
+ ;; work, unless you set PYTHONPATH presumably.
+ (when tests?
+ (let ((python-site (site-packages inputs outputs)))
+ (setenv "PYTHONPATH"
+ (string-append python-site ":"
+ (getenv "PYTHONPATH")))
+ (invoke "python" "test/run_test.py")))))
+ (add-after 'install 'remove-test-executables
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ ;; Remove test executables, but keep other executables
+ ;; such as 'torch_shm_manager' and and .so files such as
+ ;; 'libtorch_global_deps.so'.
+ (let ((python-site (site-packages inputs outputs)))
+ (for-each delete-file
+ (find-files python-site
+ "(^test_cpp_rpc|_test)$"))))))
+
+ ;; XXX: Tests attempt to download data such as
+ ;;
<https://raw.githubusercontent.com/pytorch/test-infra/master/stats/slow-tests.json>.
+ #:tests? #f))
+ (native-inputs
+ `(("cmake" ,cmake)
+ ("ninja" ,ninja)))
+ (inputs
+ `(("eigen" ,eigen)
+ ;; ("fmt" ,fmt)
+ ("fp16" ,fp16)
+ ("gemmlowp" ,gemmlowp)
+ ("googletest" ,googletest)
+ ("googlebenchmark" ,googlebenchmark)
+ ("gloo" ,gloo)
+ ("openblas" ,openblas)
+ ("openmpi" ,openmpi)
+ ("pthreadpool" ,pthreadpool)
+ ("protobuf" ,protobuf)
+ ("pybind11" ,pybind11)
+ ("sleef" ,sleef)
+ ("xnnpack" ,xnnpack)
+ ("zstd" ,zstd)))
+ (propagated-inputs
+ `(("python-astunparse" ,python-astunparse)
+ ("python-numpy" ,python-numpy)
+ ("python-pyyaml" ,python-pyyaml)
+ ("python-cffi" ,python-cffi)
+ ("python-peachpy" ,python-peachpy)
+ ("python-typing-extensions" ,python-typing-extensions)
+ ("python-future" ,python-future)
+ ("python-six" ,python-six)
+ ("python-requests" ,python-requests)
+ ("onnx" ,onnx) ;propagated for its Python modules
+ ("onnx-optimizer" ,onnx-optimizer)
+ ("cpuinfo" ,cpuinfo)))
+ (home-page "https://pytorch.org/";)
+ (synopsis "Python library for tensor computation and deep neural networks")
+ (description
+ "PyTorch is a Python package that provides two high-level features:
+
+@itemize
+@item tensor computation (like NumPy) with strong GPU acceleration;
+@item deep neural networks (DNNs) built on a tape-based autograd system.
+@end itemize
+
+You can reuse Python packages such as NumPy, SciPy, and Cython to extend
+PyTorch when needed.
+
+Note: currently this package does not provide GPU support.")
+ (license license:bsd-3)))
diff --git a/gnu/packages/patches/python-pytorch-runpath.patch
b/gnu/packages/patches/python-pytorch-runpath.patch
new file mode 100644
index 0000000..6f270ef
--- /dev/null
+++ b/gnu/packages/patches/python-pytorch-runpath.patch
@@ -0,0 +1,25 @@
+Libraries (such as 'libtorch_cpu.so') and executables (such as
'torch_shm_manager')
+get installed, quite surprisingly, to 'lib/python3.8/site-packages/{bin,lib}'.
+Make sure RUNPATH matches that.
+
+--- a/cmake/Dependencies.cmake
++++ b/cmake/Dependencies.cmake
+@@ -4,7 +4,7 @@ if(APPLE)
+ set(CMAKE_MACOSX_RPATH ON)
+ set(_rpath_portable_origin "@loader_path")
+ else()
+- set(_rpath_portable_origin $ORIGIN)
++ set(_rpath_portable_origin $ORIGIN/../lib)
+ endif(APPLE)
+ # Use separate rpaths during build and install phases
+ set(CMAKE_SKIP_BUILD_RPATH FALSE)
+
+--- a/caffe2/CMakeLists.txt
++++ b/caffe2/CMakeLists.txt
+@@ -1797,5 +1797,5 @@ if(BUILD_PYTHON)
+ if(${BUILDING_WITH_TORCH_LIBS})
+ # site-packages/caffe2/python/caffe2_pybind11_state
+ # site-packages/torch/lib
+- set(caffe2_pybind11_rpath "${_rpath_portable_origin}/../../torch/lib")
++ set(caffe2_pybind11_rpath $ORIGIN/../../torch/lib)
+ endif(${BUILDING_WITH_TORCH_LIBS})
diff --git a/gnu/packages/patches/python-pytorch-system-libraries.patch
b/gnu/packages/patches/python-pytorch-system-libraries.patch
new file mode 100644
index 0000000..c8d14b3
--- /dev/null
+++ b/gnu/packages/patches/python-pytorch-system-libraries.patch
@@ -0,0 +1,131 @@
+Use our own googletest rather than the bundled one.
+Get NNPACK to use our own PeachPy rather than the bundled one.
+
+diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
+index 5d57b9ca78..620cca4e60 100644
+--- a/cmake/Dependencies.cmake
++++ b/cmake/Dependencies.cmake
+@@ -644,11 +644,6 @@ if(BUILD_TEST OR BUILD_MOBILE_BENCHMARK OR
BUILD_MOBILE_TEST)
+ # this shouldn't be necessary anymore.
+ get_property(INC_DIR_temp DIRECTORY PROPERTY INCLUDE_DIRECTORIES)
+ set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "")
+- add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest)
+- set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES ${INC_DIR_temp})
+-
+- include_directories(BEFORE SYSTEM
${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googletest/include)
+- include_directories(BEFORE SYSTEM
${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googlemock/include)
+
+ # We will not need to test benchmark lib itself.
+ set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing as
we don't need it.")
+@@ -1485,7 +1480,7 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT
INTERN_DISABLE_ONNX)
+ endif()
+ set_property(TARGET onnx_proto PROPERTY IMPORTED_LOCATION
${ONNX_PROTO_LIBRARY})
+ message("-- Found onnx: ${ONNX_LIBRARY} ${ONNX_PROTO_LIBRARY}")
+- list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx)
++ list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx onnx_optimizer)
+ endif()
+ include_directories(${FOXI_INCLUDE_DIRS})
+ list(APPEND Caffe2_DEPENDENCY_LIBS foxi_loader)
+
+diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
+index 50ebb224ce..5953d9ddf7 100644
+--- a/caffe2/CMakeLists.txt
++++ b/caffe2/CMakeLists.txt
+@@ -1632,7 +1632,7 @@ if(BUILD_TEST)
+ if(NOT MSVC)
+ add_executable(${test_name}_${CPU_CAPABILITY} "${test_src}"
../aten/src/ATen/native/quantized/affine_quantizer_base.cpp)
+ # TODO: Get rid of c10 dependency (which is only needed for the
implementation of AT_ERROR)
+- target_link_libraries(${test_name}_${CPU_CAPABILITY} c10 sleef
gtest_main)
++ target_link_libraries(${test_name}_${CPU_CAPABILITY} c10 sleef
gtest_main gtest)
+ if(USE_FBGEMM)
+ target_link_libraries(${test_name}_${CPU_CAPABILITY} fbgemm)
+ endif()
+@@ -1655,7 +1655,7 @@ if(BUILD_TEST)
+ foreach(test_src ${Caffe2_CPU_TEST_SRCS})
+ get_filename_component(test_name ${test_src} NAME_WE)
+ add_executable(${test_name} "${test_src}")
+- target_link_libraries(${test_name} torch_library gtest_main)
++ target_link_libraries(${test_name} torch_library gtest_main gtest)
+ target_include_directories(${test_name} PRIVATE
$<INSTALL_INTERFACE:include>)
+ target_include_directories(${test_name} PRIVATE
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include>)
+ target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
+@@ -1673,7 +1673,7 @@ if(BUILD_TEST)
+ foreach(test_src ${Caffe2_GPU_TEST_SRCS})
+ get_filename_component(test_name ${test_src} NAME_WE)
+ cuda_add_executable(${test_name} "${test_src}")
+- target_link_libraries(${test_name} torch_library gtest_main)
++ target_link_libraries(${test_name} torch_library gtest_main gtest)
+ target_include_directories(${test_name} PRIVATE
$<INSTALL_INTERFACE:include>)
+ target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
+ add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
+@@ -1691,7 +1691,7 @@ if(BUILD_TEST)
+ foreach(test_src ${Caffe2_VULKAN_TEST_SRCS})
+ get_filename_component(test_name ${test_src} NAME_WE)
+ add_executable(${test_name} "${test_src}")
+- target_link_libraries(${test_name} torch_library gtest_main)
++ target_link_libraries(${test_name} torch_library gtest_main gtest)
+ target_include_directories(${test_name} PRIVATE
$<INSTALL_INTERFACE:include>)
+ target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
+ add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
+@@ -1709,7 +1709,7 @@ if(BUILD_TEST)
+ foreach(test_src ${Caffe2_HIP_TEST_SRCS})
+ get_filename_component(test_name ${test_src} NAME_WE)
+ add_executable(${test_name} "${test_src}")
+- target_link_libraries(${test_name} torch_library gtest_main)
++ target_link_libraries(${test_name} torch_library gtest_main gtest)
+ target_include_directories(${test_name} PRIVATE
$<INSTALL_INTERFACE:include>)
+ target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE}
${Caffe2_HIP_INCLUDE})
+ target_compile_options(${test_name} PRIVATE ${HIP_CXX_FLAGS})
+
+diff --git a/torch/lib/c10d/test/CMakeLists.txt
b/torch/lib/c10d/test/CMakeLists.txt
+index b74d4b65f7..fc7c207505 100644
+--- a/torch/lib/c10d/test/CMakeLists.txt
++++ b/torch/lib/c10d/test/CMakeLists.txt
+@@ -16,25 +16,25 @@ function(c10d_add_test test_src)
+ add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
+ endfunction()
+
+-c10d_add_test(FileStoreTest.cpp c10d gtest_main)
+-c10d_add_test(TCPStoreTest.cpp c10d gtest_main)
++c10d_add_test(FileStoreTest.cpp c10d gtest_main gtest)
++c10d_add_test(TCPStoreTest.cpp c10d gtest_main gtest)
+ if(NOT WIN32)
+- c10d_add_test(HashStoreTest.cpp c10d gtest_main)
++ c10d_add_test(HashStoreTest.cpp c10d gtest_main gtest)
+ endif()
+
+ if(USE_CUDA)
+ if(USE_C10D_GLOO)
+- c10d_add_test(ProcessGroupGlooTest.cpp c10d c10d_cuda_test gtest_main)
+- c10d_add_test(ProcessGroupGlooAsyncTest.cpp c10d c10d_cuda_test
gtest_main)
++ c10d_add_test(ProcessGroupGlooTest.cpp c10d c10d_cuda_test gtest_main
gtest)
++ c10d_add_test(ProcessGroupGlooAsyncTest.cpp c10d c10d_cuda_test
gtest_main gtest)
+ endif()
+ if(USE_C10D_NCCL)
+- c10d_add_test(ProcessGroupNCCLTest.cpp c10d c10d_cuda_test gtest_main)
++ c10d_add_test(ProcessGroupNCCLTest.cpp c10d c10d_cuda_test gtest_main
gtest)
+ c10d_add_test(ProcessGroupNCCLErrorsTest.cpp c10d c10d_cuda_test
+- gtest_main)
++ gtest_main gtest)
+ endif()
+ else()
+ if(USE_C10D_GLOO)
+- c10d_add_test(ProcessGroupGlooTest.cpp c10d gtest_main)
++ c10d_add_test(ProcessGroupGlooTest.cpp c10d gtest_main gtest)
+ endif()
+ endif()
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 5ecd2df..24feae3 100644
+--- a/third_party/NNPACK/CMakeLists.txt
++++ b/third_party/NNPACK/CMakeLists.txt
+@@ -427,8 +427,7 @@ IF(NNPACK_BACKEND STREQUAL "x86-64")
+ FILE(MAKE_DIRECTORY ${obj_dir})
+ ADD_CUSTOM_COMMAND(
+ OUTPUT ${obj}
+- COMMAND "PYTHONPATH=${PEACHPY_PYTHONPATH}"
+- ${PYTHON_EXECUTABLE} -m peachpy.x86_64
++ COMMAND ${PYTHON_EXECUTABLE} -m peachpy.x86_64
+ -mabi=sysv -g4 -mimage-format=${PEACHPY_IMAGE_FORMAT}
+ "-I${PROJECT_SOURCE_DIR}/src"
"-I${PROJECT_SOURCE_DIR}/src/x86_64-fma" "-I${FP16_SOURCE_DIR}/include"
+ -o ${obj} "${PROJECT_SOURCE_DIR}/${src}"
- branch master updated (48542e0 -> 6272222), guix-commits, 2021/08/04
- 09/09: gnu: Fix broken links to Arch Linux git repositories, guix-commits, 2021/08/04
- 02/09: gnu: Add python-peachpy., guix-commits, 2021/08/04
- 06/09: gnu: Add python-pytorch.,
guix-commits <=
- 01/09: gnu: Add python-opcodes., guix-commits, 2021/08/04
- 05/09: gnu: gemmlowp: Add "generic" variant., guix-commits, 2021/08/04
- 07/09: gnu: Add python-clang., guix-commits, 2021/08/04
- 08/09: import: Use PARSE-COMMAND-LINE for options., guix-commits, 2021/08/04
- 03/09: gnu: gloo: Update to c22a5cf., guix-commits, 2021/08/04
- 04/09: gnu: gloo: Honor #:tests? in 'check' phase., guix-commits, 2021/08/04