diff options
author | Nicolas Graves <ngraves@ngraves.fr> | 2024-09-09 12:08:03 +0200 |
---|---|---|
committer | Guix Patches Tester <> | 2024-09-09 13:10:12 +0200 |
commit | 770a6e1e4f5db40d34ba30550d88fc93c13618eb (patch) | |
tree | bb2d252f49b3b7e0c7568dee2c00331c169f408a | |
parent | a253e4577da7709b9633cdd8e777022266cb7b4e (diff) | |
download | guix-patches-issue-73094.tar guix-patches-issue-73094.tar.gz |
gnu: Add python-safetensors.issue-73094
* gnu/packages/machine-learning.scm (python-safetensors): New variable.
Change-Id: I90a1684d06756ce87ca0862d745a75be5919f0b2
-rw-r--r-- | gnu/packages/machine-learning.scm | 100 |
1 files changed, 100 insertions, 0 deletions
diff --git a/gnu/packages/machine-learning.scm b/gnu/packages/machine-learning.scm index 97535539be..c3467d956f 100644 --- a/gnu/packages/machine-learning.scm +++ b/gnu/packages/machine-learning.scm @@ -1086,6 +1086,106 @@ storing tensors safely, named safetensors. They aim to be safer than their @code{PyTorch} counterparts.") (license license:asl2.0))) +(define-public python-safetensors + (package + (name "python-safetensors") + (version "0.4.3") + (source + (origin + (method url-fetch) + (uri (pypi-uri "safetensors" version)) + (sha256 + (base32 "1hhiwy67jarm70l0k26fs1cjhzkgzrh79q14bklj2yp0qi8gr19g")) + (modules '((guix build utils) + (ice-9 ftw))) + (snippet + #~(begin ;; Only keeping bindings. + (for-each + (lambda (file) + (unless (member file '("." ".." "bindings" "PKG-INFO")) + (delete-file-recursively file))) + (scandir ".")) + (for-each + (lambda (file) + (unless (member file '("." "..")) + (rename-file (string-append "bindings/python/" file) + file))) + (scandir "bindings/python")))))) + (build-system cargo-build-system) + (arguments + (list + #:modules '((guix build cargo-build-system) + (guix build utils) + (ice-9 regex) + (ice-9 textual-ports) + (srfi srfi-26)) + #:phases + #~(modify-phases %standard-phases + (add-after 'unpack-rust-crates 'inject-safetensors + (lambda _ + (substitute* "Cargo.toml" + (("\\[dependencies\\]") + (format #f "[dependencies]~%safetensors = ~s" + #$(package-version rust-safetensors)))) + (call-with-input-file "Cargo.toml" + (lambda (port) + (let* ((content (get-string-all port)) + (top-match (string-match + "\\[dependencies.safetensors" + content))) + (call-with-output-file "Cargo.toml" + (cut display (match:prefix top-match) <>))))))) + (add-before 'check 'install-rust-library + (lambda _ + (copy-file "target/release/libsafetensors_rust.so" + "py_src/safetensors/_safetensors_rust.so"))) + (replace 'check + (lambda _ + (invoke "python3" + "-c" (string-append "import sys; sys.path.append" + "(\"" (getcwd) "/py_src\")") + "-m" "pytest" + "-n" "auto" + "--dist=loadfile" + "-s" "-v" "./tests/" + ;; Missing jax dependency + "--ignore=./tests/test_flax_comparison.py"))) + (add-after 'install 'install-python + (lambda _ + (let* ((pversion #$(version-major+minor + (package-version python))) + (lib (string-append #$output "/lib/python" pversion + "/site-packages/")) + (info (string-append lib "safetensors-" + #$(package-version this-package) + ".dist-info"))) + (mkdir-p info) + (copy-file "PKG-INFO" (string-append info "/METADATA")) + (copy-recursively + "py_src/safetensors" + (string-append lib "safetensors")))))) + #:cargo-inputs + `(("rust-pyo3" ,rust-pyo3-0.21) + ("rust-memmap2" ,rust-memmap2-0.9) + ("rust-safetensors" ,rust-safetensors) + ("rust-serde-json" ,rust-serde-json-1)))) + (inputs + (list rust-safetensors)) + (native-inputs + (list python-h5py + python-minimal + python-numpy + python-pytest + python-pytest-xdist + python-pytorch + tensorflow)) + (home-page "https://huggingface.co/docs/safetensors") + (synopsis "Simple and safe way to store and distribute tensors") + (description "This package provides a fast (zero-copy) and safe +(dedicated) format for storing tensors safely. This package builds upon +@code{rust-safetensors} and provides Python bindings.") + (license license:asl2.0))) + (define-public python-sentencepiece (package (name "python-sentencepiece") |