python: cythonize packages

Test to see whether we can cythonize packages. This typically fails when code is written 2 and 3 compatible. The -k keep-going flag was therefore added. By default extension modules take precedence so there is no need to remove the Python code. The store path size increases significantly, typically a factor 10.
NixOS · Aug 26, 2018 · 3ca6ba5 · 3ca6ba5
1 parent a04cf8f
commit 3ca6ba5
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 1 deletion.
diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix
@@ -12,6 +12,7 @@
 , namePrefix
 , bootstrapped-pip
 , flit
+, cython
 }:
 
 let
@@ -20,7 +21,7 @@ let
   wheel-specific = import ./build-python-package-wheel.nix { };
   common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; };
   mkPythonDerivation = import ./mk-python-derivation.nix {
-    inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook toPythonModule namePrefix;
+    inherit cython lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook toPythonModule namePrefix;
   };
 in
 

diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix
@@ -10,6 +10,7 @@
 # Whether the derivation provides a Python module or not.
 , toPythonModule
 , namePrefix
+, cython
 }:
 
 { name ? "${attrs.pname}-${attrs.version}"
@@ -50,6 +51,9 @@
 # However, some packages do provide executables with extensions, and thus bytecode is generated.
 , removeBinBytecode ? true
 
+# Compile Python code to native using Cython
+, cythonize ? false
+
 , meta ? {}
 
 , passthru ? {}
@@ -76,6 +80,7 @@ toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attrs [
   buildInputs = [ wrapPython ]
     ++ lib.optional (lib.hasSuffix "zip" (attrs.src.name or "")) unzip
     ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
+    ++ lib.optional cythonize cython
     ++ buildInputs
     ++ pythonPath;
 
@@ -99,6 +104,8 @@ toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attrs [
     # If this happens, something went wrong with the dependencies specs.
     # Intentionally kept in a subdirectory, see catch_conflicts/README.md.
     ${python.interpreter} ${./catch_conflicts}/catch_conflicts.py
+  '' + lib.optionalString cythonize ''
+    cythonize -ik $out/${python.sitePackages}/**/*.py
   '' + attrs.postFixup or '''';
 
   meta = {

diff --git a/pkgs/development/python-modules/pytoml/default.nix b/pkgs/development/python-modules/pytoml/default.nix
@@ -19,6 +19,8 @@ buildPythonPackage rec {
     sha256 = "1ip71yqxnyi4jhw5x1q7a0za61ndhpfh0vbx08jfv0w4ayng6rgv";
   };
 
+  cythonize = true;
+
   meta = with stdenv.lib; {
     description = "A TOML parser/writer for Python";
     homepage    = https://github.com/avakar/pytoml;

diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
@@ -57,6 +57,7 @@ let
     # We want Python libraries to be named like e.g. "python3.6-${name}"
     inherit namePrefix;
     inherit toPythonModule;
+    cython = self.cython;
   }));
 
   buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {