Skip to content

Commit

Permalink
Merge branch 'staging'
Browse files Browse the repository at this point in the history 
(Truly, this time :-)
  • Loading branch information
@vcunat
vcunat committed Mar 1, 2017
2 parents 150796d + 64d4bfd commit b43614a
Show file tree
Hide file tree
Showing 22 changed files with 269 additions and 87 deletions.
13 changes: 13 additions & 0 deletions doc/languages-frameworks/python.md
View file
Expand Up @@ -641,6 +641,19 @@ community to help save time. No tool is preferred at the moment.
- [pypi2nix](https://github.com/garbas/pypi2nix) by Rok Garbas
- [pypi2nix](https://github.com/offlinehacker/pypi2nix) by Jaka Hudoklin

### Deterministic builds

Python 2.7, 3.5 and 3.6 are now built deterministically and 3.4 mostly.
Minor modifications had to be made to the interpreters in order to generate
deterministic bytecode. This has security implications and is relevant for
those using Python in a `nix-shell`.

When the environment variable `DETERMINISTIC_BUILD` is set, all bytecode will have timestamp 1.
The `buildPythonPackage` function sets `DETERMINISTIC_BUILD` as well as
[PYTHONHASHSEED](https://docs.python.org/3.5/using/cmdline.html#envvar-PYTHONHASHSEED).
Both are also exported in `nix-shell`.


## FAQ

### How can I install a working Python environment?
Expand Down
10 changes: 10 additions & 0 deletions nixos/doc/manual/release-notes/rl-1703.xml
View file
Expand Up @@ -271,6 +271,16 @@ following incompatible changes:</para>
</para>
</listitem>

<listitem>
<para>
Python 2.7, 3.5 and 3.6 are now built deterministically and 3.4 mostly.
Minor modifications had to be made to the interpreters in order to generate
deterministic bytecode. This has security implications and is relevant for
those using Python in a <literal>nix-shell</literal>. See the Nixpkgs manual
for details.
</para>
</listitem>

</itemizedlist>


Expand Down
36 changes: 20 additions & 16 deletions pkgs/build-support/setup-hooks/compress-man-pages.sh
View file
Expand Up @@ -3,26 +3,30 @@ fixupOutputHooks+=('if [ -z "$dontGzipMan" ]; then compressManPages "$prefix"; f
compressManPages() {
local dir="$1"

if [ ! -d "$dir/share/man" ]; then return; fi
echo "gzipping man pages in $dir"
if [ -L "$dir"/share ] || [ -L "$dir"/share/man ] || [ ! -d "$dir/share/man" ]
then return
fi
echo "gzipping man pages under $dir/share/man/"

GLOBIGNORE=.:..:*.gz:*.bz2

for f in "$dir"/share/man/*/* "$dir"/share/man/*/*/*; do
if [ -f "$f" -a ! -L "$f" ]; then
if gzip -c -n "$f" > "$f".gz; then
rm "$f"
else
rm "$f".gz
fi
# Compress all uncompressed manpages. Don't follow symlinks, etc.
find "$dir"/share/man/ -type f -a '!' -regex '.*\.\(bz2\|gz\)$' -print0 \
| while IFS= read -r -d $'\0' f
do
if gzip -c -n "$f" > "$f".gz; then
rm "$f"
else
rm "$f".gz
fi
done

for f in "$dir"/share/man/*/* "$dir"/share/man/*/*/*; do
if [ -L "$f" -a -f `readlink -f "$f"`.gz ]; then
ln -sf `readlink "$f"`.gz "$f".gz && rm "$f"
# Point symlinks to compressed manpages.
find "$dir"/share/man/ -type l -a '!' -regex '.*\.\(bz2\|gz\)$' -print0 \
| while IFS= read -r -d $'\0' f
do
local target
target="$(readlink -f "$f")"
if [ -f "$target".gz ]; then
ln -sf "$target".gz "$f".gz && rm "$f"
fi
done

unset GLOBIGNORE
}
14 changes: 14 additions & 0 deletions pkgs/development/interpreters/python/cpython/2.7/default.nix
View file
Expand Up @@ -178,6 +178,17 @@ in stdenv.mkDerivation {
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

passthru = let
Expand Down Expand Up @@ -210,5 +221,8 @@ in stdenv.mkDerivation {
license = stdenv.lib.licenses.psfl;
platforms = stdenv.lib.platforms.all;
maintainers = with stdenv.lib.maintainers; [ chaoflow domenkozar ];
# Higher priority than Python 3.x so that `/bin/python` points to `/bin/python2`
# in case both 2 and 3 are installed.
priority = -100;
};
}
49 changes: 39 additions & 10 deletions pkgs/development/interpreters/python/cpython/3.4/default.nix
View file
@@ -1,5 +1,7 @@
{ stdenv, fetchurl
, bzip2
, expat
, libffi
, gdbm
, lzma
, ncurses
Expand Down Expand Up @@ -50,21 +52,43 @@ in stdenv.mkDerivation {

NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";

# Determinism: The interpreter is patched to write null timestamps when compiling python files.
# This way python doesn't try to update them when we freeze timestamps in nix store.
DETERMINISTIC_BUILD=1;
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED=0;

prePatch = optionalString stdenv.isDarwin ''
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
'';

postPatch = optionalString (x11Support && (tix != null)) ''
postPatch = ''
# Determinism
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
# Determinism. This is done unconditionally
substituteInPlace "Lib/importlib/_bootstrap.py" --replace "source_mtime = int(source_stats['mtime'])" "source_mtime = 1"
'' + optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
''
# Avoid picking up getentropy() from glibc >= 2.25, as that would break
# on older kernels. http://bugs.python.org/issue29157
+ optionalString stdenv.isLinux
''
+ optionalString stdenv.isLinux ''
substituteInPlace Python/random.c --replace 'defined(HAVE_GETENTROPY)' '0'
cat Python/random.c
'';
'';

CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";

configureFlags = [
"--enable-shared"
"--with-threads"
"--without-ensurepip"
"--with-system-expat"
"--with-system-ffi"
];

preConfigure = ''
for i in /usr /sw /opt /pkg; do # improve purity
Expand All @@ -74,12 +98,6 @@ in stdenv.mkDerivation {
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
export MACOSX_DEPLOYMENT_TARGET=10.6
''}
configureFlagsArray=( --enable-shared --with-threads
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
)
'';

setupHook = ./setup-hook.sh;
Expand All @@ -102,13 +120,24 @@ in stdenv.mkDerivation {
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pip3" "$out/bin/pip"
ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

postFixup = ''
Expand Down
46 changes: 38 additions & 8 deletions pkgs/development/interpreters/python/cpython/3.5/default.nix
View file
@@ -1,5 +1,7 @@
{ stdenv, fetchurl, fetchpatch
, bzip2
, expat
, libffi
, gdbm
, lzma
, ncurses
Expand Down Expand Up @@ -32,7 +34,7 @@ let
sitePackages = "lib/${libPrefix}/site-packages";

buildInputs = filter (p: p != null) [
zlib bzip2 lzma gdbm sqlite readline ncurses openssl ]
zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ]
++ optionals x11Support [ tcl tk libX11 xproto ]
++ optionals stdenv.isDarwin [ CF configd ];

Expand All @@ -50,6 +52,12 @@ in stdenv.mkDerivation {

NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";

# Determinism: The interpreter is patched to write null timestamps when compiling python files.
# This way python doesn't try to update them when we freeze timestamps in nix store.
DETERMINISTIC_BUILD=1;
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED=0;

prePatch = optionalString stdenv.isDarwin ''
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
Expand All @@ -63,10 +71,27 @@ in stdenv.mkDerivation {
})
];

postPatch = optionalString (x11Support && (tix != null)) ''
postPatch = ''
# Determinism
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
# Determinism. This is done unconditionally
substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1"
'' + optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
'';

CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";

configureFlags = [
"--enable-shared"
"--with-threads"
"--without-ensurepip"
"--with-system-expat"
"--with-system-ffi"
];

preConfigure = ''
for i in /usr /sw /opt /pkg; do # improve purity
substituteInPlace ./setup.py --replace $i /no-such-path
Expand All @@ -75,12 +100,6 @@ in stdenv.mkDerivation {
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
export MACOSX_DEPLOYMENT_TARGET=10.6
''}
configureFlagsArray=( --enable-shared --with-threads
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
)
'';

setupHook = ./setup-hook.sh;
Expand All @@ -103,13 +122,24 @@ in stdenv.mkDerivation {
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pip3" "$out/bin/pip"
ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

postFixup = ''
Expand Down
44 changes: 37 additions & 7 deletions pkgs/development/interpreters/python/cpython/3.6/default.nix
View file
@@ -1,6 +1,8 @@
{ stdenv, fetchurl, fetchpatch
, glibc
, bzip2
, expat
, libffi
, gdbm
, lzma
, ncurses
Expand Down Expand Up @@ -50,6 +52,12 @@ in stdenv.mkDerivation {

NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";

# Determinism: The interpreter is patched to write null timestamps when compiling python files.
# This way python doesn't try to update them when we freeze timestamps in nix store.
DETERMINISTIC_BUILD=1;
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED=0;

prePatch = optionalString stdenv.isDarwin ''
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
Expand All @@ -63,10 +71,27 @@ in stdenv.mkDerivation {
})
];

postPatch = optionalString (x11Support && (tix != null)) ''
postPatch = ''
# Determinism
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
# Determinism. This is done unconditionally
substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1"
'' + optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
'';

CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";

configureFlags = [
"--enable-shared"
"--with-threads"
"--without-ensurepip"
"--with-system-expat"
"--with-system-ffi"
];

preConfigure = ''
for i in /usr /sw /opt /pkg; do # improve purity
substituteInPlace ./setup.py --replace $i /no-such-path
Expand All @@ -75,12 +100,6 @@ in stdenv.mkDerivation {
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
export MACOSX_DEPLOYMENT_TARGET=10.6
''}
configureFlagsArray=( --enable-shared --with-threads
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
)
'';

setupHook = ./setup-hook.sh;
Expand All @@ -103,13 +122,24 @@ in stdenv.mkDerivation {
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pip3" "$out/bin/pip"
ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

passthru = let
Expand Down

0 comments on commit b43614a

Please sign in to comment.