Skip to content

Commit

Permalink
Python 3.x: do not regenerate _sysconfigdata
Browse files Browse the repository at this point in the history 
This commit fixes several issues:

- as reported in #24924 it was
possible that the file _sysconfigdata.pyc was generated after the actual
build of the CPython interpreter. We forgot to regenerate that file
during the build. This is now fixed

- the expression of the 3.3 interpreter now also includes some of the
determinism patches even though the output isn't yet reproducible. The
reason for adding them is that this makes the expressions of the
different interpreters more similar.

- references to -dev packages are now also removed in the 3.6 package,
thereby reducing its closure size
  • Loading branch information
@FRidh
FRidh committed Apr 16, 2017
1 parent f7a4f14 commit 5c8ffe0
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 29 deletions.
29 changes: 23 additions & 6 deletions pkgs/development/interpreters/python/cpython/3.3/default.nix
View file
Expand Up @@ -50,7 +50,17 @@ in stdenv.mkDerivation {

NIX_LDFLAGS = stdenv.lib.optionalString stdenv.isLinux "-lgcc_s";

postPatch = optionalString (x11Support && (tix != null)) ''
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
# This way python doesn't try to update them when we freeze timestamps in nix store.
DETERMINISTIC_BUILD=1;
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED=0;

postPatch = ''
# Determinism
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
# # We do not patch `Lib/importlib/_bootstrap_external.py` because it does not exist.
'' + optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
'';

Expand Down Expand Up @@ -80,31 +90,38 @@ in stdenv.mkDerivation {
fi
done
touch $out/lib/python${majorVersion}/test/__init__.py
ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}"
paxmark E $out/bin/python${majorVersion}
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pip3" "$out/bin/pip"
ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
'';
postFixup = ''
# Get rid of retained dependencies on -dev packages, and remove
# some $TMPDIR references to improve binary reproducibility.
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
done
# FIXME: should regenerate this.
rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

passthru = let
Expand Down
19 changes: 7 additions & 12 deletions pkgs/development/interpreters/python/cpython/3.4/default.nix
View file
Expand Up @@ -132,24 +132,19 @@ in stdenv.mkDerivation {
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

postFixup = ''
# Get rid of retained dependencies on -dev packages, and remove
# some $TMPDIR references to improve binary reproducibility.
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
done
# FIXME: should regenerate this.
rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

passthru = let
Expand Down
18 changes: 7 additions & 11 deletions pkgs/development/interpreters/python/cpython/3.5/default.nix
View file
Expand Up @@ -134,23 +134,19 @@ in stdenv.mkDerivation {
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

postFixup = ''
# Get rid of retained dependencies on -dev packages, and remove
# some $TMPDIR references to improve binary reproducibility.
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
done
# FIXME: should regenerate this.
rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';

passthru = let
Expand Down
7 changes: 7 additions & 0 deletions pkgs/development/interpreters/python/cpython/3.6/default.nix
View file
Expand Up @@ -134,6 +134,13 @@ in stdenv.mkDerivation {
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Get rid of retained dependencies on -dev packages, and remove
# some $TMPDIR references to improve binary reproducibility.
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
for i in $out/lib/python${majorVersion}/_sysconfigdata*.py $out/lib/python${majorVersion}/config-${majorVersion}m*/Makefile; do
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
done
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
Expand Down

0 comments on commit 5c8ffe0

Please sign in to comment.