]> code.ossystems Code Review - openembedded-core.git/commitdiff
python_pep517: use installer instead of pip
authorRoss Burton <ross@burtonini.com>
Mon, 14 Mar 2022 14:39:23 +0000 (14:39 +0000)
committerRichard Purdie <richard.purdie@linuxfoundation.org>
Tue, 15 Mar 2022 08:40:12 +0000 (08:40 +0000)
Instead of battling pip to install a wheel, use installer. Installer
does one thing, so it's faster and easier to work with.

This means setuptools, pip, and wheel are no longer part of the
bootstrap phase, so they can be built normally.  To avoid sysroot file
conflicts these three recipes can't install .pyc files to the native
sysroot.

We currently patch pypa/installer to allow us to override the interpreter
used, which means we can drop the interpreter seding.

We don't need to recompile any Python which is found in $bindir as
Python doesn't actually load those files.

Across a build of oe-core, the only differences between using pip and
installer are:

- the .dist-info/RECORD files are ordered differently
- the .dist-info/REQUESTED and INSTALLER files are not created
- the hashbang in native scripts is "/usr/bin/env nativepython" instead
  of pointing directly at the native sysroot python3.

Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
meta/classes/python_pep517.bbclass
meta/recipes-devtools/python/python3-flit-core_3.7.1.bb
meta/recipes-devtools/python/python3-pip_22.0.3.bb
meta/recipes-devtools/python/python3-setuptools_59.5.0.bb
meta/recipes-devtools/python/python3-wheel_0.37.1.bb

index 756333bcf2816708448c0fb48804ee78db13b0ee..ac7c1e44c626c4f19dbd174b1ed8b60cb666889f 100644 (file)
@@ -1,7 +1,7 @@
 # Common infrastructure for Python packages that use PEP-517 compliant packaging.
 # https://www.python.org/dev/peps/pep-0517/
 
-DEPENDS:append = " python3-pip-native"
+DEPENDS:append = " python3-installer-native"
 
 # Where to execute the build process from
 PEP517_SOURCE_PATH ?= "${S}"
@@ -11,19 +11,11 @@ PEP517_SOURCE_PATH ?= "${S}"
 # a recipe wants to install prebuilt wheels.
 PEP517_WHEEL_PATH ?= "${WORKDIR}/dist"
 
-PIP_INSTALL_ARGS = "\
-    -vvvv \
-    --ignore-installed \
-    --no-cache \
-    --no-deps \
-    --no-index \
-    --root=${D} \
-    --prefix=${prefix} \
-"
-
 PEP517_INSTALL_PYTHON = "python3"
 PEP517_INSTALL_PYTHON:class-native = "nativepython3"
 
+INSTALL_WHEEL_COMPILE_BYTECODE ?= "--compile-bytecode=0"
+
 python_pep517_do_install () {
     COUNT=$(find ${PEP517_WHEEL_PATH} -name '*.whl' | wc -l)
     if test $COUNT -eq 0; then
@@ -32,18 +24,7 @@ python_pep517_do_install () {
         bbfatal More than one wheel found in ${PEP517_WHEEL_PATH}, this should not happen
     fi
 
-    nativepython3 -m pip install ${PIP_INSTALL_ARGS} ${PEP517_WHEEL_PATH}/*.whl
-
-    cd ${D}
-    for i in ${D}${bindir}/* ${D}${sbindir}/*; do
-        if [ -f "$i" ]; then
-            sed -i -e "1s,#!.*nativepython3,#!${USRBINPATH}/env ${PEP517_INSTALL_PYTHON}," $i
-            sed -i -e "s:${PYTHON}:${USRBINPATH}/env\ ${PEP517_INSTALL_PYTHON}:g" $i
-            sed -i -e "s:${STAGING_BINDIR_NATIVE}:${bindir}:g" $i
-            # Not everything we find may be Python, so ignore errors
-            nativepython3 -mpy_compile $(realpath --relative-to=${D} $i) || true
-        fi
-    done
+    nativepython3 -m installer ${INSTALL_WHEEL_COMPILE_BYTECODE} --interpreter "${USRBINPATH}/env ${PEP517_INSTALL_PYTHON}" --destdir=${D} ${PEP517_WHEEL_PATH}/*.whl
 }
 
 # A manual do_install that just uses unzip for bootstrapping purposes. Callers should DEPEND on unzip-native.
index b12b8e42d78cc7d921821f93a9ca12b6d5124a73..dc815acf08f29c9127406c7805ae7dba82605a6e 100644 (file)
@@ -13,7 +13,7 @@ SRC_URI[sha256sum] = "3c9bd9c140515bfe62dd938c6610d10d6efb9e35cc647fc614fe5fb3a5
 inherit pypi flit_core
 
 # Need to install by hand as there's a dependency loop
-DEPENDS:remove:class-native = " python3-pip-native"
+DEPENDS:remove:class-native = " python3-installer-native"
 DEPENDS:append:class-native = " unzip-native"
 
 # We need the full flit tarball
index bfeeee789cee12034fa7471c80357f2400c615c7..9ca8fbc1e5a7688e564cfad28fbcb4eb86bcb354 100644 (file)
@@ -6,43 +6,14 @@ LIC_FILES_CHKSUM = "file://LICENSE.txt;md5=63ec52baf95163b597008bb46db68030"
 
 inherit pypi setuptools_build_meta
 
-DEPENDS += "python3"
-
-# To avoid a dependency loop; we bootstrap -native
-DEPENDS:remove:class-native = "python3-pip-native"
-DEPENDS:append:class-native = " unzip-native"
-
 SRC_URI += "file://0001-change-shebang-to-python3.patch"
 SRC_URI += "file://no_shebang_mangling.patch"
 SRC_URI += "file://reproducible.patch"
 
 SRC_URI[sha256sum] = "f29d589df8c8ab99c060e68ad294c4a9ed896624f6368c5349d70aa581b333d0"
 
-do_install:class-native() {
-    python_pep517_do_bootstrap_install
-
-    # pip install would normally generate [console_scripts] in ${bindir}
-    install -d ${D}/${bindir}
-    # We will skip the ${bindir}/pip variant as we would just remove it in the do_install:append
-    cat << EOF >> ${D}/${bindir}/pip3 | tee ${D}/${bindir}/pip${PYTHON_BASEVERSION}
-#!/bin/sh
-'''exec' ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} "\$0" "\$@"
-' '''
-# -*- coding: utf-8 -*-
-import re
-import sys
-from pip._internal.cli.main import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
-    sys.exit(main())
-EOF
-    chmod 0755 ${D}${bindir}/pip3 ${D}${bindir}/pip${PYTHON_BASEVERSION}
-}
-
 do_install:append() {
-    if [ -e ${D}/${bindir}/pip ]; then
-        rm ${D}/${bindir}/pip
-    fi
+    rm -f ${D}/${bindir}/pip
 }
 
 RDEPENDS:${PN} = "\
@@ -59,3 +30,7 @@ RDEPENDS:${PN} = "\
 "
 
 BBCLASSEXTEND = "native nativesdk"
+
+# This used to use the bootstrap install which didn't compile. Until we bump the
+# tmpdir version we can't compile the native otherwise the sysroot unpack fails
+INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode"
index d93972b9a0120f86d36a9be0ca4ce0eb129b8cdf..35bec19e33db566972553b93e47215860dfeb487 100644 (file)
@@ -17,14 +17,6 @@ SRC_URI[sha256sum] = "d144f85102f999444d06f9c0e8c737fd0194f10f2f7e5fdb77573f6e2f
 
 DEPENDS += "${PYTHON_PN}"
 
-# Avoid dependency loop; we bootstrap -native
-DEPENDS:remove:class-native = "python3-pip-native python3-setuptools-native"
-DEPENDS:append:class-native = " unzip-native"
-
-do_install:class-native() {
-    python_pep517_do_bootstrap_install
-}
-
 RDEPENDS:${PN} = "\
     ${PYTHON_PN}-2to3 \
     ${PYTHON_PN}-compile \
@@ -59,3 +51,7 @@ RDEPENDS:${PYTHON_PN}-pkg-resources = "\
     ${PYTHON_PN}-plistlib \
     ${PYTHON_PN}-pprint \
 "
+
+# This used to use the bootstrap install which didn't compile. Until we bump the
+# tmpdir version we can't compile the native otherwise the sysroot unpack fails
+INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode"
index a339e658246959364b2295e06298eb6ac8093f3b..efd6c2f968e0ebd9b12f47e7ea0e8e8bec5383ce 100644 (file)
@@ -10,26 +10,8 @@ inherit flit_core pypi
 
 SRC_URI += " file://0001-Backport-pyproject.toml-from-flit-backend-branch.patch"
 
-DEPENDS:remove:class-native = "python3-pip-native"
-
-do_install:class-native () {
-    python_pep517_do_bootstrap_install
-
-    # pip install would normally generate [project.scripts] in ${bindir}
-    install -d ${D}/${bindir}
-    cat << EOF >> ${D}/${bindir}/wheel
-#!/bin/sh
-'''exec' ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} "\$0" "\$@"
-' '''
-# -*- coding: utf-8 -*-
-import re
-import sys
-from wheel.cli import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
-    sys.exit(main())
-EOF
-    chmod 0755 ${D}${bindir}/wheel
-}
-
 BBCLASSEXTEND = "native nativesdk"
+
+# This used to use the bootstrap install which didn't compile. Until we bump the
+# tmpdir version we can't compile the native otherwise the sysroot unpack fails
+INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode"