base/bitbake.conf: Filter contents of PATH to only allow whitelisted tools

We currently have a determinism problem in that the host tools present
in PATH can influence the build. In particular, the presence of pkg-config
on the build host can mask missing pkgconfig class dependencies.

This adds in a new HOSTTOOLS variable and then uses it to set up a directory
of symlinks to the whitelisted host tools. This directory is placed as PATH
instead of the usual /usr/bin:/bin and so on.

This should improve determinism of builds and avoid the issues which have
been particularly obvious since the introduction of recipe specific sysroots.

If users find there is a tool missing, they can extend HOSTTOOLS from a global
class or global conf file.

Right now the settings should be enough to build everything in OE-Core.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 14293f83c4ff936dc215a6ac2ede590e03614ab5..fec351a890f23ded74426872995f78101d016104 100644 (file)
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -119,6 +119,25 @@ def get_lic_checksum_file_list(d):
             bb.fatal(d.getVar('PN') + ": LIC_FILES_CHKSUM contains an invalid URL: " + url)
     return " ".join(filelist)
 
+def setup_hosttools_dir(dest, toolsvar, d, fatal=True):
+    tools = d.getVar(toolsvar).split()
+    origbbenv = d.getVar("BB_ORIGENV", False)
+    path = origbbenv.getVar("PATH")
+    bb.utils.mkdirhier(dest)
+    notfound = []
+    for tool in tools:
+        desttool = os.path.join(dest, tool)
+        if not os.path.exists(desttool):
+            srctool = bb.utils.which(path, tool)
+            if "ccache" in srctool:
+                srctool = bb.utils.which(path, tool, direction=1)
+            if srctool:
+                os.symlink(srctool, desttool)
+            else:
+                notfound.append(tool)
+    if notfound and fatal:
+        bb.fatal("These tools appear to be unavailable in PATH, please install them in order to proceed:\n%s" % " ".join(notfound))
+
 addtask fetch
 do_fetch[dirs] = "${DL_DIR}"
 do_fetch[file-checksums] = "${@bb.fetch.get_checksum_file_list(d)}"
@@ -219,6 +238,9 @@ python base_eventhandler() {
         pkgarch_mapping(e.data)
         oe.utils.features_backfill("DISTRO_FEATURES", e.data)
         oe.utils.features_backfill("MACHINE_FEATURES", e.data)
+        # Works with the line in layer.conf which changes PATH to point here
+        setup_hosttools_dir(d.expand('${TMPDIR}/hosttools'), 'HOSTTOOLS', d)
+        setup_hosttools_dir(d.expand('${TMPDIR}/hosttools'), 'HOSTTOOLS_NONFATAL', d, fatal=False)
 
     if isinstance(e, bb.event.BuildStarted):
         localdata = bb.data.createCopy(e.data)

diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index f9df7cacd12f650b4b403d663e21796a9d34d7a9..bc115117c9d8afde6cbe70f2240be42efc6e5985 100644 (file)
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -453,6 +453,24 @@ export PATH
 # Build utility info.
 ##################################################################
 
+# Tools needed to run builds with OE-Core
+HOSTTOOLS += " \
+    bash sh cut sed gcc ld git rm install which find xargs cat true mktemp \
+    grep tar gzip touch cp mv basename dirname tr getopt sort awk head tail \
+    mkdir patch uniq perl python chmod python3 ar strip expr ls make as \
+    ranlib egrep echo chown cpio tee wc wget bzip2 stat date rmdir od diff \
+    md5sum dd chrpath file pod2man gunzip python2.7 ln g++ [ false true \
+    uname test hostname nm objdump objcopy cmp printf env readlink gawk fgrep \
+    expand pwd sleep diffstat chgrp flock ldd strings rpcgen du makeinfo \
+    getconf mknod cpp readelf split \
+"
+
+# Tools needed to run testimage runtime image testing
+HOSTTOOLS += "ps stty ip ssh scp ping vi"
+
+# Link to these if present
+HOSTTOOLS_NONFATAL += "ccache pip3 ld.bfd ld.gold gcc-ar gpg sftp"
+
 CCACHE ??= ""
 # Disable ccache explicitly if CCACHE is null since gcc may be a symlink
 # of ccache some distributions (e.g., Fedora 17).

diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
index 87c235fe126936933bfc2aa182d86432cb5737a3..739d82ea56401b11e711caea7a893233638bae80 100644 (file)
--- a/meta/conf/layer.conf
+++ b/meta/conf/layer.conf
@@ -59,3 +59,5 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
   oprofile->virtual/kernel \
 "
 
+# We need to keep bitbake tools in PATH
+PATH := "${@os.path.dirname(bb.utils.which(d.getVar('PATH'),'bitbake'))}:${TMPDIR}/hosttools"