siteinfo/autotools: Ensure task checksums reflect site files

Currently, if you change the site files, nothing rebuilds since they are
not accounted for in task checksums. They could/should be through the
file-checksums task flag. We need to cache all the files looked for,
whether the exist or not so that if they do exist and didn't,
the checksum also changes.

This gets complicated by the need to clean out hardcoded build
paths from the variable and that other layers can have site files.

This patch adds this functionality. A new variable, SITEINFO_PATHVARS
is added which controls which substitutions to make on the file-checksum
values to remove the hardcoded paths. Layers adding site files will need
to set this to a variable that has the layer path in it and is excluded
from task hashes (COREBASE is the one the core layer uses).

This patch will cause yocto-check-layer to fail for some layers
where site files are added yet the layer isn't a machine specific layer.
This is arguable correct since these additional site files apply to
all recipes and things from a layer like core could be changed by such
changes so it is right they should rebuild. There is a determinism issue
potentially there if not. meta-openembedded does have some such references
but looking at them they should move to core or likely just be removed as
most look obsolete anyway.

[YOCTO #13729]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/classes/autotools.bbclass b/meta/classes/autotools.bbclass
index 2c7968e6597e54a6215b01cbaefbabfc127628eb..bc0c2ea83e53ef184439109adaa55918ad6f1b4a 100644 (file)
--- a/meta/classes/autotools.bbclass
+++ b/meta/classes/autotools.bbclass
@@ -145,7 +145,13 @@ ACLOCALEXTRAPATH:class-target = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 ACLOCALEXTRAPATH:class-nativesdk = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 
 python autotools_aclocals () {
-    d.setVar("CONFIG_SITE", siteinfo_get_files(d, sysrootcache=True))
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=True)
+    d.setVar("CONFIG_SITE", " ".join(sitefiles))
+}
+
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.appendVarFlag("do_configure", "file-checksums", " " + " ".join(searched))
 }
 
 CONFIGURE_FILES = "${S}/configure.in ${S}/configure.ac ${S}/config.h.in ${S}/acinclude.m4 Makefile.am"

diff --git a/meta/classes/siteinfo.bbclass b/meta/classes/siteinfo.bbclass
index 0bd1f36805d2333d13935709b1b00ad9e000e681..c5f4dfda412eecdd11a47418fd0b95c96aba8155 100644 (file)
--- a/meta/classes/siteinfo.bbclass
+++ b/meta/classes/siteinfo.bbclass
@@ -176,17 +176,39 @@ python () {
         bb.fatal("Please add your architecture to siteinfo.bbclass")
 }
 
-def siteinfo_get_files(d, sysrootcache = False):
+# Layers with siteconfig need to add a replacement path to this variable so the
+# sstate isn't path specific
+SITEINFO_PATHVARS = "COREBASE"
+
+def siteinfo_get_files(d, sysrootcache=False):
     sitedata = siteinfo_data(d)
-    sitefiles = ""
+    sitefiles = []
+    searched = []
     for path in d.getVar("BBPATH").split(":"):
         for element in sitedata:
             filename = os.path.join(path, "site", element)
             if os.path.exists(filename):
-                sitefiles += filename + " "
+                searched.append(filename + ":True")
+                sitefiles.append(filename)
+            else:
+                searched.append(filename + ":False")
+
+    # Have to parameterise out hardcoded paths such as COREBASE for the main site files
+    for var in d.getVar("SITEINFO_PATHVARS").split():
+        searched2 = []
+        replace = os.path.normpath(d.getVar(var))
+        for s in searched:
+            searched2.append(s.replace(replace, "${" + var + "}"))
+        searched = searched2
+
+    if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
+        # We need sstate sigs for native/cross not to vary upon arch so we can't depend on the site files.
+        # In future we may want to depend upon all site files?
+        # This would show up as breaking sstatetests.SStateTests.test_sstate_32_64_same_hash for example
+        searched = []
 
     if not sysrootcache:
-        return sitefiles
+        return sitefiles, searched
 
     # Now check for siteconfig cache files in sysroots
     path_siteconfig = d.getVar('SITECONFIG_SYSROOTCACHE')
@@ -195,8 +217,8 @@ def siteinfo_get_files(d, sysrootcache = False):
             if not i.endswith("_config"):
                 continue
             filename = os.path.join(path_siteconfig, i)
-            sitefiles += filename + " "
-    return sitefiles
+            sitefiles.append(filename)
+    return sitefiles, searched
 
 #
 # Make some information available via variables

diff --git a/meta/classes/toolchain-scripts.bbclass b/meta/classes/toolchain-scripts.bbclass
index 479f3b706ea9bfdf04333fd88f9af2e77e0eae09..fb6261c91d067443eb1576cf756d73d37e0959cb 100644 (file)
--- a/meta/classes/toolchain-scripts.bbclass
+++ b/meta/classes/toolchain-scripts.bbclass
@@ -65,6 +65,7 @@ toolchain_create_sdk_env_script () {
 
 # This function creates an environment-setup-script in the TMPDIR which enables
 # a OE-core IDE to integrate with the build tree
+# Caller must ensure CONFIG_SITE is setup
 toolchain_create_tree_env_script () {
        script=${TMPDIR}/environment-setup-${REAL_MULTIMACH_TARGET_SYS}
        rm -f $script
@@ -73,7 +74,7 @@ toolchain_create_tree_env_script () {
        echo 'export PATH=${STAGING_DIR_NATIVE}/usr/bin:${STAGING_BINDIR_TOOLCHAIN}:$PATH' >> $script
        echo 'export PKG_CONFIG_SYSROOT_DIR=${PKG_CONFIG_SYSROOT_DIR}' >> $script
        echo 'export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}' >> $script
-       echo 'export CONFIG_SITE="${@siteinfo_get_files(d)}"' >> $script
+       echo 'export CONFIG_SITE="${CONFIG_SITE}"' >> $script
        echo 'export SDKTARGETSYSROOT=${STAGING_DIR_TARGET}' >> $script
        echo 'export OECORE_NATIVE_SYSROOT="${STAGING_DIR_NATIVE}"' >> $script
        echo 'export OECORE_TARGET_SYSROOT="${STAGING_DIR_TARGET}"' >> $script
@@ -161,7 +162,7 @@ EOF
 }
 
 #we get the cached site config in the runtime
-TOOLCHAIN_CONFIGSITE_NOCACHE = "${@siteinfo_get_files(d)}"
+TOOLCHAIN_CONFIGSITE_NOCACHE = "${@' '.join(siteinfo_get_files(d)[0])}"
 TOOLCHAIN_CONFIGSITE_SYSROOTCACHE = "${STAGING_DIR}/${MLPREFIX}${MACHINE}/${target_datadir}/${TARGET_SYS}_config_site.d"
 TOOLCHAIN_NEED_CONFIGSITE_CACHE ??= "virtual/${MLPREFIX}libc ncurses"
 DEPENDS += "${TOOLCHAIN_NEED_CONFIGSITE_CACHE}"

diff --git a/meta/recipes-core/meta/meta-environment.bb b/meta/recipes-core/meta/meta-environment.bb
index 27f01036657ffb964e8ab3e7816e6419bfd9ffef..7118fb2aefc2c1cf5c1553aa01252c9eb67f5626 100644 (file)
--- a/meta/recipes-core/meta/meta-environment.bb
+++ b/meta/recipes-core/meta/meta-environment.bb
@@ -47,6 +47,11 @@ python do_generate_content() {
 }
 addtask generate_content before do_install after do_compile
 
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.appendVarFlag("do_generate_content", "file-checksums", " " + " ".join(searched))
+}
+
 create_sdk_files() {
        # Setup site file for external use
        toolchain_create_sdk_siteconfig ${SDK_OUTPUT}/${SDKPATH}/site-config-${REAL_MULTIMACH_TARGET_SYS}

diff --git a/meta/recipes-core/meta/meta-ide-support.bb b/meta/recipes-core/meta/meta-ide-support.bb
index 768f6f4bb67daad44f8323cccb22fc07b50bff65..5b23f78e51e2a7b2fa5a46051ef67650368b15de 100644 (file)
--- a/meta/recipes-core/meta/meta-ide-support.bb
+++ b/meta/recipes-core/meta/meta-ide-support.bb
@@ -12,4 +12,10 @@ do_populate_ide_support () {
   toolchain_create_tree_env_script
 }
 
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.setVar("CONFIG_SITE", " ".join(sitefiles))
+    d.appendVarFlag("do_populate_ide_support", "file-checksums", " " + " ".join(searched))
+}
+
 addtask populate_ide_support before do_build after do_install