sstate: Add extra directory level

We're having speed issues on the autobuilder due to the numbers of files in sstate
directories. We previously split these by the first two characters of the hash.
This change extends this to split by the next two characters as well, creating
more layers of directories.

This should signifiantly speed up eSDK builds on the autobuilder as the current
sstate layout simply isn't scaling there but addresses a general complaint.

gen-lockedsig-cache needed to be updated for the new split level sstate.

Also update tests for new layout.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index c0329cd5d189aead59af457a223f8c13a0941847..7d2cb9eb8400b0b6765559dbf1e44cf2103aee8d 100644 (file)
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -6,7 +6,7 @@ SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
 def generate_sstatefn(spec, hash, d):
     if not hash:
         hash = "INVALID"
-    return hash[:2] + "/" + spec + hash
+    return hash[:2] + "/" + hash[2:4] + "/" + spec + hash
 
 SSTATE_PKGARCH    = "${PACKAGE_ARCH}"
 SSTATE_PKGSPEC    = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
@@ -15,7 +15,7 @@ SSTATE_PKGNAME    = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
 SSTATE_PKG        = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
 SSTATE_EXTRAPATH   = ""
 SSTATE_EXTRAPATHWILDCARD = ""
-SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
+SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}"
 
 # explicitly make PV to depend on evaluated value of PV variable
 PV[vardepvalue] = "${PV}"

diff --git a/meta/lib/oeqa/selftest/cases/signing.py b/meta/lib/oeqa/selftest/cases/signing.py
index 93b15ae6814b3872413b559bdf7c93ca73e905a3..202d54994b6fd1346890f02c9d6bfe1f619f1fa5 100644 (file)
--- a/meta/lib/oeqa/selftest/cases/signing.py
+++ b/meta/lib/oeqa/selftest/cases/signing.py
@@ -157,8 +157,8 @@ class Signing(OESelftestTestCase):
             bitbake('-c clean %s' % test_recipe)
             bitbake('-c populate_lic %s' % test_recipe)
 
-            recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz.sig')
-            recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz')
+            recipe_sig = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz.sig')
+            recipe_tgz = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz')
 
             self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.')
             self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.')

diff --git a/meta/lib/oeqa/selftest/cases/sstate.py b/meta/lib/oeqa/selftest/cases/sstate.py
index 410dec64fcd4ec90d6e3b58b7df4d1c7399e83e9..80ce9e353c08b15c412c055561f3c27e622e3609 100644 (file)
--- a/meta/lib/oeqa/selftest/cases/sstate.py
+++ b/meta/lib/oeqa/selftest/cases/sstate.py
@@ -56,11 +56,11 @@ class SStateBase(OESelftestTestCase):
     def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True):
         result = []
         for root, dirs, files in os.walk(self.sstate_path):
-            if distro_specific and re.search("%s/[a-z0-9]{2}$" % self.hostdistro, root):
+            if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root):
                 for f in files:
                     if re.search(filename_regex, f):
                         result.append(f)
-            if distro_nonspecific and re.search("%s/[a-z0-9]{2}$" % self.sstate_path, root):
+            if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root):
                 for f in files:
                     if re.search(filename_regex, f):
                         result.append(f)

diff --git a/scripts/gen-lockedsig-cache b/scripts/gen-lockedsig-cache
index 9bfae9d8323bd16d69a833485f954f4e9f37c7a2..6a7d285910465abd4d57fb17920d5126edd63945 100755 (executable)
--- a/scripts/gen-lockedsig-cache
+++ b/scripts/gen-lockedsig-cache
@@ -78,11 +78,18 @@ files = set()
 sstate_content_cache = {}
 for s in sigs:
     prefix = s[:2]
+    prefix2 = s[2:4]
     if prefix not in sstate_content_cache:
         sstate_content_cache[prefix] = build_sha_cache(prefix)
-
-    for f in sstate_content_cache[prefix][s]:
-        files.add(f)
+    if prefix2 not in sstate_content_cache[prefix]:
+        sstate_content_cache[prefix][prefix2] = build_sha_cache(prefix + "/" + prefix2)
+
+    if s in sstate_content_cache[prefix]:
+        for f in sstate_content_cache[prefix][s]:
+            files.add(f)
+    if s in sstate_content_cache[prefix][prefix2]:
+        for f in sstate_content_cache[prefix][prefix2][s]:
+            files.add(f)
 
 elapsed = time.perf_counter() - start_time
 print("Gathering file list took %.1fs" % elapsed)