package.bbclass: Optimise the per file rpm handling

Currently a process was being forked off for each individual file
this class wanted to inspect with rpmdeps. This converts it to use
rpmdeps-oecore which allows batch processing of these dependencies.

For do_package for perl, this reduced the time by about 1 minute (33%).

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 348c13ce5edc07cd153eba3f8d979b8164e23cbb..f6d6e1acee32c6a0b27198ae5a5433a8d9ec3799 100644 (file)
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1125,7 +1125,7 @@ if [ x"$D" = "x" ]; then
 fi
 }
 
-RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt"
+RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps-oecore --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmfc_magic_path ${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc' --rpmpopt ${STAGING_LIBDIR_NATIVE}/rpm/rpmpopt"
 
 # Collect perfile run-time dependency metadata
 # Output:
@@ -1136,7 +1136,7 @@ RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps --macros ${STAGING_LIBDIR_NA
 #  FILERDEPENDS_filepath_pkg - per file dep
 
 python package_do_filedeps() {
-       import os, re
+       import re
 
        pkgdest = d.getVar('PKGDEST', True)
        packages = d.getVar('PACKAGES', True)
@@ -1145,39 +1145,49 @@ python package_do_filedeps() {
        r = re.compile(r'[<>=]+ +[^ ]*')
 
        # Quick routine to process the results of the rpmdeps call...
-       def process_deps(pipe, pkg, f, provides_files, requires_files):
-               provides = []
-               requires = []
-               file = f.replace(pkgdest + "/" + pkg, "")
-               file = file.replace("@", "@at@")
-               file = file.replace(" ", "@space@")
-               file = file.replace("\t", "@tab@")
-               file = file.replace("[", "@openbrace@")
-               file = file.replace("]", "@closebrace@")
-               file = file.replace("_", "@underscore@")
+       def process_deps(pipe, pkg, provides_files, requires_files):
+               provides = {}
+               requires = {}
 
                for line in pipe:
+                       f = line.split(" ", 1)[0].strip()
+                       line = line.split(" ", 1)[1].strip()
+
                        if line.startswith("Requires:"):
                                i = requires
                        elif line.startswith("Provides:"):
                                i = provides
                        else:
                                continue
+
+                       file = f.replace(pkgdest + "/" + pkg, "")
+                       file = file.replace("@", "@at@")
+                       file = file.replace(" ", "@space@")
+                       file = file.replace("\t", "@tab@")
+                       file = file.replace("[", "@openbrace@")
+                       file = file.replace("]", "@closebrace@")
+                       file = file.replace("_", "@underscore@")
                        value = line.split(":", 1)[1].strip()
                        value = r.sub(r'(\g<0>)', value)
+
                        if value.startswith("rpmlib("):
                                continue
-                       i.append(value)
+                       if file not in i:
+                               i[file] = []
+                       i[file].append(value)
 
-               if len(provides) > 0:
+               for file in provides:
                        provides_files.append(file)
                        key = "FILERPROVIDES_" + file + "_" + pkg
-                       d.setVar(key, " ".join(provides))
+                       d.setVar(key, " ".join(provides[file]))
 
-               if len(requires) > 0:
+               for file in requires:
                        requires_files.append(file)
                        key = "FILERDEPENDS_" + file + "_" + pkg
-                       d.setVar(key, " ".join(requires))
+                       d.setVar(key, " ".join(requires[file]))
+
+       def chunks(files, n):
+               return [files[i:i+n] for i in range(0, len(files), n)]
 
        # Determine dependencies
        for pkg in packages.split():
@@ -1186,13 +1196,15 @@ python package_do_filedeps() {
 
                provides_files = []
                requires_files = []
+               rpfiles = []
                for root, dirs, files in os.walk(pkgdest + "/" + pkg):
                        for file in files:
-                               f = os.path.join(root, file)
+                               rpfiles.append(os.path.join(root, file))
 
-                               dep_pipe = os.popen(rpmdeps + " --provides --requires -v " + f)
+               for files in chunks(rpfiles, 100):
+                       dep_pipe = os.popen(rpmdeps + " " + " ".join(files))
 
-                               process_deps(dep_pipe, pkg, f, provides_files, requires_files)
+                       process_deps(dep_pipe, pkg, provides_files, requires_files)
 
                d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files))
                d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files))

diff --git a/meta/recipes-core/busybox/busybox.inc b/meta/recipes-core/busybox/busybox.inc
index f35779dc49938c0d0387c9be0345b41fd604f9c2..5fb436ee23d9d140e01818c696d9a5517e977fd5 100644 (file)
--- a/meta/recipes-core/busybox/busybox.inc
+++ b/meta/recipes-core/busybox/busybox.inc
@@ -216,9 +216,9 @@ python package_do_filedeps_append () {
        # Load/backup original set
        filerprovides = d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or ""
 
-       dep_pipe = os.popen('sed -e "s,^,Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox_links))
+       dep_pipe = os.popen('sed -e "s,^,%s/%s%s Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox, pkgdest, pkg, f_busybox_links))
 
-       process_deps(dep_pipe, pkg, "%s/%s%s" % (pkgdest, pkg, f_busybox), provides_files, requires_files)
+       process_deps(dep_pipe, pkg, provides_files, requires_files)
 
        # Add the new set
        filerprovides += d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or ""