classes/package: fix FILES_INFO serialisation in pkgdata

The FILES_INFO entry in each pkgdata file stores the list of files for
each package. Make the following improvements to how this is stored:
* Store paths as they would be seen on the target rather than
  erroneously including the full path to PKGDEST (which is specific to
  the build host the package was built on)
* For simplicity when loading the data, store complete paths for each
  entry instead of trying to break off the first part and use it as the
  dict key
* Record sizes for each file (as needed by Toaster)
* Serialise the value explicitly using json rather than just passing it
  through str().

Fixes [YOCTO #5443].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 48bb9828f5aff42eb737ede6d5767a1ab44269ba..cce24991225b553a1891c7653f6b66807fddba69 100644 (file)
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1110,6 +1110,7 @@ PKGDESTWORK = "${WORKDIR}/pkgdata"
 
 python emit_pkgdata() {
     from glob import glob
+    import json
 
     def write_if_exists(f, pkg, var):
         def encode(str):
@@ -1173,22 +1174,20 @@ python emit_pkgdata() {
     workdir = d.getVar('WORKDIR', True)
 
     for pkg in packages.split():
-        items = {}
-        for files_list in pkgfiles[pkg]:
-             item_name = os.path.basename(files_list)
-             item_path = os.path.dirname(files_list)
-             if item_path not in items:
-                 items[item_path] = []
-             items[item_path].append(item_name)
-        subdata_file = pkgdatadir + "/runtime/%s" % pkg
-
         pkgval = d.getVar('PKG_%s' % pkg, True)
         if pkgval is None:
             pkgval = pkg
             d.setVar('PKG_%s' % pkg, pkg)
 
-        d.setVar('FILES_INFO', str(items))
+        pkgdestpkg = os.path.join(pkgdest, pkg)
+        files = {}
+        for f in pkgfiles[pkg]:
+            relpth = os.path.relpath(f, pkgdestpkg)
+            fstat = os.lstat(f)
+            files[os.sep + relpth] = fstat.st_size
+        d.setVar('FILES_INFO', json.dumps(files))
 
+        subdata_file = pkgdatadir + "/runtime/%s" % pkg
         sf = open(subdata_file, 'w')
         write_if_exists(sf, pkg, 'PN')
         write_if_exists(sf, pkg, 'PV')

diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index 7dbb3844d79077df89eaa5adcd3c83866d96e232..8dc166316582b16bf3a5cb3dc0ca96786051b414 100644 (file)
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -39,8 +39,7 @@ python toaster_package_dumpdata() {
 
 
     # scan and send data for each package
-    import ast
-    import fnmatch
+    import json
 
     lpkgdata = {}
     for pkg in packages.split():
@@ -54,28 +53,11 @@ python toaster_package_dumpdata() {
             (n, v) = line.rstrip().split(":", 1)
             if pkg in n:
                 n = n.replace("_" + pkg, "")
-            lpkgdata[n] = v.strip()
-            line = sf.readline()
-            pkgsplitname = os.path.join(pkgdest, pkg)
-            # replace FILES_INFO data with a dictionary of file name - file size
             if n == 'FILES_INFO':
-                filesizedata = {}
-                val = v.strip().replace('\\\'', '\'')
-                dictval = ast.literal_eval(val)
-                for parent, dirlist in dictval.items():
-                    idx = parent.find(pkgsplitname)
-                    if idx > -1:
-                        parent = parent[idx+len(pkgsplitname):]
-                    else:
-                        bb.error("Invalid path while looking for file ", parent)
-                    for basename in dirlist:
-                        fullpath = os.path.join(parent, basename)
-                        try:
-                            filesizedata[fullpath] = os.stat(pkgsplitname + fullpath).st_size
-                        except OSError:
-                            # we may hit a symlink that is not pointing correctly over package-split
-                            filesizedata[fullpath] = 0
-                lpkgdata[n] = filesizedata
+                lpkgdata[n] = json.loads(v)
+            else:
+                lpkgdata[n] = v.strip()
+            line = sf.readline()
 
         # Fire an event containing the pkg data
         bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)

diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index 17e946e1e429423291f9120d42e5faffa78340dc..03c8f9530561ea7bd8e5d7fe7cb196a9595d0bc5 100755 (executable)
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -262,25 +262,20 @@ def find_path(args, usage, debug=False):
         print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir)
         sys.exit(1)
 
-    import ast
+    import json
     import fnmatch
 
     for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
         for fn in files:
-            pkgsplitname = '/packages-split/%s' % fn
             with open(os.path.join(root,fn)) as f:
                 for line in f:
                     if line.startswith('FILES_INFO:'):
-                        val = line.split(':', 1)[1].strip().replace('\\\'', '\'')
-                        dictval = ast.literal_eval(val)
-                        for parent, dirlist in dictval.items():
-                            idx = parent.find(pkgsplitname)
-                            if idx > -1:
-                                parent = parent[idx+len(pkgsplitname):]
-                            for basename in dirlist:
-                                fullpth = os.path.join(parent, basename)
-                                if fnmatch.fnmatchcase(fullpth, targetpath):
-                                    print("%s: %s" % (fn, fullpth))
+                        val = line.split(':', 1)[1].strip()
+                        dictval = json.loads(val)
+                        for fullpth in dictval.keys():
+                            if fnmatch.fnmatchcase(fullpth, targetpath):
+                                print("%s: %s" % (fn, fullpth))
+                        break
 
 
 def main():