python3: Improve handling of python3 manifest generation

Specifically cover detection of modules within a python package that do
not import anything within their __init__.py. This is at least the case
with the xmlrpc package which is only used via its modules xmlrpc.server
and xmlrpc.client. Other important corner cases include ctypes.utils
which depends on some modules not used by ctypes.

This is implemented by generally assuming that importing all the modules
of a package (aka *.py within a package, excluding _*.py) will provide
enough information.

Also due to this change some modules import sysconfig, resulting in
sysconfigdata being imported. Handle the conversion of its path to a
wildcard based on the platform dependent name being replaced.

Signed-off-by: Nathan Rossi <nathan@nathanrossi.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/recipes-devtools/python/python3/get_module_deps3.py b/meta/recipes-devtools/python/python3/get_module_deps3.py
index fd12baad84e4aa08de2ec7954d8f25f6c813b383..6806f23172135e472c41416cc4df21fd876b1883 100644 (file)
--- a/meta/recipes-devtools/python/python3/get_module_deps3.py
+++ b/meta/recipes-devtools/python/python3/get_module_deps3.py
@@ -9,6 +9,7 @@
 debug=False
 
 import sys
+import os
 
 # We can get a list of the modules which are currently required to run python
 # so we run python-core and get its modules, we then import what we need
@@ -48,8 +49,19 @@ current_module =  str(sys.argv[1]).rstrip()
 if(debug==True):
     log = open('log_%s' % current_module,'w')
     log.write('Module %s generated the following dependencies:\n' % current_module)
-try: 
-    importlib.import_module('%s' % current_module)
+try:
+    m = importlib.import_module(current_module)
+    # handle python packages which may not include all modules in the __init__
+    if os.path.basename(m.__file__) == "__init__.py":
+        modulepath = os.path.dirname(m.__file__)
+        for i in os.listdir(modulepath):
+            if i.startswith("_") or not(i.endswith(".py")):
+                continue
+            submodule = "{}.{}".format(current_module, i[:-3])
+            try:
+                importlib.import_module(submodule)
+            except:
+                pass # ignore all import or other exceptions raised during import
 except ImportError as e:
     if (debug==True):
         log.write('Module was not found')
@@ -107,6 +119,8 @@ for item in dif:
         dep_path = dep_path.replace(soabi,'*')
         print (dep_path)
         continue
+    if "_sysconfigdata" in dep_path:
+        dep_path = dep_path.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*")
 
     if (debug==True):
         log.write(dep_path+'\n')
@@ -140,6 +154,8 @@ for item in dif:
             log.write(cached)
         cached = fix_path(cached)
         cached = cached.replace(cpython_tag,'*')
+        if "_sysconfigdata" in cached:
+            cached = cached.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*")
         print (cached)
 
 if debug==True:

diff --git a/meta/recipes-devtools/python/python3/python3-manifest.json b/meta/recipes-devtools/python/python3/python3-manifest.json
index 3bcc9b8662ef294fe6ba6e4d16e6834eed83715a..07b084d48c75def98b4efb4d55ba38ca0914e7e7 100644 (file)
--- a/meta/recipes-devtools/python/python3/python3-manifest.json
+++ b/meta/recipes-devtools/python/python3/python3-manifest.json
@@ -324,7 +324,7 @@
             "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_compression.*.pyc",
             "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_markupbase.*.pyc",
             "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sitebuiltins.*.pyc",
-            "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sysconfigdata.*.pyc",
+            "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sysconfigdata*.*.pyc",
             "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_weakrefset.*.pyc",
             "${libdir}/python${PYTHON_MAJMIN}/__pycache__/abc.*.pyc",
             "${libdir}/python${PYTHON_MAJMIN}/__pycache__/argparse.*.pyc",