libc-package.bbclass, eglibc, glibc: enable locales generation using cross-localedef

Along with qemu this provides another option to generate locales for
[e]glibc. The new method is to use cross-localedef with appropriate arch
specific parameters.

The cross-localedef method is found to be 15 times faster than qemu's
emnualted method.

LOCALE_GENERATION_WITH_CROSS-LOCALEDEF : This is new config variable
introduced to selet qemu or cross-localedef method for locale generation.

Thanks to Mark Hatle from Windriver for providing the rich
information for cross locale generation.

Signed-off-by: Nitin A Kamble <nitin.a.kamble@intel.com>

diff --git a/meta/classes/libc-package.bbclass b/meta/classes/libc-package.bbclass
index 1c034e09bed6e01c994ba6bdf6e8f264ea94f76d..4709b33c0e9c6579c52a5e9485ceda38fd560029 100644 (file)
--- a/meta/classes/libc-package.bbclass
+++ b/meta/classes/libc-package.bbclass
@@ -9,7 +9,29 @@
 
 GLIBC_INTERNAL_USE_BINARY_LOCALE ?= "ondevice"
 
-inherit qemu
+python __anonymous () {
+    enabled = bb.data.getVar("ENABLE_BINARY_LOCALE_GENERATION", d, 1)
+
+    if enabled and int(enabled):
+        import re
+
+        target_arch = bb.data.getVar("TARGET_ARCH", d, 1)
+        binary_arches = bb.data.getVar("BINARY_LOCALE_ARCHES", d, 1) or ""
+        use_cross_localedef = bb.data.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF", d, 1) or ""
+
+        for regexp in binary_arches.split(" "):
+            r = re.compile(regexp)
+
+            if r.match(target_arch):
+                depends = bb.data.getVar("DEPENDS", d, 1)
+               if use_cross_localedef == "1" :
+                       depends = "%s cross-localedef-native" % depends
+               else:
+                       depends = "%s qemu-native" % depends
+                bb.data.setVar("DEPENDS", depends, d)
+                bb.data.setVar("GLIBC_INTERNAL_USE_BINARY_LOCALE", "compile", d)
+                break
+}
 
 def get_libc_fpu_setting(bb, d):
     if bb.data.getVar('TARGET_FPU', d, 1) in [ 'soft' ]:
@@ -105,6 +127,8 @@ do_collect_bins_from_locale_tree() {
        cp -pPR $treedir/${libdir}/locale ${PKGD}${libdir}
 }
 
+inherit qemu
+
 python package_do_split_gconvs () {
        import os, re
        if (bb.data.getVar('PACKAGE_NO_GCONV', d, 1) == '1'):
@@ -251,24 +275,49 @@ python package_do_split_gconvs () {
                bb.data.setVar('RPROVIDES_%s' % pkgname, " ".join(rprovides), d)
 
        def output_locale_binary(name, pkgname, locale, encoding):
-               qemu = qemu_target_binary(d) 
-
                treedir = base_path_join(bb.data.getVar("WORKDIR", d, 1), "locale-tree")
                ldlibdir = "%s/lib" % treedir
                path = bb.data.getVar("PATH", d, 1)
                i18npath = base_path_join(treedir, datadir, "i18n")
+               gconvpath = base_path_join(treedir, "iconvdata")
+
+               use_cross_localedef = bb.data.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF", d, 1) or "0"
+               if use_cross_localedef == "1":
+                       target_arch = bb.data.getVar('TARGET_ARCH', d, True)
+                       locale_arch_options = { \
+                               "arm":     " --uint32-align=4 --little-endian ", \
+                               "powerpc": " --uint32-align=4 --big-endian ",    \
+                               "mips":    " --uint32-align=4 --big-endian ",    \
+                               "mipsel":  " --uint32-align=4 --little-endian ", \
+                               "i586":    " --uint32-align=4 --little-endian ", \
+                               "x86_64":  " --uint32-align=4 --little-endian "  }
+
+                       if target_arch in locale_arch_options:
+                               localedef_opts = locale_arch_options[target_arch]
+                       else:
+                               bb.error("locale_arch_options not found for target_arch=" + target_arch)
+                               raise bb.build.FuncFailed("unknown arch:" + target_arch + " for locale_arch_options")
+
+                       localedef_opts += " --force --old-style --no-archive --prefix=%s \
+                               --inputfile=%s/%s/i18n/locales/%s --charmap=%s %s/usr/lib/locale/%s" \
+                               % (treedir, treedir, datadir, locale, encoding, treedir, name)
+
+                       cmd = "PATH=\"%s\" I18NPATH=\"%s\" GCONV_PATH=\"%s\" cross-localedef %s" % \
+                               (path, i18npath, gconvpath, localedef_opts)
+               else: # earlier slower qemu way 
+                       qemu = qemu_target_binary(d) 
+                       localedef_opts = "--force --old-style --no-archive --prefix=%s \
+                               --inputfile=%s/i18n/locales/%s --charmap=%s %s" \
+                               % (treedir, datadir, locale, encoding, name)
 
-               localedef_opts = "--force --old-style --no-archive --prefix=%s \
-                       --inputfile=%s/i18n/locales/%s --charmap=%s %s" \
-                       % (treedir, datadir, locale, encoding, name)
+                       qemu_options = bb.data.getVar("QEMU_OPTIONS_%s" % bb.data.getVar('PACKAGE_ARCH', d, 1), d, 1)
+                       if not qemu_options:
+                               qemu_options = bb.data.getVar('QEMU_OPTIONS', d, 1)
 
-               qemu_options = bb.data.getVar("QEMU_OPTIONS_%s" % bb.data.getVar('PACKAGE_ARCH', d, 1), d, 1)
-               if not qemu_options:
-                       qemu_options = bb.data.getVar('QEMU_OPTIONS', d, 1)
+                       cmd = "PSEUDO_RELOADED=YES PATH=\"%s\" I18NPATH=\"%s\" %s -L %s \
+                               -E LD_LIBRARY_PATH=%s %s %s/bin/localedef %s" % \
+                               (path, i18npath, qemu, treedir, ldlibdir, qemu_options, treedir, localedef_opts)
 
-               cmd = "PSEUDO_RELOADED=YES PATH=\"%s\" I18NPATH=\"%s\" %s -L %s \
-                       -E LD_LIBRARY_PATH=%s %s %s/bin/localedef %s" % \
-                       (path, i18npath, qemu, treedir, ldlibdir, qemu_options, treedir, localedef_opts)
                bb.note("generating locale %s (%s)" % (locale, encoding))
                import subprocess
                process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

diff --git a/meta/recipes-core/eglibc/eglibc-package.inc b/meta/recipes-core/eglibc/eglibc-package.inc
index 35001fd6fc0a5b264402548624404a3c0386af0c..ac0ba94bc2c408848d89889bb1084843ff9c7a58 100644 (file)
--- a/meta/recipes-core/eglibc/eglibc-package.inc
+++ b/meta/recipes-core/eglibc/eglibc-package.inc
@@ -19,13 +19,18 @@ python __anonymous () {
 # to decrease initial boot time and avoid localedef being killed by the OOM
 # killer which used to effectively break i18n on machines with < 128MB RAM.
 
-# default to disabled until qemu works for everyone
+# default to disabled 
 ENABLE_BINARY_LOCALE_GENERATION ?= "0"
 ENABLE_BINARY_LOCALE_GENERATION_pn-eglibc-nativesdk = "0"
 
+#enable locale generation on these arches
 # BINARY_LOCALE_ARCHES is a space separated list of regular expressions
 BINARY_LOCALE_ARCHES ?= "arm.* i[3-6]86 x86_64 powerpc mips"
 
+# set "1" to use cross-localedef for locale generation
+# set "0" for qemu emulation of native localedef for locale generation
+LOCALE_GENERATION_WITH_CROSS-LOCALEDEF = "1"
+
 # Set this to zero if you don't want ldconfig in the output package
 USE_LDCONFIG ?= "1"
 
@@ -79,25 +84,5 @@ DESCRIPTION_ldd = "eglibc: print shared library dependencies"
 DESCRIPTION_localedef = "eglibc: compile locale definition files"
 DESCRIPTION_eglibc-utils = "eglibc: misc utilities like iconf, local, gencat, tzselect, rpcinfo, ..."
 
-python __anonymous () {
-    enabled = bb.data.getVar("ENABLE_BINARY_LOCALE_GENERATION", d, 1)
-
-    if enabled and int(enabled):
-        import re
-
-        target_arch = bb.data.getVar("TARGET_ARCH", d, 1)
-        binary_arches = bb.data.getVar("BINARY_LOCALE_ARCHES", d, 1) or ""
-
-        for regexp in binary_arches.split(" "):
-            r = re.compile(regexp)
-
-            if r.match(target_arch):
-                depends = bb.data.getVar("DEPENDS", d, 1)
-                depends = "%s qemu-native" % depends
-                bb.data.setVar("DEPENDS", depends, d)
-                bb.data.setVar("GLIBC_INTERNAL_USE_BINARY_LOCALE", "compile", d)
-                break
-}
-
 inherit libc-package
 

diff --git a/meta/recipes-core/glibc/glibc-package.inc b/meta/recipes-core/glibc/glibc-package.inc
index 3c9132820d6f7aa2d3d63347e111d7eaf2a1496e..8faeeb7e89c17c0d8403c3f9ce5e7a09ed276285 100644 (file)
--- a/meta/recipes-core/glibc/glibc-package.inc
+++ b/meta/recipes-core/glibc/glibc-package.inc
@@ -27,6 +27,10 @@ ENABLE_BINARY_LOCALE_GENERATION_pn-glibc-nativesdk = "0"
 # BINARY_LOCALE_ARCHES is a space separated list of regular expressions
 BINARY_LOCALE_ARCHES ?= "arm.* i[3-6]86 x86_64 powerpc mips"
 
+# set "1" to use cross-localedef for locale generation
+# set "0" for qemu emulation of native localedef for locale generation
+LOCALE_GENERATION_WITH_CROSS-LOCALEDEF = "1"
+
 PKGSUFFIX = ""
 PKGSUFFIX_virtclass-nativesdk = "-nativesdk"
 
@@ -86,24 +90,4 @@ DESCRIPTION_glibc-utils = "glibc: misc utilities like iconf, local, gencat, tzse
 
 EXTRA_OEMAKE += "rootsbindir=${base_sbindir}"
 
-python __anonymous () {
-    enabled = bb.data.getVar("ENABLE_BINARY_LOCALE_GENERATION", d, 1)
-
-    if enabled and int(enabled):
-        import re
-
-        target_arch = bb.data.getVar("TARGET_ARCH", d, 1)
-        binary_arches = bb.data.getVar("BINARY_LOCALE_ARCHES", d, 1) or ""
-
-        for regexp in binary_arches.split(" "):
-            r = re.compile(regexp)
-
-            if r.match(target_arch):
-                depends = bb.data.getVar("DEPENDS", d, 1)
-                depends = "%s qemu-native" % depends
-                bb.data.setVar("DEPENDS", depends, d)
-                bb.data.setVar("GLIBC_INTERNAL_USE_BINARY_LOCALE", "compile", d)
-                break
-}
-
 inherit libc-package