classes/lib/scripts: Use shutil.move when os.rename fails

Incremental build in Docker fails with:

OSError: [Errno 18] Invalid cross-device link

when source and destination are on different overlay filesystems.

This change handles error with os.rename and retries with shutil.move.
The reason os.rename is still used is because shutil.move is too slow
for speed sensitive sections of code.

[YOCTO #14301]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index 49af61c9c5cf15aa3b603e5db91d9f40dbd1f983..554c8810ac9bfd881760036010f5666c4861c0e4 100644 (file)
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -875,7 +875,11 @@ python buildhistory_eventhandler() {
                 entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
                 bb.utils.mkdirhier(olddir)
                 for entry in entries:
-                    os.rename(os.path.join(rootdir, entry),
+                    try:
+                        os.rename(os.path.join(rootdir, entry),
+                              os.path.join(olddir, entry))
+                    except OSError:
+                        shutil.move(os.path.join(rootdir, entry),
                               os.path.join(olddir, entry))
         elif isinstance(e, bb.event.BuildCompleted):
             if reset:

diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index e3f0a7060bf2d24a965dc570743e0faa03bfa4d7..6707c2e9db5bb6aea64a6cde60edb7014caf675e 100644 (file)
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -835,6 +835,7 @@ perform_packagecopy[dirs] = "${PKGD}"
 # the fs-perms.txt files
 python fixup_perms () {
     import pwd, grp
+    import shutil
 
     # init using a string with the same format as a line as documented in
     # the fs-perms.txt file
@@ -1049,7 +1050,10 @@ python fixup_perms () {
         # Create path to move directory to, move it, and then setup the symlink
         bb.utils.mkdirhier(os.path.dirname(target))
         #bb.note("Fixup Perms: Rename %s -> %s" % (dir, ptarget))
-        os.rename(origin, target)
+        try:
+            os.rename(origin, target)
+        except OSError:
+            shutil.move(origin, target)
         #bb.note("Fixup Perms: Link %s -> %s" % (dir, link))
         os.symlink(link, origin)
 
@@ -1776,6 +1780,7 @@ python package_do_shlibs() {
     import itertools
     import re, pipes
     import subprocess
+    import shutil
 
     exclude_shlibs = d.getVar('EXCLUDE_FROM_SHLIBS', False)
     if exclude_shlibs:
@@ -1967,7 +1972,10 @@ python package_do_shlibs() {
 
         for (old, new) in renames:
             bb.note("Renaming %s to %s" % (old, new))
-            os.rename(old, new)
+            try:
+                os.rename(old, new)
+            except OSError:
+                shutil.move(old, new)
             pkgfiles[pkg].remove(old)
 
         shlibs_file = os.path.join(shlibswork_dir, pkg + ".list")

diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index 84232ed9f5a3d44756a946b60fbbeb1ad422ea17..12e758c2ad68cb230b2f51ae8a78b163b14873ad 100644 (file)
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -165,7 +165,10 @@ def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
             shutil.rmtree(temp_sdkbasepath)
         except FileNotFoundError:
             pass
-        os.rename(sdkbasepath, temp_sdkbasepath)
+        try:
+            os.rename(sdkbasepath, temp_sdkbasepath)
+        except OSError:
+            shutil.move(sdkbasepath, temp_sdkbasepath)
         cmdprefix = '. %s .; ' % conf_initpath
         logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt'
         try:
@@ -175,7 +178,10 @@ def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
             if 'attempted to execute unexpectedly and should have been setscened' in e.stdout:
                 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n'
             bb.fatal(msg)
-        os.rename(temp_sdkbasepath, sdkbasepath)
+        try:
+            os.rename(temp_sdkbasepath, sdkbasepath)
+        except OSError:
+            shutil.move(temp_sdkbasepath, sdkbasepath)
         # Clean out residue of running bitbake, which check_sstate_task_list()
         # will effectively do
         clean_esdk_builddir(d, sdkbasepath)

diff --git a/meta/classes/reproducible_build.bbclass b/meta/classes/reproducible_build.bbclass
index f06e00d70d7eaf0f8ddc3aa6615d79b549e64c94..480d58f86d40c815a6acea4430b582c212691655 100644 (file)
--- a/meta/classes/reproducible_build.bbclass
+++ b/meta/classes/reproducible_build.bbclass
@@ -57,13 +57,17 @@ do_deploy_source_date_epoch () {
 }
 
 python do_deploy_source_date_epoch_setscene () {
+    import shutil
     sstate_setscene(d)
     bb.utils.mkdirhier(d.getVar('SDE_DIR'))
     sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt')
     if os.path.exists(sde_file):
         target = d.getVar('SDE_FILE')
         bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target))
-        os.rename(sde_file, target)
+        try:
+            os.rename(sde_file, target)
+        except OSError:
+            shutil.move(sde_file, target)
     else:
         bb.debug(1, "%s not found!" % sde_file)
 }

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 8e8efd18d58dea51eb28928ede07182d4f0fc7b2..60f7a942857fa93c5ca3dab93cd0ba270e5d2862 100644 (file)
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -384,6 +384,7 @@ def sstate_installpkg(ss, d):
 def sstate_installpkgdir(ss, d):
     import oe.path
     import subprocess
+    import shutil
 
     sstateinst = d.getVar("SSTATE_INSTDIR")
     d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
@@ -401,7 +402,10 @@ def sstate_installpkgdir(ss, d):
 
     for state in ss['dirs']:
         prepdir(state[1])
-        os.rename(sstateinst + state[0], state[1])
+        try:
+            os.rename(sstateinst + state[0], state[1])
+        except OSError:
+            shutil.move(sstateinst + state[0], state[1])
     sstate_install(ss, d)
 
     for plain in ss['plaindirs']:
@@ -413,7 +417,10 @@ def sstate_installpkgdir(ss, d):
         dest = plain
         bb.utils.mkdirhier(src)
         prepdir(dest)
-        os.rename(src, dest)
+        try:
+            os.rename(src, dest)
+        except OSError:
+            shutil.move(src, dest)
 
     return True
 
@@ -638,6 +645,7 @@ python sstate_hardcode_path () {
 
 def sstate_package(ss, d):
     import oe.path
+    import shutil
 
     tmpdir = d.getVar('TMPDIR')
 
@@ -664,7 +672,10 @@ def sstate_package(ss, d):
                     continue
                 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
         bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
-        os.rename(state[1], sstatebuild + state[0])
+        try:
+            os.rename(state[1], sstatebuild + state[0])
+        except OSError:
+            shutil.move(state[1], sstatebuild + state[0])
 
     workdir = d.getVar('WORKDIR')
     sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
@@ -674,7 +685,10 @@ def sstate_package(ss, d):
             pdir = plain.replace(sharedworkdir, sstatebuild)
         bb.utils.mkdirhier(plain)
         bb.utils.mkdirhier(pdir)
-        os.rename(plain, pdir)
+        try:
+            os.rename(plain, pdir)
+        except OSError:
+            shutil.move(plain, pdir)
 
     d.setVar('SSTATE_BUILDDIR', sstatebuild)
     d.setVar('SSTATE_INSTDIR', sstatebuild)

diff --git a/meta/classes/update-alternatives.bbclass b/meta/classes/update-alternatives.bbclass
index 8c2b66e7f15d003b785ceb6e416f1ee3189603a5..7e213472e3934952a211ebb793e135990aa1b2b2 100644 (file)
--- a/meta/classes/update-alternatives.bbclass
+++ b/meta/classes/update-alternatives.bbclass
@@ -139,6 +139,7 @@ python apply_update_alternative_renames () {
        return
 
     import re
+    import shutil
 
     def update_files(alt_target, alt_target_rename, pkg, d):
         f = d.getVar('FILES_' + pkg)
@@ -184,7 +185,10 @@ python apply_update_alternative_renames () {
                         link_rename.append((alt_target, alt_target_rename))
                     else:
                         bb.note('%s: Rename %s -> %s' % (pn, alt_target, alt_target_rename))
-                        os.rename(src, dest)
+                        try:
+                            os.rename(src, dest)
+                        except OSError:
+                            shutil.move(src, dest)
                         update_files(alt_target, alt_target_rename, pkg, d)
                 else:
                     bb.warn("%s: alternative target (%s or %s) does not exist, skipping..." % (pn, alt_target, alt_target_rename))
@@ -201,7 +205,10 @@ python apply_update_alternative_renames () {
             if os.path.lexists(link_target):
                 # Ok, the link_target exists, we can rename
                 bb.note('%s: Rename (link) %s -> %s' % (pn, alt_target, alt_target_rename))
-                os.rename(src, dest)
+                try:
+                    os.rename(src, dest)
+                except OSError:
+                    shutil.move(src, dest)
             else:
                 # Try to resolve the broken link to link.${BPN}
                 link_maybe = '%s.%s' % (os.readlink(src), pn)

diff --git a/meta/lib/oe/package_manager/deb/__init__.py b/meta/lib/oe/package_manager/deb/__init__.py
index 2ee68fefb10a8200445faeb684e1fa186323027b..8edc653dff4dec923a0fe82face4d730aae5a6a9 100644 (file)
--- a/meta/lib/oe/package_manager/deb/__init__.py
+++ b/meta/lib/oe/package_manager/deb/__init__.py
@@ -5,6 +5,7 @@
 import re
 import subprocess
 from oe.package_manager import *
+import shutil
 
 class DpkgIndexer(Indexer):
     def _create_configs(self):
@@ -214,7 +215,10 @@ class DpkgPM(OpkgDpkgPM):
 
                     tmp_sf.write(status)
 
-        os.rename(status_file + ".tmp", status_file)
+        try:
+            os.rename(status_file + ".tmp", status_file)
+        except OSError:
+            shutil.move(status_file + ".tmp", status_file)
 
     def run_pre_post_installs(self, package_name=None):
         """
@@ -299,13 +303,21 @@ class DpkgPM(OpkgDpkgPM):
             for dir in dirs:
                 new_dir = re.sub(r"\.dpkg-new", "", dir)
                 if dir != new_dir:
-                    os.rename(os.path.join(root, dir),
+                    try:
+                        os.rename(os.path.join(root, dir),
+                              os.path.join(root, new_dir))
+                    except OSError:
+                        shutil.move(os.path.join(root, dir),
                               os.path.join(root, new_dir))
 
             for file in files:
                 new_file = re.sub(r"\.dpkg-new", "", file)
                 if file != new_file:
-                    os.rename(os.path.join(root, file),
+                    try:
+                        os.rename(os.path.join(root, file),
+                              os.path.join(root, new_file))
+                    except OSError:
+                        shutil.move(os.path.join(root, file),
                               os.path.join(root, new_file))
 
 

diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py
index da488c1c7f0a08a4e7584c9d8b88c4a4a935432f..2aa21949f3df647752755e66bb9b759523436ac6 100644 (file)
--- a/meta/lib/oe/package_manager/ipk/__init__.py
+++ b/meta/lib/oe/package_manager/ipk/__init__.py
@@ -213,7 +213,10 @@ class OpkgPM(OpkgDpkgPM):
 
                     tmp_sf.write(status)
 
-        os.rename(status_file + ".tmp", status_file)
+        try:
+            os.rename(status_file + ".tmp", status_file)
+        except OSError:
+            shutil.move(status_file + ".tmp", status_file)
 
     def _create_custom_config(self):
         bb.note("Building from feeds activated!")

diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 5f8102304059b065c3ae4bb7aaa00919074175e8..ef0001004733a18cbb669ba5ab578a078c6fdb30 100644 (file)
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -114,7 +114,10 @@ class Rootfs(object, metaclass=ABCMeta):
             shutil.rmtree(self.image_rootfs + '-orig')
         except:
             pass
-        os.rename(self.image_rootfs, self.image_rootfs + '-orig')
+        try:
+            os.rename(self.image_rootfs, self.image_rootfs + '-orig')
+        except OSError:
+            shutil.move(self.image_rootfs, self.image_rootfs + '-orig')
 
         bb.note("  Creating debug rootfs...")
         bb.utils.mkdirhier(self.image_rootfs)
@@ -165,10 +168,16 @@ class Rootfs(object, metaclass=ABCMeta):
             shutil.rmtree(self.image_rootfs + '-dbg')
         except:
             pass
-        os.rename(self.image_rootfs, self.image_rootfs + '-dbg')
+        try:
+            os.rename(self.image_rootfs, self.image_rootfs + '-dbg')
+        except OSError:
+            shutil.move(self.image_rootfs, self.image_rootfs + '-dbg')
 
         bb.note("  Restoreing original rootfs...")
-        os.rename(self.image_rootfs + '-orig', self.image_rootfs)
+        try:
+            os.rename(self.image_rootfs + '-orig', self.image_rootfs)
+        except OSError:
+            shutil.move(self.image_rootfs + '-orig', self.image_rootfs)
 
     def _exec_shell_cmd(self, cmd):
         fakerootcmd = self.d.getVar('FAKEROOT')

diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
index fa81584a8ccacd354fc7ef9f0ddc852b000cce11..fa93973087fe971e06dd603f55c5b39bc83de016 100644 (file)
--- a/meta/lib/oeqa/selftest/cases/wic.py
+++ b/meta/lib/oeqa/selftest/cases/wic.py
@@ -1269,6 +1269,7 @@ class Wic2(WicTestCase):
 
     def test_expand_mbr_image(self):
         """Test wic write --expand command for mbr image"""
+        import shutil
         # build an image
         config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "directdisk.wks"\n'
         self.append_config(config)
@@ -1306,8 +1307,14 @@ class Wic2(WicTestCase):
             result = runCmd("%s/usr/sbin/sfdisk -F %s" % (sysroot, new_image_path))
             self.assertTrue("0 B, 0 bytes, 0 sectors" in result.output)
 
-            os.rename(image_path, image_path + '.bak')
-            os.rename(new_image_path, image_path)
+            try:
+                os.rename(image_path, image_path + '.bak')
+            except OSError:
+                shutil.move(image_path, image_path + '.bak')
+            try:
+                os.rename(new_image_path, image_path)
+            except OSError:
+                shutil.move(new_image_path, image_path)
 
             # Check if it boots in qemu
             with runqemu('core-image-minimal', ssh=False) as qemu:
@@ -1318,7 +1325,10 @@ class Wic2(WicTestCase):
             if os.path.exists(new_image_path):
                 os.unlink(new_image_path)
             if os.path.exists(image_path + '.bak'):
-                os.rename(image_path + '.bak', image_path)
+                try:
+                    os.rename(image_path + '.bak', image_path)
+                except OSError:
+                    shutil.move(image_path + '.bak', image_path)
 
     def test_wic_ls_ext(self):
         """Test listing content of the ext partition using 'wic ls'"""

diff --git a/scripts/combo-layer b/scripts/combo-layer
index a634dd69d2a07ad775772d1ee7673469f9da44bf..1b88340b6e4bd8288b23e6eea0a6ca0fb083dc71 100755 (executable)
--- a/scripts/combo-layer
+++ b/scripts/combo-layer
@@ -508,7 +508,10 @@ def check_patch(patchfile):
     f.close()
     if of:
         of.close()
-        os.rename(patchfile + '.tmp', patchfile)
+        try:
+            os.rename(patchfile + '.tmp', patchfile)
+        except OSError:
+            shutil.move(patchfile + '.tmp', patchfile)
 
 def drop_to_shell(workdir=None):
     if not sys.stdin.isatty():

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index f364a452834e9f0edacff79b1e543b8fdc9f1464..3854dfb3e8d1099a1df50611f6edcea8b299aa85 100644 (file)
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -746,7 +746,10 @@ def _check_preserve(config, recipename):
                         os.remove(removefile)
                 else:
                     tf.write(line)
-    os.rename(newfile, origfile)
+    try:
+        os.rename(newfile, origfile)
+    except OSError:
+        shutil.move(newfile, origfile)
 
 def get_staging_kver(srcdir):
     # Kernel version from work-shared
@@ -1094,10 +1097,16 @@ def rename(args, config, basepath, workspace):
 
     # Rename bbappend
     logger.info('Renaming %s to %s' % (append, newappend))
-    os.rename(append, newappend)
+    try:
+        os.rename(append, newappend)
+    except OSError:
+        shutil.move(append, newappend)
     # Rename recipe file
     logger.info('Renaming %s to %s' % (recipefile, newfile))
-    os.rename(recipefile, newfile)
+    try:
+        os.rename(recipefile, newfile)
+    except OSError:
+        shutil.move(recipefile, newfile)
 
     # Rename source tree if it's the default path
     appendmd5 = None
@@ -1333,7 +1342,11 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
         if match_name:
             # Rename patch files
             if new_patch != match_name:
-                os.rename(os.path.join(destdir, new_patch),
+                try:
+                    os.rename(os.path.join(destdir, new_patch),
+                          os.path.join(destdir, match_name))
+                except OSError:
+                    shutil.move(os.path.join(destdir, new_patch),
                           os.path.join(destdir, match_name))
             # Need to pop it off the list now before checking changed_revs
             oldpath = existing_patches.pop(old_patch)

diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 5a057e95f58345131ee3373abf49567df3c0ccfb..2bff69f5a2cf5bb649b4b4d08810d0e3bc5a8146 100644 (file)
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -71,7 +71,12 @@ def _rename_recipe_dirs(oldpv, newpv, path):
                 if oldfile.find(oldpv) != -1:
                     newfile = oldfile.replace(oldpv, newpv)
                     if oldfile != newfile:
-                        os.rename(os.path.join(path, oldfile), os.path.join(path, newfile))
+                        try:
+                            os.rename(os.path.join(path, oldfile),
+                                  os.path.join(path, newfile))
+                        except OSError:
+                            shutil.move(os.path.join(path, oldfile),
+                                  os.path.join(path, newfile))
 
 def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path):
     oldrecipe = os.path.basename(oldrecipe)

diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py
index ea709e8c545d108f3f6c4927f86d1c7ea7f1a1d4..fb5e52b2a65113262d1689eaa971ff785796c2b4 100644 (file)
--- a/scripts/lib/wic/plugins/imager/direct.py
+++ b/scripts/lib/wic/plugins/imager/direct.py
@@ -616,5 +616,8 @@ class PartitionedImage():
                              part.start + part.size_sec - 1, part.size_sec)
 
                 partimage = self.path + '.p%d' % part.num
-                os.rename(source, partimage)
+                try:
+                    os.rename(source, partimage)
+                except OSError:
+                    shutil.move(source, partimage)
                 self.partimages.append(partimage)