oeqa/runqemu: Support RUNQEMU_TMPFS_DIR as a location to copy snapshot images to

We have a working theory that IO queues on the autobuilder are impacting
runtime testing under qemu, particularly async writes which inice does not
influence. We already pass the snapshot option to qemu which copies the
image and runs out of the copy. Add in the ability to copy the image to
a specificed location which can be a tmpfs. This means that writes to the
image would no longer be blocked by other writes to disk in the system.

Preliminary tests show that this does improve the qemu errors at the expense
of sometimes showing qemu startup timeouts as on a loaded system with a large
test image, it can take longer than 120s to copy the image to tmpfs. Having
a most consistent failure mode for loaded tests is probably desireable though.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 78da4b09bdf6bbef6156318f0972e6c671f69a13..e613759503119c7194bd395936e0154895a05242 100644 (file)
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -305,6 +305,7 @@ def testimage_main(d):
                       'dump_dir'    : d.getVar("TESTIMAGE_DUMP_DIR"),
                       'serial_ports': len(d.getVar("SERIAL_CONSOLES").split()),
                       'ovmf'        : ovmf,
+                      'tmpfsdir'    : d.getVar("RUNQEMU_TMPFS_DIR"),
                     }
 
     if d.getVar("TESTIMAGE_BOOT_PATTERNS"):

diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py
index 0f29414df542a085c587a70796278ed876762abf..792efca1f8474c69ea3bc218ec508e80cc86feb9 100644 (file)
--- a/meta/lib/oeqa/core/target/qemu.py
+++ b/meta/lib/oeqa/core/target/qemu.py
@@ -21,7 +21,7 @@ class OEQemuTarget(OESSHTarget):
             port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False,
             dump_dir='', dump_host_cmds='', display='', bootlog='',
             tmpdir='', dir_image='', boottime=60, serial_ports=2,
-            boot_patterns = defaultdict(str), ovmf=False, **kwargs):
+            boot_patterns = defaultdict(str), ovmf=False, tmpfsdir=None, **kwargs):
 
         super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout,
                 user, port)
@@ -42,7 +42,7 @@ class OEQemuTarget(OESSHTarget):
                                  use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir,
                                  dump_host_cmds=dump_host_cmds, logger=logger,
                                  serial_ports=serial_ports, boot_patterns = boot_patterns, 
-                                 use_ovmf=ovmf)
+                                 use_ovmf=ovmf, tmpfsdir=tmpfsdir)
         dump_target_cmds = kwargs.get("testimage_dump_target")
         self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner)
         self.target_dumper.create_dir("qemu")

diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py
index 19f5a4ea7e90480d704167bdf92bb23556fd4334..12057f855aedfe27127c8c17f45d162fe4b8ef8f 100644 (file)
--- a/meta/lib/oeqa/targetcontrol.py
+++ b/meta/lib/oeqa/targetcontrol.py
@@ -131,6 +131,7 @@ class QemuTarget(BaseTarget):
                             logfile = self.qemulog,
                             kernel = self.kernel,
                             boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")),
+                            tmpfsdir = d.getVar("RUNQEMU_TMPFS_DIR"),
                             logger = logger)
         else:
             self.runner = QemuRunner(machine=d.getVar("MACHINE"),
@@ -144,6 +145,7 @@ class QemuTarget(BaseTarget):
                             dump_dir = dump_dir,
                             dump_host_cmds = d.getVar("testimage_dump_host"),
                             logger = logger,
+                            tmpfsdir = d.getVar("RUNQEMU_TMPFS_DIR"),
                             serial_ports = len(d.getVar("SERIAL_CONSOLES").split()))
 
         self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner)

diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index eb23dbceb854a9fac6f89b6d593643c1af3fc3a0..278904ba0b1c942956cba4ce6f14ece9eca4056f 100644 (file)
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -32,7 +32,7 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars)))
 class QemuRunner:
 
     def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds,
-                 use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None):
+                 use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None, tmpfsdir=None):
 
         # Popen object for runqemu
         self.runqemu = None
@@ -61,6 +61,7 @@ class QemuRunner:
         self.serial_ports = serial_ports
         self.msg = ''
         self.boot_patterns = boot_patterns
+        self.tmpfsdir = tmpfsdir
 
         self.runqemutime = 120
         if not workdir:
@@ -150,6 +151,9 @@ class QemuRunner:
         else:
             env["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
 
+        if self.tmpfsdir:
+            env["RUNQEMU_TMPFS_DIR"] = self.tmpfsdir
+
         if not launch_cmd:
             launch_cmd = 'runqemu %s' % ('snapshot' if discard_writes else '')
             if self.use_kvm:

diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py
index 5c92941c0a6a70f4d9a622b66151aaf0f28c0e17..20009401ca8e2f4fa890b46cb217fdc15c6ec1fc 100644 (file)
--- a/meta/lib/oeqa/utils/qemutinyrunner.py
+++ b/meta/lib/oeqa/utils/qemutinyrunner.py
@@ -19,7 +19,7 @@ from .qemurunner import QemuRunner
 
 class QemuTinyRunner(QemuRunner):
 
-    def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger):
+    def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger, tmpfsdir=None):
 
         # Popen object for runqemu
         self.runqemu = None
@@ -37,6 +37,7 @@ class QemuTinyRunner(QemuRunner):
         self.deploy_dir_image = deploy_dir_image
         self.logfile = logfile
         self.boottime = boottime
+        self.tmpfsdir = tmpfsdir
 
         self.runqemutime = 60
         self.socketfile = "console.sock"
@@ -83,6 +84,9 @@ class QemuTinyRunner(QemuRunner):
             return False
         else:
             os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
+        if self.tmpfsdir:
+            env["RUNQEMU_TMPFS_DIR"] = self.tmpfsdir
+
 
         # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact
         # badly with screensavers.

diff --git a/scripts/runqemu b/scripts/runqemu
index 35053de368d6639edc1187b4e4a6d8c8c6c87815..ba0b701aff36ca7fb7d480bbdddc3d18a6960f5f 100755 (executable)
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -1196,6 +1196,15 @@ class BaseConfig(object):
             self.fstype = self.fstype[4:]
         rootfs_format = self.fstype if self.fstype in ('vmdk', 'vhd', 'vhdx', 'qcow2', 'vdi') else 'raw'
 
+        tmpfsdir = os.environ.get("RUNQEMU_TMPFS_DIR", None)
+        if self.snapshot and tmpfsdir:
+            newrootfs = os.path.join(tmpfsdir, os.path.basename(self.rootfs)) + "." + str(os.getpid())
+            shutil.copyfile(self.rootfs, newrootfs)
+            #print("Copying rootfs to tmpfs: %s" % newrootfs)
+            self.rootfs = newrootfs
+            # Don't need a second copy now!
+            self.snapshot = False
+
         qb_rootfs_opt = self.get('QB_ROOTFS_OPT')
         if qb_rootfs_opt:
             self.rootfs_options = qb_rootfs_opt.replace('@ROOTFS@', self.rootfs)