testsuite: Initial Avocado test implementation

Currently, we have several test cases executed from shell scripts. As test
coverage increases, adding test cases and reading the results becomes complex.
Avocado testing framework provides the necessary infrastructure to organize
test cases. This patch adds the initial Avocado test case support for future CI
migration.

Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
Tested-by: Maxim Yu. Osipov <mosipov@ilbers.de>

testsuite/README.md

@@ -0,0 +1,34 @@
+# Install Avocado
+
+The framework could be installed by using standard HOWTO:
+
+  https://github.com/avocado-framework/avocado#installing-with-standard-python-tools
+
+Then you need to install varianter yaml-to-mux plugin by following these instructions:
+
+  https://github.com/avocado-framework/avocado/tree/master/optional_plugins
+
+## For Debian 9.x
+
+        $ sudo apt-get install python-pip
+        $ pip install --user subprocess32
+        $ pip install --user avocado-framework
+        $ pip install --user avocado-framework-plugin-varianter-yaml-to-mux
+
+# Pre
+
+        $ export PATH=$PATH:~/.local/bin
+        $ cd isar
+        $ source isar-init-build-env
+
+
+# Run test
+
+Each testsuite directory contains:
+ - run.sh - script to start tests
+ - variants.yaml - set of input data
+ - *.py - test case
+
+# Other
+
+There is a tool start_vm.py which is the replacement for the bash script in isar/scripts directory.

testsuite/build_test/build_test.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess32
+import sys
+from os.path import dirname
+
+from avocado import Test
+
+class BuildTest(Test):
+
+    def test(self):
+        # TODO: add default values
+        build_dir = self.params.get('build_dir', default='.')
+        arch = self.params.get('arch', default='arm')
+        distro = self.params.get('distro', default='stretch')
+
+        self.log.info('===================================================')
+        self.log.info('Running Isar build test for (' + distro + '-' + arch + ')')
+        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('===================================================')
+
+        #isar_root = dirname(__file__) + '/..'
+        os.chdir(build_dir)
+        cmdline = ['bitbake', 'multiconfig:qemu' + arch + '-' + distro + ':isar-image-base']
+        p1 = subprocess32.run(cmdline)
+
+        if p1.returncode:
+            self.fail('Test failed')

testsuite/build_test/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+avocado run build_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR

testsuite/build_test/variant.yaml

@@ -0,0 +1,22 @@
+variants: !mux
+    stretch-amd64:
+        distro: "stretch"
+        arch:   "amd64"
+    stretch-i386:
+        distro: "stretch"
+        arch:   "i386"
+    stretch-arm:
+        distro: "stretch"
+        arch:   "arm"
+    stretch-arm64:
+        distro: "stretch"
+        arch:   "arm64"
+    buster-amd64:
+        distro: "buster"
+        arch:   "amd64"
+    buster-i386:
+        distro: "buster"
+        arch:   "i386"
+    buster-arm:
+        distro: "buster"
+        arch:   "arm"

testsuite/start_vm.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+#
+# Helper script to start QEMU with Isar image
+# Copyright (c) 2019, ilbers GmbH
+
+import argparse
+import os
+import subprocess
+import sys
+import time
+
+def get_bitbake_env(arch, distro):
+    multiconfig = 'multiconfig:qemu' + arch + '-' + distro + ':isar-image-base'
+    output = subprocess.check_output(['bitbake', '-e', str(multiconfig)])
+    return output
+
+def get_bitbake_var(output, var):
+    ret = ''
+    for line in output.splitlines():
+        if line.startswith(var):
+            ret = line.split('"')[1]
+    return ret
+
+def format_qemu_cmdline(arch, build, distro):
+    bb_output = get_bitbake_env(arch, distro).decode()
+
+    rootfs_image = ''
+    extra_args = ''
+    cpu = ['']
+
+    image_type = get_bitbake_var(bb_output, 'IMAGE_TYPE')
+    deploy_dir_image = get_bitbake_var(bb_output, 'DEPLOY_DIR_IMAGE')
+    if image_type == 'ext4-img':
+        rootfs_image = 'isar-image-base-debian-' + distro + '-qemu' + arch + '.ext4.img'
+        kernel_image = deploy_dir_image + '/' + get_bitbake_var(bb_output, 'KERNEL_IMAGE')
+        initrd_image = get_bitbake_var(bb_output, 'INITRD_IMAGE')
+
+        if not initrd_image:
+            initrd_image = '/dev/null'
+        else:
+            initrd_image = deploy_dir_image + '/' + initrd_image
+
+        serial = get_bitbake_var(bb_output, 'MACHINE_SERIAL')
+        root_dev = get_bitbake_var(bb_output, 'QEMU_ROOTFS_DEV')
+        kargs = ['-append', '"console=' + serial + ' root=/dev/' + root_dev + ' rw"']
+
+        extra_args = ['-kernel', kernel_image, '-initrd', initrd_image]
+        extra_args.extend(kargs)
+    elif image_type == 'wic-img':
+        rootfs_image = 'isar-image-base-debian-' + distro + '-qemu' + arch + '.wic.img'
+        extra_args = ['-snapshot']
+    else:
+        raise ValueError('Invalid image type: ' + str(image_type))
+
+    qemu_arch = get_bitbake_var(bb_output, 'QEMU_ARCH')
+    qemu_machine = get_bitbake_var(bb_output, 'QEMU_MACHINE')
+    qemu_cpu = get_bitbake_var(bb_output, 'QEMU_CPU')
+    qemu_disk_args = get_bitbake_var(bb_output, 'QEMU_DISK_ARGS')
+
+    qemu_disk_args = qemu_disk_args.replace('##ROOTFS_IMAGE##', deploy_dir_image + '/' + rootfs_image).split()
+
+    cmd = ['qemu-system-' + qemu_arch, '-m', '1024M']
+
+    if qemu_machine:
+        cmd.extend(['-M', qemu_machine])
+
+    if qemu_cpu:
+        cmd.extend(['-cpu', qemu_cpu])
+
+    cmd.extend(extra_args)
+    cmd.extend(qemu_disk_args)
+
+    return cmd
+
+def start_qemu(arch, build, distro):
+    cmdline = format_qemu_cmdline(arch, build, distro)
+    cmdline.insert(1, '-nographic')
+
+    p1 = subprocess.call(cmdline)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-a', '--arch', choices=['arm', 'arm64', 'amd64', 'i386'], help='set isar machine architecture.', default='arm')
+    parser.add_argument('-b', '--build', help='set path to build directory.', default=os.getcwd())
+    parser.add_argument('-d', '--distro', choices=['jessie', 'stretch'], help='set isar Debian distribution.', default='stretch')
+    args = parser.parse_args()
+
+    start_qemu(args.arch, args.build, args.distro)

testsuite/vm_boot_test/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+avocado run vm_boot_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR time_to_wait:300

testsuite/vm_boot_test/variant.yaml

@@ -0,0 +1,22 @@
+variants: !mux
+    stretch-amd64:
+        distro: "stretch"
+        arch:   "amd64"
+    stretch-i386:
+        distro: "stretch"
+        arch:   "i386"
+    stretch-arm:
+        distro: "stretch"
+        arch:   "arm"
+    stretch-arm64:
+        distro: "stretch"
+        arch:   "arm64"
+    buster-amd64:
+        distro: "buster"
+        arch:   "amd64"
+    buster-i386:
+        distro: "buster"
+        arch:   "i386"
+    buster-arm:
+        distro: "buster"
+        arch:   "arm"

testsuite/vm_boot_test/vm_boot_test.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess32
+import sys
+import time
+
+from os.path import dirname
+sys.path.append(dirname(__file__) + '/..')
+
+import start_vm
+
+from avocado import Test
+
+class VmBootTest(Test):
+
+    def test(self):
+        # TODO: add default values
+        build_dir = self.params.get('build_dir', default='.')
+        arch = self.params.get('arch', default='arm')
+        distro = self.params.get('distro', default='stretch')
+        time_to_wait = self.params.get('time_to_wait', default=60)
+
+        self.log.info('===================================================')
+        self.log.info('Running Isar VM boot test for (' + distro + '-' + arch + ')')
+        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('===================================================')
+
+        output_file = '/tmp/vm_boot_test.log'
+        if os.path.exists(output_file):
+            os.remove(output_file)
+
+        cmdline = start_vm.format_qemu_cmdline(arch, build_dir, distro)
+        cmdline.insert(1, '-nographic')
+        cmdline.append('-serial')
+        cmdline.append('file:' + output_file)
+
+        self.log.info('QEMU boot line: ' + str(cmdline))
+
+        devnull = open(os.devnull, 'w')
+
+        p1 = subprocess32.Popen(cmdline, stdout=devnull, stderr=devnull)
+        time.sleep(int(time_to_wait))
+        p1.kill()
+        p1.wait()
+
+        if os.path.exists(output_file):
+            if 'isar login:' in open(output_file).read():
+                return
+
+        self.fail('Test failed')