initial buildroot for linux 5.15

support/testing/infra/__init__.py

@@ -0,0 +1,127 @@
+import os
+import re
+import sys
+import tempfile
+import subprocess
+from urllib.request import urlopen
+from urllib.error import HTTPError, URLError
+
+ARTIFACTS_URL = "http://autobuild.buildroot.net/artefacts/"
+BASE_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "../../.."))
+
+
+def log_file_path(builddir, stage, logtofile=True):
+    """Return path to log file"""
+    return "{}-{}.log".format(builddir, stage) if logtofile else None
+
+
+def open_log_file(builddir, stage, logtofile=True):
+    """
+    Open a file for logging and return its handler.
+    If logtofile is True, returns sys.stdout. Otherwise opens a file
+    with a suitable name in the build directory.
+    """
+    return open(log_file_path(builddir, stage, logtofile), 'a+') if logtofile else sys.stdout
+
+
+def basepath(relpath=""):
+    """Return the absolute path for a file or directory relative to the Buildroot top directory."""
+    return os.path.join(BASE_DIR, relpath)
+
+
+def filepath(relpath):
+    return os.path.join(BASE_DIR, "support/testing", relpath)
+
+
+def download(dldir, filename):
+    finalpath = os.path.join(dldir, filename)
+    if os.path.exists(finalpath):
+        return finalpath
+
+    if not os.path.exists(dldir):
+        os.makedirs(dldir)
+
+    tmpfile = tempfile.mktemp(dir=dldir)
+    print("Downloading to {}".format(tmpfile))
+
+    try:
+        url_fh = urlopen(os.path.join(ARTIFACTS_URL, filename))
+        with open(tmpfile, "w+b") as tmpfile_fh:
+            tmpfile_fh.write(url_fh.read())
+    except (HTTPError, URLError) as err:
+        os.unlink(tmpfile)
+        raise err
+
+    print("Renaming from {} to {}".format(tmpfile, finalpath))
+    os.rename(tmpfile, finalpath)
+    return finalpath
+
+
+def run_cmd_on_host(builddir, cmd):
+    """Call subprocess.check_output and return the text output."""
+    out = subprocess.check_output(cmd,
+                                  stderr=open(os.devnull, "w"),
+                                  cwd=builddir,
+                                  env={"LANG": "C"},
+                                  universal_newlines=True)
+    return out
+
+
+def get_elf_arch_tag(builddir, prefix, fpath, tag):
+    """
+    Runs the cross readelf on 'fpath', then extracts the value of tag 'tag'.
+    Example:
+    >>> get_elf_arch_tag('output', 'arm-none-linux-gnueabi-',
+                         'bin/busybox', 'Tag_CPU_arch')
+    v5TEJ
+    >>>
+    """
+    cmd = ["host/bin/{}-readelf".format(prefix),
+           "-A", os.path.join("target", fpath)]
+    out = run_cmd_on_host(builddir, cmd)
+    regexp = re.compile(r"^  {}: (.*)$".format(tag))
+    for line in out.splitlines():
+        m = regexp.match(line)
+        if not m:
+            continue
+        return m.group(1)
+    return None
+
+
+def get_file_arch(builddir, prefix, fpath):
+    return get_elf_arch_tag(builddir, prefix, fpath, "Tag_CPU_arch")
+
+
+def get_elf_prog_interpreter(builddir, prefix, fpath):
+    """
+    Runs the cross readelf on 'fpath' to extract the program interpreter
+    name and returns it.
+    Example:
+    >>> get_elf_prog_interpreter('br-tests/TestExternalToolchainLinaroArm',
+                                 'arm-linux-gnueabihf',
+                                 'bin/busybox')
+    /lib/ld-linux-armhf.so.3
+    >>>
+    """
+    cmd = ["host/bin/{}-readelf".format(prefix),
+           "-l", os.path.join("target", fpath)]
+    out = run_cmd_on_host(builddir, cmd)
+    regexp = re.compile(r"^ *\[Requesting program interpreter: (.*)\]$")
+    for line in out.splitlines():
+        m = regexp.match(line)
+        if not m:
+            continue
+        return m.group(1)
+    return None
+
+
+def img_round_power2(img):
+    """
+    Rounds up the size of an image file to the next power of 2
+    """
+    sz = os.stat(img).st_size
+    pow2 = 1
+    while pow2 < sz:
+        pow2 = pow2 << 1
+    with open(img, 'ab') as f:
+        f.truncate(pow2)

support/testing/infra/basetest.py

@@ -0,0 +1,92 @@
+import unittest
+import os
+import datetime
+
+from infra.builder import Builder
+from infra.emulator import Emulator
+
+BASIC_TOOLCHAIN_CONFIG = \
+    """
+    BR2_arm=y
+    BR2_TOOLCHAIN_EXTERNAL=y
+    BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y
+    BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y
+    BR2_TOOLCHAIN_EXTERNAL_URL="https://toolchains.bootlin.com/downloads/releases/toolchains/armv5-eabi/tarballs/armv5-eabi--uclibc--bleeding-edge-2018.11-1.tar.bz2"
+    BR2_TOOLCHAIN_EXTERNAL_GCC_8=y
+    BR2_TOOLCHAIN_EXTERNAL_HEADERS_4_14=y
+    BR2_TOOLCHAIN_EXTERNAL_LOCALE=y
+    BR2_TOOLCHAIN_HAS_THREADS_DEBUG=y
+    BR2_TOOLCHAIN_EXTERNAL_CXX=y
+    """
+
+MINIMAL_CONFIG = \
+    """
+    BR2_INIT_NONE=y
+    BR2_SYSTEM_BIN_SH_NONE=y
+    # BR2_PACKAGE_BUSYBOX is not set
+    # BR2_TARGET_ROOTFS_TAR is not set
+    """
+
+
+class BRConfigTest(unittest.TestCase):
+    config = None
+    br2_external = list()
+    downloaddir = None
+    outputdir = None
+    logtofile = True
+    keepbuilds = False
+    jlevel = 0
+    timeout_multiplier = 1
+
+    def __init__(self, names):
+        super(BRConfigTest, self).__init__(names)
+        self.testname = self.__class__.__name__
+        self.builddir = self.outputdir and os.path.join(self.outputdir, self.testname)
+        self.config += '\nBR2_DL_DIR="{}"\n'.format(self.downloaddir)
+        self.config += "\nBR2_JLEVEL={}\n".format(self.jlevel)
+
+    def show_msg(self, msg):
+        print("{} {:40s} {}".format(datetime.datetime.now().strftime("%H:%M:%S"),
+                                    self.testname, msg))
+
+    def setUp(self):
+        self.show_msg("Starting")
+        self.b = Builder(self.config, self.builddir, self.logtofile)
+
+        if not self.keepbuilds:
+            self.b.delete()
+
+        if not self.b.is_finished():
+            self.b.configure(make_extra_opts=["BR2_EXTERNAL={}".format(":".join(self.br2_external))])
+
+    def tearDown(self):
+        self.show_msg("Cleaning up")
+        if self.b and not self.keepbuilds:
+            self.b.delete()
+
+
+class BRTest(BRConfigTest):
+    def __init__(self, names):
+        super(BRTest, self).__init__(names)
+        self.emulator = None
+
+    def setUp(self):
+        super(BRTest, self).setUp()
+        if not self.b.is_finished():
+            self.show_msg("Building")
+            self.b.build()
+            self.show_msg("Building done")
+
+        self.emulator = Emulator(self.builddir, self.downloaddir,
+                                 self.logtofile, self.timeout_multiplier)
+
+    def tearDown(self):
+        if self.emulator:
+            self.emulator.stop()
+        super(BRTest, self).tearDown()
+
+    # Run the given 'cmd' with a 'timeout' on the target and
+    # assert that the command succeeded
+    def assertRunOk(self, cmd, timeout=-1):
+        _, exit_code = self.emulator.run(cmd, timeout)
+        self.assertEqual(exit_code, 0)

support/testing/infra/builder.py

@@ -0,0 +1,107 @@
+import os
+import shutil
+import subprocess
+
+import infra
+
+
+class Builder(object):
+    def __init__(self, config, builddir, logtofile):
+        self.config = '\n'.join([line.lstrip() for line in
+                                 config.splitlines()]) + '\n'
+        self.builddir = builddir
+        self.logfile = infra.open_log_file(builddir, "build", logtofile)
+
+    def is_defconfig_valid(self, configfile, defconfig):
+        """Check if the .config is contains all lines present in the defconfig."""
+        with open(configfile) as configf:
+            configlines = configf.readlines()
+
+        defconfiglines = defconfig.split("\n")
+
+        # Check that all the defconfig lines are still present
+        for defconfigline in defconfiglines:
+            if defconfigline + "\n" not in configlines:
+                self.logfile.write("WARN: defconfig can't be used\n")
+                self.logfile.write("      Missing: %s\n" % defconfigline.strip())
+                self.logfile.flush()
+                return False
+
+        return True
+
+    def configure(self, make_extra_opts=[], make_extra_env={}):
+        """Configure the build.
+
+        make_extra_opts: a list of arguments to be passed to the make
+        command.
+        e.g. make_extra_opts=["BR2_EXTERNAL=/path"]
+
+        make_extra_env: a dict of variables to be appended (or replaced)
+        in the environment that calls make.
+        e.g. make_extra_env={"BR2_DL_DIR": "/path"}
+        """
+        if not os.path.isdir(self.builddir):
+            os.makedirs(self.builddir)
+
+        config_file = os.path.join(self.builddir, ".config")
+        with open(config_file, "w+") as cf:
+            cf.write(self.config)
+        # dump the defconfig to the logfile for easy debugging
+        self.logfile.write("> start defconfig\n" + self.config +
+                           "> end defconfig\n")
+        self.logfile.flush()
+
+        env = {"PATH": os.environ["PATH"]}
+        env.update(make_extra_env)
+
+        cmd = ["make",
+               "O={}".format(self.builddir)]
+        cmd += make_extra_opts
+        cmd += ["olddefconfig"]
+
+        ret = subprocess.call(cmd, stdout=self.logfile, stderr=self.logfile,
+                              cwd=infra.basepath(), env=env)
+        if ret != 0:
+            raise SystemError("Cannot olddefconfig")
+
+        if not self.is_defconfig_valid(config_file, self.config):
+            raise SystemError("The defconfig is not valid")
+
+    def build(self, make_extra_opts=[], make_extra_env={}):
+        """Perform the build.
+
+        make_extra_opts: a list of arguments to be passed to the make
+        command. It can include a make target.
+        e.g. make_extra_opts=["foo-source"]
+
+        make_extra_env: a dict of variables to be appended (or replaced)
+        in the environment that calls make.
+        e.g. make_extra_env={"BR2_DL_DIR": "/path"}
+        """
+        env = {"PATH": os.environ["PATH"]}
+        if "http_proxy" in os.environ:
+            self.logfile.write("Using system proxy: " +
+                               os.environ["http_proxy"] + "\n")
+            env['http_proxy'] = os.environ["http_proxy"]
+            env['https_proxy'] = os.environ["http_proxy"]
+        env.update(make_extra_env)
+
+        cmd = ["make", "-C", self.builddir]
+        cmd += make_extra_opts
+
+        ret = subprocess.call(cmd, stdout=self.logfile, stderr=self.logfile,
+                              env=env)
+        if ret != 0:
+            raise SystemError("Build failed")
+
+        open(self.stamp_path(), 'a').close()
+
+    def stamp_path(self):
+        return os.path.join(self.builddir, "build-done")
+
+    def is_finished(self):
+        return os.path.exists(self.stamp_path())
+
+    def delete(self):
+        if os.path.exists(self.builddir):
+            shutil.rmtree(self.builddir)

support/testing/infra/emulator.py

@@ -0,0 +1,128 @@
+import pexpect
+
+import infra
+
+
+class Emulator(object):
+
+    def __init__(self, builddir, downloaddir, logtofile, timeout_multiplier):
+        self.qemu = None
+        self.downloaddir = downloaddir
+        self.logfile = infra.open_log_file(builddir, "run", logtofile)
+        # We use elastic runners on the cloud to runs our tests. Those runners
+        # can take a long time to run the emulator. Use a timeout multiplier
+        # when running the tests to avoid sporadic failures.
+        self.timeout_multiplier = timeout_multiplier
+
+    # Start Qemu to boot the system
+    #
+    # arch: Qemu architecture to use
+    #
+    # kernel: path to the kernel image, or the special string
+    # 'builtin'. 'builtin' means a pre-built kernel image will be
+    # downloaded from ARTEFACTS_URL and suitable options are
+    # automatically passed to qemu and added to the kernel cmdline. So
+    # far only armv5, armv7 and i386 builtin kernels are available.
+    # If None, then no kernel is used, and we assume a bootable device
+    # will be specified.
+    #
+    # kernel_cmdline: array of kernel arguments to pass to Qemu -append option
+    #
+    # options: array of command line options to pass to Qemu
+    #
+    def boot(self, arch, kernel=None, kernel_cmdline=None, options=None):
+        if arch in ["armv7", "armv5"]:
+            qemu_arch = "arm"
+        else:
+            qemu_arch = arch
+
+        qemu_cmd = ["qemu-system-{}".format(qemu_arch),
+                    "-serial", "stdio",
+                    "-display", "none",
+                    "-m", "256"]
+
+        if options:
+            qemu_cmd += options
+
+        if kernel_cmdline is None:
+            kernel_cmdline = []
+
+        if kernel:
+            if kernel == "builtin":
+                if arch in ["armv7", "armv5"]:
+                    kernel_cmdline.append("console=ttyAMA0")
+
+                if arch == "armv7":
+                    kernel = infra.download(self.downloaddir,
+                                            "kernel-vexpress-5.10.7")
+                    dtb = infra.download(self.downloaddir,
+                                         "vexpress-v2p-ca9-5.10.7.dtb")
+                    qemu_cmd += ["-dtb", dtb]
+                    qemu_cmd += ["-M", "vexpress-a9"]
+                elif arch == "armv5":
+                    kernel = infra.download(self.downloaddir,
+                                            "kernel-versatile-5.10.7")
+                    dtb = infra.download(self.downloaddir,
+                                         "versatile-pb-5.10.7.dtb")
+                    qemu_cmd += ["-dtb", dtb]
+                    qemu_cmd += ["-M", "versatilepb"]
+                    qemu_cmd += ["-device", "virtio-rng-pci"]
+
+            qemu_cmd += ["-kernel", kernel]
+
+        if kernel_cmdline:
+            qemu_cmd += ["-append", " ".join(kernel_cmdline)]
+
+        self.logfile.write("> starting qemu with '%s'\n" % " ".join(qemu_cmd))
+        self.qemu = pexpect.spawn(qemu_cmd[0], qemu_cmd[1:],
+                                  timeout=5 * self.timeout_multiplier,
+                                  encoding='utf-8',
+                                  codec_errors='replace',
+                                  env={"QEMU_AUDIO_DRV": "none"})
+        # We want only stdout into the log to avoid double echo
+        self.qemu.logfile_read = self.logfile
+
+    # Wait for the login prompt to appear, and then login as root with
+    # the provided password, or no password if not specified.
+    def login(self, password=None):
+        # The login prompt can take some time to appear when running multiple
+        # instances in parallel, so set the timeout to a large value
+        index = self.qemu.expect(["buildroot login:", pexpect.TIMEOUT],
+                                 timeout=60 * self.timeout_multiplier)
+        if index != 0:
+            self.logfile.write("==> System does not boot")
+            raise SystemError("System does not boot")
+
+        self.qemu.sendline("root")
+        if password:
+            self.qemu.expect("Password:")
+            self.qemu.sendline(password)
+        index = self.qemu.expect(["# ", pexpect.TIMEOUT])
+        if index != 0:
+            raise SystemError("Cannot login")
+        self.run("dmesg -n 1")
+        # Prevent the shell from wrapping the commands at 80 columns.
+        self.run("stty columns 29999")
+
+    # Run the given 'cmd' with a 'timeout' on the target
+    # return a tuple (output, exit_code)
+    def run(self, cmd, timeout=-1):
+        self.qemu.sendline(cmd)
+        if timeout != -1:
+            timeout *= self.timeout_multiplier
+        self.qemu.expect("# ", timeout=timeout)
+        # Remove double carriage return from qemu stdout so str.splitlines()
+        # works as expected.
+        output = self.qemu.before.replace("\r\r", "\r").splitlines()[1:]
+
+        self.qemu.sendline("echo $?")
+        self.qemu.expect("# ")
+        exit_code = self.qemu.before.splitlines()[2]
+        exit_code = int(exit_code)
+
+        return output, exit_code
+
+    def stop(self):
+        if self.qemu is None:
+            return
+        self.qemu.terminate(force=True)