initial buildroot for linux 5.15
This commit is contained in:
Huan.Feng
Executable
+165
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env bash
|
||||
# A little script I whipped up to make it easy to
|
||||
# patch source trees and have sane error handling
|
||||
# -Erik
|
||||
#
|
||||
# (c) 2002 Erik Andersen <andersen@codepoet.org>
|
||||
#
|
||||
# Parameters:
|
||||
# - "-s", optional. Silent operation, don't print anything if there
|
||||
# isn't any error.
|
||||
# - the build directory, optional, default value is '.'. The place where are
|
||||
# the package sources.
|
||||
# - the patch directory, optional, default '../kernel-patches'. The place
|
||||
# where are the scripts you want to apply.
|
||||
# - other parameters are the patch name patterns, optional, default value is
|
||||
# '*'. Pattern(s) describing the patch names you want to apply.
|
||||
#
|
||||
# The script will look recursively for patches from the patch directory. If a
|
||||
# file named 'series' exists then the patches mentioned in it will be applied
|
||||
# as plain patches, regardless of their file name. If no 'series' file exists,
|
||||
# the script will look for file names matching pattern(s). If the name
|
||||
# ends with '.tar.*', '.tbz2' or '.tgz', the file is considered as an archive
|
||||
# and will be uncompressed into a directory named
|
||||
# '.patches-name_of_the_archive-unpacked'. It's the turn of this directory to
|
||||
# be scanned with '*' as pattern. Remember that scanning is recursive. Other
|
||||
# files than series file and archives are considered as a patch.
|
||||
#
|
||||
# Once a patch is found, the script will try to apply it. If its name doesn't
|
||||
# end with '.gz', '.bz', '.bz2', '.xz', '.zip', '.Z', '.diff*' or '.patch*',
|
||||
# it will be skipped. If necessary, the patch will be uncompressed before being
|
||||
# applied. The list of the patches applied is stored in '.applied_patches_list'
|
||||
# file in the build directory.
|
||||
|
||||
set -e
|
||||
|
||||
silent=
|
||||
if [ "$1" = "-s" ] ; then
|
||||
# add option to be used by the patch tool
|
||||
silent=-s
|
||||
shift
|
||||
fi
|
||||
|
||||
# Set directories from arguments, or use defaults.
|
||||
builddir=${1-.}
|
||||
patchdir=${2-../kernel-patches}
|
||||
shift 2
|
||||
patchpattern=${@-*}
|
||||
|
||||
# use a well defined sorting order
|
||||
export LC_COLLATE=C
|
||||
|
||||
if [ ! -d "${builddir}" ] ; then
|
||||
echo "Aborting. '${builddir}' is not a directory."
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -d "${patchdir}" ] ; then
|
||||
echo "Aborting. '${patchdir}' is not a directory."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remove any rejects present BEFORE patching - Because if there are
|
||||
# any, even if patches are well applied, at the end it will complain
|
||||
# about rejects in builddir.
|
||||
find ${builddir}/ '(' -name '*.rej' -o -name '.*.rej' ')' -print0 | \
|
||||
xargs -0 -r rm -f
|
||||
|
||||
function apply_patch {
|
||||
path="${1%%/}"
|
||||
patch="${2}"
|
||||
case "${path}" in
|
||||
/*) ;;
|
||||
*) path="$PWD/${path}";;
|
||||
esac
|
||||
if [ "$3" ]; then
|
||||
type="series"; uncomp="cat"
|
||||
else
|
||||
case "$patch" in
|
||||
*.gz)
|
||||
type="gzip"; uncomp="gunzip -dc"; ;;
|
||||
*.bz)
|
||||
type="bzip"; uncomp="bunzip -dc"; ;;
|
||||
*.bz2)
|
||||
type="bzip2"; uncomp="bunzip2 -dc"; ;;
|
||||
*.xz)
|
||||
type="xz"; uncomp="unxz -dc"; ;;
|
||||
*.zip)
|
||||
type="zip"; uncomp="unzip -d"; ;;
|
||||
*.Z)
|
||||
type="compress"; uncomp="uncompress -c"; ;;
|
||||
*.diff*)
|
||||
type="diff"; uncomp="cat"; ;;
|
||||
*.patch*)
|
||||
type="patch"; uncomp="cat"; ;;
|
||||
*)
|
||||
echo "Unsupported file type for ${path}/${patch}, skipping";
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
if [ -z "$silent" ] ; then
|
||||
echo ""
|
||||
echo "Applying $patch using ${type}: "
|
||||
fi
|
||||
if [ ! -e "${path}/$patch" ] ; then
|
||||
echo "Error: missing patch file ${path}/$patch"
|
||||
exit 1
|
||||
fi
|
||||
existing="$(grep -E "/${patch}\$" ${builddir}/.applied_patches_list || true)"
|
||||
if [ -n "${existing}" ]; then
|
||||
echo "Error: duplicate filename '${patch}'"
|
||||
echo "Conflicting files are:"
|
||||
echo " already applied: ${existing}"
|
||||
echo " to be applied : ${path}/${patch}"
|
||||
exit 1
|
||||
fi
|
||||
echo "${path}/${patch}" >> ${builddir}/.applied_patches_list
|
||||
${uncomp} "${path}/$patch" | patch -g0 -p1 -E --no-backup-if-mismatch -d "${builddir}" -t -N $silent
|
||||
if [ $? != 0 ] ; then
|
||||
echo "Patch failed! Please fix ${patch}!"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function scan_patchdir {
|
||||
local path=$1
|
||||
shift 1
|
||||
patches=${@-*}
|
||||
|
||||
# If there is a series file, use it instead of using ls sort order
|
||||
# to apply patches. Skip line starting with a dash.
|
||||
if [ -e "${path}/series" ] ; then
|
||||
# The format of a series file accepts a second field that is
|
||||
# used to specify the number of directory components to strip
|
||||
# when applying the patch, in the form -pN (N an integer >= 0)
|
||||
# We assume this field to always be -p1 whether it is present
|
||||
# or missing.
|
||||
series_patches="`grep -Ev "^#" ${path}/series | cut -d ' ' -f1 2> /dev/null`"
|
||||
for i in $series_patches; do
|
||||
apply_patch "$path" "$i" series
|
||||
done
|
||||
else
|
||||
for i in `cd $path; ls -d $patches 2> /dev/null` ; do
|
||||
if [ -d "${path}/$i" ] ; then
|
||||
scan_patchdir "${path}/$i"
|
||||
elif echo "$i" | grep -q -E "\.tar(\..*)?$|\.tbz2?$|\.tgz$" ; then
|
||||
unpackedarchivedir="$builddir/.patches-$(basename $i)-unpacked"
|
||||
rm -rf "$unpackedarchivedir" 2> /dev/null
|
||||
mkdir "$unpackedarchivedir"
|
||||
tar -C "$unpackedarchivedir" -xaf "${path}/$i"
|
||||
scan_patchdir "$unpackedarchivedir"
|
||||
else
|
||||
apply_patch "$path" "$i"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
touch ${builddir}/.applied_patches_list
|
||||
scan_patchdir "$patchdir" "$patchpattern"
|
||||
|
||||
# Check for rejects...
|
||||
if [ "`find $builddir/ '(' -name '*.rej' -o -name '.*.rej' ')' -print`" ] ; then
|
||||
echo "Aborting. Reject files found."
|
||||
exit 1
|
||||
fi
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This script expect to run from the Buildroot top directory.
|
||||
|
||||
import os
|
||||
import pexpect
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
if not (len(sys.argv) == 2):
|
||||
print("Error: incorrect number of arguments")
|
||||
print("""Usage: boot-qemu-image.py <qemu_arch_defconfig>""")
|
||||
sys.exit(1)
|
||||
|
||||
# Ignore non Qemu defconfig
|
||||
if not sys.argv[1].startswith('qemu_'):
|
||||
sys.exit(0)
|
||||
|
||||
if not os.path.exists('output/images/start-qemu.sh'):
|
||||
print('qemu-start.sh is missing, cannot test.')
|
||||
sys.exit(0)
|
||||
|
||||
qemu_start = os.path.join(os.getcwd(), 'output/images/start-qemu.sh')
|
||||
|
||||
child = pexpect.spawn(qemu_start, ['serial-only'],
|
||||
timeout=5, encoding='utf-8',
|
||||
env={"QEMU_AUDIO_DRV": "none"})
|
||||
|
||||
# We want only stdout into the log to avoid double echo
|
||||
child.logfile = sys.stdout
|
||||
|
||||
# Let the spawn actually try to fork+exec to the wrapper, and then
|
||||
# let the wrapper exec the qemu process.
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
child.expect(["buildroot login:"], timeout=60)
|
||||
except pexpect.EOF as e:
|
||||
# Some emulations require a fork of qemu-system, which may be
|
||||
# missing on the system, and is not provided by Buildroot.
|
||||
# In this case, spawn above will succeed at starting the wrapper
|
||||
# start-qemu.sh, but that one will fail (exit with 127) in such
|
||||
# a situation.
|
||||
exit = [int(line.split(' ')[1])
|
||||
for line in e.value.splitlines()
|
||||
if line.startswith('exitstatus: ')]
|
||||
if len(exit) and exit[0] == 127:
|
||||
print('qemu-start.sh could not find the qemu binary')
|
||||
sys.exit(0)
|
||||
print("Connection problem, exiting.")
|
||||
sys.exit(1)
|
||||
except pexpect.TIMEOUT:
|
||||
print("System did not boot in time, exiting.")
|
||||
sys.exit(1)
|
||||
|
||||
child.sendline("root\r")
|
||||
|
||||
try:
|
||||
child.expect(["# "], timeout=60)
|
||||
except pexpect.EOF:
|
||||
print("Cannot connect to shell")
|
||||
sys.exit(1)
|
||||
except pexpect.TIMEOUT:
|
||||
print("Timeout while waiting for shell")
|
||||
sys.exit(1)
|
||||
|
||||
child.sendline("poweroff\r")
|
||||
|
||||
try:
|
||||
child.expect(["System halted"], timeout=60)
|
||||
child.expect(pexpect.EOF)
|
||||
except pexpect.EOF:
|
||||
pass
|
||||
except pexpect.TIMEOUT:
|
||||
# Qemu may not exit properly after "System halted", ignore.
|
||||
print("Cannot halt machine")
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+253
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
# This script must be able to run with bash-3.1, so it can't use
|
||||
# associative arrays. Instead, it emulates them using 'eval'. It
|
||||
# can however use indexed arrays, supported since at least bash-3.0.
|
||||
|
||||
# The names of the br2-external trees, once validated.
|
||||
declare -a BR2_EXT_NAMES
|
||||
|
||||
# URL to manual for help in converting old br2-external trees.
|
||||
# Escape '#' so that make does not consider it a comment.
|
||||
MANUAL_URL='https://buildroot.org/manual.html\#br2-external-converting'
|
||||
|
||||
main() {
|
||||
local OPT OPTARG
|
||||
local br2_ext outputdir
|
||||
|
||||
while getopts :d: OPT; do
|
||||
case "${OPT}" in
|
||||
d) outputdir="${OPTARG}";;
|
||||
:) error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
|
||||
\?) error "unknown option '%s'\n" "${OPTARG}";;
|
||||
esac
|
||||
done
|
||||
# Forget options; keep only positional args
|
||||
shift $((OPTIND-1))
|
||||
|
||||
if [ -z "${outputdir}" ]; then
|
||||
error "no output directory specified (-d)\n"
|
||||
fi
|
||||
|
||||
# Trap any unexpected error to generate a meaningful error message
|
||||
trap "error 'unexpected error while generating ${ofile}\n'" ERR
|
||||
|
||||
mkdir -p "${outputdir}"
|
||||
do_validate "${outputdir}" ${@//:/ }
|
||||
do_mk "${outputdir}"
|
||||
do_kconfig "${outputdir}"
|
||||
}
|
||||
|
||||
# Validates the br2-external trees passed as arguments. Makes each of
|
||||
# them canonical and store them in the global arrays BR2_EXT_NAMES
|
||||
# and BR2_EXT_PATHS.
|
||||
#
|
||||
# Note: since this script is always first called from Makefile context
|
||||
# to generate the Makefile fragment before it is called to generate the
|
||||
# Kconfig snippet, we're sure that any error in do_validate will be
|
||||
# interpreted in Makefile context. Going up to generating the Kconfig
|
||||
# snippet means that there were no error.
|
||||
#
|
||||
do_validate() {
|
||||
local outputdir="${1}"
|
||||
local br2_ext
|
||||
shift
|
||||
|
||||
if [ ${#} -eq 0 ]; then
|
||||
# No br2-external tree is valid
|
||||
return
|
||||
fi
|
||||
|
||||
for br2_ext in "${@}"; do
|
||||
do_validate_one "${br2_ext}"
|
||||
done >"${outputdir}/.br2-external.mk"
|
||||
}
|
||||
|
||||
do_validate_one() {
|
||||
local br2_ext="${1}"
|
||||
local br2_name br2_desc n d
|
||||
|
||||
if [ ! -d "${br2_ext}" ]; then
|
||||
error "'%s': no such file or directory\n" "${br2_ext}"
|
||||
fi
|
||||
if [ ! -r "${br2_ext}" -o ! -x "${br2_ext}" ]; then
|
||||
error "'%s': permission denied\n" "${br2_ext}"
|
||||
fi
|
||||
if [ ! -f "${br2_ext}/external.desc" ]; then
|
||||
error "'%s': does not have an 'external.desc'. See %s\n" \
|
||||
"${br2_ext}" "${MANUAL_URL}"
|
||||
fi
|
||||
br2_name="$(sed -r -e '/^name: +(.*)$/!d; s//\1/' "${br2_ext}/external.desc")"
|
||||
if [ -z "${br2_name}" ]; then
|
||||
error "'%s/external.desc': does not define the name\n" "${br2_ext}"
|
||||
fi
|
||||
# Only ASCII chars in [A-Za-z0-9_] are permitted
|
||||
n="$(sed -r -e 's/[A-Za-z0-9_]//g' <<<"${br2_name}" )"
|
||||
if [ -n "${n}" ]; then
|
||||
# Escape '$' so that it gets printed
|
||||
error "'%s': name '%s' contains invalid chars: '%s'\n" \
|
||||
"${br2_ext}" "${br2_name//\$/\$\$}" "${n//\$/\$\$}"
|
||||
fi
|
||||
eval d="\"\${BR2_EXT_PATHS_${br2_name}}\""
|
||||
if [ -n "${d}" ]; then
|
||||
error "'%s': name '%s' is already used in '%s'\n" \
|
||||
"${br2_ext}" "${br2_name}" "${d}"
|
||||
fi
|
||||
br2_desc="$(sed -r -e '/^desc: +(.*)$/!d; s//\1/' "${br2_ext}/external.desc")"
|
||||
if [ ! -f "${br2_ext}/external.mk" ]; then
|
||||
error "'%s/external.mk': no such file or directory\n" "${br2_ext}"
|
||||
fi
|
||||
if [ ! -f "${br2_ext}/Config.in" ]; then
|
||||
error "'%s/Config.in': no such file or directory\n" "${br2_ext}"
|
||||
fi
|
||||
|
||||
# Register this br2-external tree, use an absolute canonical path
|
||||
br2_ext="$( cd "${br2_ext}"; pwd )"
|
||||
BR2_EXT_NAMES+=( "${br2_name}" )
|
||||
eval BR2_EXT_PATHS_${br2_name}="\"\${br2_ext}\""
|
||||
eval BR2_EXT_DESCS_${br2_name}="\"\${br2_desc:-\${br2_name}}\""
|
||||
}
|
||||
|
||||
# Generate the .mk snippet that defines makefile variables
|
||||
# for the br2-external tree
|
||||
do_mk() {
|
||||
local outputdir="${1}"
|
||||
local br2_name br2_desc br2_ext
|
||||
|
||||
{
|
||||
printf '#\n# Automatically generated file; DO NOT EDIT.\n#\n'
|
||||
printf '\n'
|
||||
|
||||
printf 'BR2_EXTERNAL ?='
|
||||
for br2_name in "${BR2_EXT_NAMES[@]}"; do
|
||||
eval br2_ext="\"\${BR2_EXT_PATHS_${br2_name}}\""
|
||||
printf ' %s' "${br2_ext}"
|
||||
done
|
||||
printf '\n'
|
||||
|
||||
printf 'BR2_EXTERNAL_NAMES = \n'
|
||||
printf 'BR2_EXTERNAL_DIRS = \n'
|
||||
printf 'BR2_EXTERNAL_MKS = \n'
|
||||
|
||||
if [ ${#BR2_EXT_NAMES[@]} -eq 0 ]; then
|
||||
printf '\n'
|
||||
printf '# No br2-external tree defined.\n'
|
||||
return
|
||||
fi
|
||||
|
||||
for br2_name in "${BR2_EXT_NAMES[@]}"; do
|
||||
eval br2_desc="\"\${BR2_EXT_DESCS_${br2_name}}\""
|
||||
eval br2_ext="\"\${BR2_EXT_PATHS_${br2_name}}\""
|
||||
printf '\n'
|
||||
printf 'BR2_EXTERNAL_NAMES += %s\n' "${br2_name}"
|
||||
printf 'BR2_EXTERNAL_DIRS += %s\n' "${br2_ext}"
|
||||
printf 'BR2_EXTERNAL_MKS += %s/external.mk\n' "${br2_ext}"
|
||||
printf 'export BR2_EXTERNAL_%s_PATH = %s\n' "${br2_name}" "${br2_ext}"
|
||||
printf 'export BR2_EXTERNAL_%s_DESC = %s\n' "${br2_name}" "${br2_desc}"
|
||||
done
|
||||
} >"${outputdir}/.br2-external.mk"
|
||||
}
|
||||
|
||||
# Generate the kconfig snippets for the br2-external tree.
|
||||
do_kconfig() {
|
||||
local outputdir="${1}"
|
||||
local br2_name br2_desc br2_ext br2
|
||||
local -a items
|
||||
|
||||
items=(
|
||||
paths
|
||||
menus
|
||||
toolchains
|
||||
jpeg
|
||||
openssl
|
||||
skeleton
|
||||
init
|
||||
)
|
||||
|
||||
for br2 in "${items[@]}"; do
|
||||
{
|
||||
printf '#\n# Automatically generated file; DO NOT EDIT.\n#\n'
|
||||
printf '\n'
|
||||
if [ ${#BR2_EXT_NAMES[@]} -eq 0 ]; then
|
||||
printf '# No br2-external tree defined.\n'
|
||||
fi
|
||||
} >"${outputdir}/.br2-external.in.${br2}"
|
||||
done
|
||||
if [ ${#BR2_EXT_NAMES[@]} -eq 0 ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
printf 'menu "External options"\n\n' >>"${outputdir}/.br2-external.in.menus"
|
||||
|
||||
for br2_name in "${BR2_EXT_NAMES[@]}"; do
|
||||
eval br2_desc="\"\${BR2_EXT_DESCS_${br2_name}}\""
|
||||
eval br2_ext="\"\${BR2_EXT_PATHS_${br2_name}}\""
|
||||
|
||||
{
|
||||
printf 'config BR2_EXTERNAL_%s_PATH\n' "${br2_name}"
|
||||
printf '\tstring\n'
|
||||
printf '\tdefault "%s"\n' "${br2_ext}"
|
||||
printf '\n'
|
||||
} >>"${outputdir}/.br2-external.in.paths"
|
||||
|
||||
{
|
||||
if [ ${#BR2_EXT_NAMES[@]} -gt 1 ]; then
|
||||
printf 'menu "%s"\n' "${br2_desc}"
|
||||
fi
|
||||
printf 'comment "%s (in %s)"\n' "${br2_desc}" "${br2_ext}"
|
||||
printf 'source "%s/Config.in"\n' "${br2_ext}"
|
||||
if [ ${#BR2_EXT_NAMES[@]} -gt 1 ]; then
|
||||
printf 'endmenu # %s\n' "${br2_name}"
|
||||
fi
|
||||
printf '\n'
|
||||
} >>"${outputdir}/.br2-external.in.menus"
|
||||
|
||||
if [ -f "${br2_ext}/provides/toolchains.in" ]; then
|
||||
printf 'comment "Toolchains from: %s"\n' "${br2_desc}"
|
||||
printf 'source "%s/provides/toolchains.in"\n' "${br2_ext}"
|
||||
printf '\n'
|
||||
else
|
||||
printf '# No toolchain from: %s\n\n' "${br2_desc}"
|
||||
fi >>"${outputdir}/.br2-external.in.toolchains"
|
||||
|
||||
if [ -f "${br2_ext}/provides/jpeg.in" ]; then
|
||||
printf 'comment "jpeg from: %s"\n' "${br2_desc}"
|
||||
printf 'source "%s/provides/jpeg.in"\n' "${br2_ext}"
|
||||
printf '\n'
|
||||
else
|
||||
printf '# No jpeg from: %s\n\n' "${br2_desc}"
|
||||
fi >>"${outputdir}/.br2-external.in.jpeg"
|
||||
|
||||
if [ -f "${br2_ext}/provides/openssl.in" ]; then
|
||||
printf 'comment "openssl from: %s"\n' "${br2_desc}"
|
||||
printf 'source "%s/provides/openssl.in"\n' "${br2_ext}"
|
||||
printf '\n'
|
||||
else
|
||||
printf '# No openssl from: %s\n\n' "${br2_desc}"
|
||||
fi >>"${outputdir}/.br2-external.in.openssl"
|
||||
|
||||
if [ -f "${br2_ext}/provides/skeleton.in" ]; then
|
||||
printf 'comment "skeleton from: %s"\n' "${br2_desc}"
|
||||
printf 'source "%s/provides/skeleton.in"\n' "${br2_ext}"
|
||||
printf '\n'
|
||||
else
|
||||
printf '# No skeleton from: %s\n\n' "${br2_desc}"
|
||||
fi >>"${outputdir}/.br2-external.in.skeleton"
|
||||
|
||||
if [ -f "${br2_ext}/provides/init.in" ]; then
|
||||
printf 'comment "init from: %s"\n' "${br2_desc}"
|
||||
printf 'source "%s/provides/init.in"\n' "${br2_ext}"
|
||||
printf '\n'
|
||||
else
|
||||
printf '# No init from: %s\n\n' "${br2_desc}"
|
||||
fi >>"${outputdir}/.br2-external.in.init"
|
||||
done
|
||||
|
||||
printf 'endmenu\n' >>"${outputdir}/.br2-external.in.menus"
|
||||
}
|
||||
|
||||
error() { local fmt="${1}"; shift; printf "BR2_EXTERNAL_ERROR = ${fmt}" "${@}"; exit 1; }
|
||||
|
||||
my_name="${0##*/}"
|
||||
main "${@}"
|
||||
@@ -0,0 +1,50 @@
|
||||
# Copyright (C) 2010-2013 Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
||||
# Copyright (C) 2019 Yann E. MORIN <yann.morin.1998@free.fr>
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
# This function returns a tuple of four dictionaries, all using package
|
||||
# names as keys:
|
||||
# - a dictionary which values are the lists of packages that are the
|
||||
# dependencies of the package used as key;
|
||||
# - a dictionary which values are the lists of packages that are the
|
||||
# reverse dependencies of the package used as key;
|
||||
# - a dictionary which values are the type of the package used as key;
|
||||
# - a dictionary which values are the version of the package used as key,
|
||||
# 'virtual' for a virtual package, or the empty string for a rootfs.
|
||||
def get_dependency_tree():
|
||||
logging.info("Getting dependency tree...")
|
||||
|
||||
deps = {}
|
||||
rdeps = defaultdict(list)
|
||||
types = {}
|
||||
versions = {}
|
||||
|
||||
# Special case for the 'all' top-level fake package
|
||||
deps['all'] = []
|
||||
types['all'] = 'target'
|
||||
versions['all'] = ''
|
||||
|
||||
cmd = ["make", "-s", "--no-print-directory", "show-info"]
|
||||
with open(os.devnull, 'wb') as devnull:
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull,
|
||||
universal_newlines=True)
|
||||
pkg_list = json.loads(p.communicate()[0])
|
||||
|
||||
for pkg in pkg_list:
|
||||
deps['all'].append(pkg)
|
||||
types[pkg] = pkg_list[pkg]["type"]
|
||||
deps[pkg] = pkg_list[pkg].get("dependencies", [])
|
||||
for p in deps[pkg]:
|
||||
rdeps[p].append(pkg)
|
||||
versions[pkg] = \
|
||||
None if pkg_list[pkg]["type"] == "rootfs" \
|
||||
else "virtual" if pkg_list[pkg]["virtual"] \
|
||||
else pkg_list[pkg]["version"]
|
||||
|
||||
return (deps, rdeps, types, versions)
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# List of hardcoded paths that should be ignored, as they may
|
||||
# contain binaries for an architecture different from the
|
||||
# architecture of the target.
|
||||
declare -a IGNORES=(
|
||||
# Skip firmware files, they could be ELF files for other
|
||||
# architectures
|
||||
"/lib/firmware"
|
||||
"/usr/lib/firmware"
|
||||
|
||||
# Skip kernel modules
|
||||
# When building a 32-bit userland on 64-bit architectures, the kernel
|
||||
# and its modules may still be 64-bit. To keep the basic
|
||||
# check-bin-arch logic simple, just skip this directory.
|
||||
"/lib/modules"
|
||||
"/usr/lib/modules"
|
||||
|
||||
# Skip files in /usr/share, several packages (qemu,
|
||||
# pru-software-support) legitimately install ELF binaries that
|
||||
# are not for the target architecture
|
||||
"/usr/share"
|
||||
|
||||
# Skip files in {/usr,}/lib/grub, since it is possible to have
|
||||
# it for a different architecture (e.g. i386 grub on x86_64).
|
||||
"/lib/grub"
|
||||
"/usr/lib/grub"
|
||||
|
||||
# Guile modules are ELF files, with a "None" machine
|
||||
"/usr/lib/guile"
|
||||
)
|
||||
|
||||
while getopts p:l:r:a:i: OPT ; do
|
||||
case "${OPT}" in
|
||||
p) package="${OPTARG}";;
|
||||
l) pkg_list="${OPTARG}";;
|
||||
r) readelf="${OPTARG}";;
|
||||
a) arch_name="${OPTARG}";;
|
||||
i)
|
||||
# Ensure we do have single '/' as separators,
|
||||
# and that we have a leading and a trailing one.
|
||||
pattern="$(sed -r -e 's:/+:/:g; s:^/*:/:; s:/*$:/:;' <<<"${OPTARG}")"
|
||||
IGNORES+=("${pattern}")
|
||||
;;
|
||||
:) error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
|
||||
\?) error "unknown option '%s'\n" "${OPTARG}";;
|
||||
esac
|
||||
done
|
||||
|
||||
if test -z "${package}" -o -z "${pkg_list}" -o -z "${readelf}" -o -z "${arch_name}" ; then
|
||||
echo "Usage: $0 -p <pkg> -l <pkg-file-list> -r <readelf> -a <arch name> [-i PATH ...]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exitcode=0
|
||||
|
||||
# Only split on new lines, for filenames-with-spaces
|
||||
IFS="
|
||||
"
|
||||
|
||||
while read f; do
|
||||
for ignore in "${IGNORES[@]}"; do
|
||||
if [[ "${f}" =~ ^"${ignore}" ]]; then
|
||||
continue 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Skip symlinks. Some symlinks may have absolute paths as
|
||||
# target, pointing to host binaries while we're building.
|
||||
if [[ -L "${TARGET_DIR}/${f}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Get architecture using readelf. We pipe through 'head -1' so
|
||||
# that when the file is a static library (.a), we only take
|
||||
# into account the architecture of the first object file.
|
||||
arch=$(LC_ALL=C ${readelf} -h "${TARGET_DIR}/${f}" 2>&1 | \
|
||||
sed -r -e '/^ Machine: +(.+)/!d; s//\1/;' | head -1)
|
||||
|
||||
# If no architecture found, assume it was not an ELF file
|
||||
if test "${arch}" = "" ; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Architecture is correct
|
||||
if test "${arch}" = "${arch_name}" ; then
|
||||
continue
|
||||
fi
|
||||
|
||||
printf 'ERROR: architecture for "%s" is "%s", should be "%s"\n' \
|
||||
"${f}" "${arch}" "${arch_name}"
|
||||
|
||||
exitcode=1
|
||||
done < <( sed -r -e "/^${package},\.(.+)$/!d; s//\1/;" ${pkg_list} )
|
||||
|
||||
exit ${exitcode}
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This scripts check that all lines present in the defconfig are
|
||||
# still present in the .config
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
if not (len(sys.argv) == 3):
|
||||
print("Error: incorrect number of arguments")
|
||||
print("""Usage: check-dotconfig <configfile> <defconfig>""")
|
||||
sys.exit(1)
|
||||
|
||||
configfile = sys.argv[1]
|
||||
defconfig = sys.argv[2]
|
||||
|
||||
# strip() to get rid of trailing \n
|
||||
with open(configfile) as configf:
|
||||
configlines = [line.strip() for line in configf.readlines()]
|
||||
|
||||
defconfiglines = []
|
||||
with open(defconfig) as defconfigf:
|
||||
# strip() to get rid of trailing \n
|
||||
for line in (line.strip() for line in defconfigf.readlines()):
|
||||
if line.startswith("BR2_"):
|
||||
defconfiglines.append(line)
|
||||
elif line.startswith('# BR2_') and line.endswith(' is not set'):
|
||||
defconfiglines.append(line)
|
||||
|
||||
# Check that all the defconfig lines are still present
|
||||
missing = [line for line in defconfiglines if line not in configlines]
|
||||
|
||||
if missing:
|
||||
print("WARN: defconfig {} can't be used:".format(defconfig))
|
||||
for m in missing:
|
||||
print(" Missing: {}".format(m))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This script scans $(HOST_DIR)/{bin,sbin} for all ELF files, and checks
|
||||
# they have an RPATH to $(HOST_DIR)/lib if they need libraries from
|
||||
# there.
|
||||
|
||||
# Override the user's locale so we are sure we can parse the output of
|
||||
# readelf(1) and file(1)
|
||||
export LC_ALL=C
|
||||
|
||||
main() {
|
||||
local pkg="${1}"
|
||||
local hostdir="${2}"
|
||||
local perpackagedir="${3}"
|
||||
local file ret
|
||||
|
||||
# Remove duplicate and trailing '/' for proper match
|
||||
hostdir="$( sed -r -e 's:/+:/:g; s:/$::;' <<<"${hostdir}" )"
|
||||
|
||||
ret=0
|
||||
while read file; do
|
||||
is_elf "${file}" || continue
|
||||
elf_needs_rpath "${file}" "${hostdir}" || continue
|
||||
check_elf_has_rpath "${file}" "${hostdir}" "${perpackagedir}" && continue
|
||||
if [ ${ret} -eq 0 ]; then
|
||||
ret=1
|
||||
printf "***\n"
|
||||
printf "*** ERROR: package %s installs executables without proper RPATH:\n" "${pkg}"
|
||||
fi
|
||||
printf "*** %s\n" "${file}"
|
||||
done < <( find "${hostdir}"/{bin,sbin} -type f 2>/dev/null )
|
||||
|
||||
return ${ret}
|
||||
}
|
||||
|
||||
is_elf() {
|
||||
local f="${1}"
|
||||
|
||||
readelf -l "${f}" 2>/dev/null \
|
||||
|grep -E 'Requesting program interpreter:' >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# This function tells whether a given ELF executable (first argument)
|
||||
# needs a RPATH pointing to the host library directory or not. It
|
||||
# needs such an RPATH if at least of the libraries used by the ELF
|
||||
# executable is available in the host library directory. This function
|
||||
# returns 0 when a RPATH is needed, 1 otherwise.
|
||||
#
|
||||
# With per-package directory support, ${hostdir} will point to the
|
||||
# current package per-package host directory, and this is where this
|
||||
# function will check if the libraries needed by the executable are
|
||||
# located (or not). In practice, the ELF executable RPATH may point to
|
||||
# another package per-package host directory, but that is fine because
|
||||
# if such an executable is within the current package per-package host
|
||||
# directory, its libraries will also have been copied into the current
|
||||
# package per-package host directory.
|
||||
elf_needs_rpath() {
|
||||
local file="${1}"
|
||||
local hostdir="${2}"
|
||||
local lib
|
||||
|
||||
while read lib; do
|
||||
[ -e "${hostdir}/lib/${lib}" ] && return 0
|
||||
done < <( readelf -d "${file}" \
|
||||
|sed -r -e '/^.* \(NEEDED\) .*Shared library: \[(.+)\]$/!d;' \
|
||||
-e 's//\1/;' \
|
||||
)
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# This function checks whether at least one of the RPATH of the given
|
||||
# ELF executable (first argument) properly points to the host library
|
||||
# directory (second argument), either through an absolute RPATH or a
|
||||
# relative RPATH. In the context of per-package directory support,
|
||||
# ${hostdir} (second argument) points to the current package host
|
||||
# directory. However, it is perfectly valid for an ELF binary to have
|
||||
# a RPATH pointing to another package per-package host directory,
|
||||
# which is why such RPATH is also accepted (the per-package directory
|
||||
# gets passed as third argument). Having a RPATH pointing to the host
|
||||
# directory will make sure the ELF executable will find at runtime the
|
||||
# shared libraries it depends on. This function returns 0 when a
|
||||
# proper RPATH was found, or 1 otherwise.
|
||||
check_elf_has_rpath() {
|
||||
local file="${1}"
|
||||
local hostdir="${2}"
|
||||
local perpackagedir="${3}"
|
||||
local rpath dir
|
||||
|
||||
while read rpath; do
|
||||
for dir in ${rpath//:/ }; do
|
||||
# Remove duplicate and trailing '/' for proper match
|
||||
dir="$( sed -r -e 's:/+:/:g; s:/$::;' <<<"${dir}" )"
|
||||
[ "${dir}" = "${hostdir}/lib" ] && return 0
|
||||
[ "${dir}" = "\$ORIGIN/../lib" ] && return 0
|
||||
# This check is done even for builds where
|
||||
# BR2_PER_PACKAGE_DIRECTORIES is disabled. In this case,
|
||||
# PER_PACKAGE_DIR and therefore ${perpackagedir} points to
|
||||
# a non-existent directory, and this check will always be
|
||||
# false.
|
||||
[[ ${dir} =~ ${perpackagedir}/[^/]+/host/lib ]] && return 0
|
||||
done
|
||||
done < <( readelf -d "${file}" \
|
||||
|sed -r -e '/.* \(R(UN)?PATH\) +Library r(un)?path: \[(.+)\]$/!d' \
|
||||
-e 's//\3/;' \
|
||||
)
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
main "${@}"
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/sh
|
||||
|
||||
# This script (and the embedded C code) will check that the actual
|
||||
# headers version match the user told us they were:
|
||||
#
|
||||
# - if both versions are the same, all is well.
|
||||
#
|
||||
# - if the actual headers are older than the user told us, this is
|
||||
# an error.
|
||||
#
|
||||
# - if the actual headers are more recent than the user told us, and
|
||||
# we are doing a strict check, then this is an error.
|
||||
#
|
||||
# - if the actual headers are more recent than the user told us, and
|
||||
# we are doing a loose check, then a warning is printed, but this is
|
||||
# not an error.
|
||||
|
||||
BUILDDIR="${1}"
|
||||
SYSROOT="${2}"
|
||||
# Make sure we have enough version components
|
||||
HDR_VER="${3}.0.0"
|
||||
CHECK="${4}" # 'strict' or 'loose'
|
||||
|
||||
HDR_M="${HDR_VER%%.*}"
|
||||
HDR_V="${HDR_VER#*.}"
|
||||
HDR_m="${HDR_V%%.*}"
|
||||
|
||||
# Exit on any error, so we don't try to run an unexisting program if the
|
||||
# compilation fails.
|
||||
set -e
|
||||
|
||||
# Set the clean-up trap in advance to prevent a race condition in which we
|
||||
# create the file but get a SIGTERM before setting it. Notice that we don't
|
||||
# need to care about EXEC being empty, since 'rm -f ""' does nothing.
|
||||
trap 'rm -f "${EXEC}"' EXIT
|
||||
|
||||
EXEC="$(mktemp -p "${BUILDDIR}" -t .check-headers.XXXXXX)"
|
||||
|
||||
# We do not want to account for the patch-level, since headers are
|
||||
# not supposed to change for different patchlevels, so we mask it out.
|
||||
# This only applies to kernels >= 3.0, but those are the only one
|
||||
# we actually care about; we treat all 2.6.x kernels equally.
|
||||
${HOSTCC} -imacros "${SYSROOT}/usr/include/linux/version.h" \
|
||||
-x c -o "${EXEC}" - <<_EOF_
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
int main(int argc __attribute__((unused)),
|
||||
char** argv __attribute__((unused)))
|
||||
{
|
||||
int l = LINUX_VERSION_CODE & ~0xFF;
|
||||
int h = KERNEL_VERSION(${HDR_M},${HDR_m},0);
|
||||
|
||||
if ((l >= h) && !strcmp("${CHECK}", "loose"))
|
||||
return 0;
|
||||
|
||||
if (l != h) {
|
||||
printf("Incorrect selection of kernel headers: ");
|
||||
printf("expected %d.%d.x, got %d.%d.x\n", ${HDR_M}, ${HDR_m},
|
||||
((LINUX_VERSION_CODE>>16) & 0xFF),
|
||||
((LINUX_VERSION_CODE>>8) & 0xFF));
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
_EOF_
|
||||
|
||||
"${EXEC}"
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Check if a given custom skeleton or overlay complies to the merged /usr
|
||||
# requirements:
|
||||
# /
|
||||
# /bin -> usr/bin
|
||||
# /lib -> usr/lib
|
||||
# /sbin -> usr/sbin
|
||||
# /usr/bin/
|
||||
# /usr/lib/
|
||||
# /usr/sbin/
|
||||
#
|
||||
# Output: the list of non-compliant paths (empty if compliant).
|
||||
#
|
||||
|
||||
# Extract the inode numbers for all of those directories. In case any is
|
||||
# a symlink, we want to get the inode of the pointed-to directory, so we
|
||||
# append '/.' to be sure we get the target directory. Since the symlinks
|
||||
# can be anyway (/bin -> /usr/bin or /usr/bin -> /bin), we do that for
|
||||
# all of them.
|
||||
#
|
||||
lib_inode=$(stat -c '%i' "${1}/lib/." 2>/dev/null)
|
||||
bin_inode=$(stat -c '%i' "${1}/bin/." 2>/dev/null)
|
||||
sbin_inode=$(stat -c '%i' "${1}/sbin/." 2>/dev/null)
|
||||
usr_lib_inode=$(stat -c '%i' "${1}/usr/lib/." 2>/dev/null)
|
||||
usr_bin_inode=$(stat -c '%i' "${1}/usr/bin/." 2>/dev/null)
|
||||
usr_sbin_inode=$(stat -c '%i' "${1}/usr/sbin/." 2>/dev/null)
|
||||
|
||||
not_merged_dirs=""
|
||||
test -z "$lib_inode" || \
|
||||
test "$lib_inode" = "$usr_lib_inode" || \
|
||||
not_merged_dirs="/lib"
|
||||
test -z "$bin_inode" || \
|
||||
test "$bin_inode" = "$usr_bin_inode" || \
|
||||
not_merged_dirs="$not_merged_dirs /bin"
|
||||
test -z "$sbin_inode" || \
|
||||
test "$sbin_inode" = "$usr_sbin_inode" || \
|
||||
not_merged_dirs="$not_merged_dirs /sbin"
|
||||
echo "${not_merged_dirs# }"
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from xml.etree.ElementTree import Element, SubElement
|
||||
import gzip
|
||||
import os
|
||||
import requests
|
||||
import time
|
||||
from xml.dom import minidom
|
||||
|
||||
VALID_REFS = ['VENDOR', 'VERSION', 'CHANGE_LOG', 'PRODUCT', 'PROJECT', 'ADVISORY']
|
||||
|
||||
CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
|
||||
|
||||
ns = {
|
||||
'': 'http://cpe.mitre.org/dictionary/2.0',
|
||||
'cpe-23': 'http://scap.nist.gov/schema/cpe-extension/2.3',
|
||||
'xml': 'http://www.w3.org/XML/1998/namespace'
|
||||
}
|
||||
|
||||
|
||||
class CPE:
|
||||
def __init__(self, cpe_str, titles, refs):
|
||||
self.cpe_str = cpe_str
|
||||
self.titles = titles
|
||||
self.references = refs
|
||||
self.cpe_cur_ver = "".join(self.cpe_str.split(":")[5:6])
|
||||
|
||||
def update_xml_dict(self):
|
||||
ET.register_namespace('', 'http://cpe.mitre.org/dictionary/2.0')
|
||||
cpes = Element('cpe-list')
|
||||
cpes.set('xmlns:cpe-23', "http://scap.nist.gov/schema/cpe-extension/2.3")
|
||||
cpes.set('xmlns:ns6', "http://scap.nist.gov/schema/scap-core/0.1")
|
||||
cpes.set('xmlns:scap-core', "http://scap.nist.gov/schema/scap-core/0.3")
|
||||
cpes.set('xmlns:config', "http://scap.nist.gov/schema/configuration/0.1")
|
||||
cpes.set('xmlns:xsi', "http://www.w3.org/2001/XMLSchema-instance")
|
||||
cpes.set('xmlns:meta', "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2")
|
||||
cpes.set('xsi:schemaLocation', " ".join(["http://scap.nist.gov/schema/cpe-extension/2.3",
|
||||
"https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd",
|
||||
"http://cpe.mitre.org/dictionary/2.0",
|
||||
"https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd",
|
||||
"http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2",
|
||||
"https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd",
|
||||
"http://scap.nist.gov/schema/scap-core/0.3",
|
||||
"https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd",
|
||||
"http://scap.nist.gov/schema/configuration/0.1",
|
||||
"https://scap.nist.gov/schema/nvd/configuration_0.1.xsd",
|
||||
"http://scap.nist.gov/schema/scap-core/0.1",
|
||||
"https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"]))
|
||||
item = SubElement(cpes, 'cpe-item')
|
||||
cpe_short_name = CPE.short_name(self.cpe_str)
|
||||
cpe_new_ver = CPE.version_update(self.cpe_str)
|
||||
|
||||
item.set('name', 'cpe:/' + cpe_short_name)
|
||||
self.titles[0].text.replace(self.cpe_cur_ver, cpe_new_ver)
|
||||
for title in self.titles:
|
||||
item.append(title)
|
||||
if self.references:
|
||||
item.append(self.references)
|
||||
cpe23item = SubElement(item, 'cpe-23:cpe23-item')
|
||||
cpe23item.set('name', self.cpe_str)
|
||||
|
||||
# Generate the XML as a string
|
||||
xmlstr = ET.tostring(cpes)
|
||||
|
||||
# And use minidom to pretty print the XML
|
||||
return minidom.parseString(xmlstr).toprettyxml(encoding="utf-8").decode("utf-8")
|
||||
|
||||
@staticmethod
|
||||
def version(cpe):
|
||||
return cpe.split(":")[5]
|
||||
|
||||
@staticmethod
|
||||
def product(cpe):
|
||||
return cpe.split(":")[4]
|
||||
|
||||
@staticmethod
|
||||
def short_name(cpe):
|
||||
return ":".join(cpe.split(":")[2:6])
|
||||
|
||||
@staticmethod
|
||||
def version_update(cpe):
|
||||
return ":".join(cpe.split(":")[5:6])
|
||||
|
||||
@staticmethod
|
||||
def no_version(cpe):
|
||||
return ":".join(cpe.split(":")[:5])
|
||||
|
||||
|
||||
class CPEDB:
|
||||
def __init__(self, nvd_path):
|
||||
self.all_cpes = dict()
|
||||
self.all_cpes_no_version = dict()
|
||||
self.nvd_path = nvd_path
|
||||
|
||||
def get_xml_dict(self):
|
||||
print("CPE: Setting up NIST dictionary")
|
||||
if not os.path.exists(os.path.join(self.nvd_path, "cpe")):
|
||||
os.makedirs(os.path.join(self.nvd_path, "cpe"))
|
||||
|
||||
cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(CPEDB_URL))
|
||||
if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400:
|
||||
print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]")
|
||||
cpe_dict = requests.get(CPEDB_URL)
|
||||
open(cpe_dict_local, "wb").write(cpe_dict.content)
|
||||
|
||||
print("CPE: Unzipping xml manifest...")
|
||||
nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb'))
|
||||
print("CPE: Converting xml manifest to dict...")
|
||||
tree = ET.parse(nist_cpe_file)
|
||||
all_cpedb = tree.getroot()
|
||||
self.parse_dict(all_cpedb)
|
||||
|
||||
def parse_dict(self, all_cpedb):
|
||||
# Cycle through the dict and build two dict to be used for custom
|
||||
# lookups of partial and complete CPE objects
|
||||
# The objects are then used to create new proposed XML updates if
|
||||
# if is determined one is required
|
||||
# Out of the different language titles, select English
|
||||
for cpe in all_cpedb.findall(".//{http://cpe.mitre.org/dictionary/2.0}cpe-item"):
|
||||
cpe_titles = []
|
||||
for title in cpe.findall('.//{http://cpe.mitre.org/dictionary/2.0}title[@xml:lang="en-US"]', ns):
|
||||
title.tail = None
|
||||
cpe_titles.append(title)
|
||||
|
||||
# Some older CPE don't include references, if they do, make
|
||||
# sure we handle the case of one ref needing to be packed
|
||||
# in a list
|
||||
cpe_ref = cpe.find(".//{http://cpe.mitre.org/dictionary/2.0}references")
|
||||
if cpe_ref:
|
||||
for ref in cpe_ref.findall(".//{http://cpe.mitre.org/dictionary/2.0}reference"):
|
||||
ref.tail = None
|
||||
ref.text = ref.text.upper()
|
||||
if ref.text not in VALID_REFS:
|
||||
ref.text = ref.text + "-- UPDATE this entry, here are some examples and just one word should be used -- " + ' '.join(VALID_REFS) # noqa E501
|
||||
cpe_ref.tail = None
|
||||
cpe_ref.text = None
|
||||
|
||||
cpe_str = cpe.find(".//{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item").get('name')
|
||||
item = CPE(cpe_str, cpe_titles, cpe_ref)
|
||||
cpe_str_no_version = CPE.no_version(cpe_str)
|
||||
# This dict must have a unique key for every CPE version
|
||||
# which allows matching to the specific obj data of that
|
||||
# NIST dict entry
|
||||
self.all_cpes.update({cpe_str: item})
|
||||
# This dict has one entry for every CPE (w/o version) to allow
|
||||
# partial match (no valid version) check (the obj is saved and
|
||||
# used as seed for suggested xml updates. By updating the same
|
||||
# non-version'd entry, it assumes the last update here is the
|
||||
# latest version in the NIST dict)
|
||||
self.all_cpes_no_version.update({cpe_str_no_version: item})
|
||||
|
||||
def find_partial(self, cpe_str):
|
||||
cpe_str_no_version = CPE.no_version(cpe_str)
|
||||
if cpe_str_no_version in self.all_cpes_no_version:
|
||||
return cpe_str_no_version
|
||||
|
||||
def find_partial_obj(self, cpe_str):
|
||||
cpe_str_no_version = CPE.no_version(cpe_str)
|
||||
if cpe_str_no_version in self.all_cpes_no_version:
|
||||
return self.all_cpes_no_version[cpe_str_no_version]
|
||||
|
||||
def find_partial_latest_version(self, cpe_str_partial):
|
||||
cpe_obj = self.find_partial_obj(cpe_str_partial)
|
||||
return cpe_obj.cpe_cur_ver
|
||||
|
||||
def find(self, cpe_str):
|
||||
if self.find_partial(cpe_str):
|
||||
if cpe_str in self.all_cpes:
|
||||
return cpe_str
|
||||
|
||||
def gen_update_xml(self, cpe_str):
|
||||
cpe = self.find_partial_obj(cpe_str)
|
||||
return cpe.update_xml_dict()
|
||||
Executable
+273
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
||||
# Copyright (C) 2020 by Gregory CLEMENT <gregory.clement@bootlin.com>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import requests # URL checking
|
||||
import distutils.version
|
||||
import time
|
||||
import gzip
|
||||
import sys
|
||||
import operator
|
||||
|
||||
try:
|
||||
import ijson
|
||||
# backend is a module in < 2.5, a string in >= 2.5
|
||||
if 'python' in getattr(ijson.backend, '__name__', ijson.backend):
|
||||
try:
|
||||
import ijson.backends.yajl2_cffi as ijson
|
||||
except ImportError:
|
||||
sys.stderr.write('Warning: Using slow ijson python backend\n')
|
||||
except ImportError:
|
||||
sys.stderr.write("You need ijson to parse NVD for CVE check\n")
|
||||
exit(1)
|
||||
|
||||
sys.path.append('utils/')
|
||||
|
||||
NVD_START_YEAR = 2002
|
||||
NVD_JSON_VERSION = "1.1"
|
||||
NVD_BASE_URL = "https://nvd.nist.gov/feeds/json/cve/" + NVD_JSON_VERSION
|
||||
|
||||
ops = {
|
||||
'>=': operator.ge,
|
||||
'>': operator.gt,
|
||||
'<=': operator.le,
|
||||
'<': operator.lt,
|
||||
'=': operator.eq
|
||||
}
|
||||
|
||||
|
||||
# Check if two CPE IDs match each other
|
||||
def cpe_matches(cpe1, cpe2):
|
||||
cpe1_elems = cpe1.split(":")
|
||||
cpe2_elems = cpe2.split(":")
|
||||
|
||||
remains = filter(lambda x: x[0] not in ["*", "-"] and x[1] not in ["*", "-"] and x[0] != x[1],
|
||||
zip(cpe1_elems, cpe2_elems))
|
||||
return len(list(remains)) == 0
|
||||
|
||||
|
||||
def cpe_product(cpe):
|
||||
return cpe.split(':')[4]
|
||||
|
||||
|
||||
def cpe_version(cpe):
|
||||
return cpe.split(':')[5]
|
||||
|
||||
|
||||
class CVE:
|
||||
"""An accessor class for CVE Items in NVD files"""
|
||||
CVE_AFFECTS = 1
|
||||
CVE_DOESNT_AFFECT = 2
|
||||
CVE_UNKNOWN = 3
|
||||
|
||||
def __init__(self, nvd_cve):
|
||||
"""Initialize a CVE from its NVD JSON representation"""
|
||||
self.nvd_cve = nvd_cve
|
||||
|
||||
@staticmethod
|
||||
def download_nvd_year(nvd_path, year):
|
||||
metaf = "nvdcve-%s-%s.meta" % (NVD_JSON_VERSION, year)
|
||||
path_metaf = os.path.join(nvd_path, metaf)
|
||||
jsonf_gz = "nvdcve-%s-%s.json.gz" % (NVD_JSON_VERSION, year)
|
||||
path_jsonf_gz = os.path.join(nvd_path, jsonf_gz)
|
||||
|
||||
# If the database file is less than a day old, we assume the NVD data
|
||||
# locally available is recent enough.
|
||||
if os.path.exists(path_jsonf_gz) and os.stat(path_jsonf_gz).st_mtime >= time.time() - 86400:
|
||||
return path_jsonf_gz
|
||||
|
||||
# If not, we download the meta file
|
||||
url = "%s/%s" % (NVD_BASE_URL, metaf)
|
||||
print("Getting %s" % url)
|
||||
page_meta = requests.get(url)
|
||||
page_meta.raise_for_status()
|
||||
|
||||
# If the meta file already existed, we compare the existing
|
||||
# one with the data newly downloaded. If they are different,
|
||||
# we need to re-download the database.
|
||||
# If the database does not exist locally, we need to redownload it in
|
||||
# any case.
|
||||
if os.path.exists(path_metaf) and os.path.exists(path_jsonf_gz):
|
||||
meta_known = open(path_metaf, "r").read()
|
||||
if page_meta.text == meta_known:
|
||||
return path_jsonf_gz
|
||||
|
||||
# Grab the compressed JSON NVD, and write files to disk
|
||||
url = "%s/%s" % (NVD_BASE_URL, jsonf_gz)
|
||||
print("Getting %s" % url)
|
||||
page_json = requests.get(url)
|
||||
page_json.raise_for_status()
|
||||
open(path_jsonf_gz, "wb").write(page_json.content)
|
||||
open(path_metaf, "w").write(page_meta.text)
|
||||
return path_jsonf_gz
|
||||
|
||||
@classmethod
|
||||
def read_nvd_dir(cls, nvd_dir):
|
||||
"""
|
||||
Iterate over all the CVEs contained in NIST Vulnerability Database
|
||||
feeds since NVD_START_YEAR. If the files are missing or outdated in
|
||||
nvd_dir, a fresh copy will be downloaded, and kept in .json.gz
|
||||
"""
|
||||
for year in range(NVD_START_YEAR, datetime.datetime.now().year + 1):
|
||||
filename = CVE.download_nvd_year(nvd_dir, year)
|
||||
try:
|
||||
content = ijson.items(gzip.GzipFile(filename), 'CVE_Items.item')
|
||||
except: # noqa: E722
|
||||
print("ERROR: cannot read %s. Please remove the file then rerun this script" % filename)
|
||||
raise
|
||||
for cve in content:
|
||||
yield cls(cve)
|
||||
|
||||
def each_product(self):
|
||||
"""Iterate over each product section of this cve"""
|
||||
for vendor in self.nvd_cve['cve']['affects']['vendor']['vendor_data']:
|
||||
for product in vendor['product']['product_data']:
|
||||
yield product
|
||||
|
||||
def parse_node(self, node):
|
||||
"""
|
||||
Parse the node inside the configurations section to extract the
|
||||
cpe information usefull to know if a product is affected by
|
||||
the CVE. Actually only the product name and the version
|
||||
descriptor are needed, but we also provide the vendor name.
|
||||
"""
|
||||
|
||||
# The node containing the cpe entries matching the CVE can also
|
||||
# contain sub-nodes, so we need to manage it.
|
||||
for child in node.get('children', ()):
|
||||
for parsed_node in self.parse_node(child):
|
||||
yield parsed_node
|
||||
|
||||
for cpe in node.get('cpe_match', ()):
|
||||
if not cpe['vulnerable']:
|
||||
return
|
||||
product = cpe_product(cpe['cpe23Uri'])
|
||||
version = cpe_version(cpe['cpe23Uri'])
|
||||
# ignore when product is '-', which means N/A
|
||||
if product == '-':
|
||||
return
|
||||
op_start = ''
|
||||
op_end = ''
|
||||
v_start = ''
|
||||
v_end = ''
|
||||
|
||||
if version != '*' and version != '-':
|
||||
# Version is defined, this is a '=' match
|
||||
op_start = '='
|
||||
v_start = version
|
||||
else:
|
||||
# Parse start version, end version and operators
|
||||
if 'versionStartIncluding' in cpe:
|
||||
op_start = '>='
|
||||
v_start = cpe['versionStartIncluding']
|
||||
|
||||
if 'versionStartExcluding' in cpe:
|
||||
op_start = '>'
|
||||
v_start = cpe['versionStartExcluding']
|
||||
|
||||
if 'versionEndIncluding' in cpe:
|
||||
op_end = '<='
|
||||
v_end = cpe['versionEndIncluding']
|
||||
|
||||
if 'versionEndExcluding' in cpe:
|
||||
op_end = '<'
|
||||
v_end = cpe['versionEndExcluding']
|
||||
|
||||
yield {
|
||||
'id': cpe['cpe23Uri'],
|
||||
'v_start': v_start,
|
||||
'op_start': op_start,
|
||||
'v_end': v_end,
|
||||
'op_end': op_end
|
||||
}
|
||||
|
||||
def each_cpe(self):
|
||||
for node in self.nvd_cve['configurations']['nodes']:
|
||||
for cpe in self.parse_node(node):
|
||||
yield cpe
|
||||
|
||||
@property
|
||||
def identifier(self):
|
||||
"""The CVE unique identifier"""
|
||||
return self.nvd_cve['cve']['CVE_data_meta']['ID']
|
||||
|
||||
@property
|
||||
def affected_products(self):
|
||||
"""The set of CPE products referred by this CVE definition"""
|
||||
return set(cpe_product(p['id']) for p in self.each_cpe())
|
||||
|
||||
def affects(self, name, version, cve_ignore_list, cpeid=None):
|
||||
"""
|
||||
True if the Buildroot Package object passed as argument is affected
|
||||
by this CVE.
|
||||
"""
|
||||
if self.identifier in cve_ignore_list:
|
||||
return self.CVE_DOESNT_AFFECT
|
||||
|
||||
pkg_version = distutils.version.LooseVersion(version)
|
||||
if not hasattr(pkg_version, "version"):
|
||||
print("Cannot parse package '%s' version '%s'" % (name, version))
|
||||
pkg_version = None
|
||||
|
||||
# if we don't have a cpeid, build one based on name and version
|
||||
if not cpeid:
|
||||
cpeid = "cpe:2.3:*:*:%s:%s:*:*:*:*:*:*:*" % (name, version)
|
||||
# if we have a cpeid, use its version instead of the package
|
||||
# version, as they might be different due to
|
||||
# <pkg>_CPE_ID_VERSION
|
||||
else:
|
||||
pkg_version = distutils.version.LooseVersion(cpe_version(cpeid))
|
||||
|
||||
for cpe in self.each_cpe():
|
||||
if not cpe_matches(cpe['id'], cpeid):
|
||||
continue
|
||||
if not cpe['v_start'] and not cpe['v_end']:
|
||||
return self.CVE_AFFECTS
|
||||
if not pkg_version:
|
||||
continue
|
||||
|
||||
if cpe['v_start']:
|
||||
try:
|
||||
cve_affected_version = distutils.version.LooseVersion(cpe['v_start'])
|
||||
inrange = ops.get(cpe['op_start'])(pkg_version, cve_affected_version)
|
||||
except TypeError:
|
||||
return self.CVE_UNKNOWN
|
||||
|
||||
# current package version is before v_start, so we're
|
||||
# not affected by the CVE
|
||||
if not inrange:
|
||||
continue
|
||||
|
||||
if cpe['v_end']:
|
||||
try:
|
||||
cve_affected_version = distutils.version.LooseVersion(cpe['v_end'])
|
||||
inrange = ops.get(cpe['op_end'])(pkg_version, cve_affected_version)
|
||||
except TypeError:
|
||||
return self.CVE_UNKNOWN
|
||||
|
||||
# current package version is after v_end, so we're
|
||||
# not affected by the CVE
|
||||
if not inrange:
|
||||
continue
|
||||
|
||||
# We're in the version range affected by this CVE
|
||||
return self.CVE_AFFECTS
|
||||
|
||||
return self.CVE_DOESNT_AFFECT
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/sh
|
||||
|
||||
# This script registers the toolchain of a Buildroot project into the
|
||||
# Eclipse plugin. To do so, it adds a new line for the Buildroot
|
||||
# toolchain into the $HOME/.buildroot-eclipse.toolchains file, which
|
||||
# the Eclipse Buildroot plugin reads to discover automatically the
|
||||
# available Buildroot toolchains on the system.
|
||||
#
|
||||
# This script should typically not be called manually. Instead, one
|
||||
# should enable the BR2_ECLIPSE_REGISTER configuration option, which
|
||||
# will lead Buildroot to automatically call this script with the
|
||||
# appropriate arguments.
|
||||
#
|
||||
# Usage:
|
||||
# eclipse-register-toolchain project-directory toolchain-prefix architecture
|
||||
#
|
||||
# project-directory is the absolute path to the Buildroot project
|
||||
# output directory (which contains the host/, target/, build/,
|
||||
# images/, etc. subdirectories). It should be an absolute and
|
||||
# canonical path.
|
||||
#
|
||||
# toolchain-prefix is the prefix of the cross-compilation tools, i.e
|
||||
# 'arm-linux-' if the cross-compiler executable is 'arm-linux-gcc'.
|
||||
#
|
||||
# architecture is the lower-cased name of the architecture targetted
|
||||
# by the Buildroot project.
|
||||
|
||||
if test $# -ne 3; then
|
||||
echo "Invalid number of arguments."
|
||||
echo "Usage: $0 project-directory toolchain-prefix architecture"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
project_directory=$1
|
||||
toolchain_prefix=$2
|
||||
architecture=$3
|
||||
|
||||
if test ! -d ${project_directory} ; then
|
||||
echo "Non-existing project directory ${project_directory}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if test ! -d ${project_directory}/host ; then
|
||||
echo "Your project directory does not look like a Buildroot output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if test ! -e ${project_directory}/host/bin/${toolchain_prefix}gcc ; then
|
||||
echo "Cannot find the cross-compiler in the project directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOOLCHAIN_ECLIPSE_FILE=${HOME}/.buildroot-eclipse.toolchains
|
||||
|
||||
# First, we remove all lines from the ${TOOLCHAIN_ECLISPE_FILE} that
|
||||
# correspond to toolchains that no longer exist.
|
||||
if test -f ${TOOLCHAIN_ECLIPSE_FILE} ; then
|
||||
mv ${TOOLCHAIN_ECLIPSE_FILE} ${TOOLCHAIN_ECLIPSE_FILE}.tmp
|
||||
cat ${TOOLCHAIN_ECLIPSE_FILE}.tmp | while read toolchain ; do
|
||||
path=$(echo ${toolchain} | cut -f1 -d ':')
|
||||
# Filter lines corresponding to still existing projects
|
||||
echo "Testing ${path} ..."
|
||||
if ! test -d ${path} ; then
|
||||
continue
|
||||
fi
|
||||
# .. and the current project
|
||||
if test ${path} = ${project_directory} ; then
|
||||
continue
|
||||
fi
|
||||
echo ${toolchain} >> ${TOOLCHAIN_ECLIPSE_FILE}
|
||||
done
|
||||
rm ${TOOLCHAIN_ECLIPSE_FILE}.tmp
|
||||
fi
|
||||
|
||||
# Add the toolchain
|
||||
echo "${project_directory}:${toolchain_prefix}:${architecture}" >> ${TOOLCHAIN_ECLIPSE_FILE}
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This script is used to generate a gconv-modules file that takes into
|
||||
# account only the gconv modules installed by Buildroot. It receives
|
||||
# on its standard input the original complete gconv-modules file from
|
||||
# the toolchain, and as arguments the list of gconv modules that were
|
||||
# actually installed, and writes on its standard output the new
|
||||
# gconv-modules file.
|
||||
|
||||
# The format of gconv-modules is precisely documented in the
|
||||
# file itself. It consists of two different directives:
|
||||
# module FROMSET TOSET FILENAME COST
|
||||
# alias ALIAS REALNAME
|
||||
# and that's what this script parses and generates.
|
||||
#
|
||||
# There are two kinds of 'module' directives:
|
||||
# - the first defines conversion of a charset to/from INTERNAL representation
|
||||
# - the second defines conversion of a charset to/from another charset
|
||||
# we handle each with slightly different code, since the second never has
|
||||
# associated aliases.
|
||||
|
||||
gawk -v files="${1}" '
|
||||
$1 == "alias" {
|
||||
aliases[$3] = aliases[$3] " " $2;
|
||||
}
|
||||
$1 == "module" && $2 != "INTERNAL" && $3 == "INTERNAL" {
|
||||
file2internals[$4] = file2internals[$4] " " $2;
|
||||
mod2cost[$2] = $5;
|
||||
}
|
||||
$1 == "module" && $2 != "INTERNAL" && $3 != "INTERNAL" {
|
||||
file2cset[$4] = file2cset[$4] " " $2 ":" $3;
|
||||
mod2cost[$2] = $5;
|
||||
}
|
||||
|
||||
END {
|
||||
nb_files = split(files, all_files);
|
||||
for(f = 1; f <= nb_files; f++) {
|
||||
file = all_files[f];
|
||||
printf("# Modules and aliases for: %s\n", file);
|
||||
nb_mods = split(file2internals[file], mods);
|
||||
for(i = 1; i <= nb_mods; i++) {
|
||||
nb_aliases = split(aliases[mods[i]], mod_aliases);
|
||||
for(j = 1; j <= nb_aliases; j++) {
|
||||
printf("alias\t%s\t%s\n", mod_aliases[j], mods[i]);
|
||||
}
|
||||
printf("module\t%s\t%s\t%s\t%d\n", mods[i], "INTERNAL", file, mod2cost[mods[i]]);
|
||||
printf("module\t%s\t%s\t%s\t%d\n", "INTERNAL", mods[i], file, mod2cost[mods[i]]);
|
||||
printf("\n" );
|
||||
}
|
||||
printf("%s", nb_mods != 0 ? "\n" : "");
|
||||
nb_csets = split(file2cset[file], csets);
|
||||
for(i = 1; i <= nb_csets; i++) {
|
||||
split(csets[i], cs, ":");
|
||||
printf("module\t%s\t%s\t%s\t%d\n", cs[1], cs[2], file, mod2cost[cs[1]]);
|
||||
}
|
||||
printf("%s", nb_csets != 0 ? "\n\n" : "");
|
||||
}
|
||||
}
|
||||
'
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This is a script to find, and correct, a problem with old versions of
|
||||
# configure that affect powerpc64 and powerpc64le.
|
||||
|
||||
# The issue causes configure to incorrectly determine that shared library
|
||||
# support is not present in the linker. This causes the package to build a
|
||||
# static library rather than a dynamic one and although the build will succeed,
|
||||
# it may cause packages that link with the static library it to fail due to
|
||||
# undefined symbols.
|
||||
|
||||
# This script searches for files named 'configure' that appear to have this
|
||||
# issue (by searching for a known bad pattern) and patching them.
|
||||
|
||||
set -e
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 <package build directory>"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
srcdir="$1"
|
||||
files=$(cd "$srcdir" && find . -name configure \
|
||||
-exec grep -qF 'Generated by GNU Autoconf' {} \; \
|
||||
-exec grep -qF 'ppc*-*linux*|powerpc*-*linux*)' {} \; -print)
|
||||
|
||||
# --ignore-whitespace is needed because some packages have included
|
||||
# copies of configure scripts where tabs have been replaced with spaces.
|
||||
for c in $files; do
|
||||
patch --ignore-whitespace "$srcdir"/"$c" <<'EOF'
|
||||
--- a/configure 2016-11-16 15:31:46.097447271 +1100
|
||||
+++ b/configure 2008-07-21 12:17:23.000000000 +1000
|
||||
@@ -4433,7 +4433,10 @@
|
||||
x86_64-*linux*)
|
||||
LD="${LD-ld} -m elf_x86_64"
|
||||
;;
|
||||
- ppc*-*linux*|powerpc*-*linux*)
|
||||
+ powerpcle-*linux*)
|
||||
+ LD="${LD-ld} -m elf64lppc"
|
||||
+ ;;
|
||||
+ powerpc-*linux*)
|
||||
LD="${LD-ld} -m elf64ppc"
|
||||
;;
|
||||
s390*-*linux*)
|
||||
EOF
|
||||
done
|
||||
|
||||
Executable
+162
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (C) 2016 Samuel Martin <s.martin49@gmail.com>
|
||||
# Copyright (C) 2017 Wolfgang Grandegger <wg@grandegger.com>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
usage() {
|
||||
cat <<EOF >&2
|
||||
Usage: ${0} TREE_KIND
|
||||
|
||||
Description:
|
||||
|
||||
This script scans a tree and sanitize ELF files' RPATH found in there.
|
||||
|
||||
Sanitization behaves the same whatever the kind of the processed tree,
|
||||
but the resulting RPATH differs. The rpath sanitization is done using
|
||||
"patchelf --make-rpath-relative".
|
||||
|
||||
Arguments:
|
||||
|
||||
TREE_KIND Kind of tree to be processed.
|
||||
Allowed values: host, target, staging
|
||||
|
||||
Environment:
|
||||
|
||||
PATCHELF patchelf program to use
|
||||
(default: HOST_DIR/bin/patchelf)
|
||||
|
||||
HOST_DIR host directory
|
||||
STAGING_DIR staging directory
|
||||
TARGET_DIR target directory
|
||||
|
||||
TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR
|
||||
(default HOST_DIR/opt/ext-toolchain)
|
||||
|
||||
Returns: 0 if success or 1 in case of error
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
: ${PATCHELF:=${HOST_DIR}/bin/patchelf}
|
||||
|
||||
# ELF files should not be in these sub-directories
|
||||
HOST_EXCLUDEPATHS="/share/terminfo"
|
||||
STAGING_EXCLUDEPATHS="/usr/include /usr/share/terminfo"
|
||||
TARGET_EXCLUDEPATHS="/lib/firmware"
|
||||
|
||||
main() {
|
||||
local rootdir
|
||||
local tree="${1}"
|
||||
local find_args=( )
|
||||
local sanitize_extra_args=( )
|
||||
|
||||
if ! "${PATCHELF}" --version > /dev/null 2>&1; then
|
||||
echo "Error: can't execute patchelf utility '${PATCHELF}'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "${tree}" in
|
||||
host)
|
||||
rootdir="${HOST_DIR}"
|
||||
|
||||
# do not process the sysroot (only contains target binaries)
|
||||
find_args+=( "-path" "${STAGING_DIR}" "-prune" "-o" )
|
||||
|
||||
# do not process the external toolchain installation directory to
|
||||
# avoid breaking it.
|
||||
test "${TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR}" != "" && \
|
||||
find_args+=( "-path" "${TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR}" "-prune" "-o" )
|
||||
|
||||
for excludepath in ${HOST_EXCLUDEPATHS}; do
|
||||
find_args+=( "-path" "${HOST_DIR}""${excludepath}" "-prune" "-o" )
|
||||
done
|
||||
|
||||
# do not process the patchelf binary but a copy to work-around "file in use"
|
||||
find_args+=( "-path" "${PATCHELF}" "-prune" "-o" )
|
||||
cp "${PATCHELF}" "${PATCHELF}.__to_be_patched"
|
||||
|
||||
# we always want $ORIGIN-based rpaths to make it relocatable.
|
||||
sanitize_extra_args+=( "--relative-to-file" )
|
||||
;;
|
||||
|
||||
staging)
|
||||
rootdir="${STAGING_DIR}"
|
||||
|
||||
# ELF files should not be in these sub-directories
|
||||
for excludepath in ${STAGING_EXCLUDEPATHS}; do
|
||||
find_args+=( "-path" "${STAGING_DIR}""${excludepath}" "-prune" "-o" )
|
||||
done
|
||||
|
||||
# should be like for the target tree below
|
||||
sanitize_extra_args+=( "--no-standard-lib-dirs" )
|
||||
;;
|
||||
|
||||
target)
|
||||
rootdir="${TARGET_DIR}"
|
||||
|
||||
for excludepath in ${TARGET_EXCLUDEPATHS}; do
|
||||
find_args+=( "-path" "${TARGET_DIR}""${excludepath}" "-prune" "-o" )
|
||||
done
|
||||
|
||||
# we don't want $ORIGIN-based rpaths but absolute paths without rootdir.
|
||||
# we also want to remove rpaths pointing to /lib or /usr/lib.
|
||||
sanitize_extra_args+=( "--no-standard-lib-dirs" )
|
||||
;;
|
||||
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
find_args+=( "-type" "f" "-print" )
|
||||
|
||||
while read file ; do
|
||||
# check if it's an ELF file
|
||||
rpath=$(${PATCHELF} --print-rpath "${file}" 2>&1)
|
||||
if test $? -ne 0 ; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# make files writable if necessary
|
||||
changed=$(chmod -c u+w "${file}")
|
||||
|
||||
# With per-package directory support, most RPATH of host
|
||||
# binaries will point to per-package directories. This won't
|
||||
# work with the --make-rpath-relative ${rootdir} invocation as
|
||||
# the per-package host directory is not within ${rootdir}. So,
|
||||
# we rewrite all RPATHs pointing to per-package directories so
|
||||
# that they point to the global host directry.
|
||||
changed_rpath=$(echo ${rpath} | sed "s@${PER_PACKAGE_DIR}/[^/]\+/host@${HOST_DIR}@")
|
||||
if test "${rpath}" != "${changed_rpath}" ; then
|
||||
${PATCHELF} --set-rpath ${changed_rpath} "${file}"
|
||||
fi
|
||||
|
||||
# call patchelf to sanitize the rpath
|
||||
${PATCHELF} --make-rpath-relative "${rootdir}" ${sanitize_extra_args[@]} "${file}"
|
||||
# restore the original permission
|
||||
test "${changed}" != "" && chmod u-w "${file}"
|
||||
done < <(find "${rootdir}" ${find_args[@]})
|
||||
|
||||
# Restore patched patchelf utility
|
||||
test "${tree}" = "host" && mv "${PATCHELF}.__to_be_patched" "${PATCHELF}"
|
||||
|
||||
# ignore errors
|
||||
return 0
|
||||
}
|
||||
|
||||
main ${@}
|
||||
Executable
+496
@@ -0,0 +1,496 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os.path
|
||||
import re
|
||||
import requests
|
||||
import textwrap
|
||||
|
||||
BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
|
||||
|
||||
AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
|
||||
# Do not edit
|
||||
"""
|
||||
|
||||
# In the below dict:
|
||||
|
||||
# - 'conditions' indicate the cumulative conditions under which the
|
||||
# toolchain will be made available. In several situations, a given
|
||||
# toolchain is usable on several architectures variants (for
|
||||
# example, an ARMv6 toolchain can be used on ARMv7)
|
||||
# - 'test_options' indicate one specific configuration where the
|
||||
# toolchain can be used. It is used to create the runtime test
|
||||
# cases. If 'test_options' does not exist, the code assumes it can
|
||||
# be made equal to 'conditions'
|
||||
# - 'prefix' is the prefix of the cross-compilation toolchain tools
|
||||
|
||||
arches = {
|
||||
'aarch64': {
|
||||
'conditions': ['BR2_aarch64'],
|
||||
'prefix': 'aarch64',
|
||||
},
|
||||
'aarch64be': {
|
||||
'conditions': ['BR2_aarch64_be'],
|
||||
'prefix': 'aarch64_be',
|
||||
},
|
||||
'arcle-750d': {
|
||||
'conditions': ['BR2_arcle', 'BR2_arc750d'],
|
||||
'prefix': 'arc',
|
||||
},
|
||||
'arcle-hs38': {
|
||||
'conditions': ['BR2_arcle', 'BR2_archs38'],
|
||||
'prefix': 'arc',
|
||||
},
|
||||
'armv5-eabi': {
|
||||
'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
|
||||
'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
|
||||
'prefix': 'arm',
|
||||
},
|
||||
'armv6-eabihf': {
|
||||
'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
|
||||
'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
|
||||
'prefix': 'arm',
|
||||
},
|
||||
'armv7-eabihf': {
|
||||
'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
|
||||
'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
|
||||
'prefix': 'arm',
|
||||
},
|
||||
'armv7m': {
|
||||
'conditions': ['BR2_ARM_CPU_ARMV7M'],
|
||||
'test_options': ['BR2_arm', 'BR2_cortex_m4'],
|
||||
'prefix': 'arm',
|
||||
},
|
||||
'm68k-68xxx': {
|
||||
'conditions': ['BR2_m68k_m68k'],
|
||||
'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
|
||||
'prefix': 'm68k',
|
||||
},
|
||||
'm68k-coldfire': {
|
||||
'conditions': ['BR2_m68k_cf'],
|
||||
'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
|
||||
'prefix': 'm68k',
|
||||
},
|
||||
'microblazebe': {
|
||||
'conditions': ['BR2_microblazebe'],
|
||||
'prefix': 'microblaze',
|
||||
},
|
||||
'microblazeel': {
|
||||
'conditions': ['BR2_microblazeel'],
|
||||
'prefix': 'microblazeel',
|
||||
},
|
||||
'mips32': {
|
||||
# Not sure it could be used by other mips32 variants?
|
||||
'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
|
||||
'prefix': 'mips',
|
||||
},
|
||||
'mips32el': {
|
||||
# Not sure it could be used by other mips32el variants?
|
||||
'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
|
||||
'prefix': 'mipsel',
|
||||
},
|
||||
'mips32r5el': {
|
||||
'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
|
||||
'prefix': 'mipsel',
|
||||
},
|
||||
'mips32r6el': {
|
||||
'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
|
||||
'prefix': 'mipsel',
|
||||
},
|
||||
'mips64': {
|
||||
# Not sure it could be used by other mips64 variants?
|
||||
'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
|
||||
'prefix': 'mips64',
|
||||
},
|
||||
'mips64-n32': {
|
||||
# Not sure it could be used by other mips64 variants?
|
||||
'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
|
||||
'prefix': 'mips64',
|
||||
},
|
||||
'mips64el-n32': {
|
||||
# Not sure it could be used by other mips64el variants?
|
||||
'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
|
||||
'prefix': 'mips64el',
|
||||
},
|
||||
'mips64r6el-n32': {
|
||||
'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
|
||||
'prefix': 'mips64el',
|
||||
},
|
||||
'nios2': {
|
||||
'conditions': ['BR2_nios2'],
|
||||
'prefix': 'nios2',
|
||||
},
|
||||
'openrisc': {
|
||||
'conditions': ['BR2_or1k'],
|
||||
'prefix': 'or1k',
|
||||
},
|
||||
'powerpc-440fp': {
|
||||
# Not sure it could be used by other powerpc variants?
|
||||
'conditions': ['BR2_powerpc', 'BR2_powerpc_440fp'],
|
||||
'prefix': 'powerpc',
|
||||
},
|
||||
'powerpc-e300c3': {
|
||||
# Not sure it could be used by other powerpc variants?
|
||||
'conditions': ['BR2_powerpc', 'BR2_powerpc_e300c3'],
|
||||
'prefix': 'powerpc',
|
||||
},
|
||||
'powerpc-e500mc': {
|
||||
# Not sure it could be used by other powerpc variants?
|
||||
'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
|
||||
'prefix': 'powerpc',
|
||||
},
|
||||
'powerpc64-e5500': {
|
||||
'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
|
||||
'prefix': 'powerpc64',
|
||||
},
|
||||
'powerpc64-e6500': {
|
||||
'conditions': ['BR2_powerpc64', 'BR2_powerpc_e6500'],
|
||||
'prefix': 'powerpc64',
|
||||
},
|
||||
'powerpc64-power8': {
|
||||
'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
|
||||
'prefix': 'powerpc64',
|
||||
},
|
||||
'powerpc64le-power8': {
|
||||
'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
|
||||
'prefix': 'powerpc64le',
|
||||
},
|
||||
'riscv32-ilp32d': {
|
||||
'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
|
||||
'prefix': 'riscv32',
|
||||
},
|
||||
'riscv64': {
|
||||
'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
|
||||
'prefix': 'riscv64',
|
||||
},
|
||||
'sh-sh4': {
|
||||
'conditions': ['BR2_sh', 'BR2_sh4'],
|
||||
'prefix': 'sh4',
|
||||
},
|
||||
'sh-sh4aeb': {
|
||||
'conditions': ['BR2_sh', 'BR2_sh4aeb'],
|
||||
'prefix': 'sh4aeb',
|
||||
},
|
||||
'sparc64': {
|
||||
'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
|
||||
'prefix': 'sparc64',
|
||||
},
|
||||
'sparcv8': {
|
||||
'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
|
||||
'prefix': 'sparc',
|
||||
},
|
||||
'x86-64-core-i7': {
|
||||
'conditions': ['BR2_x86_64',
|
||||
'BR2_X86_CPU_HAS_MMX',
|
||||
'BR2_X86_CPU_HAS_SSE',
|
||||
'BR2_X86_CPU_HAS_SSE2',
|
||||
'BR2_X86_CPU_HAS_SSE3',
|
||||
'BR2_X86_CPU_HAS_SSSE3',
|
||||
'BR2_X86_CPU_HAS_SSE4',
|
||||
'BR2_X86_CPU_HAS_SSE42'],
|
||||
'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
|
||||
'prefix': 'x86_64',
|
||||
},
|
||||
'x86-core2': {
|
||||
'conditions': ['BR2_i386',
|
||||
'BR2_X86_CPU_HAS_MMX',
|
||||
'BR2_X86_CPU_HAS_SSE',
|
||||
'BR2_X86_CPU_HAS_SSE2',
|
||||
'BR2_X86_CPU_HAS_SSE3',
|
||||
'BR2_X86_CPU_HAS_SSSE3'],
|
||||
'test_options': ['BR2_i386', 'BR2_x86_core2'],
|
||||
'prefix': 'i686',
|
||||
},
|
||||
'x86-i686': {
|
||||
'conditions': ['BR2_i386',
|
||||
'!BR2_x86_i486',
|
||||
'!BR2_x86_i586',
|
||||
'!BR2_x86_x1000'],
|
||||
'test_options': ['BR2_i386',
|
||||
'BR2_x86_i686'],
|
||||
'prefix': 'i686',
|
||||
},
|
||||
'xtensa-lx60': {
|
||||
'conditions': ['BR2_xtensa', 'BR2_XTENSA_CUSTOM', 'BR2_XTENSA_LITTLE_ENDIAN'],
|
||||
'prefix': 'xtensa',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Toolchain:
|
||||
def __init__(self, arch, libc, variant, version):
|
||||
self.arch = arch
|
||||
self.libc = libc
|
||||
self.variant = variant
|
||||
self.version = version
|
||||
self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
|
||||
self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
|
||||
(self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
|
||||
self.fragment = requests.get(self.fragment_url).text.split("\n")
|
||||
self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
|
||||
|
||||
@property
|
||||
def tarball_url(self):
|
||||
return os.path.join(BASE_URL, self.arch, "tarballs",
|
||||
self.fname_prefix + ".tar.bz2")
|
||||
|
||||
@property
|
||||
def hash_url(self):
|
||||
return os.path.join(BASE_URL, self.arch, "tarballs",
|
||||
self.fname_prefix + ".sha256")
|
||||
|
||||
@property
|
||||
def fragment_url(self):
|
||||
return os.path.join(BASE_URL, self.arch, "fragments",
|
||||
self.fname_prefix + ".frag")
|
||||
|
||||
def gen_config_in_options(self, f):
|
||||
f.write("config %s\n" % self.option_name)
|
||||
f.write("\tbool \"%s %s %s %s\"\n" %
|
||||
(self.arch, self.libc, self.variant, self.version))
|
||||
depends = []
|
||||
selects = []
|
||||
|
||||
for c in arches[self.arch]['conditions']:
|
||||
depends.append(c)
|
||||
|
||||
for frag in self.fragment:
|
||||
# libc type
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
|
||||
selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
|
||||
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
|
||||
# glibc needs mmu support
|
||||
depends.append("BR2_USE_MMU")
|
||||
# glibc doesn't support static only configuration
|
||||
depends.append("!BR2_STATIC_LIBS")
|
||||
selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
|
||||
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
|
||||
# musl needs mmu support
|
||||
depends.append("BR2_USE_MMU")
|
||||
selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
|
||||
|
||||
# gcc version
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
|
||||
m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
|
||||
assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
|
||||
selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
|
||||
|
||||
# kernel headers version
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
|
||||
m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
|
||||
assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
|
||||
selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
|
||||
|
||||
# C++
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
|
||||
selects.append("BR2_INSTALL_LIBSTDCPP")
|
||||
|
||||
# SSP
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
|
||||
selects.append("BR2_TOOLCHAIN_HAS_SSP")
|
||||
|
||||
# wchar
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
|
||||
selects.append("BR2_USE_WCHAR")
|
||||
|
||||
# locale
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
|
||||
# locale implies the availability of wchar
|
||||
selects.append("BR2_USE_WCHAR")
|
||||
selects.append("BR2_ENABLE_LOCALE")
|
||||
|
||||
# thread support
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
|
||||
selects.append("BR2_TOOLCHAIN_HAS_THREADS")
|
||||
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
|
||||
selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
|
||||
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
|
||||
selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
|
||||
|
||||
# RPC
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
|
||||
selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
|
||||
|
||||
# D language
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
|
||||
selects.append("BR2_TOOLCHAIN_HAS_DLANG")
|
||||
|
||||
# fortran
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
|
||||
selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
|
||||
|
||||
# OpenMP
|
||||
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
|
||||
selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
|
||||
|
||||
for depend in depends:
|
||||
f.write("\tdepends on %s\n" % depend)
|
||||
|
||||
for select in selects:
|
||||
f.write("\tselect %s\n" % select)
|
||||
|
||||
f.write("\thelp\n")
|
||||
|
||||
desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
|
||||
(self.arch, self.libc)
|
||||
|
||||
if self.variant == "stable":
|
||||
desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
|
||||
else:
|
||||
desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
|
||||
|
||||
f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n")
|
||||
f.write("\n")
|
||||
f.write("\t https://toolchains.bootlin.com/\n")
|
||||
|
||||
f.write("\n")
|
||||
|
||||
def gen_mk(self, f):
|
||||
f.write("ifeq ($(%s),y)\n" % self.option_name)
|
||||
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
|
||||
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
|
||||
(self.arch, self.libc, self.variant))
|
||||
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
|
||||
os.path.join(BASE_URL, self.arch, "tarballs"))
|
||||
f.write("endif\n\n")
|
||||
pass
|
||||
|
||||
def gen_hash(self, f):
|
||||
f.write("# From %s\n" % self.hash_url)
|
||||
f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
|
||||
|
||||
def gen_test(self, f):
|
||||
if self.variant == "stable":
|
||||
variant = "Stable"
|
||||
else:
|
||||
variant = "BleedingEdge"
|
||||
testname = "TestExternalToolchainBootlin" + \
|
||||
self.arch.replace("-", "").capitalize() + \
|
||||
self.libc.capitalize() + variant
|
||||
f.write("\n\n")
|
||||
f.write("class %s(TestExternalToolchain):\n" % testname)
|
||||
f.write(" config = \"\"\"\n")
|
||||
if 'test_options' in arches[self.arch]:
|
||||
test_options = arches[self.arch]['test_options']
|
||||
else:
|
||||
test_options = arches[self.arch]['conditions']
|
||||
for opt in test_options:
|
||||
if opt.startswith("!"):
|
||||
f.write(" # %s is not set\n" % opt[1:])
|
||||
else:
|
||||
f.write(" %s=y\n" % opt)
|
||||
f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n")
|
||||
f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
|
||||
f.write(" %s=y\n" % self.option_name)
|
||||
f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n")
|
||||
f.write(" \"\"\"\n")
|
||||
f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
|
||||
f.write("\n")
|
||||
f.write(" def test_run(self):\n")
|
||||
f.write(" TestExternalToolchain.common_check(self)\n")
|
||||
|
||||
def __repr__(self):
|
||||
return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
|
||||
(self.arch, self.libc, self.variant, self.version, self.option_name)
|
||||
|
||||
|
||||
def get_toolchains():
|
||||
toolchains = list()
|
||||
for arch, details in arches.items():
|
||||
print(arch)
|
||||
url = os.path.join(BASE_URL, arch, "available_toolchains")
|
||||
page = requests.get(url).text
|
||||
fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
|
||||
# This dict will allow us to keep only the latest version for
|
||||
# each toolchain.
|
||||
tmp = dict()
|
||||
for fname in fnames:
|
||||
parts = fname.split('--')
|
||||
assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
|
||||
libc = parts[1]
|
||||
if parts[2].startswith("stable-"):
|
||||
variant = "stable"
|
||||
version = parts[2][len("stable-"):]
|
||||
elif parts[2].startswith("bleeding-edge-"):
|
||||
variant = "bleeding-edge"
|
||||
version = parts[2][len("bleeding-edge-"):]
|
||||
tmp[(arch, libc, variant)] = version
|
||||
|
||||
toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
|
||||
|
||||
return toolchains
|
||||
|
||||
|
||||
def gen_config_in_options(toolchains, fpath):
|
||||
with open(fpath, "w") as f:
|
||||
f.write(AUTOGENERATED_COMMENT)
|
||||
|
||||
f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
|
||||
f.write("\tbool\n")
|
||||
for arch, details in arches.items():
|
||||
f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
|
||||
f.write("\n")
|
||||
|
||||
f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
|
||||
|
||||
f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
|
||||
f.write("\tdefault \"$(ARCH)-linux\"\n")
|
||||
|
||||
f.write("\n")
|
||||
|
||||
f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
|
||||
f.write("\tdefault \"toolchain-external-bootlin\"\n")
|
||||
|
||||
f.write("\n")
|
||||
|
||||
f.write("choice\n")
|
||||
f.write("\tprompt \"Bootlin toolchain variant\"\n")
|
||||
|
||||
for toolchain in toolchains:
|
||||
toolchain.gen_config_in_options(f)
|
||||
|
||||
f.write("endchoice\n")
|
||||
f.write("endif\n")
|
||||
|
||||
|
||||
def gen_mk(toolchains, fpath):
|
||||
with open(fpath, "w") as f:
|
||||
f.write("#" * 80 + "\n")
|
||||
f.write("#\n")
|
||||
f.write("# toolchain-external-bootlin\n")
|
||||
f.write("#\n")
|
||||
f.write("#" * 80 + "\n")
|
||||
f.write("\n")
|
||||
f.write(AUTOGENERATED_COMMENT)
|
||||
for toolchain in toolchains:
|
||||
toolchain.gen_mk(f)
|
||||
f.write("$(eval $(toolchain-external-package))\n")
|
||||
|
||||
|
||||
def gen_hash(toolchains, fpath):
|
||||
with open(fpath, "w") as f:
|
||||
f.write(AUTOGENERATED_COMMENT)
|
||||
for toolchain in toolchains:
|
||||
toolchain.gen_hash(f)
|
||||
|
||||
|
||||
def gen_runtime_test(toolchains, fpath):
|
||||
with open(fpath, "w") as f:
|
||||
f.write(AUTOGENERATED_COMMENT)
|
||||
f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
|
||||
for toolchain in toolchains:
|
||||
toolchain.gen_test(f)
|
||||
|
||||
|
||||
def gen_toolchains(toolchains):
|
||||
maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
|
||||
gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
|
||||
gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
|
||||
gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
|
||||
gen_runtime_test(toolchains,
|
||||
os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
|
||||
|
||||
|
||||
toolchains = get_toolchains()
|
||||
gen_toolchains(toolchains)
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
from cpedb import CPEDB, CPE
|
||||
|
||||
|
||||
def gen_update_xml_reports(cpeids, cpedb, output):
|
||||
cpe_need_update = []
|
||||
|
||||
for cpe in cpeids:
|
||||
result = cpedb.find(cpe)
|
||||
if not result:
|
||||
result = cpedb.find_partial(CPE.no_version(cpe))
|
||||
if result:
|
||||
cpe_need_update.append(cpe)
|
||||
else:
|
||||
print("WARNING: no match found for '%s'" % cpe)
|
||||
|
||||
for cpe in cpe_need_update:
|
||||
xml = cpedb.gen_update_xml(cpe)
|
||||
fname = CPE.product(cpe) + '-' + CPE.version(cpe) + '.xml'
|
||||
print("Generating %s" % fname)
|
||||
with open(os.path.join(output, fname), 'w+') as fp:
|
||||
fp.write(xml)
|
||||
|
||||
print("Generated %d update files out of %d CPEs" % (len(cpe_need_update), len(cpeids)))
|
||||
|
||||
|
||||
def get_cpe_ids():
|
||||
print("Getting list of CPE for enabled packages")
|
||||
cmd = ["make", "--no-print-directory", "show-info"]
|
||||
js = json.loads(subprocess.check_output(cmd).decode("utf-8"))
|
||||
return set([v["cpe-id"] for k, v in js.items() if "cpe-id" in v])
|
||||
|
||||
|
||||
def resolvepath(path):
|
||||
return os.path.abspath(os.path.expanduser(path))
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output', dest='output',
|
||||
help='Path to the output CPE update files', type=resolvepath, required=True)
|
||||
parser.add_argument('--nvd-path', dest='nvd_path',
|
||||
help='Path to the local NVD database', type=resolvepath, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def __main__():
|
||||
args = parse_args()
|
||||
if not os.path.isdir(args.output):
|
||||
print("ERROR: output directory %s does not exist" % args.output)
|
||||
sys.exit(1)
|
||||
cpedb = CPEDB(args.nvd_path)
|
||||
cpedb.get_xml_dict()
|
||||
cpeids = get_cpe_ids()
|
||||
gen_update_xml_reports(cpeids, cpedb, args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
__main__()
|
||||
Executable
+138
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
main() {
|
||||
local template="${1}"
|
||||
|
||||
preamble "${template}"
|
||||
gen_tests
|
||||
}
|
||||
|
||||
preamble() {
|
||||
local template="${1}"
|
||||
|
||||
cat - "${template}" <<-_EOF_
|
||||
# This file is generated; do not edit!
|
||||
# Builds appear on https://gitlab.com/buildroot.org/buildroot/pipelines
|
||||
|
||||
image: ${CI_JOB_IMAGE}
|
||||
|
||||
_EOF_
|
||||
}
|
||||
|
||||
gen_tests() {
|
||||
local -a basics defconfigs runtimes
|
||||
local do_basics do_defconfigs do_runtime do_testpkg
|
||||
local defconfigs_ext cfg tst
|
||||
|
||||
basics=( DEVELOPERS flake8 package )
|
||||
|
||||
defconfigs=( $(cd configs; LC_ALL=C ls -1 *_defconfig) )
|
||||
|
||||
runtimes=( $(./support/testing/run-tests -l 2>&1 \
|
||||
| sed -r -e '/^test_run \((.*)\).*/!d; s//\1/' \
|
||||
| LC_ALL=C sort)
|
||||
)
|
||||
|
||||
if [ -n "${CI_COMMIT_TAG}" ]; then
|
||||
# When a tag is added to the Buildroot git tree, we want
|
||||
# to run the runtime tests and only test Qemu defconfigs.
|
||||
defconfigs=( $(cd configs; LC_ALL=C ls -1 qemu_*_defconfig) )
|
||||
do_basics=true
|
||||
do_defconfigs=base
|
||||
do_runtime=true
|
||||
elif [ "${CI_PIPELINE_SOURCE}" = "trigger" ]; then
|
||||
case "${BR_SCHEDULE_JOBS}" in
|
||||
(basic)
|
||||
do_basics=true
|
||||
do_defconfigs=check
|
||||
defconfigs_ext=_check
|
||||
;;
|
||||
(defconfig)
|
||||
do_defconfigs=base
|
||||
;;
|
||||
(runtime)
|
||||
do_runtime=true
|
||||
;;
|
||||
esac
|
||||
else
|
||||
case "${CI_COMMIT_REF_NAME}" in
|
||||
(*-basics)
|
||||
do_basics=true
|
||||
do_defconfigs=check
|
||||
defconfigs_ext=_check
|
||||
;;
|
||||
(*-defconfigs)
|
||||
do_defconfigs=base
|
||||
;;
|
||||
(*-*_defconfig)
|
||||
defconfigs=( "${CI_COMMIT_REF_NAME##*-}" )
|
||||
do_defconfigs=base
|
||||
;;
|
||||
(*-runtime-tests)
|
||||
do_runtime=true
|
||||
;;
|
||||
(*-tests.*)
|
||||
runtimes=( $(./support/testing/run-tests -l 2>&1 \
|
||||
| sed -r -e '/^test_run \((.*)\).*/!d; s//\1/' \
|
||||
| LC_ALL=C sort \
|
||||
| grep "^${CI_COMMIT_REF_NAME##*-}")
|
||||
)
|
||||
do_runtime=true
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Retrieve defconfig for test-pkg from the git commit message (if any)
|
||||
if grep -q -E '^test-pkg config:$' <<<"${CI_COMMIT_DESCRIPTION}"; then
|
||||
sed -r -n -e '/^test-pkg config:$/{:a;n;p;ba;}' \
|
||||
<<<"${CI_COMMIT_DESCRIPTION}" \
|
||||
>defconfig.frag
|
||||
if [ ! -s defconfig.frag ]; then
|
||||
printf "Empty configuration fragment.\n" >&2; exit 1
|
||||
fi
|
||||
# Use --all since we expect the user having already pre-tested the
|
||||
# new package with the default subset of toolchains.
|
||||
./utils/test-pkg \
|
||||
--all --prepare-only \
|
||||
--config-snippet defconfig.frag \
|
||||
--build-dir br-test-pkg >&2
|
||||
do_testpkg=( $(ls -1 br-test-pkg/*/.config 2>/dev/null |xargs -r dirname ) )
|
||||
if [ "${#do_testpkg[@]}" -eq 0 ]; then
|
||||
printf "Configuration fragment enables no test.\n" >&2; exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# If nothing else, at least do the basics to generate a valid pipeline
|
||||
if [ -z "${do_defconfigs}" \
|
||||
-a -z "${do_runtime}" \
|
||||
-a -z "${do_testpkg}" \
|
||||
]
|
||||
then
|
||||
do_basics=true
|
||||
fi
|
||||
|
||||
if ${do_basics:-false}; then
|
||||
for tst in "${basics[@]}"; do
|
||||
printf 'check-%s: { extends: .check-%s_base }\n' "${tst}" "${tst}"
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -n "${do_defconfigs}" ]; then
|
||||
for cfg in "${defconfigs[@]}"; do
|
||||
printf '%s%s: { extends: .defconfig_%s }\n' \
|
||||
"${cfg}" "${defconfigs_ext}" "${do_defconfigs}"
|
||||
done
|
||||
fi
|
||||
|
||||
if ${do_runtime:-false}; then
|
||||
printf '%s: { extends: .runtime_test_base }\n' "${runtimes[@]}"
|
||||
fi
|
||||
|
||||
if [ -n "${do_testpkg}" ]; then
|
||||
printf '%s: { extends: .test_pkg }\n' "${do_testpkg[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
main "${@}"
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
die() {
|
||||
cat <<EOF >&2
|
||||
Error: $@
|
||||
|
||||
Usage: ${0} -c GENIMAGE_CONFIG_FILE
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse arguments and put into argument list of the script
|
||||
opts="$(getopt -n "${0##*/}" -o c: -- "$@")" || exit $?
|
||||
eval set -- "$opts"
|
||||
|
||||
GENIMAGE_TMP="${BUILD_DIR}/genimage.tmp"
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-c)
|
||||
GENIMAGE_CFG="${2}";
|
||||
shift 2 ;;
|
||||
--) # Discard all non-option parameters
|
||||
shift 1;
|
||||
break ;;
|
||||
*)
|
||||
die "unknown option '${1}'" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -n "${GENIMAGE_CFG}" ] || die "Missing argument"
|
||||
|
||||
# Pass an empty rootpath. genimage makes a full copy of the given rootpath to
|
||||
# ${GENIMAGE_TMP}/root so passing TARGET_DIR would be a waste of time and disk
|
||||
# space. We don't rely on genimage to build the rootfs image, just to insert a
|
||||
# pre-built one in the disk image.
|
||||
|
||||
trap 'rm -rf "${ROOTPATH_TMP}"' EXIT
|
||||
ROOTPATH_TMP="$(mktemp -d)"
|
||||
|
||||
rm -rf "${GENIMAGE_TMP}"
|
||||
|
||||
genimage \
|
||||
--rootpath "${ROOTPATH_TMP}" \
|
||||
--tmppath "${GENIMAGE_TMP}" \
|
||||
--inputpath "${BINARIES_DIR}" \
|
||||
--outputpath "${BINARIES_DIR}" \
|
||||
--config "${GENIMAGE_CFG}"
|
||||
Executable
+312
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) 2011 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
||||
# Copyright (C) 2013 by Yann E. MORIN <yann.morin.1998@free.fr>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
# This script generates graphs of packages build time, from the timing
|
||||
# data generated by Buildroot in the $(O)/build-time.log file.
|
||||
#
|
||||
# Example usage:
|
||||
#
|
||||
# cat $(O)/build-time.log | ./support/scripts/graph-build-time --type=histogram --output=foobar.pdf
|
||||
#
|
||||
# Three graph types are available :
|
||||
#
|
||||
# * histogram, which creates an histogram of the build time for each
|
||||
# package, decomposed by each step (extract, patch, configure,
|
||||
# etc.). The order in which the packages are shown is
|
||||
# configurable: by package name, by build order, or by duration
|
||||
# order. See the --order option.
|
||||
#
|
||||
# * pie-packages, which creates a pie chart of the build time of
|
||||
# each package (without decomposition in steps). Packages that
|
||||
# contributed to less than 1% of the overall build time are all
|
||||
# grouped together in an "Other" entry.
|
||||
#
|
||||
# * pie-steps, which creates a pie chart of the time spent globally
|
||||
# on each step (extract, patch, configure, etc...)
|
||||
#
|
||||
# The default is to generate an histogram ordered by package name.
|
||||
#
|
||||
# Requirements:
|
||||
#
|
||||
# * matplotlib (python-matplotlib on Debian/Ubuntu systems)
|
||||
# * numpy (python-numpy on Debian/Ubuntu systems)
|
||||
# * argparse (by default in Python 2.7, requires python-argparse if
|
||||
# Python 2.6 is used)
|
||||
|
||||
import sys
|
||||
|
||||
try:
|
||||
import matplotlib as mpl
|
||||
import numpy
|
||||
except ImportError:
|
||||
sys.stderr.write("You need python-matplotlib and python-numpy to generate build graphs\n")
|
||||
exit(1)
|
||||
|
||||
# Use the Agg backend (which produces a PNG output, see
|
||||
# http://matplotlib.org/faq/usage_faq.html#what-is-a-backend),
|
||||
# otherwise an incorrect backend is used on some host machines).
|
||||
# Note: matplotlib.use() must be called *before* matplotlib.pyplot.
|
||||
mpl.use('Agg')
|
||||
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
import matplotlib.font_manager as fm # noqa: E402
|
||||
import csv # noqa: E402
|
||||
import argparse # noqa: E402
|
||||
|
||||
steps = ['download', 'extract', 'patch', 'configure', 'build',
|
||||
'install-target', 'install-staging', 'install-images',
|
||||
'install-host']
|
||||
|
||||
default_colors = ['#8d02ff', '#e60004', '#009836', '#2e1d86', '#ffed00',
|
||||
'#0068b5', '#f28e00', '#940084', '#97c000']
|
||||
|
||||
alternate_colors = ['#ffbe0a', '#96bdff', '#3f7f7f', '#ff0000', '#00c000',
|
||||
'#0080ff', '#c000ff', '#00eeee', '#e0e000']
|
||||
|
||||
|
||||
class Package:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.steps_duration = {}
|
||||
self.steps_start = {}
|
||||
self.steps_end = {}
|
||||
|
||||
def add_step(self, step, state, time):
|
||||
if state == "start":
|
||||
self.steps_start[step] = time
|
||||
else:
|
||||
self.steps_end[step] = time
|
||||
if step in self.steps_start and step in self.steps_end:
|
||||
self.steps_duration[step] = self.steps_end[step] - self.steps_start[step]
|
||||
|
||||
def get_duration(self, step=None):
|
||||
if step is None:
|
||||
duration = 0
|
||||
for step in list(self.steps_duration.keys()):
|
||||
duration += self.steps_duration[step]
|
||||
return duration
|
||||
if step in self.steps_duration:
|
||||
return self.steps_duration[step]
|
||||
return 0
|
||||
|
||||
|
||||
# Generate an histogram of the time spent in each step of each
|
||||
# package.
|
||||
def pkg_histogram(data, output, order="build"):
|
||||
n_pkgs = len(data)
|
||||
ind = numpy.arange(n_pkgs)
|
||||
|
||||
if order == "duration":
|
||||
data = sorted(data, key=lambda p: p.get_duration(), reverse=True)
|
||||
elif order == "name":
|
||||
data = sorted(data, key=lambda p: p.name, reverse=False)
|
||||
|
||||
# Prepare the vals array, containing one entry for each step
|
||||
vals = []
|
||||
for step in steps:
|
||||
val = []
|
||||
for p in data:
|
||||
val.append(p.get_duration(step))
|
||||
vals.append(val)
|
||||
|
||||
bottom = [0] * n_pkgs
|
||||
legenditems = []
|
||||
|
||||
plt.figure()
|
||||
|
||||
# Draw the bars, step by step
|
||||
for i in range(0, len(vals)):
|
||||
b = plt.bar(ind+0.1, vals[i], width=0.8, color=colors[i], bottom=bottom, linewidth=0.25)
|
||||
legenditems.append(b[0])
|
||||
bottom = [bottom[j] + vals[i][j] for j in range(0, len(vals[i]))]
|
||||
|
||||
# Draw the package names
|
||||
plt.xticks(ind + .6, [p.name for p in data], rotation=-60, rotation_mode="anchor", fontsize=8, ha='left')
|
||||
|
||||
# Adjust size of graph depending on the number of packages
|
||||
# Ensure a minimal size twice as the default
|
||||
# Magic Numbers do Magic Layout!
|
||||
ratio = max(((n_pkgs + 10) / 48, 2))
|
||||
borders = 0.1 / ratio
|
||||
sz = plt.gcf().get_figwidth()
|
||||
plt.gcf().set_figwidth(sz * ratio)
|
||||
|
||||
# Adjust space at borders, add more space for the
|
||||
# package names at the bottom
|
||||
plt.gcf().subplots_adjust(bottom=0.2, left=borders, right=1-borders)
|
||||
|
||||
# Remove ticks in the graph for each package
|
||||
axes = plt.gcf().gca()
|
||||
for line in axes.get_xticklines():
|
||||
line.set_markersize(0)
|
||||
|
||||
axes.set_ylabel('Time (seconds)')
|
||||
|
||||
# Reduce size of legend text
|
||||
leg_prop = fm.FontProperties(size=6)
|
||||
|
||||
# Draw legend
|
||||
plt.legend(legenditems, steps, prop=leg_prop)
|
||||
|
||||
if order == "name":
|
||||
plt.title('Build time of packages\n')
|
||||
elif order == "build":
|
||||
plt.title('Build time of packages, by build order\n')
|
||||
elif order == "duration":
|
||||
plt.title('Build time of packages, by duration order\n')
|
||||
|
||||
# Save graph
|
||||
plt.savefig(output)
|
||||
|
||||
|
||||
# Generate a pie chart with the time spent building each package.
|
||||
def pkg_pie_time_per_package(data, output):
|
||||
# Compute total build duration
|
||||
total = 0
|
||||
for p in data:
|
||||
total += p.get_duration()
|
||||
|
||||
# Build the list of labels and values, and filter the packages
|
||||
# that account for less than 1% of the build time.
|
||||
labels = []
|
||||
values = []
|
||||
other_value = 0
|
||||
for p in sorted(data, key=lambda p: p.get_duration()):
|
||||
if p.get_duration() < (total * 0.01):
|
||||
other_value += p.get_duration()
|
||||
else:
|
||||
labels.append(p.name)
|
||||
values.append(p.get_duration())
|
||||
|
||||
labels.append('Other')
|
||||
values.append(other_value)
|
||||
|
||||
plt.figure()
|
||||
|
||||
# Draw pie graph
|
||||
patches, texts, autotexts = plt.pie(values, labels=labels,
|
||||
autopct='%1.1f%%', shadow=True,
|
||||
colors=colors)
|
||||
|
||||
# Reduce text size
|
||||
proptease = fm.FontProperties()
|
||||
proptease.set_size('xx-small')
|
||||
plt.setp(autotexts, fontproperties=proptease)
|
||||
plt.setp(texts, fontproperties=proptease)
|
||||
|
||||
plt.title('Build time per package')
|
||||
plt.savefig(output)
|
||||
|
||||
|
||||
# Generate a pie chart with a portion for the overall time spent in
|
||||
# each step for all packages.
|
||||
def pkg_pie_time_per_step(data, output):
|
||||
steps_values = []
|
||||
for step in steps:
|
||||
val = 0
|
||||
for p in data:
|
||||
val += p.get_duration(step)
|
||||
steps_values.append(val)
|
||||
|
||||
plt.figure()
|
||||
|
||||
# Draw pie graph
|
||||
patches, texts, autotexts = plt.pie(steps_values, labels=steps,
|
||||
autopct='%1.1f%%', shadow=True,
|
||||
colors=colors)
|
||||
|
||||
# Reduce text size
|
||||
proptease = fm.FontProperties()
|
||||
proptease.set_size('xx-small')
|
||||
plt.setp(autotexts, fontproperties=proptease)
|
||||
plt.setp(texts, fontproperties=proptease)
|
||||
|
||||
plt.title('Build time per step')
|
||||
plt.savefig(output)
|
||||
|
||||
|
||||
# Parses the csv file passed on standard input and returns a list of
|
||||
# Package objects, filed with the duration of each step and the total
|
||||
# duration of the package.
|
||||
def read_data(input_file):
|
||||
if input_file is None:
|
||||
input_file = sys.stdin
|
||||
else:
|
||||
input_file = open(input_file)
|
||||
reader = csv.reader(input_file, delimiter=':')
|
||||
pkgs = []
|
||||
|
||||
# Auxilliary function to find a package by name in the list.
|
||||
def getpkg(name):
|
||||
for p in pkgs:
|
||||
if p.name == name:
|
||||
return p
|
||||
return None
|
||||
|
||||
for row in reader:
|
||||
time = float(row[0].strip())
|
||||
state = row[1].strip()
|
||||
step = row[2].strip()
|
||||
pkg = row[3].strip()
|
||||
|
||||
p = getpkg(pkg)
|
||||
if p is None:
|
||||
p = Package(pkg)
|
||||
pkgs.append(p)
|
||||
|
||||
p.add_step(step, state, time)
|
||||
|
||||
return pkgs
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description='Draw build time graphs')
|
||||
parser.add_argument("--type", '-t', metavar="GRAPH_TYPE",
|
||||
help="Type of graph (histogram, pie-packages, pie-steps)")
|
||||
parser.add_argument("--order", '-O', metavar="GRAPH_ORDER",
|
||||
help="Ordering of packages: build or duration (for histogram only)")
|
||||
parser.add_argument("--alternate-colors", '-c', action="store_true",
|
||||
help="Use alternate colour-scheme")
|
||||
parser.add_argument("--input", '-i', metavar="INPUT",
|
||||
help="Input file (usually $(O)/build/build-time.log)")
|
||||
parser.add_argument("--output", '-o', metavar="OUTPUT", required=True,
|
||||
help="Output file (.pdf or .png extension)")
|
||||
args = parser.parse_args()
|
||||
|
||||
d = read_data(args.input)
|
||||
|
||||
if args.alternate_colors:
|
||||
colors = alternate_colors
|
||||
else:
|
||||
colors = default_colors
|
||||
|
||||
if args.type == "histogram" or args.type is None:
|
||||
if args.order == "build" or args.order == "duration" or args.order == "name":
|
||||
pkg_histogram(d, args.output, args.order)
|
||||
elif args.order is None:
|
||||
pkg_histogram(d, args.output, "name")
|
||||
else:
|
||||
sys.stderr.write("Unknown ordering: %s\n" % args.order)
|
||||
exit(1)
|
||||
elif args.type == "pie-packages":
|
||||
pkg_pie_time_per_package(d, args.output)
|
||||
elif args.type == "pie-steps":
|
||||
pkg_pie_time_per_step(d, args.output)
|
||||
else:
|
||||
sys.stderr.write("Unknown type: %s\n" % args.type)
|
||||
exit(1)
|
||||
Executable
+357
@@ -0,0 +1,357 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Usage (the graphviz package must be installed in your distribution)
|
||||
# ./support/scripts/graph-depends [-p package-name] > test.dot
|
||||
# dot -Tpdf test.dot -o test.pdf
|
||||
#
|
||||
# With no arguments, graph-depends will draw a complete graph of
|
||||
# dependencies for the current configuration.
|
||||
# If '-p <package-name>' is specified, graph-depends will draw a graph
|
||||
# of dependencies for the given package name.
|
||||
# If '-d <depth>' is specified, graph-depends will limit the depth of
|
||||
# the dependency graph to 'depth' levels.
|
||||
#
|
||||
# Limitations
|
||||
#
|
||||
# * Some packages have dependencies that depend on the Buildroot
|
||||
# configuration. For example, many packages have a dependency on
|
||||
# openssl if openssl has been enabled. This tool will graph the
|
||||
# dependencies as they are with the current Buildroot
|
||||
# configuration.
|
||||
#
|
||||
# Copyright (C) 2010-2013 Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
||||
# Copyright (C) 2019 Yann E. MORIN <yann.morin.1998@free.fr>
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import argparse
|
||||
from fnmatch import fnmatch
|
||||
|
||||
import brpkgutil
|
||||
|
||||
# Modes of operation:
|
||||
MODE_FULL = 1 # draw full dependency graph for all selected packages
|
||||
MODE_PKG = 2 # draw dependency graph for a given package
|
||||
|
||||
allpkgs = []
|
||||
|
||||
|
||||
# The Graphviz "dot" utility doesn't like dashes in node names. So for
|
||||
# node names, we strip all dashes. Also, nodes can't start with a number,
|
||||
# so we prepend an underscore.
|
||||
def pkg_node_name(pkg):
|
||||
return "_" + pkg.replace("-", "")
|
||||
|
||||
|
||||
# Basic cache for the results of the is_dep() function, in order to
|
||||
# optimize the execution time. The cache is a dict of dict of boolean
|
||||
# values. The key to the primary dict is "pkg", and the key of the
|
||||
# sub-dicts is "pkg2".
|
||||
is_dep_cache = {}
|
||||
|
||||
|
||||
def is_dep_cache_insert(pkg, pkg2, val):
|
||||
try:
|
||||
is_dep_cache[pkg].update({pkg2: val})
|
||||
except KeyError:
|
||||
is_dep_cache[pkg] = {pkg2: val}
|
||||
|
||||
|
||||
# Retrieves from the cache whether pkg2 is a transitive dependency
|
||||
# of pkg.
|
||||
# Note: raises a KeyError exception if the dependency is not known.
|
||||
def is_dep_cache_lookup(pkg, pkg2):
|
||||
return is_dep_cache[pkg][pkg2]
|
||||
|
||||
|
||||
# This function return True if pkg is a dependency (direct or
|
||||
# transitive) of pkg2, dependencies being listed in the deps
|
||||
# dictionary. Returns False otherwise.
|
||||
# This is the un-cached version.
|
||||
def is_dep_uncached(pkg, pkg2, deps):
|
||||
try:
|
||||
for p in deps[pkg2]:
|
||||
if pkg == p:
|
||||
return True
|
||||
if is_dep(pkg, p, deps):
|
||||
return True
|
||||
except KeyError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
# See is_dep_uncached() above; this is the cached version.
|
||||
def is_dep(pkg, pkg2, deps):
|
||||
try:
|
||||
return is_dep_cache_lookup(pkg, pkg2)
|
||||
except KeyError:
|
||||
val = is_dep_uncached(pkg, pkg2, deps)
|
||||
is_dep_cache_insert(pkg, pkg2, val)
|
||||
return val
|
||||
|
||||
|
||||
# This function eliminates transitive dependencies; for example, given
|
||||
# these dependency chain: A->{B,C} and B->{C}, the A->{C} dependency is
|
||||
# already covered by B->{C}, so C is a transitive dependency of A, via B.
|
||||
# The functions does:
|
||||
# - for each dependency d[i] of the package pkg
|
||||
# - if d[i] is a dependency of any of the other dependencies d[j]
|
||||
# - do not keep d[i]
|
||||
# - otherwise keep d[i]
|
||||
def remove_transitive_deps(pkg, deps):
|
||||
d = deps[pkg]
|
||||
new_d = []
|
||||
for i in range(len(d)):
|
||||
keep_me = True
|
||||
for j in range(len(d)):
|
||||
if j == i:
|
||||
continue
|
||||
if is_dep(d[i], d[j], deps):
|
||||
keep_me = False
|
||||
if keep_me:
|
||||
new_d.append(d[i])
|
||||
return new_d
|
||||
|
||||
|
||||
# List of dependencies that all/many packages have, and that we want
|
||||
# to trim when generating the dependency graph.
|
||||
MANDATORY_DEPS = ['toolchain', 'skeleton', 'host-skeleton', 'host-tar', 'host-gzip', 'host-ccache']
|
||||
|
||||
|
||||
# This function removes the dependency on some 'mandatory' package, like the
|
||||
# 'toolchain' package, or the 'skeleton' package
|
||||
def remove_mandatory_deps(pkg, deps):
|
||||
return [p for p in deps[pkg] if p not in MANDATORY_DEPS]
|
||||
|
||||
|
||||
# This function returns all dependencies of pkg that are part of the
|
||||
# mandatory dependencies:
|
||||
def get_mandatory_deps(pkg, deps):
|
||||
return [p for p in deps[pkg] if p in MANDATORY_DEPS]
|
||||
|
||||
|
||||
# This function will check that there is no loop in the dependency chain
|
||||
# As a side effect, it builds up the dependency cache.
|
||||
def check_circular_deps(deps):
|
||||
def recurse(pkg):
|
||||
if pkg not in list(deps.keys()):
|
||||
return
|
||||
if pkg in not_loop:
|
||||
return
|
||||
not_loop.append(pkg)
|
||||
chain.append(pkg)
|
||||
for p in deps[pkg]:
|
||||
if p in chain:
|
||||
logging.warning("\nRecursion detected for : %s" % (p))
|
||||
while True:
|
||||
_p = chain.pop()
|
||||
logging.warning("which is a dependency of: %s" % (_p))
|
||||
if p == _p:
|
||||
sys.exit(1)
|
||||
recurse(p)
|
||||
chain.pop()
|
||||
|
||||
not_loop = []
|
||||
chain = []
|
||||
for pkg in list(deps.keys()):
|
||||
recurse(pkg)
|
||||
|
||||
|
||||
# This functions trims down the dependency list of all packages.
|
||||
# It applies in sequence all the dependency-elimination methods.
|
||||
def remove_extra_deps(deps, rootpkg, transitive, arrow_dir):
|
||||
# For the direct dependencies, find and eliminate mandatory
|
||||
# deps, and add them to the root package. Don't do it for a
|
||||
# reverse graph, because mandatory deps are only direct deps.
|
||||
if arrow_dir == "forward":
|
||||
for pkg in list(deps.keys()):
|
||||
if not pkg == rootpkg:
|
||||
for d in get_mandatory_deps(pkg, deps):
|
||||
if d not in deps[rootpkg]:
|
||||
deps[rootpkg].append(d)
|
||||
deps[pkg] = remove_mandatory_deps(pkg, deps)
|
||||
for pkg in list(deps.keys()):
|
||||
if not transitive or pkg == rootpkg:
|
||||
deps[pkg] = remove_transitive_deps(pkg, deps)
|
||||
return deps
|
||||
|
||||
|
||||
# Print the attributes of a node: label and fill-color
|
||||
def print_attrs(outfile, pkg, pkg_type, pkg_version, depth, colors):
|
||||
name = pkg_node_name(pkg)
|
||||
if pkg == 'all':
|
||||
label = 'ALL'
|
||||
else:
|
||||
label = pkg
|
||||
if depth == 0:
|
||||
color = colors[0]
|
||||
else:
|
||||
if pkg_type == "host":
|
||||
color = colors[2]
|
||||
else:
|
||||
color = colors[1]
|
||||
if pkg_version == "virtual":
|
||||
outfile.write("%s [label = <<I>%s</I>>]\n" % (name, label))
|
||||
else:
|
||||
outfile.write("%s [label = \"%s\"]\n" % (name, label))
|
||||
outfile.write("%s [color=%s,style=filled]\n" % (name, color))
|
||||
|
||||
|
||||
done_deps = []
|
||||
|
||||
|
||||
# Print the dependency graph of a package
|
||||
def print_pkg_deps(outfile, dict_deps, dict_types, dict_versions, stop_list, exclude_list,
|
||||
arrow_dir, draw_graph, depth, max_depth, pkg, colors):
|
||||
if pkg in done_deps:
|
||||
return
|
||||
done_deps.append(pkg)
|
||||
if draw_graph:
|
||||
print_attrs(outfile, pkg, dict_types[pkg], dict_versions[pkg], depth, colors)
|
||||
elif depth != 0:
|
||||
outfile.write("%s " % pkg)
|
||||
if pkg not in dict_deps:
|
||||
return
|
||||
for p in stop_list:
|
||||
if fnmatch(pkg, p):
|
||||
return
|
||||
if dict_versions[pkg] == "virtual" and "virtual" in stop_list:
|
||||
return
|
||||
if dict_types[pkg] == "host" and "host" in stop_list:
|
||||
return
|
||||
if max_depth == 0 or depth < max_depth:
|
||||
for d in dict_deps[pkg]:
|
||||
if dict_versions[d] == "virtual" and "virtual" in exclude_list:
|
||||
continue
|
||||
if dict_types[d] == "host" and "host" in exclude_list:
|
||||
continue
|
||||
add = True
|
||||
for p in exclude_list:
|
||||
if fnmatch(d, p):
|
||||
add = False
|
||||
break
|
||||
if add:
|
||||
if draw_graph:
|
||||
outfile.write("%s -> %s [dir=%s]\n" % (pkg_node_name(pkg), pkg_node_name(d), arrow_dir))
|
||||
print_pkg_deps(outfile, dict_deps, dict_types, dict_versions, stop_list, exclude_list,
|
||||
arrow_dir, draw_graph, depth + 1, max_depth, d, colors)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Graph packages dependencies")
|
||||
parser.add_argument("--check-only", "-C", dest="check_only", action="store_true", default=False,
|
||||
help="Only do the dependency checks (circular deps...)")
|
||||
parser.add_argument("--outfile", "-o", metavar="OUT_FILE", dest="outfile",
|
||||
help="File in which to generate the dot representation")
|
||||
parser.add_argument("--package", '-p', metavar="PACKAGE",
|
||||
help="Graph the dependencies of PACKAGE")
|
||||
parser.add_argument("--depth", '-d', metavar="DEPTH", dest="depth", type=int, default=0,
|
||||
help="Limit the dependency graph to DEPTH levels; 0 means no limit.")
|
||||
parser.add_argument("--stop-on", "-s", metavar="PACKAGE", dest="stop_list", action="append",
|
||||
help="Do not graph past this package (can be given multiple times)." +
|
||||
" Can be a package name or a glob, " +
|
||||
" 'virtual' to stop on virtual packages, or " +
|
||||
"'host' to stop on host packages.")
|
||||
parser.add_argument("--exclude", "-x", metavar="PACKAGE", dest="exclude_list", action="append",
|
||||
help="Like --stop-on, but do not add PACKAGE to the graph.")
|
||||
parser.add_argument("--exclude-mandatory", "-X", action="store_true",
|
||||
help="Like if -x was passed for all mandatory dependencies.")
|
||||
parser.add_argument("--colors", "-c", metavar="COLOR_LIST", dest="colors",
|
||||
default="lightblue,grey,gainsboro",
|
||||
help="Comma-separated list of the three colors to use" +
|
||||
" to draw the top-level package, the target" +
|
||||
" packages, and the host packages, in this order." +
|
||||
" Defaults to: 'lightblue,grey,gainsboro'")
|
||||
parser.add_argument("--transitive", dest="transitive", action='store_true',
|
||||
default=False)
|
||||
parser.add_argument("--no-transitive", dest="transitive", action='store_false',
|
||||
help="Draw (do not draw) transitive dependencies")
|
||||
parser.add_argument("--direct", dest="direct", action='store_true', default=True,
|
||||
help="Draw direct dependencies (the default)")
|
||||
parser.add_argument("--reverse", dest="direct", action='store_false',
|
||||
help="Draw reverse dependencies")
|
||||
parser.add_argument("--quiet", '-q', dest="quiet", action='store_true',
|
||||
help="Quiet")
|
||||
parser.add_argument("--flat-list", '-f', dest="flat_list", action='store_true', default=False,
|
||||
help="Do not draw graph, just print a flat list")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
check_only = args.check_only
|
||||
|
||||
logging.basicConfig(stream=sys.stderr, format='%(message)s',
|
||||
level=logging.WARNING if args.quiet else logging.INFO)
|
||||
|
||||
if args.outfile is None:
|
||||
outfile = sys.stdout
|
||||
else:
|
||||
if check_only:
|
||||
logging.error("don't specify outfile and check-only at the same time")
|
||||
sys.exit(1)
|
||||
outfile = open(args.outfile, "w")
|
||||
|
||||
if args.package is None:
|
||||
mode = MODE_FULL
|
||||
rootpkg = 'all'
|
||||
else:
|
||||
mode = MODE_PKG
|
||||
rootpkg = args.package
|
||||
|
||||
if args.stop_list is None:
|
||||
stop_list = []
|
||||
else:
|
||||
stop_list = args.stop_list
|
||||
|
||||
if args.exclude_list is None:
|
||||
exclude_list = []
|
||||
else:
|
||||
exclude_list = args.exclude_list
|
||||
|
||||
if args.exclude_mandatory:
|
||||
exclude_list += MANDATORY_DEPS
|
||||
|
||||
if args.direct:
|
||||
arrow_dir = "forward"
|
||||
else:
|
||||
if mode == MODE_FULL:
|
||||
logging.error("--reverse needs a package")
|
||||
sys.exit(1)
|
||||
arrow_dir = "back"
|
||||
|
||||
draw_graph = not args.flat_list
|
||||
|
||||
# Get the colors: we need exactly three colors,
|
||||
# so no need not split more than 4
|
||||
# We'll let 'dot' validate the colors...
|
||||
colors = args.colors.split(',', 4)
|
||||
if len(colors) != 3:
|
||||
logging.error("Error: incorrect color list '%s'" % args.colors)
|
||||
sys.exit(1)
|
||||
|
||||
deps, rdeps, dict_types, dict_versions = brpkgutil.get_dependency_tree()
|
||||
dict_deps = deps if args.direct else rdeps
|
||||
|
||||
check_circular_deps(dict_deps)
|
||||
if check_only:
|
||||
sys.exit(0)
|
||||
|
||||
dict_deps = remove_extra_deps(dict_deps, rootpkg, args.transitive, arrow_dir)
|
||||
|
||||
# Start printing the graph data
|
||||
if draw_graph:
|
||||
outfile.write("digraph G {\n")
|
||||
|
||||
print_pkg_deps(outfile, dict_deps, dict_types, dict_versions, stop_list, exclude_list,
|
||||
arrow_dir, draw_graph, 0, args.depth, rootpkg, colors)
|
||||
|
||||
if draw_graph:
|
||||
outfile.write("}\n")
|
||||
else:
|
||||
outfile.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Try to hardlink a file into a directory, fallback to copy on failure.
|
||||
#
|
||||
# Hardlink-or-copy the source file in the first argument into the
|
||||
# destination directory in the second argument, using the basename in
|
||||
# the third argument as basename for the destination file. If the third
|
||||
# argument is missing, use the basename of the source file as basename
|
||||
# for the destination file.
|
||||
#
|
||||
# In either case, remove the destination prior to doing the
|
||||
# hardlink-or-copy.
|
||||
#
|
||||
# Note that this is NOT an atomic operation.
|
||||
|
||||
set -e
|
||||
|
||||
main() {
|
||||
local src_file="${1}"
|
||||
local dst_dir="${2}"
|
||||
local dst_file="${3}"
|
||||
|
||||
if [ -n "${dst_file}" ]; then
|
||||
dst_file="${dst_dir}/${dst_file}"
|
||||
else
|
||||
dst_file="${dst_dir}/${src_file##*/}"
|
||||
fi
|
||||
|
||||
mkdir -p "${dst_dir}"
|
||||
rm -f "${dst_file}"
|
||||
ln -f "${src_file}" "${dst_file}" 2>/dev/null \
|
||||
|| cp -f "${src_file}" "${dst_file}"
|
||||
}
|
||||
|
||||
main "${@}"
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/bin/sh
|
||||
# Generates a small Makefile used in the root of the output
|
||||
# directory, to allow make to be started from there.
|
||||
# The Makefile also allow for more convenient build of external modules
|
||||
|
||||
# Usage
|
||||
# $1 - Kernel src directory
|
||||
# $2 - Output directory
|
||||
|
||||
|
||||
test ! -r $2/Makefile -o -O $2/Makefile || exit 0
|
||||
# Only overwrite automatically generated Makefiles
|
||||
# (so we do not overwrite buildroot Makefile)
|
||||
if test -e $2/Makefile && ! grep -q Automatically $2/Makefile
|
||||
then
|
||||
exit 0
|
||||
fi
|
||||
echo " GEN $2/Makefile"
|
||||
|
||||
cat << EOF > $2/Makefile
|
||||
# Automatically generated by $0: don't edit
|
||||
|
||||
ifeq ("\$(origin V)", "command line")
|
||||
VERBOSE := \$(V)
|
||||
endif
|
||||
ifneq (\$(VERBOSE),1)
|
||||
Q := @
|
||||
endif
|
||||
|
||||
lastword = \$(word \$(words \$(1)),\$(1))
|
||||
makedir := \$(dir \$(call lastword,\$(MAKEFILE_LIST)))
|
||||
|
||||
MAKEARGS := -C $1
|
||||
MAKEARGS += O=\$(if \$(patsubst /%,,\$(makedir)),\$(CURDIR)/)\$(patsubst %/,%,\$(makedir))
|
||||
|
||||
MAKEFLAGS += --no-print-directory
|
||||
|
||||
.PHONY: _all \$(MAKECMDGOALS)
|
||||
|
||||
all := \$(filter-out Makefile,\$(MAKECMDGOALS))
|
||||
|
||||
_all:
|
||||
\$(Q)umask 0022 && \$(MAKE) \$(MAKEARGS) \$(all)
|
||||
|
||||
Makefile:;
|
||||
|
||||
\$(all): _all
|
||||
@:
|
||||
|
||||
%/: _all
|
||||
@:
|
||||
EOF
|
||||
Executable
+443
@@ -0,0 +1,443 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
myname="${0##*/}"
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
# Configurable items
|
||||
MIN_UID=1000
|
||||
MAX_UID=1999
|
||||
MIN_GID=1000
|
||||
MAX_GID=1999
|
||||
# No more is configurable below this point
|
||||
#----------------------------------------------------------------------------
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
error() {
|
||||
local fmt="${1}"
|
||||
shift
|
||||
|
||||
printf "%s: " "${myname}" >&2
|
||||
printf "${fmt}" "${@}" >&2
|
||||
}
|
||||
fail() {
|
||||
error "$@"
|
||||
exit 1
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
if [ ${#} -ne 2 ]; then
|
||||
fail "usage: %s USERS_TABLE TARGET_DIR\n"
|
||||
fi
|
||||
USERS_TABLE="${1}"
|
||||
TARGET_DIR="${2}"
|
||||
shift 2
|
||||
PASSWD="${TARGET_DIR}/etc/passwd"
|
||||
SHADOW="${TARGET_DIR}/etc/shadow"
|
||||
GROUP="${TARGET_DIR}/etc/group"
|
||||
# /etc/gshadow is not part of the standard skeleton, so not everybody
|
||||
# will have it, but some may have it, and its content must be in sync
|
||||
# with /etc/group, so any use of gshadow must be conditional.
|
||||
GSHADOW="${TARGET_DIR}/etc/gshadow"
|
||||
|
||||
# We can't simply source ${BR2_CONFIG} as it may contains constructs
|
||||
# such as:
|
||||
# BR2_DEFCONFIG="$(CONFIG_DIR)/defconfig"
|
||||
# which when sourced from a shell script will eventually try to execute
|
||||
# a command named 'CONFIG_DIR', which is plain wrong for virtually every
|
||||
# systems out there.
|
||||
# So, we have to scan that file instead. Sigh... :-(
|
||||
PASSWD_METHOD="$( sed -r -e '/^BR2_TARGET_GENERIC_PASSWD_METHOD="(.*)"$/!d;' \
|
||||
-e 's//\1/;' \
|
||||
"${BR2_CONFIG}" \
|
||||
)"
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
get_uid() {
|
||||
local username="${1}"
|
||||
|
||||
awk -F: -v username="${username}" \
|
||||
'$1 == username { printf( "%d\n", $3 ); }' "${PASSWD}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
get_ugid() {
|
||||
local username="${1}"
|
||||
|
||||
awk -F: -v username="${username}" \
|
||||
'$1 == username { printf( "%d\n", $4 ); }' "${PASSWD}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
get_gid() {
|
||||
local group="${1}"
|
||||
|
||||
awk -F: -v group="${group}" \
|
||||
'$1 == group { printf( "%d\n", $3 ); }' "${GROUP}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
get_members() {
|
||||
local group="${1}"
|
||||
|
||||
awk -F: -v group="${group}" \
|
||||
'$1 == group { printf( "%s\n", $4 ); }' "${GROUP}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
get_username() {
|
||||
local uid="${1}"
|
||||
|
||||
awk -F: -v uid="${uid}" \
|
||||
'$3 == uid { printf( "%s\n", $1 ); }' "${PASSWD}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
get_group() {
|
||||
local gid="${1}"
|
||||
|
||||
awk -F: -v gid="${gid}" \
|
||||
'$3 == gid { printf( "%s\n", $1 ); }' "${GROUP}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
get_ugroup() {
|
||||
local username="${1}"
|
||||
local ugid
|
||||
|
||||
ugid="$( get_ugid "${username}" )"
|
||||
if [ -n "${ugid}" ]; then
|
||||
get_group "${ugid}"
|
||||
fi
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
# Sanity-check the new user/group:
|
||||
# - check the gid is not already used for another group
|
||||
# - check the group does not already exist with another gid
|
||||
# - check the user does not already exist with another gid
|
||||
# - check the uid is not already used for another user
|
||||
# - check the user does not already exist with another uid
|
||||
# - check the user does not already exist in another group
|
||||
check_user_validity() {
|
||||
local username="${1}"
|
||||
local uid="${2}"
|
||||
local group="${3}"
|
||||
local gid="${4}"
|
||||
local _uid _ugid _gid _username _group _ugroup
|
||||
|
||||
_group="$( get_group "${gid}" )"
|
||||
_gid="$( get_gid "${group}" )"
|
||||
_ugid="$( get_ugid "${username}" )"
|
||||
_username="$( get_username "${uid}" )"
|
||||
_uid="$( get_uid "${username}" )"
|
||||
_ugroup="$( get_ugroup "${username}" )"
|
||||
|
||||
if [ "${username}" = "root" ]; then
|
||||
fail "invalid username '%s\n'" "${username}"
|
||||
fi
|
||||
|
||||
if [ ${gid} -lt -1 -o ${gid} -eq 0 ]; then
|
||||
fail "invalid gid '%d' for '%s'\n" ${gid} "${username}"
|
||||
elif [ ${gid} -ne -1 ]; then
|
||||
# check the gid is not already used for another group
|
||||
if [ -n "${_group}" -a "${_group}" != "${group}" ]; then
|
||||
fail "gid '%d' for '%s' is already used by group '%s'\n" \
|
||||
${gid} "${username}" "${_group}"
|
||||
fi
|
||||
|
||||
# check the group does not already exists with another gid
|
||||
# Need to split the check in two, otherwise '[' complains it
|
||||
# is missing arguments when _gid is empty
|
||||
if [ -n "${_gid}" ] && [ ${_gid} -ne ${gid} ]; then
|
||||
fail "group '%s' for '%s' already exists with gid '%d' (wants '%d')\n" \
|
||||
"${group}" "${username}" ${_gid} ${gid}
|
||||
fi
|
||||
|
||||
# check the user does not already exists with another gid
|
||||
# Need to split the check in two, otherwise '[' complains it
|
||||
# is missing arguments when _ugid is empty
|
||||
if [ -n "${_ugid}" ] && [ ${_ugid} -ne ${gid} ]; then
|
||||
fail "user '%s' already exists with gid '%d' (wants '%d')\n" \
|
||||
"${username}" ${_ugid} ${gid}
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${uid} -lt -1 -o ${uid} -eq 0 ]; then
|
||||
fail "invalid uid '%d' for '%s'\n" ${uid} "${username}"
|
||||
elif [ ${uid} -ne -1 ]; then
|
||||
# check the uid is not already used for another user
|
||||
if [ -n "${_username}" -a "${_username}" != "${username}" ]; then
|
||||
fail "uid '%d' for '%s' already used by user '%s'\n" \
|
||||
${uid} "${username}" "${_username}"
|
||||
fi
|
||||
|
||||
# check the user does not already exists with another uid
|
||||
# Need to split the check in two, otherwise '[' complains it
|
||||
# is missing arguments when _uid is empty
|
||||
if [ -n "${_uid}" ] && [ ${_uid} -ne ${uid} ]; then
|
||||
fail "user '%s' already exists with uid '%d' (wants '%d')\n" \
|
||||
"${username}" ${_uid} ${uid}
|
||||
fi
|
||||
fi
|
||||
|
||||
# check the user does not already exist in another group
|
||||
if [ -n "${_ugroup}" -a "${_ugroup}" != "${group}" ]; then
|
||||
fail "user '%s' already exists with group '%s' (wants '%s')\n" \
|
||||
"${username}" "${_ugroup}" "${group}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
# Generate a unique GID for given group. If the group already exists,
|
||||
# then simply report its current GID. Otherwise, generate the lowest GID
|
||||
# that is:
|
||||
# - not 0
|
||||
# - comprised in [MIN_GID..MAX_GID]
|
||||
# - not already used by a group
|
||||
generate_gid() {
|
||||
local group="${1}"
|
||||
local gid
|
||||
|
||||
gid="$( get_gid "${group}" )"
|
||||
if [ -z "${gid}" ]; then
|
||||
for(( gid=MIN_GID; gid<=MAX_GID; gid++ )); do
|
||||
if [ -z "$( get_group "${gid}" )" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ ${gid} -gt ${MAX_GID} ]; then
|
||||
fail "can not allocate a GID for group '%s'\n" "${group}"
|
||||
fi
|
||||
fi
|
||||
printf "%d\n" "${gid}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
# Add a group; if it does already exist, remove it first
|
||||
add_one_group() {
|
||||
local group="${1}"
|
||||
local gid="${2}"
|
||||
local members
|
||||
|
||||
# Generate a new GID if needed
|
||||
if [ ${gid} -eq -1 ]; then
|
||||
gid="$( generate_gid "${group}" )"
|
||||
fi
|
||||
|
||||
members=$(get_members "$group")
|
||||
# Remove any previous instance of this group, and re-add the new one
|
||||
sed -i --follow-symlinks -e '/^'"${group}"':.*/d;' "${GROUP}"
|
||||
printf "%s:x:%d:%s\n" "${group}" "${gid}" "${members}" >>"${GROUP}"
|
||||
|
||||
# Ditto for /etc/gshadow if it exists
|
||||
if [ -f "${GSHADOW}" ]; then
|
||||
sed -i --follow-symlinks -e '/^'"${group}"':.*/d;' "${GSHADOW}"
|
||||
printf "%s:*::\n" "${group}" >>"${GSHADOW}"
|
||||
fi
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
# Generate a unique UID for given username. If the username already exists,
|
||||
# then simply report its current UID. Otherwise, generate the lowest UID
|
||||
# that is:
|
||||
# - not 0
|
||||
# - comprised in [MIN_UID..MAX_UID]
|
||||
# - not already used by a user
|
||||
generate_uid() {
|
||||
local username="${1}"
|
||||
local uid
|
||||
|
||||
uid="$( get_uid "${username}" )"
|
||||
if [ -z "${uid}" ]; then
|
||||
for(( uid=MIN_UID; uid<=MAX_UID; uid++ )); do
|
||||
if [ -z "$( get_username "${uid}" )" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ ${uid} -gt ${MAX_UID} ]; then
|
||||
fail "can not allocate a UID for user '%s'\n" "${username}"
|
||||
fi
|
||||
fi
|
||||
printf "%d\n" "${uid}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
# Add given user to given group, if not already the case
|
||||
add_user_to_group() {
|
||||
local username="${1}"
|
||||
local group="${2}"
|
||||
local _f
|
||||
|
||||
for _f in "${GROUP}" "${GSHADOW}"; do
|
||||
[ -f "${_f}" ] || continue
|
||||
sed -r -i --follow-symlinks \
|
||||
-e 's/^('"${group}"':.*:)(([^:]+,)?)'"${username}"'(,[^:]+*)?$/\1\2\4/;' \
|
||||
-e 's/^('"${group}"':.*)$/\1,'"${username}"'/;' \
|
||||
-e 's/,+/,/' \
|
||||
-e 's/:,/:/' \
|
||||
"${_f}"
|
||||
done
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
# Encode a password
|
||||
encode_password() {
|
||||
local passwd="${1}"
|
||||
|
||||
mkpasswd -m "${PASSWD_METHOD}" "${passwd}"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
# Add a user; if it does already exist, remove it first
|
||||
add_one_user() {
|
||||
local username="${1}"
|
||||
local uid="${2}"
|
||||
local group="${3}"
|
||||
local gid="${4}"
|
||||
local passwd="${5}"
|
||||
local home="${6}"
|
||||
local shell="${7}"
|
||||
local groups="${8}"
|
||||
local comment="${9}"
|
||||
local _f _group _home _shell _gid _passwd
|
||||
|
||||
# First, sanity-check the user
|
||||
check_user_validity "${username}" "${uid}" "${group}" "${gid}"
|
||||
|
||||
# Generate a new UID if needed
|
||||
if [ ${uid} -eq -1 ]; then
|
||||
uid="$( generate_uid "${username}" )"
|
||||
fi
|
||||
|
||||
# Remove any previous instance of this user
|
||||
for _f in "${PASSWD}" "${SHADOW}"; do
|
||||
sed -r -i --follow-symlinks -e '/^'"${username}"':.*/d;' "${_f}"
|
||||
done
|
||||
|
||||
_gid="$( get_gid "${group}" )"
|
||||
_shell="${shell}"
|
||||
if [ "${shell}" = "-" ]; then
|
||||
_shell="/bin/false"
|
||||
fi
|
||||
case "${home}" in
|
||||
-) _home="/";;
|
||||
/) fail "home can not explicitly be '/'\n";;
|
||||
/*) _home="${home}";;
|
||||
*) fail "home must be an absolute path\n";;
|
||||
esac
|
||||
case "${passwd}" in
|
||||
-)
|
||||
_passwd=""
|
||||
;;
|
||||
!=*)
|
||||
_passwd='!'"$( encode_password "${passwd#!=}" )"
|
||||
;;
|
||||
=*)
|
||||
_passwd="$( encode_password "${passwd#=}" )"
|
||||
;;
|
||||
*)
|
||||
_passwd="${passwd}"
|
||||
;;
|
||||
esac
|
||||
|
||||
printf "%s:x:%d:%d:%s:%s:%s\n" \
|
||||
"${username}" "${uid}" "${_gid}" \
|
||||
"${comment}" "${_home}" "${_shell}" \
|
||||
>>"${PASSWD}"
|
||||
printf "%s:%s:::::::\n" \
|
||||
"${username}" "${_passwd}" \
|
||||
>>"${SHADOW}"
|
||||
|
||||
# Add the user to its additional groups
|
||||
if [ "${groups}" != "-" ]; then
|
||||
for _group in ${groups//,/ }; do
|
||||
add_user_to_group "${username}" "${_group}"
|
||||
done
|
||||
fi
|
||||
|
||||
# If the user has a home, chown it
|
||||
# (Note: stdout goes to the fakeroot-script)
|
||||
if [ "${home}" != "-" ]; then
|
||||
mkdir -p "${TARGET_DIR}/${home}"
|
||||
printf "chown -h -R %d:%d '%s'\n" "${uid}" "${_gid}" "${TARGET_DIR}/${home}"
|
||||
fi
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
main() {
|
||||
local username uid group gid passwd home shell groups comment
|
||||
local line
|
||||
local -a ENTRIES
|
||||
|
||||
# Some sanity checks
|
||||
if [ ${MIN_UID} -le 0 ]; then
|
||||
fail "MIN_UID must be >0 (currently %d)\n" ${MIN_UID}
|
||||
fi
|
||||
if [ ${MIN_GID} -le 0 ]; then
|
||||
fail "MIN_GID must be >0 (currently %d)\n" ${MIN_GID}
|
||||
fi
|
||||
|
||||
# Read in all the file in memory, exclude empty lines and comments
|
||||
while read line; do
|
||||
ENTRIES+=( "${line}" )
|
||||
done < <( sed -r -e 's/#.*//; /^[[:space:]]*$/d;' "${USERS_TABLE}" )
|
||||
|
||||
# We first create groups whose gid is not -1, and then we create groups
|
||||
# whose gid is -1 (automatic), so that, if a group is defined both with
|
||||
# a specified gid and an automatic gid, we ensure the specified gid is
|
||||
# used, rather than a different automatic gid is computed.
|
||||
|
||||
# First, create all the main groups which gid is *not* automatic
|
||||
for line in "${ENTRIES[@]}"; do
|
||||
read username uid group gid passwd home shell groups comment <<<"${line}"
|
||||
[ ${gid} -ge 0 ] || continue # Automatic gid
|
||||
add_one_group "${group}" "${gid}"
|
||||
done
|
||||
|
||||
# Then, create all the main groups which gid *is* automatic
|
||||
for line in "${ENTRIES[@]}"; do
|
||||
read username uid group gid passwd home shell groups comment <<<"${line}"
|
||||
[ ${gid} -eq -1 ] || continue # Non-automatic gid
|
||||
add_one_group "${group}" "${gid}"
|
||||
done
|
||||
|
||||
# Then, create all the additional groups
|
||||
# If any additional group is already a main group, we should use
|
||||
# the gid of that main group; otherwise, we can use any gid
|
||||
for line in "${ENTRIES[@]}"; do
|
||||
read username uid group gid passwd home shell groups comment <<<"${line}"
|
||||
if [ "${groups}" != "-" ]; then
|
||||
for g in ${groups//,/ }; do
|
||||
add_one_group "${g}" -1
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
# When adding users, we do as for groups, in case two packages create
|
||||
# the same user, one with an automatic uid, the other with a specified
|
||||
# uid, to ensure the specified uid is used, rather than an incompatible
|
||||
# uid be generated.
|
||||
|
||||
# Now, add users whose uid is *not* automatic
|
||||
for line in "${ENTRIES[@]}"; do
|
||||
read username uid group gid passwd home shell groups comment <<<"${line}"
|
||||
[ "${username}" != "-" ] || continue # Magic string to skip user creation
|
||||
[ ${uid} -ge 0 ] || continue # Automatic uid
|
||||
add_one_user "${username}" "${uid}" "${group}" "${gid}" "${passwd}" \
|
||||
"${home}" "${shell}" "${groups}" "${comment}"
|
||||
done
|
||||
|
||||
# Finally, add users whose uid *is* automatic
|
||||
for line in "${ENTRIES[@]}"; do
|
||||
read username uid group gid passwd home shell groups comment <<<"${line}"
|
||||
[ "${username}" != "-" ] || continue # Magic string to skip user creation
|
||||
[ ${uid} -eq -1 ] || continue # Non-automatic uid
|
||||
add_one_user "${username}" "${uid}" "${group}" "${gid}" "${passwd}" \
|
||||
"${home}" "${shell}" "${groups}" "${comment}"
|
||||
done
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------
|
||||
main "${@}"
|
||||
Executable
+1170
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Byte compile all .py files from provided directories. This script is an
|
||||
alternative implementation of compileall.compile_dir written with
|
||||
cross-compilation in mind.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import py_compile
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def compile_one(host_path, strip_root=None, verbose=False):
|
||||
"""
|
||||
Compile a .py file into a .pyc file located next to it.
|
||||
|
||||
:arg host_path:
|
||||
Absolute path to the file to compile on the host running the build.
|
||||
:arg strip_root:
|
||||
Prefix to remove from the original source paths encoded in compiled
|
||||
files.
|
||||
:arg verbose:
|
||||
Print compiled file paths.
|
||||
"""
|
||||
if os.path.islink(host_path) or not os.path.isfile(host_path):
|
||||
return # only compile real files
|
||||
|
||||
if not re.match(r"^[_A-Za-z][_A-Za-z0-9]*\.py$",
|
||||
os.path.basename(host_path)):
|
||||
return # only compile "importable" python modules
|
||||
|
||||
if strip_root is not None:
|
||||
# determine the runtime path of the file (i.e.: relative path to root
|
||||
# dir prepended with "/").
|
||||
runtime_path = os.path.join("/", os.path.relpath(host_path, strip_root))
|
||||
else:
|
||||
runtime_path = host_path
|
||||
|
||||
if verbose:
|
||||
print(" PYC {}".format(runtime_path))
|
||||
|
||||
# will raise an error if the file cannot be compiled
|
||||
py_compile.compile(host_path, cfile=host_path + "c",
|
||||
dfile=runtime_path, doraise=True)
|
||||
|
||||
|
||||
def existing_dir_abs(arg):
|
||||
"""
|
||||
argparse type callback that checks that argument is a directory and returns
|
||||
its absolute path.
|
||||
"""
|
||||
if not os.path.isdir(arg):
|
||||
raise argparse.ArgumentTypeError('no such directory: {!r}'.format(arg))
|
||||
return os.path.abspath(arg)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("dirs", metavar="DIR", nargs="+", type=existing_dir_abs,
|
||||
help="Directory to recursively scan and compile")
|
||||
parser.add_argument("--strip-root", metavar="ROOT", type=existing_dir_abs,
|
||||
help="""
|
||||
Prefix to remove from the original source paths encoded
|
||||
in compiled files
|
||||
""")
|
||||
parser.add_argument("--verbose", action="store_true",
|
||||
help="Print compiled files")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
for d in args.dirs:
|
||||
if args.strip_root and ".." in os.path.relpath(d, args.strip_root):
|
||||
parser.error("DIR: not inside ROOT dir: {!r}".format(d))
|
||||
for parent, _, files in os.walk(d):
|
||||
for f in files:
|
||||
compile_one(os.path.join(parent, f), args.strip_root,
|
||||
args.verbose)
|
||||
|
||||
except Exception as e:
|
||||
print("error: {}".format(e))
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# This scripts adds local version information from the version
|
||||
# control systems git, mercurial (hg) and subversion (svn).
|
||||
#
|
||||
# If something goes wrong, send a mail the kernel build mailinglist
|
||||
# (see MAINTAINERS) and CC Nico Schottelius
|
||||
# <nico-linuxsetlocalversion -at- schottelius.org>.
|
||||
#
|
||||
#
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [srctree]" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
cd "${1:-.}" || usage
|
||||
|
||||
# Check for git and a git repo.
|
||||
if head=`git rev-parse --verify --short HEAD 2>/dev/null`; then
|
||||
|
||||
atag="`git describe 2>/dev/null`"
|
||||
|
||||
# Show -g<commit> if we have no tag, or just the tag
|
||||
# otherwise.
|
||||
if [ -z "${atag}" ] ; then
|
||||
printf "%s%s" -g ${head}
|
||||
else
|
||||
printf ${atag}
|
||||
fi
|
||||
|
||||
# Is this git on svn?
|
||||
if git config --get svn-remote.svn.url >/dev/null; then
|
||||
printf -- '-svn%s' "`git svn find-rev $head`"
|
||||
fi
|
||||
|
||||
# Update index only on r/w media
|
||||
[ -w . ] && git update-index --refresh --unmerged > /dev/null
|
||||
|
||||
# Check for uncommitted changes
|
||||
if git diff-index --name-only HEAD | grep -v "^scripts/package" \
|
||||
| read dummy; then
|
||||
printf '%s' -dirty
|
||||
fi
|
||||
|
||||
# All done with git
|
||||
exit
|
||||
fi
|
||||
|
||||
# Check for mercurial and a mercurial repo.
|
||||
# In the git case, 'git describe' will show the latest tag, and unless we are
|
||||
# exactly on that tag, the number of commits since then, and last commit id.
|
||||
# Mimic something similar in the Mercurial case.
|
||||
if hgid=`HGRCPATH= hg id --id --tags 2>/dev/null`; then
|
||||
tag=`printf '%s' "$hgid" | cut -d' ' -f2 --only-delimited`
|
||||
|
||||
# Do we have an untagged version?
|
||||
if [ -z "$tag" -o "$tag" = tip ]; then
|
||||
# current revision is not tagged, determine latest tag
|
||||
latesttag=`HGRCPATH= hg log -r. -T '{latesttag}' 2>/dev/null`
|
||||
# In case there is more than one tag on the latest tagged commit,
|
||||
# 'latesttag' will separate them by colon (:). We'll retain this.
|
||||
# In case there is no tag at all, 'null' will be returned.
|
||||
if [ "$latesttag" = "null" ]; then
|
||||
latesttag=''
|
||||
fi
|
||||
|
||||
# add the commit id
|
||||
id=`printf '%s' "$hgid" | sed 's/[+ ].*//'`
|
||||
printf '%s%s%s' "${latesttag}" -hg "$id"
|
||||
else
|
||||
# current revision is tagged, just print the tag
|
||||
printf ${tag}
|
||||
fi
|
||||
|
||||
# Are there uncommitted changes?
|
||||
# These are represented by + after the changeset id.
|
||||
case "$hgid" in
|
||||
*+|*+\ *) printf '%s' -dirty ;;
|
||||
esac
|
||||
|
||||
# All done with mercurial
|
||||
exit
|
||||
fi
|
||||
|
||||
# Check for svn and a svn repo.
|
||||
if rev=`LC_ALL=C svn info 2>/dev/null | grep '^Last Changed Rev'`; then
|
||||
rev=`echo $rev | awk '{print $NF}'`
|
||||
printf -- '-svn%s' "$rev"
|
||||
|
||||
# All done with svn
|
||||
exit
|
||||
fi
|
||||
Executable
+308
@@ -0,0 +1,308 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) 2014 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
||||
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import argparse
|
||||
import csv
|
||||
import collections
|
||||
import math
|
||||
|
||||
try:
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.font_manager as fm
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
sys.stderr.write("You need python-matplotlib to generate the size graph\n")
|
||||
exit(1)
|
||||
|
||||
|
||||
class Config:
|
||||
biggest_first = False
|
||||
iec = False
|
||||
size_limit = 0.01
|
||||
colors = ['#e60004', '#f28e00', '#ffed00', '#940084',
|
||||
'#2e1d86', '#0068b5', '#009836', '#97c000']
|
||||
|
||||
|
||||
#
|
||||
# This function adds a new file to 'filesdict', after checking its
|
||||
# size. The 'filesdict' contain the relative path of the file as the
|
||||
# key, and as the value a tuple containing the name of the package to
|
||||
# which the file belongs and the size of the file.
|
||||
#
|
||||
# filesdict: the dict to which the file is added
|
||||
# relpath: relative path of the file
|
||||
# fullpath: absolute path to the file
|
||||
# pkg: package to which the file belongs
|
||||
#
|
||||
def add_file(filesdict, relpath, abspath, pkg):
|
||||
if not os.path.exists(abspath):
|
||||
return
|
||||
if os.path.islink(abspath):
|
||||
return
|
||||
sz = os.stat(abspath).st_size
|
||||
filesdict[relpath] = (pkg, sz)
|
||||
|
||||
|
||||
#
|
||||
# This function returns a dict where each key is the path of a file in
|
||||
# the root filesystem, and the value is a tuple containing two
|
||||
# elements: the name of the package to which this file belongs and the
|
||||
# size of the file.
|
||||
#
|
||||
# builddir: path to the Buildroot output directory
|
||||
#
|
||||
def build_package_dict(builddir):
|
||||
filesdict = {}
|
||||
with open(os.path.join(builddir, "build", "packages-file-list.txt")) as f:
|
||||
for line in f.readlines():
|
||||
pkg, fpath = line.split(",", 1)
|
||||
# remove the initial './' in each file path
|
||||
fpath = fpath.strip()[2:]
|
||||
fullpath = os.path.join(builddir, "target", fpath)
|
||||
add_file(filesdict, fpath, fullpath, pkg)
|
||||
return filesdict
|
||||
|
||||
|
||||
#
|
||||
# This function builds a dictionary that contains the name of a
|
||||
# package as key, and the size of the files installed by this package
|
||||
# as the value.
|
||||
#
|
||||
# filesdict: dictionary with the name of the files as key, and as
|
||||
# value a tuple containing the name of the package to which the files
|
||||
# belongs, and the size of the file. As returned by
|
||||
# build_package_dict.
|
||||
#
|
||||
# builddir: path to the Buildroot output directory
|
||||
#
|
||||
def build_package_size(filesdict, builddir):
|
||||
pkgsize = collections.defaultdict(int)
|
||||
|
||||
seeninodes = set()
|
||||
for root, _, files in os.walk(os.path.join(builddir, "target")):
|
||||
for f in files:
|
||||
fpath = os.path.join(root, f)
|
||||
if os.path.islink(fpath):
|
||||
continue
|
||||
|
||||
st = os.stat(fpath)
|
||||
if st.st_ino in seeninodes:
|
||||
# hard link
|
||||
continue
|
||||
else:
|
||||
seeninodes.add(st.st_ino)
|
||||
|
||||
frelpath = os.path.relpath(fpath, os.path.join(builddir, "target"))
|
||||
if frelpath not in filesdict:
|
||||
print("WARNING: %s is not part of any package" % frelpath)
|
||||
pkg = "unknown"
|
||||
else:
|
||||
pkg = filesdict[frelpath][0]
|
||||
|
||||
pkgsize[pkg] += st.st_size
|
||||
|
||||
return pkgsize
|
||||
|
||||
|
||||
#
|
||||
# Given a dict returned by build_package_size(), this function
|
||||
# generates a pie chart of the size installed by each package.
|
||||
#
|
||||
# pkgsize: dictionary with the name of the package as a key, and the
|
||||
# size as the value, as returned by build_package_size.
|
||||
#
|
||||
# outputf: output file for the graph
|
||||
#
|
||||
def draw_graph(pkgsize, outputf):
|
||||
def size2string(sz):
|
||||
if Config.iec:
|
||||
divider = 1024.0
|
||||
prefixes = ['', 'Ki', 'Mi', 'Gi', 'Ti']
|
||||
else:
|
||||
divider = 1000.0
|
||||
prefixes = ['', 'k', 'M', 'G', 'T']
|
||||
while sz > divider and len(prefixes) > 1:
|
||||
prefixes = prefixes[1:]
|
||||
sz = sz/divider
|
||||
# precision is made so that there are always at least three meaningful
|
||||
# digits displayed (e.g. '3.14' and '10.4', not just '3' and '10')
|
||||
precision = int(2-math.floor(math.log10(sz))) if sz < 1000 else 0
|
||||
return '{:.{prec}f} {}B'.format(sz, prefixes[0], prec=precision)
|
||||
|
||||
total = sum(pkgsize.values())
|
||||
labels = []
|
||||
values = []
|
||||
other_value = 0
|
||||
unknown_value = 0
|
||||
for (p, sz) in sorted(pkgsize.items(), key=lambda x: x[1],
|
||||
reverse=Config.biggest_first):
|
||||
if sz < (total * Config.size_limit):
|
||||
other_value += sz
|
||||
elif p == "unknown":
|
||||
unknown_value = sz
|
||||
else:
|
||||
labels.append("%s (%s)" % (p, size2string(sz)))
|
||||
values.append(sz)
|
||||
if unknown_value != 0:
|
||||
labels.append("Unknown (%s)" % (size2string(unknown_value)))
|
||||
values.append(unknown_value)
|
||||
if other_value != 0:
|
||||
labels.append("Other (%s)" % (size2string(other_value)))
|
||||
values.append(other_value)
|
||||
|
||||
plt.figure()
|
||||
patches, texts, autotexts = plt.pie(values, labels=labels,
|
||||
autopct='%1.1f%%', shadow=True,
|
||||
colors=Config.colors)
|
||||
# Reduce text size
|
||||
proptease = fm.FontProperties()
|
||||
proptease.set_size('xx-small')
|
||||
plt.setp(autotexts, fontproperties=proptease)
|
||||
plt.setp(texts, fontproperties=proptease)
|
||||
|
||||
plt.suptitle("Filesystem size per package", fontsize=18, y=.97)
|
||||
plt.title("Total filesystem size: %s" % (size2string(total)), fontsize=10,
|
||||
y=.96)
|
||||
plt.savefig(outputf)
|
||||
|
||||
|
||||
#
|
||||
# Generate a CSV file with statistics about the size of each file, its
|
||||
# size contribution to the package and to the overall system.
|
||||
#
|
||||
# filesdict: dictionary with the name of the files as key, and as
|
||||
# value a tuple containing the name of the package to which the files
|
||||
# belongs, and the size of the file. As returned by
|
||||
# build_package_dict.
|
||||
#
|
||||
# pkgsize: dictionary with the name of the package as a key, and the
|
||||
# size as the value, as returned by build_package_size.
|
||||
#
|
||||
# outputf: output CSV file
|
||||
#
|
||||
def gen_files_csv(filesdict, pkgsizes, outputf):
|
||||
total = 0
|
||||
for (p, sz) in pkgsizes.items():
|
||||
total += sz
|
||||
with open(outputf, 'w') as csvfile:
|
||||
wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
|
||||
wr.writerow(["File name",
|
||||
"Package name",
|
||||
"File size",
|
||||
"Package size",
|
||||
"File size in package (%)",
|
||||
"File size in system (%)"])
|
||||
for f, (pkgname, filesize) in filesdict.items():
|
||||
pkgsize = pkgsizes[pkgname]
|
||||
|
||||
if pkgsize == 0:
|
||||
percent_pkg = 0
|
||||
else:
|
||||
percent_pkg = float(filesize) / pkgsize * 100
|
||||
|
||||
percent_total = float(filesize) / total * 100
|
||||
|
||||
wr.writerow([f, pkgname, filesize, pkgsize,
|
||||
"%.1f" % percent_pkg,
|
||||
"%.1f" % percent_total])
|
||||
|
||||
|
||||
#
|
||||
# Generate a CSV file with statistics about the size of each package,
|
||||
# and their size contribution to the overall system.
|
||||
#
|
||||
# pkgsize: dictionary with the name of the package as a key, and the
|
||||
# size as the value, as returned by build_package_size.
|
||||
#
|
||||
# outputf: output CSV file
|
||||
#
|
||||
def gen_packages_csv(pkgsizes, outputf):
|
||||
total = sum(pkgsizes.values())
|
||||
with open(outputf, 'w') as csvfile:
|
||||
wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
|
||||
wr.writerow(["Package name", "Package size",
|
||||
"Package size in system (%)"])
|
||||
for (pkg, size) in pkgsizes.items():
|
||||
wr.writerow([pkg, size, "%.1f" % (float(size) / total * 100)])
|
||||
|
||||
|
||||
#
|
||||
# Our special action for --iec, --binary, --si, --decimal
|
||||
#
|
||||
class PrefixAction(argparse.Action):
|
||||
def __init__(self, option_strings, dest, **kwargs):
|
||||
for key in ["type", "nargs"]:
|
||||
if key in kwargs:
|
||||
raise ValueError('"{}" not allowed'.format(key))
|
||||
super(PrefixAction, self).__init__(option_strings, dest, nargs=0,
|
||||
type=bool, **kwargs)
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
setattr(namespace, self.dest, option_string in ["--iec", "--binary"])
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Draw size statistics graphs')
|
||||
|
||||
parser.add_argument("--builddir", '-i', metavar="BUILDDIR", required=True,
|
||||
help="Buildroot output directory")
|
||||
parser.add_argument("--graph", '-g', metavar="GRAPH",
|
||||
help="Graph output file (.pdf or .png extension)")
|
||||
parser.add_argument("--file-size-csv", '-f', metavar="FILE_SIZE_CSV",
|
||||
help="CSV output file with file size statistics")
|
||||
parser.add_argument("--package-size-csv", '-p', metavar="PKG_SIZE_CSV",
|
||||
help="CSV output file with package size statistics")
|
||||
parser.add_argument("--biggest-first", action='store_true',
|
||||
help="Sort packages in decreasing size order, " +
|
||||
"rather than in increasing size order")
|
||||
parser.add_argument("--iec", "--binary", "--si", "--decimal",
|
||||
action=PrefixAction,
|
||||
help="Use IEC (binary, powers of 1024) or SI (decimal, "
|
||||
"powers of 1000, the default) prefixes")
|
||||
parser.add_argument("--size-limit", "-l", type=float,
|
||||
help='Under this size ratio, files are accounted to ' +
|
||||
'the generic "Other" package. Default: 0.01 (1%%)')
|
||||
args = parser.parse_args()
|
||||
|
||||
Config.biggest_first = args.biggest_first
|
||||
Config.iec = args.iec
|
||||
if args.size_limit is not None:
|
||||
if args.size_limit < 0.0 or args.size_limit > 1.0:
|
||||
parser.error("--size-limit must be in [0.0..1.0]")
|
||||
Config.size_limit = args.size_limit
|
||||
|
||||
# Find out which package installed what files
|
||||
pkgdict = build_package_dict(args.builddir)
|
||||
|
||||
# Collect the size installed by each package
|
||||
pkgsize = build_package_size(pkgdict, args.builddir)
|
||||
|
||||
if args.graph:
|
||||
draw_graph(pkgsize, args.graph)
|
||||
if args.file_size_csv:
|
||||
gen_files_csv(pkgdict, pkgsize, args.file_size_csv)
|
||||
if args.package_size_csv:
|
||||
gen_packages_csv(pkgsize, args.package_size_csv)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user