initial buildroot for linux 5.15

support/download/bzr

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# We want to catch any unexpected failure, and exit immediately
+set -e
+
+# Download helper for bzr, to be called from the download wrapper script
+#
+# Options:
+#   -q          Be quiet
+#   -o FILE     Generate archive in FILE.
+#   -u URI      Clone from repository at URI.
+#   -c CSET     Use changeset (or revision) CSET.
+#   -n NAME     Use basename NAME.
+#
+# Environment:
+#   BZR      : the bzr command to call
+
+
+quiet=
+while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
+    case "${OPT}" in
+    q)  quiet=-q;;
+    o)  output="${OPTARG}";;
+    u)  uri="${OPTARG}";;
+    c)  cset="${OPTARG}";;
+    n)  basename="${OPTARG}";;
+    :)  printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
+    \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1)) # Get rid of our options
+
+# Caller needs to single-quote its arguments to prevent them from
+# being expanded a second time (in case there are spaces in them)
+_bzr() {
+    if [ -z "${quiet}" ]; then
+        printf '%s ' ${BZR} "${@}"; printf '\n'
+    fi
+    _plain_bzr "$@"
+}
+# Note: please keep command below aligned with what is printed above
+_plain_bzr() {
+    eval ${BZR} "${@}"
+}
+
+# --per-file-timestamps comes with bzr-2.2 (released August 2010),
+# so only pass it if bzr is recent enough. We compute versions as:
+# major*1000 + minor
+bzr_min_version=2002
+bzr_version=$(($(bzr --version |
+                 sed -r -n 's/^Bazaar \(bzr\) ([[:digit:]]+)\.([[:digit:]]+)\..*$/\1*1000+\2/p')
+             ))
+
+# If the version is recent enough, we can generate reproducible
+# archives; otherwise, we just hope for the best (as it would
+# be downloaded from the BR mirror if what we generate here does
+# not match the hash we have for it).
+if [ ${bzr_version} -ge ${bzr_min_version} ]; then
+    timestamp_opt="--per-file-timestamps"
+fi
+
+_bzr export ${quiet} --root="'${basename}/'" --format=tgz \
+    ${timestamp_opt} - "${@}" "'${uri}'" -r "'${cset}'" \
+    >"${output}"

support/download/check-hash

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+set -e
+
+# Helper to check a file matches its known hash
+# Call it with:
+#   $1: the path of the file containing all the expected hashes
+#   $2: the full path to the temporary file that was downloaded, and
+#       that is to be checked
+#   $3: the final basename of the file, to which it will be ultimately
+#       saved as, to be able to match it to the corresponding hashes
+#       in the .hash file
+#
+# Exit codes:
+#   0:  the hash file exists and the file to check matches all its hashes,
+#       or the hash file does not exist
+#   1:  unknown command-line option
+#   2:  the hash file exists and the file to check does not match at least
+#       one of its hashes
+#   3:  the hash file exists and there was no hash to check the file against
+#   4:  the hash file exists and at least one hash type is unknown
+
+while getopts :q OPT; do
+    case "${OPT}" in
+    q)  exec >/dev/null;;
+    \?) exit 1;;
+    esac
+done
+shift $((OPTIND-1))
+
+h_file="${1}"
+file="${2}"
+base="${3}"
+
+# Bail early if no hash to check
+if [ -z "${h_file}" ]; then
+    exit 0
+fi
+# Does the hash-file exist?
+if [ ! -f "${h_file}" ]; then
+    printf "WARNING: no hash file for %s\n" "${base}" >&2
+    exit 0
+fi
+
+# Check one hash for a file
+# $1: algo hash
+# $2: known hash
+# $3: file (full path)
+check_one_hash() {
+    _h="${1}"
+    _known="${2}"
+    _file="${3}"
+
+    # Note: md5 is supported, but undocumented on purpose.
+    # Note: sha3 is not supported, since there is currently no implementation
+    #       (the NIST has yet to publish the parameters).
+    # Note: 'none' means there is explicitly no hash for that file.
+    case "${_h}" in
+        none)
+            return 0
+            ;;
+        md5|sha1)                       ;;
+        sha224|sha256|sha384|sha512)    ;;
+        *) # Unknown hash, exit with error
+            printf "ERROR: unknown hash '%s' for '%s'\n"  \
+                   "${_h}" "${base}" >&2
+            exit 4
+            ;;
+    esac
+
+    # Do the hashes match?
+    _hash=$( ${_h}sum "${_file}" |cut -d ' ' -f 1 )
+    if [ "${_hash}" = "${_known}" ]; then
+        printf "%s: OK (%s: %s)\n" "${base}" "${_h}" "${_hash}"
+        return 0
+    fi
+
+    printf "ERROR: %s has wrong %s hash:\n" "${base}" "${_h}" >&2
+    printf "ERROR: expected: %s\n" "${_known}" >&2
+    printf "ERROR: got     : %s\n" "${_hash}" >&2
+    printf "ERROR: Incomplete download, or man-in-the-middle (MITM) attack\n" >&2
+
+    exit 2
+}
+
+# Do we know one or more hashes for that file?
+nb_checks=0
+while read t h f; do
+    case "${t}" in
+        ''|'#'*)
+            # Skip comments and empty lines
+            continue
+            ;;
+        *)
+            if [ "${f}" = "${base}" ]; then
+                check_one_hash "${t}" "${h}" "${file}"
+                : $((nb_checks++))
+            fi
+            ;;
+    esac
+done <"${h_file}"
+
+if [ ${nb_checks} -eq 0 ]; then
+    case " ${BR_NO_CHECK_HASH_FOR} " in
+    *" ${base} "*)
+        # File explicitly has no hash
+        exit 0
+        ;;
+    esac
+    printf "ERROR: No hash found for %s\n" "${base}" >&2
+    exit 3
+fi

support/download/cvs

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# We want to catch any unexpected failure, and exit immediately
+set -e
+
+# Download helper for cvs, to be called from the download wrapper script
+#
+# Options:
+#   -q          Be quiet
+#   -o FILE     Generate archive in FILE.
+#   -u URI      Checkout from repository at URI.
+#   -c REV      Use revision REV.
+#   -N RAWNAME  Use rawname (aka module name) RAWNAME.
+#   -n NAME     Use basename NAME.
+#
+# Environment:
+#   CVS      : the cvs command to call
+
+quiet=
+while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
+    case "${OPT}" in
+    q)  quiet=-Q;;
+    o)  output="${OPTARG}";;
+    u)  uri="${OPTARG#*://}";;
+    c)  rev="${OPTARG}";;
+    N)  rawname="${OPTARG}";;
+    n)  basename="${OPTARG}";;
+    :)  printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
+    \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1)) # Get rid of our options
+
+# Caller needs to single-quote its arguments to prevent them from
+# being expanded a second time (in case there are spaces in them).
+# If the CVS server is deadlocked, the client will never return (cfr.
+# http://autobuild.buildroot.net/results/23d/23d1034b33d0354de15de2ec4a8ccd0603e8db78/build-end.log
+# ). Since nobody sane will put large code bases in CVS, a timeout of
+# 10 minutes should do the trick.
+_cvs() {
+    if [ -z "${quiet}" ]; then
+        printf '%s ' timeout 10m ${CVS} "${@}"; printf '\n'
+    fi
+    _plain_cvs "$@"
+}
+# Note: please keep command below aligned with what is printed above
+_plain_cvs() {
+    eval timeout 10m ${CVS} "${@}"
+}
+
+if [[ ${rev} =~ ^[0-9] ]]; then
+    # Date, because a tag or a branch cannot begin with a number
+    select="-D"
+else
+    # Tag or branch
+    select="-r"
+fi
+
+# The absence of an initial : on ${uri} means access method undefined
+if [[ ! "${uri}" =~ ^: ]]; then
+   # defaults to anonymous pserver
+   uri=":pserver:anonymous@${uri}"
+fi
+
+export TZ=UTC
+_cvs ${quiet} -z3 -d"'${uri}'" \
+     co "${@}" -d "'${basename}'" ${select} "'${rev}'" -P "'${rawname}'"
+
+tar czf "${output}" "${basename}"

support/download/dl-wrapper

@@ -0,0 +1,219 @@
+#!/usr/bin/env bash
+
+# This script is a wrapper to the other download backends.
+# Its role is to ensure atomicity when saving downloaded files
+# back to BR2_DL_DIR, and not clutter BR2_DL_DIR with partial,
+# failed downloads.
+
+# To avoid cluttering BR2_DL_DIR, we download to a trashable
+# location, namely in $(BUILD_DIR).
+# Then, we move the downloaded file to a temporary file in the
+# same directory as the final output file.
+# This allows us to finally atomically rename it to its final
+# name.
+# If anything goes wrong, we just remove all the temporaries
+# created so far.
+
+# We want to catch any unexpected failure, and exit immediately.
+set -e
+
+export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:ru:qf:e"
+
+main() {
+    local OPT OPTARG
+    local backend output hfile recurse quiet rc
+    local -a uris
+
+    # Parse our options; anything after '--' is for the backend
+    while getopts ":c:d:D:o:n:N:H:rf:u:q" OPT; do
+        case "${OPT}" in
+        c)  cset="${OPTARG}";;
+        d)  dl_dir="${OPTARG}";;
+        D)  old_dl_dir="${OPTARG}";;
+        o)  output="${OPTARG}";;
+        n)  raw_base_name="${OPTARG}";;
+        N)  base_name="${OPTARG}";;
+        H)  hfile="${OPTARG}";;
+        r)  recurse="-r";;
+        f)  filename="${OPTARG}";;
+        u)  uris+=( "${OPTARG}" );;
+        q)  quiet="-q";;
+        :)  error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
+        \?) error "unknown option '%s'\n" "${OPTARG}";;
+        esac
+    done
+
+    # Forget our options, and keep only those for the backend
+    shift $((OPTIND-1))
+
+    if [ -z "${output}" ]; then
+        error "no output specified, use -o\n"
+    fi
+
+    # Legacy handling: check if the file already exists in the global
+    # download directory. If it does, hard-link it. If it turns out it
+    # was an incorrect download, we'd still check it below anyway.
+    # If we can neither link nor copy, fallback to doing a download.
+    # NOTE! This is not atomic, is subject to TOCTTOU, but the whole
+    # dl-wrapper runs under an flock, so we're safe.
+    if [ ! -e "${output}" -a -e "${old_dl_dir}/${filename}" ]; then
+        ln "${old_dl_dir}/${filename}" "${output}" || \
+        cp "${old_dl_dir}/${filename}" "${output}" || \
+        true
+    fi
+
+    # If the output file already exists and:
+    # - there's no .hash file: do not download it again and exit promptly
+    # - matches all its hashes: do not download it again and exit promptly
+    # - fails at least one of its hashes: force a re-download
+    # - there's no hash (but a .hash file): consider it a hard error
+    if [ -e "${output}" ]; then
+        if support/download/check-hash ${quiet} "${hfile}" "${output}" "${output##*/}"; then
+            exit 0
+        elif [ ${?} -ne 2 ]; then
+            # Do not remove the file, otherwise it might get re-downloaded
+            # from a later location (i.e. primary -> upstream -> mirror).
+            # Do not print a message, check-hash already did.
+            exit 1
+        fi
+        rm -f "${output}"
+        warn "Re-downloading '%s'...\n" "${output##*/}"
+    fi
+
+    # Look through all the uris that we were given to download the package
+    # source
+    download_and_check=0
+    rc=1
+    for uri in "${uris[@]}"; do
+        backend_urlencode="${uri%%+*}"
+        backend="${backend_urlencode%|*}"
+        case "${backend}" in
+            git|svn|cvs|bzr|file|scp|hg) ;;
+            *) backend="wget" ;;
+        esac
+        uri=${uri#*+}
+
+        urlencode=${backend_urlencode#*|}
+        # urlencode must be "urlencode"
+        [ "${urlencode}" != "urlencode" ] && urlencode=""
+
+        # tmpd is a temporary directory in which backends may store
+        # intermediate by-products of the download.
+        # tmpf is the file in which the backends should put the downloaded
+        # content.
+        # tmpd is located in $(BUILD_DIR), so as not to clutter the (precious)
+        # $(BR2_DL_DIR)
+        # We let the backends create tmpf, so they are able to set whatever
+        # permission bits they want (although we're only really interested in
+        # the executable bit.)
+        tmpd="$(mktemp -d "${BUILD_DIR}/.${output##*/}.XXXXXX")"
+        tmpf="${tmpd}/output"
+
+        # Helpers expect to run in a directory that is *really* trashable, so
+        # they are free to create whatever files and/or sub-dirs they might need.
+        # Doing the 'cd' here rather than in all backends is easier.
+        cd "${tmpd}"
+
+        # If the backend fails, we can just remove the content of the temporary
+        # directory to remove all the cruft it may have left behind, and try
+        # the next URI until it succeeds. Once out of URI to try, we need to
+        # cleanup and exit.
+        if ! "${OLDPWD}/support/download/${backend}" \
+                $([ -n "${urlencode}" ] && printf %s '-e') \
+                -c "${cset}" \
+                -d "${dl_dir}" \
+                -n "${raw_base_name}" \
+                -N "${base_name}" \
+                -f "${filename}" \
+                -u "${uri}" \
+                -o "${tmpf}" \
+                ${quiet} ${recurse} -- "${@}"
+        then
+            # cd back to keep path coherence
+            cd "${OLDPWD}"
+            rm -rf "${tmpd}"
+            continue
+        fi
+
+        # cd back to free the temp-dir, so we can remove it later
+        cd "${OLDPWD}"
+
+        # Check if the downloaded file is sane, and matches the stored hashes
+        # for that file
+        if support/download/check-hash ${quiet} "${hfile}" "${tmpf}" "${output##*/}"; then
+            rc=0
+        else
+            if [ ${?} -ne 3 ]; then
+                rm -rf "${tmpd}"
+                continue
+            fi
+
+            # the hash file exists and there was no hash to check the file
+            # against
+            rc=1
+        fi
+        download_and_check=1
+        break
+    done
+
+    # We tried every URI possible, none seems to work or to check against the
+    # available hash. *ABORT MISSION*
+    if [ "${download_and_check}" -eq 0 ]; then
+        rm -rf "${tmpd}"
+        exit 1
+    fi
+
+    # tmp_output is in the same directory as the final output, so we can
+    # later move it atomically.
+    tmp_output="$(mktemp "${output}.XXXXXX")"
+
+    # 'mktemp' creates files with 'go=-rwx', so the files are not accessible
+    # to users other than the one doing the download (and root, of course).
+    # This can be problematic when a shared BR2_DL_DIR is used by different
+    # users (e.g. on a build server), where all users may write to the shared
+    # location, since other users would not be allowed to read the files
+    # another user downloaded.
+    # So, we restore the 'go' access rights to a more sensible value, while
+    # still abiding by the current user's umask. We must do that before the
+    # final 'mv', so just do it now.
+    # Some backends (cp and scp) may create executable files, so we need to
+    # carry the executable bit if needed.
+    [ -x "${tmpf}" ] && new_mode=755 || new_mode=644
+    new_mode=$(printf "%04o" $((0${new_mode} & ~0$(umask))))
+    chmod ${new_mode} "${tmp_output}"
+
+    # We must *not* unlink tmp_output, otherwise there is a small window
+    # during which another download process may create the same tmp_output
+    # name (very, very unlikely; but not impossible.)
+    # Using 'cp' is not reliable, since 'cp' may unlink the destination file
+    # if it is unable to open it with O_WRONLY|O_TRUNC; see:
+    #   http://pubs.opengroup.org/onlinepubs/9699919799/utilities/cp.html
+    # Since the destination filesystem can be anything, it might not support
+    # O_TRUNC, so 'cp' would unlink it first.
+    # Use 'cat' and append-redirection '>>' to save to the final location,
+    # since that is the only way we can be 100% sure of the behaviour.
+    if ! cat "${tmpf}" >>"${tmp_output}"; then
+        rm -rf "${tmpd}" "${tmp_output}"
+        exit 1
+    fi
+    rm -rf "${tmpd}"
+
+    # tmp_output and output are on the same filesystem, so POSIX guarantees
+    # that 'mv' is atomic, because it then uses rename() that POSIX mandates
+    # to be atomic, see:
+    #   http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html
+    if ! mv -f "${tmp_output}" "${output}"; then
+        rm -f "${tmp_output}"
+        exit 1
+    fi
+
+    return ${rc}
+}
+
+trace()  { local msg="${1}"; shift; printf "%s: ${msg}" "${my_name}" "${@}"; }
+warn()   { trace "${@}" >&2; }
+errorN() { local ret="${1}"; shift; warn "${@}"; exit ${ret}; }
+error()  { errorN 1 "${@}"; }
+
+my_name="${0##*/}"
+main "${@}"

support/download/file

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# We want to catch any unexpected failure, and exit immediately
+set -e
+
+# Download helper for cp, to be called from the download wrapper script
+#
+# Options:
+#   -q          Be quiet.
+#   -o FILE     Copy to file FILE.
+#   -f FILE     Copy from basename file FILE.
+#   -u DIR      Copy from FILE in DIR.
+#
+# Environment:
+#   LOCALFILES: the cp command to call
+
+# 'cp' usually does not print anything on its stdout, whereas the
+# other download backends, even if not verbose, at least print some
+# progress information.
+# Make 'cp' verbose by default, so it behaves a bit like the others.
+verbose=-v
+
+while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
+    case "${OPT}" in
+    q)  verbose=;;
+    o)  output="${OPTARG}";;
+    f)  file="${OPTARG}";;
+    u)  dir="${OPTARG}";;
+    :)  printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
+    \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1)) # Get rid of our options
+
+# Caller needs to single-quote its arguments to prevent them from
+# being expanded a second time (in case there are spaces in them)
+_localfiles() {
+    if [ -n "${verbose}" ]; then
+        printf '%s ' ${LOCALFILES} "${@}"; printf '\n'
+    fi
+    _plain_localfiles "$@"
+}
+# Note: please keep command below aligned with what is printed above
+_plain_localfiles() {
+    eval ${LOCALFILES} "${@}"
+}
+
+_localfiles ${verbose} "'${dir##file://}/${file}'" "'${output}'"

support/download/git

@@ -0,0 +1,216 @@
+#!/usr/bin/env bash
+
+# NOTE: if the output of this backend has to change (e.g. we change what gets
+# included in the archive (e.g. LFS), or we change the format of the archive
+# (e.g. tar options, compression ratio or method)), we MUST update the format
+# version in the variable BR_FMT_VERSION_git, in package/pkg-download.mk.
+
+# We want to catch any unexpected failure, and exit immediately
+set -E
+
+# Download helper for git, to be called from the download wrapper script
+#
+# Options:
+#   -q          Be quiet.
+#   -r          Clone and archive sub-modules.
+#   -o FILE     Generate archive in FILE.
+#   -u URI      Clone from repository at URI.
+#   -c CSET     Use changeset CSET.
+#   -n NAME     Use basename NAME.
+#
+# Environment:
+#   GIT      : the git command to call
+
+. "${0%/*}/helpers"
+
+# Save our path and options in case we need to call ourselves again
+myname="${0}"
+declare -a OPTS=("${@}")
+
+# This function is called when an error occurs. Its job is to attempt a
+# clone from scratch (only once!) in case the git tree is borked, or in
+# case an unexpected and unsupported situation arises with submodules
+# or uncommitted stuff (e.g. if the user manually mucked around in the
+# git cache).
+_on_error() {
+    local ret=${?}
+
+    printf "Detected a corrupted git cache.\n" >&2
+    if ${BR_GIT_BACKEND_FIRST_FAULT:-false}; then
+        printf "This is the second time in a row; bailing out\n" >&2
+        exit ${ret}
+    fi
+    export BR_GIT_BACKEND_FIRST_FAULT=true
+
+    printf "Removing it and starting afresh.\n" >&2
+
+    popd >/dev/null
+    rm -rf "${git_cache}"
+
+    exec "${myname}" "${OPTS[@]}" || exit ${ret}
+}
+
+quiet=
+recurse=0
+while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
+    case "${OPT}" in
+    q)  quiet=-q; exec >/dev/null;;
+    r)  recurse=1;;
+    o)  output="${OPTARG}";;
+    u)  uri="${OPTARG}";;
+    c)  cset="${OPTARG}";;
+    d)  dl_dir="${OPTARG}";;
+    n)  basename="${OPTARG}";;
+    :)  printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
+    \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1)) # Get rid of our options
+
+# Create and cd into the directory that will contain the local git cache
+git_cache="${dl_dir}/git"
+mkdir -p "${git_cache}"
+pushd "${git_cache}" >/dev/null
+
+# Any error now should try to recover
+trap _on_error ERR
+
+# Caller needs to single-quote its arguments to prevent them from
+# being expanded a second time (in case there are spaces in them)
+_git() {
+    if [ -z "${quiet}" ]; then
+        printf '%s ' GIT_DIR="${git_cache}/.git" ${GIT} "${@}"; printf '\n'
+    fi
+    _plain_git "$@"
+}
+# Note: please keep command below aligned with what is printed above
+_plain_git() {
+    eval GIT_DIR="${git_cache}/.git" ${GIT} "${@}"
+}
+
+# Create a warning file, that the user should not use the git cache.
+# It's ours. Our precious.
+cat <<-_EOF_ >"${dl_dir}/git.readme"
+	IMPORTANT NOTE!
+
+	The git tree located in this directory is for the exclusive use
+	by Buildroot, which uses it as a local cache to reduce bandwidth
+	usage.
+
+	Buildroot *will* trash any changes in that tree whenever it needs
+	to use it. Buildroot may even remove it in case it detects the
+	repository may have been damaged or corrupted.
+
+	Do *not* work in that directory; your changes will eventually get
+	lost. Do *not* even use it as a remote, or as the source for new
+	worktrees; your commits will eventually get lost.
+_EOF_
+
+# Initialise a repository in the git cache. If the repository already
+# existed, this is a noop, unless the repository was broken, in which
+# case this magically restores it to working conditions. In the latter
+# case, we might be missing blobs, but that's not a problem: we'll
+# fetch what we need later anyway.
+#
+# We can still go through the wrapper, because 'init' does not use the
+# path pointed to by GIT_DIR, but really uses the directory passed as
+# argument.
+_git init .
+
+# Ensure the repo has an origin (in case a previous run was killed).
+if ! _plain_git remote |grep -q -E '^origin$'; then
+    _git remote add origin "'${uri}'"
+fi
+
+_git remote set-url origin "'${uri}'"
+
+printf "Fetching all references\n"
+_git fetch origin
+_git fetch origin -t
+
+# Try to get the special refs exposed by some forges (pull-requests for
+# github, changes for gerrit...). There is no easy way to know whether
+# the cset the user passed us is such a special ref or a tag or a sha1
+# or whatever else. We'll eventually fail at checking out that cset,
+# below, if there is an issue anyway. Since most of the cset we're gonna
+# have to clone are not such special refs, consign the output to oblivion
+# so as not to alarm unsuspecting users, but still trace it as a warning.
+if ! _git fetch origin "'${cset}:${cset}'" >/dev/null 2>&1; then
+    printf "Could not fetch special ref '%s'; assuming it is not special.\n" "${cset}"
+fi
+
+# Check that the changeset does exist. If it does not, re-cloning from
+# scratch won't help, so we don't want to trash the repository for a
+# missing commit. We just exit without going through the ERR trap.
+if ! _git rev-parse --quiet --verify "'${cset}^{commit}'" >/dev/null 2>&1; then
+    printf "Commit '%s' does not exist in this repository.\n" "${cset}"
+    exit 1
+fi
+
+# The new cset we want to checkout might have different submodules, or
+# have sub-dirs converted to/from a submodule. So we would need to
+# deregister _current_ submodules before we checkout.
+#
+# Using "git submodule deinit --all" would remove all the files for
+# all submodules, including the corresponding .git files or directories.
+# However, it  was only introduced with git-1.8.3, which is too recent
+# for some enterprise-grade distros.
+#
+# So, we fall-back to just removing all submodules directories. We do
+# not need to be recursive, as removing a submodule will de-facto remove
+# its own submodules.
+#
+# For recent git versions, the repository for submodules is stored
+# inside the repository of the super repository, so the following will
+# only remove the working copies of submodules, effectively caching the
+# submodules.
+#
+# For older versions however, the repository is stored in the .git/ of
+# the submodule directory, so the following will effectively remove the
+# the working copy as well as the repository, which means submodules
+# will not be cached for older versions.
+#
+cmd='printf "Deregistering submodule \"%s\"\n" "${path}" && cd .. && rm -rf "${path##*/}"'
+_git submodule --quiet foreach "'${cmd}'"
+
+# Checkout the required changeset, so that we can update the required
+# submodules.
+_git checkout -f -q "'${cset}'"
+
+# Get rid of now-untracked directories (in case a git operation was
+# interrupted in a previous run, or to get rid of empty directories
+# that were parents of submodules removed above).
+_git clean -ffdx
+
+# Get date of commit to generate a reproducible archive.
+# %ci is ISO 8601, so it's fully qualified, with TZ and all.
+date="$( _plain_git log -1 --pretty=format:%ci )"
+
+# There might be submodules, so fetch them.
+if [ ${recurse} -eq 1 ]; then
+    _git submodule update --init --recursive
+
+    # Older versions of git will store the absolute path of the git tree
+    # in the .git of submodules, while newer versions just use relative
+    # paths. Detect and fix the older variants to use relative paths, so
+    # that the archives are reproducible across a wider range of git
+    # versions. However, we can't do that if git is too old and uses
+    # full repositories for submodules.
+    cmd='printf "%s\n" "${path}/"'
+    for module_dir in $( _plain_git submodule --quiet foreach "'${cmd}'" ); do
+        [ -f "${module_dir}/.git" ] || continue
+        relative_dir="$( sed -r -e 's,/+,/,g; s,[^/]+/,../,g' <<<"${module_dir}" )"
+        sed -r -i -e "s:^gitdir\: $(pwd)/:gitdir\: "${relative_dir}":" "${module_dir}/.git"
+    done
+fi
+
+popd >/dev/null
+
+# Generate the archive.
+# We do not want the .git dir; we keep other .git files, in case they are the
+# only files in their directory.
+# The .git dir would generate non reproducible tarballs as it depends on
+# the state of the remote server. It also would generate large tarballs
+# (gigabytes for some linux trees) when a full clone took place.
+mk_tar_gz "${git_cache}" "${basename}" "${date}" "${output}" ".git/*"

support/download/helpers

@@ -0,0 +1,76 @@
+# Generate a reproducible archive from the content of a directory
+#
+# $1    : input directory
+# $2    : leading component in archive
+# $3    : ISO8601 date: YYYY-MM-DDThh:mm:ssZZ
+# $4    : output file
+# $5... : globs of filenames to exclude from the archive, suitable for
+#         find's -path option, and relative to the input directory $1
+#
+# Notes :
+#   - the timestamp is internally rounded to the highest entire second
+#     less than or equal to the timestamp (i.e. any sub-second fractional
+#     part is ignored)
+#   - must not be called with CWD as, or below, the input directory
+#   - some temporary files are created in CWD, and removed at the end
+#
+# Example:
+#   $ find /path/to/temp/dir
+#   /path/to/temp/dir/
+#   /path/to/temp/dir/some-file
+#   /path/to/temp/dir/some-dir/
+#   /path/to/temp/dir/some-dir/some-other-file
+#
+#   $ mk_tar_gz /path/to/some/dir \
+#               foo_bar-1.2.3 \
+#               1970-01-01T00:00:00Z \
+#               /path/to/foo.tar.gz \
+#               '.git/*' '.svn/*'
+#
+#   $ tar tzf /path/to/foo.tar.gz
+#   foo_bar-1.2.3/some-file
+#   foo_bar-1.2.3/some-dir/some-other-file
+#
+mk_tar_gz() {
+    local in_dir="${1}"
+    local base_dir="${2}"
+    local date="${3}"
+    local out="${4}"
+    shift 4
+    local glob tmp pax_options
+    local -a find_opts
+
+    for glob; do
+        find_opts+=( -or -path "./${glob#./}" )
+    done
+
+    # Drop sub-second precision to play nice with GNU tar's valid_timespec check
+    date="$(date -d "${date}" -u +%Y-%m-%dT%H:%M:%S+00:00)"
+
+    pax_options="delete=atime,delete=ctime,delete=mtime"
+    pax_options+=",exthdr.name=%d/PaxHeaders/%f,exthdr.mtime={${date}}"
+
+    tmp="$(mktemp --tmpdir="$(pwd)")"
+    pushd "${in_dir}" >/dev/null
+
+    # Establish list
+    find . -not -type d -and -not \( -false "${find_opts[@]}" \) >"${tmp}.list"
+    # Sort list for reproducibility
+    LC_ALL=C sort <"${tmp}.list" >"${tmp}.sorted"
+
+    # Create POSIX tarballs, since that's the format the most reproducible
+    tar cf - --transform="s#^\./#${base_dir}/#S" \
+             --numeric-owner --owner=0 --group=0 --mtime="${date}" \
+             --format=posix --pax-option="${pax_options}" \
+             -T "${tmp}.sorted" >"${tmp}.tar"
+
+    # Compress the archive
+    gzip -6 -n <"${tmp}.tar" >"${out}"
+
+    rm -f "${tmp}"{.list,.sorted,.tar}
+
+    popd >/dev/null
+}
+
+# Keep this line and the following as last lines in this file.
+# vim: ft=bash

support/download/hg

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# We want to catch any unexpected failure, and exit immediately
+set -e
+
+# Download helper for hg, to be called from the download wrapper script
+#
+# Options:
+#   -q          Be quiet.
+#   -o FILE     Generate archive in FILE.
+#   -u URI      Clone from repository at URI.
+#   -c CSET     Use changeset (or revision) CSET.
+#   -n NAME     Use basename NAME.
+#
+# Environment:
+#   HG       : the hg command to call
+
+quiet=
+while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
+    case "${OPT}" in
+    q)  quiet=-q;;
+    o)  output="${OPTARG}";;
+    u)  uri="${OPTARG}";;
+    c)  cset="${OPTARG}";;
+    n)  basename="${OPTARG}";;
+    :)  printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
+    \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1)) # Get rid of our options
+
+# Caller needs to single-quote its arguments to prevent them from
+# being expanded a second time (in case there are spaces in them)
+_hg() {
+    if [ -z "${quiet}" ]; then
+        printf '%s ' ${HG} "${@}"; printf '\n'
+    fi
+    _plain_hg "$@"
+}
+# Note: please keep command below aligned with what is printed above
+_plain_hg() {
+    eval ${HG} "${@}"
+}
+
+_hg clone ${quiet} "${@}" --noupdate "'${uri}'" "'${basename}'"
+
+_plain_hg archive ${quiet} --repository "'${basename}'" --type tgz \
+            --prefix "'${basename}'" --rev "'${cset}'" \
+            - >"${output}"

support/download/scp

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# We want to catch any unexpected failure, and exit immediately
+set -e
+
+# Download helper for scp, to be called from the download wrapper script
+#
+# Options:
+#   -q          Be quiet.
+#   -o FILE     Copy to local file FILE.
+#   -f FILE     Copy from remote file FILE.
+#   -u URI      Download file at URI.
+#
+# Environment:
+#   SCP       : the scp command to call
+
+quiet=
+while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
+    case "${OPT}" in
+    q)  quiet=-q;;
+    o)  output="${OPTARG}";;
+    f)  filename="${OPTARG}";;
+    u)  uri="${OPTARG}";;
+    :)  printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
+    \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1)) # Get rid of our options
+
+# Caller needs to single-quote its arguments to prevent them from
+# being expanded a second time (in case there are spaces in them)
+_scp() {
+    if [ -z "${quiet}" ]; then
+        printf '%s ' ${SCP} "${@}"; printf '\n'
+    fi
+    _plain_scp "$@"
+}
+# Note: please keep command below aligned with what is printed above
+_plain_scp() {
+    eval ${SCP} "${@}"
+}
+
+# Remove any scheme prefix
+uri="${uri##scp://}"
+
+_scp ${quiet} "${@}" "'${uri}/${filename}'" "'${output}'"

support/download/svn

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# NOTE: if the output of this backend has to change (e.g. we change what gets
+# included in the archive, or we change the format of the archive (e.g. tar
+# options, compression ratio or method)), we MUST update the format version
+# in the variable BR_FTM_VERSION_svn, in package/pkg-download.mk.
+
+# We want to catch any unexpected failure, and exit immediately
+set -e
+
+# Download helper for svn, to be called from the download wrapper script
+#
+# Options:
+#   -q          Be quiet.
+#   -o FILE     Generate archive in FILE.
+#   -u URI      Checkout from repository at URI.
+#   -c REV      Use revision REV.
+#   -n NAME     Use basename NAME.
+#
+# Environment:
+#   SVN      : the svn command to call
+
+. "${0%/*}/helpers"
+
+quiet=
+while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
+    case "${OPT}" in
+    q)  quiet=-q;;
+    o)  output="${OPTARG}";;
+    u)  uri="${OPTARG}";;
+    c)  rev="${OPTARG}";;
+    n)  basename="${OPTARG}";;
+    :)  printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
+    \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1)) # Get rid of our options
+
+# Caller needs to single-quote its arguments to prevent them from
+# being expanded a second time (in case there are spaces in them)
+_svn() {
+    if [ -z "${quiet}" ]; then
+        printf '%s ' ${SVN} "${@}"; printf '\n'
+    fi
+    _plain_svn "$@"
+}
+# Note: please keep command below aligned with what is printed above
+_plain_svn() {
+    eval ${SVN} "${@}"
+}
+
+_svn export --ignore-keywords ${quiet} "${@}" "'${uri}@${rev}'" "'${basename}'"
+
+# Get the date of the revision, to generate reproducible archives.
+# The output format is YYYY-MM-DDTHH:MM:SS.mmmuuuZ (i.e. always in the
+# UTC timezone), which we can feed as-is to the --mtime option for tar.
+# In case there is a redirection (e.g. http -> https), just keep the
+# last line (svn outputs everything on stdout)
+date="$( _plain_svn info "'${uri}@${rev}'" \
+        |sed -r -e '/^Last Changed Date: /!d; s///'
+       )"
+
+# Generate the archive.
+# We did a 'svn export' above, so it's not a working copy (there is no .svn
+# directory or file to ignore).
+mk_tar_gz "${basename}" "${basename}" "${date}" "${output}"

support/download/wget

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# We want to catch any unexpected failure, and exit immediately
+set -e
+
+# Download helper for wget, to be called from the download wrapper script
+#
+# Options:
+#   -q          Be quiet.
+#   -o FILE     Save into file FILE.
+#   -f FILENAME The filename of the tarball to get at URL
+#   -u URL      Download file at URL.
+#   -e ENCODE   Tell wget to urlencode the filename passed to it
+#
+# Environment:
+#   WGET     : the wget command to call
+
+quiet=
+while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
+    case "${OPT}" in
+    q)  quiet=-q;;
+    o)  output="${OPTARG}";;
+    f)  filename="${OPTARG}";;
+    u)  url="${OPTARG}";;
+    e)  encode="-e";;
+    :)  printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
+    \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1)) # Get rid of our options
+
+# Caller needs to single-quote its arguments to prevent them from
+# being expanded a second time (in case there are spaces in them)
+_wget() {
+    if [ -z "${quiet}" ]; then
+        printf '%s ' ${WGET} "${@}"; printf '\n'
+    fi
+    _plain_wget "$@"
+}
+# Note: please keep command below aligned with what is printed above
+_plain_wget() {
+    eval ${WGET} "${@}"
+}
+
+# Replace every '?' with '%3F' in the filename; only for the PRIMARY and BACKUP
+# mirror
+[ -n "${encode}" ] && filename=${filename//\?/%3F}
+
+_wget ${quiet} "${@}" -O "'${output}'" "'${url}/${filename}'"