Merge pull request #837 from stilor/download-verify

Verification of the downloads

8 files changed, 226 insertions, 62 deletions

diff --git a/scripts/build/cc/gcc.sh b/scripts/build/cc/gcc.sh
index 54303410..b5ae7b10 100644
--- a/scripts/build/cc/gcc.sh
+++ b/scripts/build/cc/gcc.sh
@@ -14,7 +14,8 @@ do_cc_get() {
     # GCC source tree, which will not be there unless we get it and
     # put it there ourselves
     if [ "${CT_CC_LANG_JAVA_USE_ECJ}" = "y" ]; then
-        if ! CT_GetFile ecj ecj-latest .jar $(CT_Mirrors sourceware java); then
+        if ! CT_GetFile package=ecj basename=ecj-latest extensions=.jar \
+                mirrors=$(CT_Mirrors sourceware java); then
             # Should be a package, too - but with Java retirement in GCC,
             # it may not make sense.
             CT_Abort "Failed to download ecj-latest.jar"
diff --git a/scripts/build/companion_libs/200-libelf.sh b/scripts/build/companion_libs/200-libelf.sh
index d1ac0b42..5f1a8d73 100644
--- a/scripts/build/companion_libs/200-libelf.sh
+++ b/scripts/build/companion_libs/200-libelf.sh
@@ -81,6 +81,7 @@ do_libelf_for_target() {
     libelf_opts+=( "destdir=${CT_SYSROOT_DIR}" )
     libelf_opts+=( "host=${CT_TARGET}" )
 
+    libelf_opts+=( "cflags=${CT_TARGET_CFLAGS}" )
     libelf_opts+=( "prefix=${prefix}" )
     libelf_opts+=( "shared=${CT_SHARED_LIBS}" )
     do_libelf_backend "${libelf_opts[@]}"
diff --git a/scripts/build/companion_libs/210-expat.sh b/scripts/build/companion_libs/210-expat.sh
index dcb72081..f8485410 100644
--- a/scripts/build/companion_libs/210-expat.sh
+++ b/scripts/build/companion_libs/210-expat.sh
@@ -54,6 +54,7 @@ do_expat_for_target() {
             prefix="/usr"
             ;;
     esac
+    expat_opts+=( "cflags=${CT_TARGET_CFLAGS}" )
     expat_opts+=( "prefix=${prefix}" )
     expat_opts+=( "destdir=${CT_SYSROOT_DIR}" )
     expat_opts+=( "shared=${CT_SHARED_LIBS}" )
@@ -103,7 +104,7 @@ do_expat_backend() {
     CT_DoLog EXTRA "Building expat"
     CT_DoExecLog ALL make ${JOBSFLAGS}
     CT_DoLog EXTRA "Installing expat"
-    CT_DoExecLog ALL make install INSTALL_ROOT="${destdir}"
+    CT_DoExecLog ALL make install DESTDIR="${destdir}"
 }
 
 fi
diff --git a/scripts/build/companion_libs/220-ncurses.sh b/scripts/build/companion_libs/220-ncurses.sh
index 815cf4b0..a32df424 100644
--- a/scripts/build/companion_libs/220-ncurses.sh
+++ b/scripts/build/companion_libs/220-ncurses.sh
@@ -96,6 +96,7 @@ do_ncurses_for_target() {
                        prefix="${prefix}" \
                        destdir="${CT_SYSROOT_DIR}" \
                        shared="${CT_SHARED_LIBS}" \
+                       cflags="${CT_TARGET_CFLAGS}" \
                        "${opts[@]}"
     CT_Popd
     CT_EndStep
diff --git a/scripts/build/debug/200-duma.sh b/scripts/build/debug/200-duma.sh
index bd350304..0d98c386 100644
--- a/scripts/build/debug/200-duma.sh
+++ b/scripts/build/debug/200-duma.sh
@@ -19,8 +19,8 @@ do_debug_duma_build() {
     make_args=(
         prefix="${CT_DEBUGROOT_DIR}/usr"
         HOSTCC="${CT_BUILD}-gcc"
-        CC="${CT_TARGET}-${CT_CC}"
-        CXX="${CT_TARGET}-g++"
+        CC="${CT_TARGET}-${CT_CC} ${CT_TARGET_CFLAGS}"
+        CXX="${CT_TARGET}-g++ ${CT_TARGET_CFLAGS}"
         RANLIB="${CT_TARGET}-ranlib"
         OS="${CT_KERNEL}"
     )
diff --git a/scripts/build/debug/300-gdb.sh b/scripts/build/debug/300-gdb.sh
index e3a40d07..11378ecb 100644
--- a/scripts/build/debug/300-gdb.sh
+++ b/scripts/build/debug/300-gdb.sh
@@ -13,11 +13,6 @@ do_debug_gdb_build() {
 
     gdb_src_dir="${CT_SRC_DIR}/gdb"
 
-    # Version 6.3 and below behave badly with gdbmi
-    case "${CT_GDB_VERSION}" in
-        6.2*|6.3)   extra_config+=("--disable-gdbmi");;
-    esac
-
     if [ "${CT_GDB_HAS_PKGVERSION_BUGURL}" = "y" ]; then
         [ -n "${CT_PKGVERSION}" ] && extra_config+=("--with-pkgversion=${CT_PKGVERSION}")
         [ -n "${CT_TOOLCHAIN_BUGURL}" ] && extra_config+=("--with-bugurl=${CT_TOOLCHAIN_BUGURL}")
@@ -219,15 +214,19 @@ do_debug_gdb_build() {
         [ "${CT_TOOLCHAIN_ENABLE_NLS}" != "y" ] &&    \
         native_extra_config+=("--disable-nls")
 
-        CPP_for_gdb="${CT_TARGET}-cpp"
-        CC_for_gdb="${CT_TARGET}-${CT_CC}"
-        CXX_for_gdb="${CT_TARGET}-g++"
-        LD_for_gdb="${CT_TARGET}-ld"
+        CPP_for_gdb="${CT_TARGET}-cpp ${CT_TARGET_CFLAGS}"
+        CC_for_gdb="${CT_TARGET}-${CT_CC} ${CT_TARGET_CFLAGS} ${CT_TARGET_LDFLAGS}"
+        CXX_for_gdb="${CT_TARGET}-g++ ${CT_TARGET_CFLAGS} ${CT_TARGET_LDFLAGS}"
+        LD_for_gdb="${CT_TARGET}-ld ${CT_TARGET_LDFLAGS}"
         if [ "${CT_GDB_NATIVE_STATIC}" = "y" ]; then
             CC_for_gdb+=" -static"
             CXX_for_gdb+=" -static"
             LD_for_gdb+=" -static"
         fi
+        CPP_for_gdb=`echo $CPP_for_gdb`
+        CC_for_gdb=`echo $CC_for_gdb`
+        CXX_for_gdb=`echo $CXX_for_gdb`
+        LD_for_gdb=`echo $LD_for_gdb`
 
         export ac_cv_func_strncmp_works=yes
 
@@ -321,13 +320,23 @@ do_debug_gdb_build() {
         gdbserver_extra_config+=("--disable-ld")
         gdbserver_extra_config+=("--disable-gas")
 
+        CPP_for_gdb="${CT_TARGET}-cpp ${CT_TARGET_CFLAGS}"
+        CC_for_gdb="${CT_TARGET}-${CT_CC} ${CT_TARGET_CFLAGS} ${CT_TARGET_LDFLAGS}"
+        CXX_for_gdb="${CT_TARGET}-g++ ${CT_TARGET_CFLAGS} ${CT_TARGET_LDFLAGS}"
+        LD_for_gdb="${CT_TARGET}-ld ${CT_TARGET_LDFLAGS}"
+        CPP_for_gdb=`echo $CPP_for_gdb`
+        CC_for_gdb=`echo $CC_for_gdb`
+        CXX_for_gdb=`echo $CXX_for_gdb`
+        LD_for_gdb=`echo $LD_for_gdb`
+
         CT_DoExecLog CFG                                \
         CC_FOR_BUILD="${CT_BUILD}-gcc"                  \
         CFLAGS_FOR_BUILD="${CT_CFLAGS_FOR_BUILD}"       \
         LDFLAGS_FOR_BUILD="${CT_LDFLAGS_FOR_BUILD}"     \
-        CC="${CT_TARGET}-${CT_CC}"                      \
-        CPP="${CT_TARGET}-cpp"                          \
-        LD="${CT_TARGET}-ld"                            \
+        CPP="${CPP_for_gdb}"                            \
+        CC="${CC_for_gdb}"                              \
+        CXX="${CXX_for_gdb}"                            \
+        LD="${LD_for_gdb}"                              \
         LDFLAGS="${gdbserver_LDFLAGS}"                  \
         ${CONFIG_SHELL}                                 \
         "${gdb_src_dir}/gdb/gdbserver/configure"        \
diff --git a/scripts/build/debug/500-strace.sh b/scripts/build/debug/500-strace.sh
index cb4643af..305a76c2 100644
--- a/scripts/build/debug/500-strace.sh
+++ b/scripts/build/debug/500-strace.sh
@@ -1,21 +1,36 @@
 # Build script for strace
 
-do_debug_strace_get() {
+do_debug_strace_get()
+{
     CT_Fetch STRACE
 }
 
-do_debug_strace_extract() {
+do_debug_strace_extract()
+{
     CT_ExtractPatch STRACE
 }
 
-do_debug_strace_build() {
+do_debug_strace_build()
+{
+    local cflags="${CT_TARGET_CFLAGS}"
+
     CT_DoStep INFO "Installing strace"
 
+    if [ "${CT_LIBC_MUSL}" = "y" ]; then
+        # Otherwise kernel headers cause errors when included, e.g.
+        # <netinet/in.h> and <linux/in6.h>. Kernel's libc-compat.h
+        # only cares about GLIBC.  uClibc-ng does the same
+        # internally, pretending it's GLIBC for kernel headers inclusion.
+        cflags+=" -D__GLIBC__"
+    fi
+
     CT_mkdir_pushd "${CT_BUILD_DIR}/build-strace"
 
     CT_DoLog EXTRA "Configuring strace"
     CT_DoExecLog CFG                                           \
     CC="${CT_TARGET}-${CT_CC}"                                 \
+    CFLAGS="${cflags}"                                         \
+    LDFLAGS="${CT_TARGET_LDFLAGS}"                             \
     CPP="${CT_TARGET}-cpp"                                     \
     LD="${CT_TARGET}-ld"                                       \
     ${CONFIG_SHELL}                                            \
diff --git a/scripts/functions b/scripts/functions
index b8b49133..c3ef29e0 100644
--- a/scripts/functions
+++ b/scripts/functions
@@ -382,6 +382,7 @@ CT_DoExecLog() {
             break
         fi
     done
+    CT_DoLog DEBUG "==> Return status ${ret}"
     exit ${ret}
     )
     # Catch failure of the sub-shell
@@ -702,7 +703,8 @@ CT_GetFileBasename()
 # This functions always returns true (0), as it can be legitimate not
 # to find the requested URL (think about snapshots, different layouts
 # for different gcc versions, etc...).
-CT_DoGetFile() {
+CT_DoGetFile()
+{
     local url="${1}"
     local dest="${CT_TARBALLS_DIR}/${url##*/}"
     local tmp="${dest}.tmp-dl"
@@ -750,7 +752,8 @@ CT_DoGetFile() {
 # This function saves the specified to local storage if possible,
 # and if so, symlinks it for later usage
 # Usage: CT_SaveLocal </full/path/file.name>
-CT_SaveLocal() {
+CT_SaveLocal()
+{
     local file="$1"
     local basename="${file##*/}"
 
@@ -763,42 +766,172 @@ CT_SaveLocal() {
     fi
 }
 
+# Verify the file against a known digest.
+# Usage: CT_DoVerifyDigest <local-file-path> <package-directory>
+CT_DoVerifyDigest()
+{
+    local path="$1"
+    local file="${path##*/}"
+    local dir="${path%/*}"
+    local pkgdir="$2"
+    local alg="${CT_VERIFY_DOWNLOAD_DIGEST_ALG}"
+    local chksum a f c
+
+    if [ ! -r "${pkgdir}/chksum" ]; then
+        CT_DoLog WARN "Not verifying '${file}': digest missing"
+        return 0
+    fi
+    CT_DoLog EXTRA "Verifying ${alg^^} checksum for '${file}'"
+    chksum=`"${alg}sum" "${path}"`
+    chksum="${chksum%%[[:space:]]*}"
+    while read a f c; do
+        if [ "${a}" != "${alg}" -o "${f}" != "${file}" ]; then
+            continue
+        fi
+        if [ "${c}" = "${chksum}" ]; then
+            CT_DoLog DEBUG "Correct ${alg} digest for ${file}: ${chksum}"
+            return 0
+        else
+            CT_DoLog ERROR "Bad ${alg} digest for ${file}: ${chksum}, expect ${c}"
+            return 1
+        fi
+    done < "${pkgdir}/chksum"
+    CT_DoLog WARN "Downloaded file ${file} reference digest not available"
+    return 0
+}
+
+# Decompress a file to stdout
+CT_ZCat()
+{
+    local file="$1"
+
+    case "${file}" in
+    *.tar.xz)
+        xz -fdc "${file}"
+        ;;
+    *.tar.lzma)
+        xz -fdc --format=lzma "${file}"
+        ;;
+    *.tar.lz)
+        lzip -fdc "${file}"
+        ;;
+    *.tar.bz2)
+        bzip2 -dc "${file}"
+        ;;
+    *.tar.gz|*.tgz)
+        gzip -dc "${file}"
+        ;;
+    *.tar)
+        cat "${file}"
+        ;;
+    *)
+        CT_Abort "Unsupported archive file name '${file}'"
+    esac
+}
+
+# Verify the file against a detached signature.
+# Fetched from the URL, or obtained from the package directory.
+# Usage: CT_DoVerifySignature <local-file-path> <URL-used-for-download> <signature-format>
+CT_DoVerifySignature()
+{
+    local path="$1"
+    local file="${path##*/}"
+    local dir="${path%/*}"
+    local url="$2"
+    local urldir="${url%/*}"
+    local format="$3"
+    local method="${format%/*}"
+    local ext="${format#*/}"
+    local sigfile
+    local cat
+
+    CT_DoLog EXTRA "Verifying detached signature for '${file}'"
+    case "${method}" in
+    packed)
+        # Typical case: release is packed, then signed
+        sigfile="${file}"
+        cat=cat
+        ;;
+    unpacked)
+        # Linux kernel: uncompressed tarball is signed, them compressed by various methods
+        case "${file}" in
+        *.tar.*)
+            sigfile="${file%.tar.*}.tar"
+            cat=CT_ZCat
+            ;;
+        *)
+            CT_Abort "'unpacked' signature method only supported for tar archives"
+            ;;
+        esac
+        ;;
+    *)
+        CT_Abort "Unsupported signature method ${method}"
+        ;;
+    esac
+
+    # No recursion, as we don't pass signature_format argument
+    if ! CT_DoGetFile "${urldir}/${sigfile}${ext}"; then
+        CT_DoLog WARN "Failed to download the signature '${sigfile}${ext}'"
+        return 1
+    fi
+
+    CT_Pushd "${dir}"
+    if ! ${cat} "${file}" | CT_DoExecLog ALL gpg --verify "${sigfile}${ext}" -; then
+        # Remove the signature so it's re-downloaded next time
+        CT_DoExecLog ALL rm "${sigfile}${ext}"
+        CT_Popd
+        return 1
+    fi
+    CT_Popd
+
+    # If we get here, verification succeeded.
+    CT_SaveLocal "${CT_TARBALLS_DIR}/${sigfile}${ext}"
+}
+
 # Download the file from one of the URLs passed as argument
-# Usage: CT_GetFile <packagename> <basename> <extensions> <url> [url ...]
-CT_GetFile() {
-    local ext
+CT_GetFile()
+{
+    local -a argnames=(
+        package             # Name of the package
+        version             # Version of the package
+        basename            # Base name of file/archive
+        extensions          # Extension(s) for the file/archive
+        digest              # If 'y', verify the digest
+        signature_format    # Format of the signature
+        mirrors             # Mirrors to download from
+    )
     local -a URLS
-    local url
-    local package="$1"
-    local file="$2"
-    local extensions="$3"
-    shift 3
+    local ext url
+
+    for arg in "${argnames[@]/%/=}" "$@"; do
+        eval "local ${arg//[[:space:]]/\\ }"
+    done
 
     # Does any of the requested files exist localy?
     for ext in ${extensions}; do
         # Do we already have it in *our* tarballs dir?
-        if [ -r "${CT_TARBALLS_DIR}/${file}${ext}" ]; then
-            CT_DoLog DEBUG "Already have '${CT_TARBALLS_DIR}/${file}${ext}'"
+        if [ -r "${CT_TARBALLS_DIR}/${basename}${ext}" ]; then
+            CT_DoLog DEBUG "Already have '${CT_TARBALLS_DIR}/${basename}${ext}'"
             return 0
         fi
 
         if [ -n "${CT_LOCAL_TARBALLS_DIR}" -a "${CT_FORCE_DOWNLOAD}" != "y" -a \
-                -r "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" ]; then
-            CT_DoLog DEBUG "Got '${file}' from local storage"
-            CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" \
-                    "${CT_TARBALLS_DIR}/${file}${ext}"
+                -r "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" ]; then
+            CT_DoLog DEBUG "Got '${basename}' from local storage"
+            CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" \
+                    "${CT_TARBALLS_DIR}/${basename}${ext}"
             return 0
         fi
     done
 
     # No, it does not...  If not allowed to download from the Internet, don't.
     if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then
-        CT_DoLog DEBUG "Not allowed to download from the Internet, aborting ${file} download"
+        CT_DoLog DEBUG "Not allowed to download from the Internet, aborting ${basename} download"
         return 1
     fi
 
     # Try to retrieve the file
-    CT_DoLog EXTRA "Retrieving '${file}'"
+    CT_DoLog EXTRA "Retrieving '${basename}'"
 
     # Add URLs on the LAN mirror
     if [ "${CT_USE_MIRROR}" = "y" ]; then
@@ -810,18 +943,31 @@ CT_GetFile() {
     fi
 
     if [ "${CT_FORCE_MIRROR}" != "y" ]; then
-        URLS+=( "${@}" )
+        URLS+=( ${mirrors} )
     fi
 
     # Scan all URLs in turn, and try to grab a tarball from there
-    # Do *not* try git trees (ext=/.git), this is handled in a specific
-    # wrapper, below
     for ext in ${extensions}; do
         # Try all urls in turn
         for url in "${URLS[@]}"; do
             [ -n "${url}" ] || continue
-            if CT_DoGetFile "${url}/${file}${ext}"; then
-                CT_SaveLocal "${CT_TARBALLS_DIR}/${file}${ext}"
+            if CT_DoGetFile "${url}/${basename}${ext}"; then
+                if [ -n "${digest}" ] && ! CT_DoVerifyDigest \
+                            "${CT_TARBALLS_DIR}/${basename}${ext}" \
+                            "${CT_LIB_DIR}/packages/${package}/${version}"; then
+                    CT_DoLog ERROR "Digest verification failed; removing the download"
+                    CT_DoExecLog ALL rm "${CT_TARBALLS_DIR}/${basename}${ext}"
+                    return 1
+                fi
+                if [ -n "${signature_format}" ] && ! CT_DoVerifySignature \
+                            "${CT_TARBALLS_DIR}/${basename}${ext}" \
+                            "${url}/${basename}${ext}" \
+                            "${signature_format}"; then
+                    CT_DoLog ERROR "Signature verification failed; removing the download"
+                    CT_DoExecLog ALL rm "${CT_TARBALLS_DIR}/${basename}${ext}"
+                    return 1
+                fi
+                CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}${ext}"
                 return 0
             fi
         done
@@ -1742,7 +1888,7 @@ CT_PackageRun()
 
     # Variables that are per-fork
     for v in basename pkg_name version \
-            src_release mirrors archive_filename archive_dirname archive_formats \
+            src_release mirrors archive_filename archive_dirname archive_formats signature_format \
             src_devel devel_vcs devel_url devel_branch devel_revision devel_subdir devel_bootstrap \
             src_custom custom_location; do
         eval "local ${v}=\${CT_${use}_${v^^}}"
@@ -1780,7 +1926,11 @@ CT_DoFetch()
         else
             basename="${pkg_name}-${version}"
         fi
-        if ! CT_GetFile "${pkg_name}" "${archive_filename}" "${archive_formats}" ${mirrors}; then
+        if ! CT_GetFile package="${pkg_name}" version="${version}" \
+                basename="${archive_filename}" extensions="${archive_formats}" \
+                digest="${CT_VERIFY_DOWNLOAD_DIGEST}" \
+                signature_format="${CT_VERIFY_DOWNLOAD_SIGNATURE:+${signature_format}}" \
+                mirrors="${mirrors}"; then
             CT_Abort "${pkg_name}: download failed"
         fi
 
@@ -1811,7 +1961,8 @@ CT_DoFetch()
         # attempt getting it from local storage or from the mirror if configured.
         # Bzip2 offers a reasonable compromise between compression speed and size.
         if [ "${unique_id}" != "to.be.determined" ] && \
-                CT_GetFile "${pkg_name}" "${basename}" '.tar.bz2'; then
+                CT_GetFile package="${pkg_name}" version="${version}" \
+                        basename="${basename}" extensions='.tar.bz2'; then
             return 0
         fi
 
@@ -1862,23 +2013,8 @@ CT_Extract()
 
     CT_DoExecLog ALL mkdir -p "${dir}"
     case "${file}" in
-        *.tar.xz)
-            xz -fdc "${file}" | CT_DoExecLog FILE tar x -v -f - -C "${dir}" ${components}
-            ;;
-        *.tar.lzma)
-            xz -fdc "${file}" | CT_DoExecLog FILE tar x -v -f - -C "${dir}" ${components}
-            ;;
-        *.tar.lz)
-            lzip -fdc "${file}" | CT_DoExecLog FILE tar x -v -f - -C "${dir}" ${components}
-            ;;
-        *.tar.bz2)
-            bzip2 -dc "${file}" | CT_DoExecLog FILE tar x -v -f - -C "${dir}" ${components}
-            ;;
-        *.tar.gz|*.tgz)
-            gzip -dc "${file}" | CT_DoExecLog FILE tar x -v -f - -C "${dir}" ${components}
-            ;;
-        *.tar)
-            CT_DoExecLog FILE tar x -v -f "${file}" -C "${dir}" ${components}
+        *.tar.*|*.tar)
+            CT_ZCat "${file}" | CT_DoExecLog FILE tar x -v -f - -C "${dir}" ${components}
             ;;
         *.zip)
             CT_Pushd "${dir}"