From d26dc5d5f5509232e7bd162baa70697c86307079 Mon Sep 17 00:00:00 2001 From: Alexey Neyman Date: Sat, 19 Aug 2017 00:17:49 -0700 Subject: Use per-package list of formats to determine downloads Fixes #789 Signed-off-by: Alexey Neyman diff --git a/maintainer/kconfig-versions.template b/maintainer/kconfig-versions.template index e7e0121..4692a25 100644 --- a/maintainer/kconfig-versions.template +++ b/maintainer/kconfig-versions.template @@ -167,21 +167,6 @@ endif endchoice -#!if [ -n "@@all_versions@@" ] -config @@pfx@@_MIRRORS - string - default "@@mirrors@@" - -config @@pfx@@_ARCHIVE_FILENAME - string - default "@@archive_filename@@" - -config @@pfx@@_ARCHIVE_DIRNAME - string - default "@@archive_dirname@@" - -#!end-if - #!// Below, we explicitly select all milestones to which a given version #!// compares greater-or-equal. We don't select just the latest applicable #!// (and letting milestones chain-select each other, with FOO_6_or_later @@ -260,6 +245,28 @@ config @@pfx@@_VERSION #!end-foreach default "unknown" +#!if [ -n "@@all_versions@@" ] +config @@pfx@@_MIRRORS + string + default "@@mirrors@@" + +config @@pfx@@_ARCHIVE_FILENAME + string + default "@@archive_filename@@" + +config @@pfx@@_ARCHIVE_DIRNAME + string + default "@@archive_dirname@@" + +config @@pfx@@_ARCHIVE_FORMATS + string +#!foreach version + default "@@archive_formats@@" if @@pfx@@_V_@@kcfg@@ +#!end-foreach + default "@@archive_formats@@" + +#!end-if + #!if [ "@@nforks@@" -ge 2 ] endif #!end-if diff --git a/scripts/build/cc/gcc.sh b/scripts/build/cc/gcc.sh index 6019b35..163a6a2 100644 --- a/scripts/build/cc/gcc.sh +++ b/scripts/build/cc/gcc.sh @@ -14,7 +14,7 @@ do_cc_get() { # GCC source tree, which will not be there unless we get it and # put it there ourselves if [ "${CT_CC_LANG_JAVA_USE_ECJ}" = "y" ]; then - if ! CT_GetFile ecj-latest .jar $(CT_Mirrors sourceware java); then + if ! CT_GetFile ecj ecj-latest .jar $(CT_Mirrors sourceware java); then # Should be a package, too - but with Java retirement in GCC, # it may not make sense. CT_Abort "Failed to download ecj-latest.jar" diff --git a/scripts/functions b/scripts/functions index ba2e062..147de34 100644 --- a/scripts/functions +++ b/scripts/functions @@ -639,16 +639,13 @@ CT_GetFileExtension() { local ext local file="$1" - shift - local first_ext="$1" - for ext in ${first_ext} $(CT_DoListTarballExt); do + for ext in $(CT_DoListTarballExt); do if [ -e "${file}${ext}" -o -L "${file}${ext}" ]; then echo "${ext}" exit 0 fi done - exit 1 } @@ -686,6 +683,7 @@ CT_DoGetFile() { T="${CT_CONNECT_TIMEOUT}" fi + CT_DoLog DEBUG "Trying '${url}'" if [ "${CT_DOWNLOAD_AGENT_WGET}" = "y" ]; then if CT_DoExecLog ALL wget ${CT_DOWNLOAD_WGET_OPTIONS} \ ${T:+-T ${T}} \ @@ -706,43 +704,15 @@ CT_DoGetFile() { # Success, we got it, good! mv "${tmp}" "${dest}" CT_DoLog DEBUG "Got it from: \"${url}\"" + return 0 else - # Woops... + # Whoops... rm -f "${tmp}" CT_DoLog DEBUG "Not at this location: \"${url}\"" + return 1 fi } -# This function tries to retrieve a tarball form a local directory -# Usage: CT_GetLocal [.extension] -CT_GetLocal() { - local basename="$1" - local first_ext="$2" - local ext - - # Do we already have it in *our* tarballs dir? - if ext="$( CT_GetFileExtension "${CT_TARBALLS_DIR}/${basename}" ${first_ext} )"; then - CT_DoLog DEBUG "Already have '${CT_TARBALLS_DIR}/${basename}${ext}'" - return 0 - fi - - if [ -n "${CT_LOCAL_TARBALLS_DIR}" ]; then - CT_DoLog DEBUG "Trying to retrieve an already downloaded copy of '${basename}'" - # We'd rather have a bzip2'ed tarball, then gzipped tarball, plain tarball, - # or, as a failover, a file without extension. - for ext in ${first_ext} $(CT_DoListTarballExt) ''; do - CT_DoLog DEBUG "Trying '${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}'" - if [ -r "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" -a \ - "${CT_FORCE_DOWNLOAD}" != "y" ]; then - CT_DoLog DEBUG "Got '${basename}' from local storage" - CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" "${CT_TARBALLS_DIR}/${basename}${ext}" - return 0 - fi - done - fi - return 1 -} - # This function saves the specified to local storage if possible, # and if so, symlinks it for later usage # Usage: CT_SaveLocal @@ -760,29 +730,34 @@ CT_SaveLocal() { } # Download the file from one of the URLs passed as argument -# Usage: CT_GetFile [.extension] [url ...] +# Usage: CT_GetFile [url ...] CT_GetFile() { local ext local -a URLS local url - local file="$1" - local first_ext - shift - # If next argument starts with a dot, then this is not an URL, - # and we can consider that it is a preferred extension. - case "$1" in - .*) first_ext="$1" - shift - ;; - esac + local package="$1" + local file="$2" + local extensions="$3" + shift 3 + + # Does any of the requested files exist localy? + for ext in ${extensions}; do + # Do we already have it in *our* tarballs dir? + if [ -r "${CT_TARBALLS_DIR}/${file}${ext}" ]; then + CT_DoLog DEBUG "Already have '${CT_TARBALLS_DIR}/${file}${ext}'" + return 0 + fi - # Does it exist localy? - if CT_GetLocal "${file}" ${first_ext}; then - return 0 - fi - # No, it does not... + if [ -n "${CT_LOCAL_TARBALLS_DIR}" -a "${CT_FORCE_DOWNLOAD}" != "y" -a \ + -r "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" ]; then + CT_DoLog DEBUG "Got '${file}' from local storage" + CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" \ + "${CT_TARBALLS_DIR}/${file}${ext}" + return 0 + fi + done - # If not allowed to download from the Internet, don't + # No, it does not... If not allowed to download from the Internet, don't. if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then CT_DoLog DEBUG "Not allowed to download from the Internet, aborting ${file} download" return 1 @@ -794,7 +769,9 @@ CT_GetFile() { # Add URLs on the LAN mirror if [ "${CT_USE_MIRROR}" = "y" ]; then CT_TestOrAbort "Please set the mirror base URL" -n "${CT_MIRROR_BASE_URL}" - URLS+=( "${CT_MIRROR_BASE_URL}/${file%-*}" ) + if [ -n "${package}" ]; then + URLS+=( "${CT_MIRROR_BASE_URL}/${package}" ) + fi URLS+=( "${CT_MIRROR_BASE_URL}" ) fi @@ -805,14 +782,11 @@ CT_GetFile() { # Scan all URLs in turn, and try to grab a tarball from there # Do *not* try git trees (ext=/.git), this is handled in a specific # wrapper, below - for ext in ${first_ext} $(CT_DoListTarballExt) ''; do + for ext in ${extensions}; do # Try all urls in turn for url in "${URLS[@]}"; do [ -n "${url}" ] || continue - CT_DoLog DEBUG "Trying '${url}/${file}${ext}'" - CT_DoGetFile "${url}/${file}${ext}" - if [ -f "${CT_TARBALLS_DIR}/${file}${ext}" ]; then - CT_DoLog DEBUG "Got '${file}' from the Internet" + if CT_DoGetFile "${url}/${file}${ext}"; then CT_SaveLocal "${CT_TARBALLS_DIR}/${file}${ext}" return 0 fi @@ -1683,7 +1657,7 @@ CT_PackageRun() # Variables that are per-fork for v in basename pkg_name version \ - src_release mirrors archive_filename archive_dirname \ + src_release mirrors archive_filename archive_dirname archive_formats \ src_devel devel_vcs devel_url devel_branch devel_revision devel_subdir devel_bootstrap \ src_custom custom_location; do eval "local ${v}=\${CT_${use}_${v^^}}" @@ -1720,7 +1694,7 @@ CT_DoFetch() else basename="${pkg_name}-${version}" fi - if ! CT_GetFile "${archive_filename}" ${mirrors}; then + if ! CT_GetFile "${pkg_name}" "${archive_filename}" "${archive_formats}" ${mirrors}; then CT_Abort "${pkg_name}: download failed" fi @@ -1749,8 +1723,9 @@ CT_DoFetch() # Try getting the tarball with empty list of URLs: it will only # attempt getting it from local storage or from the mirror if configured. + # Bzip2 offers a reasonable compromise between compression speed and size. if [ "${unique_id}" != "to.be.determined" ] && \ - CT_GetFile "${basename}" .tar.bz2; then + CT_GetFile "${pkg_name}" "${basename}" '.tar.bz2'; then return 0 fi -- cgit v0.10.2-6-g49f6