scripts/functions
author "Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
Wed May 04 00:04:23 2011 +0200 (2011-05-04)
changeset 2608 aa09a36c3d36
parent 2605 09b546182d59
child 2609 409fe6b9ede2
permissions -rw-r--r--
scripts/functions: test for decompressors before use

./configure does check for the presence of gz and bzip2, so we can
safely use them in the build scripts.

On the other hand, more recent formats (eg. XZ) are not yet widely
available, and we do not want, and can't, force the user to install
them as a pre-requisite.

So, build up a list of allowed tarball formats based on the available
decompressors. For no, this is a static list, but the upcoming XZ
support will conditionnaly add to this list.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
     1 # This file contains some usefull common functions -*- sh -*-
     2 # Copyright 2007 Yann E. MORIN
     3 # Licensed under the GPL v2. See COPYING in the root of this package
     4 
     5 # Prepare the fault handler
     6 CT_OnError() {
     7     local ret=$?
     8     local intro
     9     local file
    10     local line
    11     local func
    12     local step_depth
    13 
    14     # Bail out early in subshell, the upper level shell will act accordingly.
    15     [ ${BASH_SUBSHELL} -eq 0 ] || exit $ret
    16 
    17     # Print steps backtrace
    18     step_depth=${CT_STEP_COUNT}
    19     CT_STEP_COUNT=2
    20     CT_DoLog ERROR ""
    21     intro="Build failed"
    22     for((step=step_depth; step>1; step--)); do
    23         CT_DoLog ERROR ">>  ${intro} in step '${CT_STEP_MESSAGE[${step}]}'"
    24         intro="      called"
    25     done
    26 
    27     # Print functions backtrace
    28     intro="Error happened in"
    29     offset=1
    30     CT_DoLog ERROR ">>"
    31     for((depth=1; ${BASH_LINENO[$((${depth}-1))]}>0; depth++)); do
    32         file="${BASH_SOURCE[${depth}]#${CT_LIB_DIR}/}"
    33         case "${depth}" in
    34             1)  line="";;
    35             *)  line="@${BASH_LINENO[${depth}-1]}"
    36         esac
    37         func="${FUNCNAME[${depth}]}"
    38         CT_DoLog ERROR ">>  ${intro}: ${func}[${file}${line}]"
    39         intro="      called from"
    40     done
    41 
    42     # Help diagnose the error
    43     CT_DoLog ERROR ">>"
    44     if [ "${CT_LOG_TO_FILE}" = "y" ]; then
    45         CT_DoLog ERROR ">>  For more info on this error, look at the file: '${tmp_log_file#${CT_TOP_DIR}/}'"
    46     fi
    47     CT_DoLog ERROR ">>  There is a list of known issues, some with workarounds, in:"
    48     CT_DoLog ERROR ">>      '${CT_DOC_DIR#${CT_TOP_DIR}/}/B - Known issues.txt'"
    49 
    50     CT_DoLog ERROR ""
    51     CT_DoLog ERROR "Build failed in step '${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}'"
    52 
    53     CT_DoLog ERROR ""
    54     CT_DoEnd ERROR
    55     exit $ret
    56 }
    57 
    58 # Install the fault handler
    59 trap CT_OnError ERR
    60 
    61 # Inherit the fault handler in subshells and functions
    62 set -E
    63 
    64 # Make pipes fail on the _first_ failed command
    65 # Not supported on bash < 3.x, but we need it, so drop the obsoleting bash-2.x
    66 set -o pipefail
    67 
    68 # Don't hash commands' locations, and search every time it is requested.
    69 # This is slow, but needed because of the static/shared core gcc which shall
    70 # always match to shared if it exists, and only fallback to static if the
    71 # shared is not found
    72 set +o hashall
    73 
    74 # Log policy:
    75 #  - first of all, save stdout so we can see the live logs: fd #6
    76 exec 6>&1
    77 #  - then point stdout to the log file
    78 tmp_log_file="${CT_TOP_DIR}/build.log"
    79 rm -f "${tmp_log_file}"
    80 exec >>"${tmp_log_file}"
    81 
    82 # The different log levels:
    83 CT_LOG_LEVEL_ERROR=0
    84 CT_LOG_LEVEL_WARN=1
    85 CT_LOG_LEVEL_INFO=2
    86 CT_LOG_LEVEL_EXTRA=3
    87 CT_LOG_LEVEL_CFG=4
    88 CT_LOG_LEVEL_FILE=5
    89 CT_LOG_LEVEL_STATE=6
    90 CT_LOG_LEVEL_ALL=7
    91 CT_LOG_LEVEL_DEBUG=8
    92 
    93 # Make it easy to use \n and !
    94 CR=$(printf "\n")
    95 BANG='!'
    96 
    97 # A function to log what is happening
    98 # Different log level are available:
    99 #   - ERROR:   A serious, fatal error occurred
   100 #   - WARN:    A non fatal, non serious error occurred, take your responsbility with the generated build
   101 #   - INFO:    Informational messages
   102 #   - EXTRA:   Extra informational messages
   103 #   - CFG:     Output of various "./configure"-type scripts
   104 #   - FILE:    File / archive unpacking.
   105 #   - STATE:   State save & restore
   106 #   - ALL:     Component's build messages
   107 #   - DEBUG:   Internal debug messages
   108 # Usage: CT_DoLog <level> [message]
   109 # If message is empty, then stdin will be logged.
   110 CT_DoLog() {
   111     local max_level LEVEL level cur_l cur_L
   112     local l
   113     eval max_level="\${CT_LOG_LEVEL_${CT_LOG_LEVEL_MAX}}"
   114     # Set the maximum log level to DEBUG if we have none
   115     [ -z "${max_level}" ] && max_level=${CT_LOG_LEVEL_DEBUG}
   116 
   117     LEVEL="$1"; shift
   118     eval level="\${CT_LOG_LEVEL_${LEVEL}}"
   119 
   120     if [ $# -eq 0 ]; then
   121         cat -
   122     else
   123         printf "%s\n" "${*}"
   124     fi |( IFS="${CR}" # We want the full lines, even leading spaces
   125           _prog_bar_cpt=0
   126           _prog_bar[0]='/'
   127           _prog_bar[1]='-'
   128           _prog_bar[2]='\'
   129           _prog_bar[3]='|'
   130           indent=$((2*CT_STEP_COUNT))
   131           while read line; do
   132               case "${CT_LOG_SEE_TOOLS_WARN},${line}" in
   133                 y,*"warning:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
   134                 y,*"WARNING:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
   135                 *"error:"*)             cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
   136                 *"make["*"]: *** ["*)   cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
   137                 *)                      cur_L="${LEVEL}"; cur_l="${level}";;
   138               esac
   139               # There will always be a log file (stdout, fd #1), be it /dev/null
   140               printf "[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}"
   141               if [ ${cur_l} -le ${max_level} ]; then
   142                   # Only print to console (fd #6) if log level is high enough.
   143                   printf "${CT_LOG_PROGRESS_BAR:+\r}[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}" >&6
   144               fi
   145               if [ "${CT_LOG_PROGRESS_BAR}" = "y" ]; then
   146                   printf "\r[%02d:%02d] %s " $((SECONDS/60)) $((SECONDS%60)) "${_prog_bar[$((_prog_bar_cpt/10))]}" >&6
   147                   _prog_bar_cpt=$(((_prog_bar_cpt+1)%40))
   148               fi
   149           done
   150         )
   151 
   152     return 0
   153 }
   154 
   155 # Execute an action, and log its messages
   156 # It is possible to even log local variable assignments (a-la: var=val ./cmd opts)
   157 # Usage: CT_DoExecLog <level> [VAR=val...] <command> [parameters...]
   158 CT_DoExecLog() {
   159     local level="$1"
   160     shift
   161     (
   162     for i in "$@"; do
   163         tmp_log+="'${i}' "
   164     done
   165     while true; do
   166         case "${1}" in
   167             *=*)    eval export "'${1}'"; shift;;
   168             *)      break;;
   169         esac
   170     done
   171     CT_DoLog DEBUG "==> Executing: ${tmp_log}"
   172     "${@}" 2>&1 |CT_DoLog "${level}"
   173     )
   174     # Catch failure of the sub-shell
   175     [ $? -eq 0 ]
   176 }
   177 
   178 # Tail message to be logged whatever happens
   179 # Usage: CT_DoEnd <level>
   180 CT_DoEnd()
   181 {
   182     local level="$1"
   183     CT_STOP_DATE=$(CT_DoDate +%s%N)
   184     CT_STOP_DATE_HUMAN=$(CT_DoDate +%Y%m%d.%H%M%S)
   185     if [ "${level}" != "ERROR" ]; then
   186         CT_DoLog "${level:-INFO}" "Build completed at ${CT_STOP_DATE_HUMAN}"
   187     fi
   188     elapsed=$((CT_STOP_DATE-CT_STAR_DATE))
   189     elapsed_min=$((elapsed/(60*1000*1000*1000)))
   190     elapsed_sec=$(printf "%02d" $(((elapsed%(60*1000*1000*1000))/(1000*1000*1000))))
   191     elapsed_csec=$(printf "%02d" $(((elapsed%(1000*1000*1000))/(10*1000*1000))))
   192     CT_DoLog ${level:-INFO} "(elapsed: ${elapsed_min}:${elapsed_sec}.${elapsed_csec})"
   193 }
   194 
   195 # Remove entries referring to . and other relative paths
   196 # Usage: CT_SanitizePath
   197 CT_SanitizePath() {
   198     local new
   199     local p
   200     local IFS=:
   201     for p in $PATH; do
   202         # Only accept absolute paths;
   203         # Note: as a special case the empty string in PATH is equivalent to .
   204         if [ -n "${p}" -a -z "${p%%/*}" ]; then
   205             new="${new}${new:+:}${p}"
   206         fi
   207     done
   208     PATH="${new}"
   209 }
   210 
   211 # Sanitise the directory name contained in the variable passed as argument:
   212 # - remove duplicate /
   213 # Usage: CT_SanitiseVarDir CT_PREFIX_DIR
   214 CT_SanitiseVarDir() {
   215     local var
   216     local old_dir
   217     local new_dir
   218 
   219     for var in "$@"; do
   220         eval "old_dir=\"\${${var}}\""
   221         new_dir="$( printf "${old_dir}"     \
   222                     |sed -r -e 's:/+:/:g;'  \
   223                   )"
   224         eval "${var}=\"${new_dir}\""
   225         CT_DoLog DEBUG "Sanitised '${var}': '${old_dir}' -> '${new_dir}'"
   226     done
   227 }
   228 
   229 # Abort the execution with an error message
   230 # Usage: CT_Abort <message>
   231 CT_Abort() {
   232     CT_DoLog ERROR "$1"
   233     exit 1
   234 }
   235 
   236 # Test a condition, and print a message if satisfied
   237 # Usage: CT_Test <message> <tests>
   238 CT_Test() {
   239     local ret
   240     local m="$1"
   241     shift
   242     CT_DoLog DEBUG "Testing '! ( $* )'"
   243     test "$@" && CT_DoLog WARN "$m"
   244     return 0
   245 }
   246 
   247 # Test a condition, and abort with an error message if satisfied
   248 # Usage: CT_TestAndAbort <message> <tests>
   249 CT_TestAndAbort() {
   250     local m="$1"
   251     shift
   252     CT_DoLog DEBUG "Testing '! ( $* )'"
   253     test "$@" && CT_Abort "$m"
   254     return 0
   255 }
   256 
   257 # Test a condition, and abort with an error message if not satisfied
   258 # Usage: CT_TestAndAbort <message> <tests>
   259 CT_TestOrAbort() {
   260     local m="$1"
   261     shift
   262     CT_DoLog DEBUG "Testing '$*'"
   263     test "$@" || CT_Abort "$m"
   264     return 0
   265 }
   266 
   267 # Test the presence of a tool, or abort if not found
   268 # Usage: CT_HasOrAbort <tool>
   269 CT_HasOrAbort() {
   270     CT_TestAndAbort "'${1}' not found and needed for successful toolchain build." -z "$(CT_Which "${1}")"
   271     return 0
   272 }
   273 
   274 # Search a program: wrap "which" for those system where
   275 # "which" verbosely says there is no match (Mandriva is
   276 # such a sucker...)
   277 # Usage: CT_Which <filename>
   278 CT_Which() {
   279   which "$1" 2>/dev/null || true
   280 }
   281 
   282 # Get current date with nanosecond precision
   283 # On those system not supporting nanosecond precision, faked with rounding down
   284 # to the highest entire second
   285 # Usage: CT_DoDate <fmt>
   286 CT_DoDate() {
   287     date "$1" |sed -r -e 's/%?N$/000000000/;'
   288 }
   289 
   290 CT_STEP_COUNT=1
   291 CT_STEP_MESSAGE[${CT_STEP_COUNT}]="<none>"
   292 # Memorise a step being done so that any error is caught
   293 # Usage: CT_DoStep <loglevel> <message>
   294 CT_DoStep() {
   295     local start=$(CT_DoDate +%s%N)
   296     CT_DoLog "$1" "================================================================="
   297     CT_DoLog "$1" "$2"
   298     CT_STEP_COUNT=$((CT_STEP_COUNT+1))
   299     CT_STEP_LEVEL[${CT_STEP_COUNT}]="$1"; shift
   300     CT_STEP_START[${CT_STEP_COUNT}]="${start}"
   301     CT_STEP_MESSAGE[${CT_STEP_COUNT}]="$1"
   302     return 0
   303 }
   304 
   305 # End the step just being done
   306 # Usage: CT_EndStep
   307 CT_EndStep() {
   308     local stop=$(CT_DoDate +%s%N)
   309     local duration=$(printf "%032d" $((stop-${CT_STEP_START[${CT_STEP_COUNT}]})) |sed -r -e 's/([[:digit:]]{2})[[:digit:]]{7}$/\.\1/; s/^0+//; s/^\./0\./;')
   310     local elapsed=$(printf "%02d:%02d" $((SECONDS/60)) $((SECONDS%60)))
   311     local level="${CT_STEP_LEVEL[${CT_STEP_COUNT}]}"
   312     local message="${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}"
   313     CT_STEP_COUNT=$((CT_STEP_COUNT-1))
   314     CT_DoLog "${level}" "${message}: done in ${duration}s (at ${elapsed})"
   315     return 0
   316 }
   317 
   318 # Pushes into a directory, and pops back
   319 CT_Pushd() {
   320     pushd "$1" >/dev/null 2>&1
   321 }
   322 CT_Popd() {
   323     popd >/dev/null 2>&1
   324 }
   325 
   326 # Creates a temporary directory
   327 # $1: variable to assign to
   328 # Usage: CT_MktempDir foo
   329 CT_MktempDir() {
   330     # Some mktemp do not allow more than 6 Xs
   331     eval "$1"=$(mktemp -q -d "${CT_BUILD_DIR}/tmp.XXXXXX")
   332     CT_TestOrAbort "Could not make temporary directory" -n "${!1}" -a -d "${!1}"
   333     CT_DoLog DEBUG "Made temporary directory '${!1}'"
   334     return 0
   335 }
   336 
   337 # Removes one or more directories, even if it is read-only, or its parent is
   338 # Usage: CT_DoForceRmdir dir [...]
   339 CT_DoForceRmdir() {
   340     local dir
   341     local mode
   342     for dir in "${@}"; do
   343         [ -d "${dir}" ] || continue
   344         case "$CT_SYS_OS" in
   345             Linux|CYGWIN*)
   346                 mode="$(stat -c '%a' "$(dirname "${dir}")")"
   347                 ;;
   348             Darwin|*BSD)
   349                 mode="$(stat -f '%Lp' "$(dirname "${dir}")")"
   350                 ;;
   351             *)
   352                 CT_Abort "Unhandled host OS $CT_SYS_OS"
   353                 ;;
   354         esac
   355         CT_DoExecLog ALL chmod u+w "$(dirname "${dir}")"
   356         CT_DoExecLog ALL chmod -R u+w "${dir}"
   357         CT_DoExecLog ALL rm -rf "${dir}"
   358         CT_DoExecLog ALL chmod ${mode} "$(dirname "${dir}")"
   359     done
   360 }
   361 
   362 # Echoes the specified string on stdout until the pipe breaks.
   363 # Doesn't fail
   364 # $1: string to echo
   365 # Usage: CT_DoYes "" |make oldconfig
   366 CT_DoYes() {
   367     yes "$1" || true
   368 }
   369 
   370 # Add the specified directory to LD_LIBRARY_PATH, and export it
   371 # If the specified patch is already present, just export
   372 # $1: path to add
   373 # $2: add as 'first' or 'last' path, 'first' is assumed if $2 is empty
   374 # Usage CT_SetLibPath /some/where/lib [first|last]
   375 CT_SetLibPath() {
   376     local path="$1"
   377     local pos="$2"
   378 
   379     case ":${LD_LIBRARY_PATH}:" in
   380         *:"${path}":*)  ;;
   381         *)  case "${pos}" in
   382                 last)
   383                     CT_DoLog DEBUG "Adding '${path}' at end of LD_LIBRARY_PATH"
   384                     LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${path}"
   385                     ;;
   386                 first|"")
   387                     CT_DoLog DEBUG "Adding '${path}' at start of LD_LIBRARY_PATH"
   388                     LD_LIBRARY_PATH="${path}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
   389                     ;;
   390                 *)
   391                     CT_Abort "Incorrect position '${pos}' to add '${path}' to LD_LIBRARY_PATH"
   392                     ;;
   393             esac
   394             ;;
   395     esac
   396     CT_DoLog DEBUG "==> LD_LIBRARY_PATH='${LD_LIBRARY_PATH}'"
   397     export LD_LIBRARY_PATH
   398 }
   399 
   400 # Build up the list of allowed tarball extensions
   401 # Add them in the prefered order; most preferred comes first
   402 CT_DoListTarballExt() {
   403     printf ".tar.bz2\n"
   404     printf ".tar.gz\n.tgz\n"
   405     printf ".tar\n"
   406 }
   407 
   408 # Get the file name extension of a component
   409 # Usage: CT_GetFileExtension <component_name-component_version> [extension]
   410 # If found, echoes the extension to stdout, and return 0
   411 # If not found, echoes nothing on stdout, and return !0.
   412 CT_GetFileExtension() {
   413     local ext
   414     local file="$1"
   415     shift
   416     local first_ext="$1"
   417 
   418     # we need to also check for an empty extension for those very
   419     # peculiar components that don't have one (such as sstrip from
   420     # buildroot).
   421     for ext in ${first_ext} $(CT_DoListTarballExt) /.git ''; do
   422         if [ -e "${CT_TARBALLS_DIR}/${file}${ext}" ]; then
   423             echo "${ext}"
   424             exit 0
   425         fi
   426     done
   427 
   428     exit 1
   429 }
   430 
   431 # Try to retrieve the specified URL (HTTP or FTP)
   432 # Usage: CT_DoGetFile <URL>
   433 # This functions always returns true (0), as it can be legitimate not
   434 # to find the requested URL (think about snapshots, different layouts
   435 # for different gcc versions, etc...).
   436 CT_DoGetFile() {
   437     local url="${1}"
   438     local dest="${CT_TARBALLS_DIR}/${url##*/}"
   439     local tmp="${dest}.tmp-dl"
   440     # OK, just look if we have them...
   441     # We are sure at least one is available, ./configure checked for it.
   442     local _curl=$(CT_Which curl)
   443     local _wget=$(CT_Which wget)
   444     _curl="${_curl:-false}"
   445     _wget="${_wget:-false}"
   446 
   447     # Remove potential left-over from a previous run
   448     rm -f "${tmp}"
   449 
   450     # Some (very old!) FTP server might not support the passive mode, thus
   451     # retry without.
   452     # We also retry a few times, in case there is a transient error (eg. behind
   453     # a dynamic IP that changes during the transfer...)
   454     # With automated download as we are doing, it can be very dangerous to
   455     # continue the downloads. It's far better to simply overwrite the
   456     # destination file.
   457     # Some company networks have firewalls to connect to the internet, but it's
   458     # not easy to detect them, and wget does not timeout by default while
   459     # connecting, so force a global ${CT_CONNECT_TIMEOUT}-second timeout.
   460     # For curl, no good progress indicator is available. So, be silent.
   461     if CT_DoExecLog ALL "${_curl}" --ftp-pasv    --retry 3 --connect-timeout ${CT_CONNECT_TIMEOUT} -L -f -s -o "${tmp}"   "${url}"  \
   462     || CT_DoExecLog ALL "${_curl}"               --retry 3 --connect-timeout ${CT_CONNECT_TIMEOUT} -L -f -s -o "${tmp}"   "${url}"  \
   463     || CT_DoExecLog ALL "${_wget}" --passive-ftp --tries=3 -T ${CT_CONNECT_TIMEOUT} -nc --progress=dot:binary -O "${tmp}" "${url}"  \
   464     || CT_DoExecLog ALL "${_wget}"               --tries=3 -T ${CT_CONNECT_TIMEOUT} -nc --progress=dot:binary -O "${tmp}" "${url}"  \
   465     ; then
   466         # One of them succeeded, good!
   467         mv "${tmp}" "${dest}"
   468     else
   469         # Woops...
   470         rm -f "${tmp}"
   471     fi
   472 }
   473 
   474 # This function tries to retrieve a tarball form a local directory
   475 # Usage: CT_GetLocal <basename> [.extension]
   476 CT_GetLocal() {
   477     local basename="$1"
   478     local first_ext="$2"
   479     local ext
   480 
   481     # Do we already have it in *our* tarballs dir?
   482     if ext="$( CT_GetFileExtension "${basename}" ${first_ext} )"; then
   483         CT_DoLog DEBUG "Already have '${basename}'"
   484         return 0
   485     fi
   486 
   487     if [ -n "${CT_LOCAL_TARBALLS_DIR}" ]; then
   488         CT_DoLog DEBUG "Trying to retrieve an already downloaded copy of '${basename}'"
   489         # We'd rather have a bzip2'ed tarball, then gzipped tarball, plain tarball,
   490         # or, as a failover, a file without extension.
   491         for ext in ${first_ext} $(CT_DoListTarballExt) ''; do
   492             CT_DoLog DEBUG "Trying '${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}'"
   493             if [ -r "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" -a \
   494                  "${CT_FORCE_DOWNLOAD}" != "y" ]; then
   495                 CT_DoLog DEBUG "Got '${basename}' from local storage"
   496                 CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" "${CT_TARBALLS_DIR}/${basename}${ext}"
   497                 return 0
   498             fi
   499         done
   500     fi
   501     return 1
   502 }
   503 
   504 # This function saves the specified to local storage if possible,
   505 # and if so, symlinks it for later usage
   506 # Usage: CT_SaveLocal </full/path/file.name>
   507 CT_SaveLocal() {
   508     local file="$1"
   509     local basename="${file##*/}"
   510 
   511     if [ "${CT_SAVE_TARBALLS}" = "y" ]; then
   512         CT_DoLog EXTRA "Saving '${basename}' to local storage"
   513         # The file may already exist if downloads are forced: remove it first
   514         CT_DoExecLog ALL rm -f "${CT_LOCAL_TARBALLS_DIR}/${basename}"
   515         CT_DoExecLog ALL mv -f "${file}" "${CT_LOCAL_TARBALLS_DIR}"
   516         CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}" "${file}"
   517     fi
   518 }
   519 
   520 # Download the file from one of the URLs passed as argument
   521 # Usage: CT_GetFile <basename> [.extension] <url> [url ...]
   522 CT_GetFile() {
   523     local ext
   524     local -a URLS
   525     local url
   526     local file="$1"
   527     local first_ext
   528     shift
   529     # If next argument starts with a dot, then this is not an URL,
   530     # and we can consider that it is a preferred extension.
   531     case "$1" in
   532         .*) first_ext="$1"
   533             shift
   534             ;;
   535     esac
   536 
   537     # Does it exist localy?
   538     if CT_GetLocal "${file}" ${first_ext}; then
   539         return 0
   540     fi
   541     # No, it does not...
   542 
   543     # Try to retrieve the file
   544     CT_DoLog EXTRA "Retrieving '${file}'"
   545 
   546     # Add URLs on the LAN mirror
   547     if [ "${CT_USE_MIRROR}" = "y" ]; then
   548         CT_TestOrAbort "Please set the mirror base URL" -n "${CT_MIRROR_BASE_URL}"
   549         URLS+=( "${CT_MIRROR_BASE_URL}/${file%-*}" )
   550         URLS+=( "${CT_MIRROR_BASE_URL}" )
   551     fi
   552 
   553     if [ "${CT_FORBID_DOWNLOAD}" != "y" ]; then
   554         URLS+=( "${@}" )
   555     fi
   556 
   557     # Scan all URLs in turn, and try to grab a tarball from there
   558     # Do *not* try git trees (ext=/.git), this is handled in a specific
   559     # wrapper, below
   560     for ext in ${first_ext} $(CT_DoListTarballExt) ''; do
   561         # Try all urls in turn
   562         for url in "${URLS[@]}"; do
   563             [ -n "${url}" ] || continue
   564             CT_DoLog DEBUG "Trying '${url}/${file}${ext}'"
   565             CT_DoGetFile "${url}/${file}${ext}"
   566             if [ -f "${CT_TARBALLS_DIR}/${file}${ext}" ]; then
   567                 CT_DoLog DEBUG "Got '${file}' from the Internet"
   568                 CT_SaveLocal "${CT_TARBALLS_DIR}/${file}${ext}"
   569                 return 0
   570             fi
   571         done
   572     done
   573 
   574     # Just return error, someone may want to catch and handle the error
   575     # (eg. glibc/eglibc add-ons can be missing).
   576     return 1
   577 }
   578 
   579 # Checkout from CVS, and build the associated tarball
   580 # The tarball will be called ${basename}.tar.bz2
   581 # Prerequisite: either the server does not require password,
   582 # or the user must already be logged in.
   583 # 'tag' is the tag to retrieve. Must be specified, but can be empty.
   584 # If dirname is specified, then module will be renamed to dirname
   585 # prior to building the tarball.
   586 # Usage: CT_GetCVS <basename> <url> <module> <tag> [dirname[=subdir]]
   587 # Note: if '=subdir' is given, then it is used instead of 'module'.
   588 CT_GetCVS() {
   589     local basename="$1"
   590     local uri="$2"
   591     local module="$3"
   592     local tag="${4:+-r ${4}}"
   593     local dirname="$5"
   594     local tmp_dir
   595 
   596     # First try locally, then the mirror
   597     if CT_GetFile "${basename}"; then
   598         # Got it! Return early! :-)
   599         return 0
   600     fi
   601 
   602     if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then
   603         CT_DoLog WARN "Downloads forbidden, not trying cvs retrieval"
   604         return 1
   605     fi
   606 
   607     CT_MktempDir tmp_dir
   608     CT_Pushd "${tmp_dir}"
   609 
   610     CT_DoExecLog ALL cvs -z 9 -d "${uri}" co -P ${tag} "${module}"
   611     if [ -n "${dirname}" ]; then
   612         case "${dirname}" in
   613             *=*)
   614                 CT_DoExecLog DEBUG mv "${dirname#*=}" "${dirname%%=*}"
   615                 CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${dirname%%=*}"
   616                 ;;
   617             *)
   618                 CT_DoExecLog ALL mv "${module}" "${dirname}"
   619                 CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${dirname:-${module}}"
   620                 ;;
   621         esac
   622     fi
   623     CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}.tar.bz2"
   624 
   625     CT_Popd
   626     CT_DoExecLog ALL rm -rf "${tmp_dir}"
   627 }
   628 
   629 # Check out from SVN, and build the associated tarball
   630 # The tarball will be called ${basename}.tar.bz2
   631 # Prerequisite: either the server does not require password,
   632 # or the user must already be logged in.
   633 # 'rev' is the revision to retrieve
   634 # Usage: CT_GetSVN <basename> <url> [rev]
   635 CT_GetSVN() {
   636     local basename="$1"
   637     local uri="$2"
   638     local rev="$3"
   639 
   640     # First try locally, then the mirror
   641     if CT_GetFile "${basename}"; then
   642         # Got it! Return early! :-)
   643         return 0
   644     fi
   645 
   646     if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then
   647         CT_DoLog WARN "Downloads forbidden, not trying svn retrieval"
   648         return 1
   649     fi
   650 
   651     CT_MktempDir tmp_dir
   652     CT_Pushd "${tmp_dir}"
   653 
   654     if ! CT_DoExecLog ALL svn export ${rev:+-r ${rev}} "${uri}" "${basename}"; then
   655         CT_DoLog WARN "Could not retrieve '${basename}'"
   656         return 1
   657     fi
   658     CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${basename}"
   659     CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}.tar.bz2"
   660 
   661     CT_Popd
   662     CT_DoExecLog ALL rm -rf "${tmp_dir}"
   663 }
   664 
   665 # Clone a git tree
   666 # Tries the given URLs in turn until one can get cloned. No tarball will be created.
   667 # Prerequisites: either the server does not require password,
   668 # or the user has already taken any action to authenticate to the server.
   669 # The cloned tree will *not* be stored in the local tarballs dir!
   670 # Usage: CT_GetGit <basename> <url [url ...]>
   671 CT_GetGit() {
   672     local basename="$1"; shift
   673     local url
   674     local cloned=0
   675 
   676     if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then
   677         CT_DoLog WARN "Downloads forbidden, not trying git retrieval"
   678         return 1
   679     fi
   680 
   681     # Do we have it in our tarballs dir?
   682     if [ -d "${CT_TARBALLS_DIR}/${basename}/.git" ]; then
   683         CT_DoLog EXTRA "Updating git tree '${basename}'"
   684         CT_Pushd "${CT_TARBALLS_DIR}/${basename}"
   685         CT_DoExecLog ALL git pull
   686         CT_Popd
   687     else
   688         CT_DoLog EXTRA "Retrieving git tree '${basename}'"
   689         for url in "${@}"; do
   690             CT_DoLog ALL "Trying to clone from '${url}'"
   691             CT_DoForceRmdir "${CT_TARBALLS_DIR}/${basename}"
   692             if git clone "${url}" "${CT_TARBALLS_DIR}/${basename}" 2>&1 |CT_DoLog ALL; then
   693                 cloned=1
   694                 break
   695             fi
   696         done
   697         CT_TestOrAbort "Could not clone '${basename}'" ${cloned} -ne 0
   698     fi
   699 }
   700 
   701 # Extract a tarball
   702 # Some tarballs need to be extracted in specific places. Eg.: glibc addons
   703 # must be extracted in the glibc directory; uCLibc locales must be extracted
   704 # in the extra/locale sub-directory of uClibc. This is taken into account
   705 # by the caller, that did a 'cd' into the correct path before calling us
   706 # and sets nochdir to 'nochdir'.
   707 # Note also that this function handles the git trees!
   708 # Usage: CT_Extract <basename> [nochdir] [options]
   709 # where 'options' are dependent on the source (eg. git branch/tag...)
   710 CT_Extract() {
   711     local nochdir="$1"
   712     local basename
   713     local ext
   714     local -a tar_opts
   715 
   716     if [ "${nochdir}" = "nochdir" ]; then
   717         shift
   718         nochdir="$(pwd)"
   719     else
   720         nochdir="${CT_SRC_DIR}"
   721     fi
   722 
   723     basename="$1"
   724     shift
   725 
   726     if ! ext="$(CT_GetFileExtension "${basename}")"; then
   727         CT_DoLog WARN "'${basename}' not found in '${CT_TARBALLS_DIR}'"
   728         return 1
   729     fi
   730     local full_file="${CT_TARBALLS_DIR}/${basename}${ext}"
   731 
   732     # Check if already extracted
   733     if [ -e "${CT_SRC_DIR}/.${basename}.extracted" ]; then
   734         CT_DoLog DEBUG "Already extracted '${basename}'"
   735         return 0
   736     fi
   737 
   738     # Check if previously partially extracted
   739     if [ -e "${CT_SRC_DIR}/.${basename}.extracting" ]; then
   740         CT_DoLog ERROR "The '${basename}' sources were partially extracted."
   741         CT_DoLog ERROR "Please remove first:"
   742         CT_DoLog ERROR " - the source dir for '${basename}', in '${CT_SRC_DIR}'"
   743         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${basename}.extracting'"
   744         CT_Abort "I'll stop now to avoid any carnage..."
   745     fi
   746     CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${basename}.extracting"
   747 
   748     CT_Pushd "${nochdir}"
   749 
   750     CT_DoLog EXTRA "Extracting '${basename}'"
   751     CT_DoExecLog FILE mkdir -p "${basename}"
   752     tar_opts=( "--strip-components=1" )
   753     tar_opts+=( "-C" "${basename}" )
   754     tar_opts+=( "-xv" )
   755     case "${ext}" in
   756         .tar.bz2)     CT_DoExecLog FILE tar "${tar_opts[@]}" -j -f "${full_file}";;
   757         .tar.gz|.tgz) CT_DoExecLog FILE tar "${tar_opts[@]}" -z -f "${full_file}";;
   758         .tar)         CT_DoExecLog FILE tar "${tar_opts[@]}" -f "${full_file}";;
   759         /.git)        CT_ExtractGit "${basename}" "${@}";;
   760         *)            CT_DoLog WARN "Don't know how to handle '${basename}${ext}': unknown extension"
   761                       return 1
   762                       ;;
   763     esac
   764 
   765     # Don't mark as being extracted for git
   766     case "${ext}" in
   767         /.git)  ;;
   768         *)      CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${basename}.extracted";;
   769     esac
   770     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/.${basename}.extracting"
   771 
   772     CT_Popd
   773 }
   774 
   775 # Create a working git clone
   776 # Usage: CT_ExtractGit <basename> [ref]
   777 # where 'ref' is the reference to use:
   778 #   the full name of a branch, like "remotes/origin/branch_name"
   779 #   a date as understandable by git, like "YYYY-MM-DD[ hh[:mm[:ss]]]"
   780 #   a tag name
   781 CT_ExtractGit() {
   782     local basename="${1}"
   783     local ref="${2}"
   784     local clone_dir
   785     local ref_type
   786 
   787     # pushd now to be able to get git revlist in case ref is a date
   788     clone_dir="${CT_TARBALLS_DIR}/${basename}"
   789     CT_Pushd "${clone_dir}"
   790 
   791     # What kind of reference is ${ref} ?
   792     if [ -z "${ref}" ]; then
   793         # Don't update the clone, keep as-is
   794         ref_type=none
   795     elif git tag |grep -E "^${ref}$" >/dev/null 2>&1; then
   796         ref_type=tag
   797     elif git branch -a --no-color |grep -E "^. ${ref}$" >/dev/null 2>&1; then
   798         ref_type=branch
   799     elif date -d "${ref}" >/dev/null 2>&1; then
   800         ref_type=date
   801         ref=$(git rev-list -n1 --before="${ref}")
   802     else
   803         CT_Abort "Reference '${ref}' is an incorrect git reference: neither tag, branch nor date"
   804     fi
   805 
   806     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/${basename}"
   807     CT_DoExecLog ALL ln -sf "${clone_dir}" "${CT_SRC_DIR}/${basename}"
   808 
   809     case "${ref_type}" in
   810         none)   ;;
   811         *)      CT_DoExecLog FILE git checkout "${ref}";;
   812     esac
   813 
   814     CT_Popd
   815 }
   816 
   817 # Patches the specified component
   818 # See CT_Extract, above, for explanations on 'nochdir'
   819 # Usage: CT_Patch [nochdir] <packagename> <packageversion>
   820 # If the package directory is *not* packagename-packageversion, then
   821 # the caller must cd into the proper directory first, and call us
   822 # with nochdir
   823 CT_Patch() {
   824     local nochdir="$1"
   825     local pkgname
   826     local version
   827     local pkgdir
   828     local base_file
   829     local ver_file
   830     local d
   831     local -a patch_dirs
   832     local bundled_patch_dir
   833     local local_patch_dir
   834 
   835     if [ "${nochdir}" = "nochdir" ]; then
   836         shift
   837         pkgname="$1"
   838         version="$2"
   839         pkgdir="${pkgname}-${version}"
   840         nochdir="$(pwd)"
   841     else
   842         pkgname="$1"
   843         version="$2"
   844         pkgdir="${pkgname}-${version}"
   845         nochdir="${CT_SRC_DIR}/${pkgdir}"
   846     fi
   847 
   848     # Check if already patched
   849     if [ -e "${CT_SRC_DIR}/.${pkgdir}.patched" ]; then
   850         CT_DoLog DEBUG "Already patched '${pkgdir}'"
   851         return 0
   852     fi
   853 
   854     # Check if already partially patched
   855     if [ -e "${CT_SRC_DIR}/.${pkgdir}.patching" ]; then
   856         CT_DoLog ERROR "The '${pkgdir}' sources were partially patched."
   857         CT_DoLog ERROR "Please remove first:"
   858         CT_DoLog ERROR " - the source dir for '${pkgdir}', in '${CT_SRC_DIR}'"
   859         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${pkgdir}.extracted'"
   860         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${pkgdir}.patching'"
   861         CT_Abort "I'll stop now to avoid any carnage..."
   862     fi
   863     touch "${CT_SRC_DIR}/.${pkgdir}.patching"
   864 
   865     CT_Pushd "${nochdir}"
   866 
   867     CT_DoLog EXTRA "Patching '${pkgdir}'"
   868 
   869     bundled_patch_dir="${CT_LIB_DIR}/patches/${pkgname}/${version}"
   870     local_patch_dir="${CT_LOCAL_PATCH_DIR}/${pkgname}/${version}"
   871 
   872     case "${CT_PATCH_ORDER}" in
   873         bundled)        patch_dirs=("${bundled_patch_dir}");;
   874         local)          patch_dirs=("${local_patch_dir}");;
   875         bundled,local)  patch_dirs=("${bundled_patch_dir}" "${local_patch_dir}");;
   876         local,bundled)  patch_dirs=("${local_patch_dir}" "${bundled_patch_dir}");;
   877         none)           patch_dirs=;;
   878     esac
   879 
   880     for d in "${patch_dirs[@]}"; do
   881         CT_DoLog DEBUG "Looking for patches in '${d}'..."
   882         if [ -n "${d}" -a -d "${d}" ]; then
   883             for p in "${d}"/*.patch; do
   884                 if [ -f "${p}" ]; then
   885                     CT_DoLog DEBUG "Applying patch '${p}'"
   886                     CT_DoExecLog ALL patch --no-backup-if-mismatch -g0 -F1 -p1 -f <"${p}"
   887                 fi
   888             done
   889             if [ "${CT_PATCH_SINGLE}" = "y" ]; then
   890                 break
   891             fi
   892         fi
   893     done
   894 
   895     if [ "${CT_OVERIDE_CONFIG_GUESS_SUB}" = "y" ]; then
   896         CT_DoLog ALL "Overiding config.guess and config.sub"
   897         for cfg in config_guess config_sub; do
   898             eval ${cfg}="${CT_LIB_DIR}/scripts/${cfg/_/.}"
   899             [ -e "${CT_TOP_DIR}/scripts/${cfg/_/.}" ] && eval ${cfg}="${CT_TOP_DIR}/scripts/${cfg/_/.}"
   900             # Can't use CT_DoExecLog because of the '{} \;' to be passed un-mangled to find
   901             find . -type f -name "${cfg/_/.}" -exec cp -v "${!cfg}" {} \; |CT_DoLog ALL
   902         done
   903     fi
   904 
   905     CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${pkgdir}.patched"
   906     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/.${pkgdir}.patching"
   907 
   908     CT_Popd
   909 }
   910 
   911 # Two wrappers to call config.(guess|sub) either from CT_TOP_DIR or CT_LIB_DIR.
   912 # Those from CT_TOP_DIR, if they exist, will be be more recent than those from CT_LIB_DIR.
   913 CT_DoConfigGuess() {
   914     if [ -x "${CT_TOP_DIR}/scripts/config.guess" ]; then
   915         "${CT_TOP_DIR}/scripts/config.guess"
   916     else
   917         "${CT_LIB_DIR}/scripts/config.guess"
   918     fi
   919 }
   920 
   921 CT_DoConfigSub() {
   922     if [ -x "${CT_TOP_DIR}/scripts/config.sub" ]; then
   923         "${CT_TOP_DIR}/scripts/config.sub" "$@"
   924     else
   925         "${CT_LIB_DIR}/scripts/config.sub" "$@"
   926     fi
   927 }
   928 
   929 # Compute the target tuple from what is provided by the user
   930 # Usage: CT_DoBuildTargetTuple
   931 # In fact this function takes the environment variables to build the target
   932 # tuple. It is needed both by the normal build sequence, as well as the
   933 # sample saving sequence.
   934 CT_DoBuildTargetTuple() {
   935     # Set the endianness suffix, and the default endianness gcc option
   936     case "${CT_ARCH_BE},${CT_ARCH_LE}" in
   937         y,) target_endian_eb=eb
   938             target_endian_el=
   939             CT_ARCH_ENDIAN_CFLAG="-mbig-endian"
   940             CT_ARCH_ENDIAN_LDFLAG="-EB"
   941             ;;
   942         ,y) target_endian_eb=
   943             target_endian_el=el
   944             CT_ARCH_ENDIAN_CFLAG="-mlittle-endian"
   945             CT_ARCH_ENDIAN_LDFLAG="-EL"
   946             ;;
   947     esac
   948 
   949     # Build the default architecture tuple part
   950     CT_TARGET_ARCH="${CT_ARCH}"
   951 
   952     # Set defaults for the system part of the tuple. Can be overriden
   953     # by architecture-specific values.
   954     case "${CT_LIBC}" in
   955         *glibc) CT_TARGET_SYS=gnu;;
   956         uClibc) CT_TARGET_SYS=uclibc;;
   957         *)      CT_TARGET_SYS=elf;;
   958     esac
   959 
   960     # Set the default values for ARCH, ABI, CPU, TUNE, FPU and FLOAT
   961     unset CT_ARCH_ARCH_CFLAG CT_ARCH_ABI_CFLAG CT_ARCH_CPU_CFLAG CT_ARCH_TUNE_CFLAG CT_ARCH_FPU_CFLAG CT_ARCH_FLOAT_CFLAG
   962     unset CT_ARCH_WITH_ARCH CT_ARCH_WITH_ABI CT_ARCH_WITH_CPU CT_ARCH_WITH_TUNE CT_ARCH_WITH_FPU CT_ARCH_WITH_FLOAT
   963     [ "${CT_ARCH_ARCH}"     ] && { CT_ARCH_ARCH_CFLAG="-march=${CT_ARCH_ARCH}";  CT_ARCH_WITH_ARCH="--with-arch=${CT_ARCH_ARCH}"; }
   964     [ "${CT_ARCH_ABI}"      ] && { CT_ARCH_ABI_CFLAG="-mabi=${CT_ARCH_ABI}";     CT_ARCH_WITH_ABI="--with-abi=${CT_ARCH_ABI}";    }
   965     [ "${CT_ARCH_CPU}"      ] && { CT_ARCH_CPU_CFLAG="-mcpu=${CT_ARCH_CPU}";     CT_ARCH_WITH_CPU="--with-cpu=${CT_ARCH_CPU}";    }
   966     [ "${CT_ARCH_TUNE}"     ] && { CT_ARCH_TUNE_CFLAG="-mtune=${CT_ARCH_TUNE}";  CT_ARCH_WITH_TUNE="--with-tune=${CT_ARCH_TUNE}"; }
   967     [ "${CT_ARCH_FPU}"      ] && { CT_ARCH_FPU_CFLAG="-mfpu=${CT_ARCH_FPU}";     CT_ARCH_WITH_FPU="--with-fpu=${CT_ARCH_FPU}";    }
   968     [ "${CT_ARCH_FLOAT_SW}" ] && { CT_ARCH_FLOAT_CFLAG="-msoft-float";           CT_ARCH_WITH_FLOAT="--with-float=soft";          }
   969 
   970     # Build the default kernel tuple part
   971     CT_TARGET_KERNEL="${CT_KERNEL}"
   972 
   973     # Overide the default values with the components specific settings
   974     CT_DoArchTupleValues
   975     CT_DoKernelTupleValues
   976 
   977     # Finish the target tuple construction
   978     CT_TARGET="${CT_TARGET_ARCH}"
   979     CT_TARGET="${CT_TARGET}${CT_TARGET_VENDOR:+-${CT_TARGET_VENDOR}}"
   980     CT_TARGET="${CT_TARGET}${CT_TARGET_KERNEL:+-${CT_TARGET_KERNEL}}"
   981     CT_TARGET="${CT_TARGET}${CT_TARGET_SYS:+-${CT_TARGET_SYS}}"
   982 
   983     # Sanity checks
   984     __sed_alias=""
   985     if [ -n "${CT_TARGET_ALIAS_SED_EXPR}" ]; then
   986         __sed_alias=$(echo "${CT_TARGET}" |sed -r -e "${CT_TARGET_ALIAS_SED_EXPR}")
   987     fi
   988     case ":${CT_TARGET_VENDOR}:${CT_TARGET_ALIAS}:${__sed_alias}:" in
   989       :*" "*:*:*:) CT_Abort "Don't use spaces in the vendor string, it breaks things.";;
   990       :*"-"*:*:*:) CT_Abort "Don't use dashes in the vendor string, it breaks things.";;
   991       :*:*" "*:*:) CT_Abort "Don't use spaces in the target alias, it breaks things.";;
   992       :*:*:*" "*:) CT_Abort "Don't use spaces in the target sed transform, it breaks things.";;
   993     esac
   994 
   995     # Canonicalise it
   996     CT_TARGET=$(CT_DoConfigSub "${CT_TARGET}")
   997     # Prepare the target CFLAGS
   998     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ENDIAN_CFLAG}"
   999     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ARCH_CFLAG}"
  1000     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ABI_CFLAG}"
  1001     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_CPU_CFLAG}"
  1002     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_TUNE_CFLAG}"
  1003     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FPU_CFLAG}"
  1004     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FLOAT_CFLAG}"
  1005 
  1006     # Now on for the target LDFLAGS
  1007     CT_ARCH_TARGET_LDFLAGS="${CT_ARCH_TARGET_LDFLAGS} ${CT_ARCH_ENDIAN_LDFLAG}"
  1008 }
  1009 
  1010 # This function does pause the build until the user strikes "Return"
  1011 # Usage: CT_DoPause [optional_message]
  1012 CT_DoPause() {
  1013     local foo
  1014     local message="${1:-Pausing for your pleasure}"
  1015     CT_DoLog INFO "${message}"
  1016     read -p "Press 'Enter' to continue, or Ctrl-C to stop..." foo >&6
  1017     return 0
  1018 }
  1019 
  1020 # This function creates a tarball of the specified directory, but
  1021 # only if it exists
  1022 # Usage: CT_DoTarballIfExists <dir> <tarball_basename> [extra_tar_options [...]]
  1023 CT_DoTarballIfExists() {
  1024     local dir="$1"
  1025     local tarball="$2"
  1026     shift 2
  1027     local -a extra_tar_opts=( "$@" )
  1028     local -a compress
  1029 
  1030     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1031         y)  compress=( gzip -c -3 - ); tar_ext=.gz;;
  1032         *)  compress=( cat - );        tar_ext=;;
  1033     esac
  1034 
  1035     if [ -d "${dir}" ]; then
  1036         CT_DoLog DEBUG "  Saving '${dir}'"
  1037         { tar c -C "${dir}" -v -f - "${extra_tar_opts[@]}" .    \
  1038           |"${compress[@]}" >"${tarball}.tar${tar_ext}"         ;
  1039         } 2>&1 |sed -r -e 's/^/    /;' |CT_DoLog STATE
  1040     else
  1041         CT_DoLog STATE "  Not saving '${dir}': does not exist"
  1042     fi
  1043 }
  1044 
  1045 # This function extracts a tarball to the specified directory, but
  1046 # only if the tarball exists
  1047 # Usage: CT_DoExtractTarballIfExists <tarball_basename> <dir> [extra_tar_options [...]]
  1048 CT_DoExtractTarballIfExists() {
  1049     local tarball="$1"
  1050     local dir="$2"
  1051     shift 2
  1052     local -a extra_tar_opts=( "$@" )
  1053     local -a uncompress
  1054 
  1055     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1056         y)  uncompress=( gzip -c -d ); tar_ext=.gz;;
  1057         *)  uncompress=( cat );        tar_ext=;;
  1058     esac
  1059 
  1060     if [ -f "${tarball}.tar${tar_ext}" ]; then
  1061         CT_DoLog DEBUG "  Restoring '${dir}'"
  1062         CT_DoForceRmdir "${dir}"
  1063         CT_DoExecLog DEBUG mkdir -p "${dir}"
  1064         { "${uncompress[@]}" "${tarball}.tar${tar_ext}"     \
  1065           |tar x -C "${dir}" -v -f - "${extra_tar_opts[@]}" ;
  1066         } 2>&1 |sed -r -e 's/^/    /;' |CT_DoLog STATE
  1067     else
  1068         CT_DoLog STATE "  Not restoring '${dir}': does not exist"
  1069     fi
  1070 }
  1071 
  1072 # This function saves the state of the toolchain to be able to restart
  1073 # at any one point
  1074 # Usage: CT_DoSaveState <next_step_name>
  1075 CT_DoSaveState() {
  1076 	[ "${CT_DEBUG_CT_SAVE_STEPS}" = "y" ] || return 0
  1077     local state_name="$1"
  1078     local state_dir="${CT_STATE_DIR}/${state_name}"
  1079 
  1080     # Log this to the log level required by the user
  1081     CT_DoLog ${CT_LOG_LEVEL_MAX} "Saving state to restart at step '${state_name}'..."
  1082 
  1083     rm -rf "${state_dir}"
  1084     mkdir -p "${state_dir}"
  1085 
  1086     CT_DoLog STATE "  Saving environment and aliases"
  1087     # We must omit shell functions, and some specific bash variables
  1088     # that break when restoring the environment, later. We could do
  1089     # all the processing in the awk script, but a sed is easier...
  1090     set |awk '
  1091               BEGIN { _p = 1; }
  1092               $0~/^[^ ]+ \(\)/ { _p = 0; }
  1093               _p == 1
  1094               $0 == "}" { _p = 1; }
  1095               ' |sed -r -e '/^BASH_(ARGC|ARGV|LINENO|SOURCE|VERSINFO)=/d;
  1096                            /^(UID|EUID)=/d;
  1097                            /^(FUNCNAME|GROUPS|PPID|SHELLOPTS)=/d;' >"${state_dir}/env.sh"
  1098 
  1099     CT_DoTarballIfExists "${CT_BUILDTOOLS_PREFIX_DIR}" "${state_dir}/buildtools_dir"
  1100     CT_DoTarballIfExists "${CT_COMPLIBS_DIR}" "${state_dir}/complibs_dir"
  1101     CT_DoTarballIfExists "${CT_CONFIG_DIR}" "${state_dir}/config_dir"
  1102     CT_DoTarballIfExists "${CT_CC_CORE_STATIC_PREFIX_DIR}" "${state_dir}/cc_core_static_prefix_dir"
  1103     CT_DoTarballIfExists "${CT_CC_CORE_SHARED_PREFIX_DIR}" "${state_dir}/cc_core_shared_prefix_dir"
  1104     CT_DoTarballIfExists "${CT_PREFIX_DIR}" "${state_dir}/prefix_dir" --exclude '*.log'
  1105 
  1106     CT_DoLog STATE "  Saving log file"
  1107     exec >/dev/null
  1108     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1109         y)  gzip -3 -c "${tmp_log_file}"  >"${state_dir}/log.gz";;
  1110         *)  cat "${tmp_log_file}" >"${state_dir}/log";;
  1111     esac
  1112     exec >>"${tmp_log_file}"
  1113 }
  1114 
  1115 # This function restores a previously saved state
  1116 # Usage: CT_DoLoadState <state_name>
  1117 CT_DoLoadState(){
  1118     local state_name="$1"
  1119     local state_dir="${CT_STATE_DIR}/${state_name}"
  1120     local old_RESTART="${CT_RESTART}"
  1121     local old_STOP="${CT_STOP}"
  1122 
  1123     CT_TestOrAbort "The previous build did not reach the point where it could be restarted at '${CT_RESTART}'" -d "${state_dir}"
  1124 
  1125     # We need to do something special with the log file!
  1126     if [ "${CT_LOG_TO_FILE}" = "y" ]; then
  1127         exec >"${state_dir}/tail.log"
  1128     fi
  1129 
  1130     # Log this to the log level required by the user
  1131     CT_DoLog ${CT_LOG_LEVEL_MAX} "Restoring state at step '${state_name}', as requested."
  1132 
  1133     CT_DoExtractTarballIfExists "${state_dir}/prefix_dir" "${CT_PREFIX_DIR}"
  1134     CT_DoExtractTarballIfExists "${state_dir}/cc_core_shared_prefix_dir" "${CT_CC_CORE_SHARED_PREFIX_DIR}"
  1135     CT_DoExtractTarballIfExists "${state_dir}/cc_core_static_prefix_dir" "${CT_CC_CORE_STATIC_PREFIX_DIR}"
  1136     CT_DoExtractTarballIfExists "${state_dir}/config_dir" "${CT_CONFIG_DIR}"
  1137     CT_DoExtractTarballIfExists "${state_dir}/complibs_dir" "${CT_COMPLIBS_DIR}"
  1138     CT_DoExtractTarballIfExists "${state_dir}/buildtools_dir" "${CT_BUILDTOOLS_PREFIX_DIR}"
  1139 
  1140     # Restore the environment, discarding any error message
  1141     # (for example, read-only bash internals)
  1142     CT_DoLog STATE "  Restoring environment"
  1143     . "${state_dir}/env.sh" >/dev/null 2>&1 || true
  1144 
  1145     # Restore the new RESTART and STOP steps
  1146     CT_RESTART="${old_RESTART}"
  1147     CT_STOP="${old_STOP}"
  1148     unset old_stop old_restart
  1149 
  1150     CT_DoLog STATE "  Restoring log file"
  1151     exec >/dev/null
  1152     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1153         y)  zcat "${state_dir}/log.gz" >"${tmp_log_file}";;
  1154         *)  cat "${state_dir}/log" >"${tmp_log_file}";;
  1155     esac
  1156     cat "${state_dir}/tail.log" >>"${tmp_log_file}"
  1157     exec >>"${tmp_log_file}"
  1158     rm -f "${state_dir}/tail.log"
  1159 }