scripts/functions
author "Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
Tue Aug 02 18:28:10 2011 +0200 (2011-08-02)
changeset 2590 b64cfb67944e
parent 2589 2289c546c8e1
child 2592 4908eb2b6f17
permissions -rw-r--r--
scripts/functions: svn retrieval first tries the mirror for tarballs

The svn download helper looks for the local tarballs dir to see if it
can find a pre-downloaded tarball, and if it does not find it, does
the actual fetch to upstream via svn.

In the process, it does not even try to get a tarball from the local
mirror, which can be useful if the mirror has been pre-populated
manually (or with a previously downloaded tree).

Fake a tarball get with the standard tarball-download helper, but
without specifying any upstream URL, which makes the helper directly
try the LAN mirror.

Of course, if no mirror is specified, no URL wil be available, and
the standard svn retrieval will kick in.

Reported-by: ANDY KENNEDY <ANDY.KENNEDY@adtran.com>
Signed-off-by: "Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
     1 # This file contains some usefull common functions -*- sh -*-
     2 # Copyright 2007 Yann E. MORIN
     3 # Licensed under the GPL v2. See COPYING in the root of this package
     4 
     5 # Prepare the fault handler
     6 CT_OnError() {
     7     local ret=$?
     8     local intro
     9     local file
    10     local line
    11     local func
    12     local step_depth
    13 
    14     # Bail out early in subshell, the upper level shell will act accordingly.
    15     [ ${BASH_SUBSHELL} -eq 0 ] || exit $ret
    16 
    17     # Print steps backtrace
    18     step_depth=${CT_STEP_COUNT}
    19     CT_STEP_COUNT=2
    20     CT_DoLog ERROR ""
    21     intro="Build failed"
    22     for((step=step_depth; step>1; step--)); do
    23         CT_DoLog ERROR ">>  ${intro} in step '${CT_STEP_MESSAGE[${step}]}'"
    24         intro="      called"
    25     done
    26 
    27     # Print functions backtrace
    28     intro="Error happened in"
    29     offset=1
    30     CT_DoLog ERROR ">>"
    31     for((depth=1; ${BASH_LINENO[$((${depth}-1))]}>0; depth++)); do
    32         file="${BASH_SOURCE[${depth}]#${CT_LIB_DIR}/}"
    33         case "${depth}" in
    34             1)  line="";;
    35             *)  line="@${BASH_LINENO[${depth}-1]}"
    36         esac
    37         func="${FUNCNAME[${depth}]}"
    38         CT_DoLog ERROR ">>  ${intro}: ${func}[${file}${line}]"
    39         intro="      called from"
    40     done
    41 
    42     # Help diagnose the error
    43     CT_DoLog ERROR ">>"
    44     if [ "${CT_LOG_TO_FILE}" = "y" ]; then
    45         CT_DoLog ERROR ">>  For more info on this error, look at the file: '${tmp_log_file#${CT_TOP_DIR}/}'"
    46     fi
    47     CT_DoLog ERROR ">>  There is a list of known issues, some with workarounds, in:"
    48     CT_DoLog ERROR ">>      '${CT_DOC_DIR#${CT_TOP_DIR}/}/B - Known issues.txt'"
    49 
    50     CT_DoLog ERROR ""
    51     CT_DoLog ERROR "Build failed in step '${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}'"
    52 
    53     CT_DoLog ERROR ""
    54     CT_DoEnd ERROR
    55     exit $ret
    56 }
    57 
    58 # Install the fault handler
    59 trap CT_OnError ERR
    60 
    61 # Inherit the fault handler in subshells and functions
    62 set -E
    63 
    64 # Make pipes fail on the _first_ failed command
    65 # Not supported on bash < 3.x, but we need it, so drop the obsoleting bash-2.x
    66 set -o pipefail
    67 
    68 # Don't hash commands' locations, and search every time it is requested.
    69 # This is slow, but needed because of the static/shared core gcc which shall
    70 # always match to shared if it exists, and only fallback to static if the
    71 # shared is not found
    72 set +o hashall
    73 
    74 # Log policy:
    75 #  - first of all, save stdout so we can see the live logs: fd #6
    76 exec 6>&1
    77 #  - then point stdout to the log file
    78 tmp_log_file="${CT_TOP_DIR}/build.log"
    79 rm -f "${tmp_log_file}"
    80 exec >>"${tmp_log_file}"
    81 
    82 # The different log levels:
    83 CT_LOG_LEVEL_ERROR=0
    84 CT_LOG_LEVEL_WARN=1
    85 CT_LOG_LEVEL_INFO=2
    86 CT_LOG_LEVEL_EXTRA=3
    87 CT_LOG_LEVEL_CFG=4
    88 CT_LOG_LEVEL_FILE=5
    89 CT_LOG_LEVEL_STATE=6
    90 CT_LOG_LEVEL_ALL=7
    91 CT_LOG_LEVEL_DEBUG=8
    92 
    93 # Make it easy to use \n and !
    94 CR=$(printf "\n")
    95 BANG='!'
    96 
    97 # A function to log what is happening
    98 # Different log level are available:
    99 #   - ERROR:   A serious, fatal error occurred
   100 #   - WARN:    A non fatal, non serious error occurred, take your responsbility with the generated build
   101 #   - INFO:    Informational messages
   102 #   - EXTRA:   Extra informational messages
   103 #   - CFG:     Output of various "./configure"-type scripts
   104 #   - FILE:    File / archive unpacking.
   105 #   - STATE:   State save & restore
   106 #   - ALL:     Component's build messages
   107 #   - DEBUG:   Internal debug messages
   108 # Usage: CT_DoLog <level> [message]
   109 # If message is empty, then stdin will be logged.
   110 CT_DoLog() {
   111     local max_level LEVEL level cur_l cur_L
   112     local l
   113     eval max_level="\${CT_LOG_LEVEL_${CT_LOG_LEVEL_MAX}}"
   114     # Set the maximum log level to DEBUG if we have none
   115     [ -z "${max_level}" ] && max_level=${CT_LOG_LEVEL_DEBUG}
   116 
   117     LEVEL="$1"; shift
   118     eval level="\${CT_LOG_LEVEL_${LEVEL}}"
   119 
   120     if [ $# -eq 0 ]; then
   121         cat -
   122     else
   123         printf "%s\n" "${*}"
   124     fi |( IFS="${CR}" # We want the full lines, even leading spaces
   125           _prog_bar_cpt=0
   126           _prog_bar[0]='/'
   127           _prog_bar[1]='-'
   128           _prog_bar[2]='\'
   129           _prog_bar[3]='|'
   130           indent=$((2*CT_STEP_COUNT))
   131           while read line; do
   132               case "${CT_LOG_SEE_TOOLS_WARN},${line}" in
   133                 y,*"warning:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
   134                 y,*"WARNING:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
   135                 *"error:"*)             cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
   136                 *"make["*"]: *** ["*)   cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
   137                 *)                      cur_L="${LEVEL}"; cur_l="${level}";;
   138               esac
   139               # There will always be a log file (stdout, fd #1), be it /dev/null
   140               printf "[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}"
   141               if [ ${cur_l} -le ${max_level} ]; then
   142                   # Only print to console (fd #6) if log level is high enough.
   143                   printf "${CT_LOG_PROGRESS_BAR:+\r}[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}" >&6
   144               fi
   145               if [ "${CT_LOG_PROGRESS_BAR}" = "y" ]; then
   146                   printf "\r[%02d:%02d] %s " $((SECONDS/60)) $((SECONDS%60)) "${_prog_bar[$((_prog_bar_cpt/10))]}" >&6
   147                   _prog_bar_cpt=$(((_prog_bar_cpt+1)%40))
   148               fi
   149           done
   150         )
   151 
   152     return 0
   153 }
   154 
   155 # Execute an action, and log its messages
   156 # It is possible to even log local variable assignments (a-la: var=val ./cmd opts)
   157 # Usage: CT_DoExecLog <level> [VAR=val...] <command> [parameters...]
   158 CT_DoExecLog() {
   159     local level="$1"
   160     shift
   161     (
   162     for i in "$@"; do
   163         tmp_log+="'${i}' "
   164     done
   165     while true; do
   166         case "${1}" in
   167             *=*)    eval export "'${1}'"; shift;;
   168             *)      break;;
   169         esac
   170     done
   171     CT_DoLog DEBUG "==> Executing: ${tmp_log}"
   172     "${@}" 2>&1 |CT_DoLog "${level}"
   173     )
   174     # Catch failure of the sub-shell
   175     [ $? -eq 0 ]
   176 }
   177 
   178 # Tail message to be logged whatever happens
   179 # Usage: CT_DoEnd <level>
   180 CT_DoEnd()
   181 {
   182     local level="$1"
   183     CT_STOP_DATE=$(CT_DoDate +%s%N)
   184     CT_STOP_DATE_HUMAN=$(CT_DoDate +%Y%m%d.%H%M%S)
   185     if [ "${level}" != "ERROR" ]; then
   186         CT_DoLog "${level:-INFO}" "Build completed at ${CT_STOP_DATE_HUMAN}"
   187     fi
   188     elapsed=$((CT_STOP_DATE-CT_STAR_DATE))
   189     elapsed_min=$((elapsed/(60*1000*1000*1000)))
   190     elapsed_sec=$(printf "%02d" $(((elapsed%(60*1000*1000*1000))/(1000*1000*1000))))
   191     elapsed_csec=$(printf "%02d" $(((elapsed%(1000*1000*1000))/(10*1000*1000))))
   192     CT_DoLog ${level:-INFO} "(elapsed: ${elapsed_min}:${elapsed_sec}.${elapsed_csec})"
   193 }
   194 
   195 # Remove entries referring to . and other relative paths
   196 # Usage: CT_SanitizePath
   197 CT_SanitizePath() {
   198     local new
   199     local p
   200     local IFS=:
   201     for p in $PATH; do
   202         # Only accept absolute paths;
   203         # Note: as a special case the empty string in PATH is equivalent to .
   204         if [ -n "${p}" -a -z "${p%%/*}" ]; then
   205             new="${new}${new:+:}${p}"
   206         fi
   207     done
   208     PATH="${new}"
   209 }
   210 
   211 # Sanitise the directory name contained in the variable passed as argument:
   212 # - remove duplicate /
   213 # Usage: CT_SanitiseVarDir CT_PREFIX_DIR
   214 CT_SanitiseVarDir() {
   215     local var
   216     local old_dir
   217     local new_dir
   218 
   219     for var in "$@"; do
   220         eval "old_dir=\"\${${var}}\""
   221         new_dir="$( printf "${old_dir}"     \
   222                     |sed -r -e 's:/+:/:g;'  \
   223                   )"
   224         eval "${var}=\"${new_dir}\""
   225         CT_DoLog DEBUG "Sanitised '${var}': '${old_dir}' -> '${new_dir}'"
   226     done
   227 }
   228 
   229 # Abort the execution with an error message
   230 # Usage: CT_Abort <message>
   231 CT_Abort() {
   232     CT_DoLog ERROR "$1"
   233     exit 1
   234 }
   235 
   236 # Test a condition, and print a message if satisfied
   237 # Usage: CT_Test <message> <tests>
   238 CT_Test() {
   239     local ret
   240     local m="$1"
   241     shift
   242     CT_DoLog DEBUG "Testing '! ( $* )'"
   243     test "$@" && CT_DoLog WARN "$m"
   244     return 0
   245 }
   246 
   247 # Test a condition, and abort with an error message if satisfied
   248 # Usage: CT_TestAndAbort <message> <tests>
   249 CT_TestAndAbort() {
   250     local m="$1"
   251     shift
   252     CT_DoLog DEBUG "Testing '! ( $* )'"
   253     test "$@" && CT_Abort "$m"
   254     return 0
   255 }
   256 
   257 # Test a condition, and abort with an error message if not satisfied
   258 # Usage: CT_TestAndAbort <message> <tests>
   259 CT_TestOrAbort() {
   260     local m="$1"
   261     shift
   262     CT_DoLog DEBUG "Testing '$*'"
   263     test "$@" || CT_Abort "$m"
   264     return 0
   265 }
   266 
   267 # Test the presence of a tool, or abort if not found
   268 # Usage: CT_HasOrAbort <tool>
   269 CT_HasOrAbort() {
   270     CT_TestAndAbort "'${1}' not found and needed for successful toolchain build." -z "$(CT_Which "${1}")"
   271     return 0
   272 }
   273 
   274 # Search a program: wrap "which" for those system where
   275 # "which" verbosely says there is no match (Mandriva is
   276 # such a sucker...)
   277 # Usage: CT_Which <filename>
   278 CT_Which() {
   279   which "$1" 2>/dev/null || true
   280 }
   281 
   282 # Get current date with nanosecond precision
   283 # On those system not supporting nanosecond precision, faked with rounding down
   284 # to the highest entire second
   285 # Usage: CT_DoDate <fmt>
   286 CT_DoDate() {
   287     date "$1" |sed -r -e 's/%?N$/000000000/;'
   288 }
   289 
   290 CT_STEP_COUNT=1
   291 CT_STEP_MESSAGE[${CT_STEP_COUNT}]="<none>"
   292 # Memorise a step being done so that any error is caught
   293 # Usage: CT_DoStep <loglevel> <message>
   294 CT_DoStep() {
   295     local start=$(CT_DoDate +%s%N)
   296     CT_DoLog "$1" "================================================================="
   297     CT_DoLog "$1" "$2"
   298     CT_STEP_COUNT=$((CT_STEP_COUNT+1))
   299     CT_STEP_LEVEL[${CT_STEP_COUNT}]="$1"; shift
   300     CT_STEP_START[${CT_STEP_COUNT}]="${start}"
   301     CT_STEP_MESSAGE[${CT_STEP_COUNT}]="$1"
   302     return 0
   303 }
   304 
   305 # End the step just being done
   306 # Usage: CT_EndStep
   307 CT_EndStep() {
   308     local stop=$(CT_DoDate +%s%N)
   309     local duration=$(printf "%032d" $((stop-${CT_STEP_START[${CT_STEP_COUNT}]})) |sed -r -e 's/([[:digit:]]{2})[[:digit:]]{7}$/\.\1/; s/^0+//; s/^\./0\./;')
   310     local elapsed=$(printf "%02d:%02d" $((SECONDS/60)) $((SECONDS%60)))
   311     local level="${CT_STEP_LEVEL[${CT_STEP_COUNT}]}"
   312     local message="${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}"
   313     CT_STEP_COUNT=$((CT_STEP_COUNT-1))
   314     CT_DoLog "${level}" "${message}: done in ${duration}s (at ${elapsed})"
   315     return 0
   316 }
   317 
   318 # Pushes into a directory, and pops back
   319 CT_Pushd() {
   320     pushd "$1" >/dev/null 2>&1
   321 }
   322 CT_Popd() {
   323     popd >/dev/null 2>&1
   324 }
   325 
   326 # Creates a temporary directory
   327 # $1: variable to assign to
   328 # Usage: CT_MktempDir foo
   329 CT_MktempDir() {
   330     # Some mktemp do not allow more than 6 Xs
   331     eval "$1"=$(mktemp -q -d "${CT_BUILD_DIR}/tmp.XXXXXX")
   332     CT_TestOrAbort "Could not make temporary directory" -n "${!1}" -a -d "${!1}"
   333     CT_DoLog DEBUG "Made temporary directory '${!1}'"
   334     return 0
   335 }
   336 
   337 # Removes one or more directories, even if it is read-only, or its parent is
   338 # Usage: CT_DoForceRmdir dir [...]
   339 CT_DoForceRmdir() {
   340     local dir
   341     local mode
   342     for dir in "${@}"; do
   343         [ -d "${dir}" ] || continue
   344         case "$CT_SYS_OS" in
   345             Linux|CYGWIN*)
   346                 mode="$(stat -c '%a' "$(dirname "${dir}")")"
   347                 ;;
   348             Darwin|*BSD)
   349                 mode="$(stat -f '%Lp' "$(dirname "${dir}")")"
   350                 ;;
   351             *)
   352                 CT_Abort "Unhandled host OS $CT_SYS_OS"
   353                 ;;
   354         esac
   355         CT_DoExecLog ALL chmod u+w "$(dirname "${dir}")"
   356         CT_DoExecLog ALL chmod -R u+w "${dir}"
   357         CT_DoExecLog ALL rm -rf "${dir}"
   358         CT_DoExecLog ALL chmod ${mode} "$(dirname "${dir}")"
   359     done
   360 }
   361 
   362 # Echoes the specified string on stdout until the pipe breaks.
   363 # Doesn't fail
   364 # $1: string to echo
   365 # Usage: CT_DoYes "" |make oldconfig
   366 CT_DoYes() {
   367     yes "$1" || true
   368 }
   369 
   370 # Add the specified directory to LD_LIBRARY_PATH, and export it
   371 # If the specified patch is already present, just export
   372 # $1: path to add
   373 # $2: add as 'first' or 'last' path, 'first' is assumed if $2 is empty
   374 # Usage CT_SetLibPath /some/where/lib [first|last]
   375 CT_SetLibPath() {
   376     local path="$1"
   377     local pos="$2"
   378 
   379     case ":${LD_LIBRARY_PATH}:" in
   380         *:"${path}":*)  ;;
   381         *)  case "${pos}" in
   382                 last)
   383                     CT_DoLog DEBUG "Adding '${path}' at end of LD_LIBRARY_PATH"
   384                     LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${path}"
   385                     ;;
   386                 first|"")
   387                     CT_DoLog DEBUG "Adding '${path}' at start of LD_LIBRARY_PATH"
   388                     LD_LIBRARY_PATH="${path}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
   389                     ;;
   390                 *)
   391                     CT_Abort "Incorrect position '${pos}' to add '${path}' to LD_LIBRARY_PATH"
   392                     ;;
   393             esac
   394             ;;
   395     esac
   396     CT_DoLog DEBUG "==> LD_LIBRARY_PATH='${LD_LIBRARY_PATH}'"
   397     export LD_LIBRARY_PATH
   398 }
   399 
   400 # Get the file name extension of a component
   401 # Usage: CT_GetFileExtension <component_name-component_version> [extension]
   402 # If found, echoes the extension to stdout, and return 0
   403 # If not found, echoes nothing on stdout, and return !0.
   404 CT_GetFileExtension() {
   405     local ext
   406     local file="$1"
   407     shift
   408     local first_ext="$1"
   409 
   410     # we need to also check for an empty extension for those very
   411     # peculiar components that don't have one (such as sstrip from
   412     # buildroot).
   413     for ext in ${first_ext} .tar.gz .tar.bz2 .tgz .tar /.git ''; do
   414         if [ -e "${CT_TARBALLS_DIR}/${file}${ext}" ]; then
   415             echo "${ext}"
   416             exit 0
   417         fi
   418     done
   419 
   420     exit 1
   421 }
   422 
   423 # Try to retrieve the specified URL (HTTP or FTP)
   424 # Usage: CT_DoGetFile <URL>
   425 # This functions always returns true (0), as it can be legitimate not
   426 # to find the requested URL (think about snapshots, different layouts
   427 # for different gcc versions, etc...).
   428 CT_DoGetFile() {
   429     local url="${1}"
   430     local dest="${CT_TARBALLS_DIR}/${url##*/}"
   431     local tmp="${dest}.tmp-dl"
   432     # OK, just look if we have them...
   433     # We are sure at least one is available, ./configure checked for it.
   434     local _curl=$(CT_Which curl)
   435     local _wget=$(CT_Which wget)
   436     _curl="${_curl:-false}"
   437     _wget="${_wget:-false}"
   438 
   439     # Remove potential left-over from a previous run
   440     rm -f "${tmp}"
   441 
   442     # Some (very old!) FTP server might not support the passive mode, thus
   443     # retry without.
   444     # We also retry a few times, in case there is a transient error (eg. behind
   445     # a dynamic IP that changes during the transfer...)
   446     # With automated download as we are doing, it can be very dangerous to
   447     # continue the downloads. It's far better to simply overwrite the
   448     # destination file.
   449     # Some company networks have firewalls to connect to the internet, but it's
   450     # not easy to detect them, and wget does not timeout by default while
   451     # connecting, so force a global ${CT_CONNECT_TIMEOUT}-second timeout.
   452     # For curl, no good progress indicator is available. So, be silent.
   453     if CT_DoExecLog ALL "${_curl}" --ftp-pasv    --retry 3 --connect-timeout ${CT_CONNECT_TIMEOUT} -L -f -s -o "${tmp}"   "${url}"  \
   454     || CT_DoExecLog ALL "${_curl}"               --retry 3 --connect-timeout ${CT_CONNECT_TIMEOUT} -L -f -s -o "${tmp}"   "${url}"  \
   455     || CT_DoExecLog ALL "${_wget}" --passive-ftp --tries=3 -T ${CT_CONNECT_TIMEOUT} -nc --progress=dot:binary -O "${tmp}" "${url}"  \
   456     || CT_DoExecLog ALL "${_wget}"               --tries=3 -T ${CT_CONNECT_TIMEOUT} -nc --progress=dot:binary -O "${tmp}" "${url}"  \
   457     ; then
   458         # One of them succeeded, good!
   459         mv "${tmp}" "${dest}"
   460     else
   461         # Woops...
   462         rm -f "${tmp}"
   463     fi
   464 }
   465 
   466 # This function tries to retrieve a tarball form a local directory
   467 # Usage: CT_GetLocal <basename> [.extension]
   468 CT_GetLocal() {
   469     local basename="$1"
   470     local first_ext="$2"
   471     local ext
   472 
   473     # Do we already have it in *our* tarballs dir?
   474     if ext="$( CT_GetFileExtension "${basename}" ${first_ext} )"; then
   475         CT_DoLog DEBUG "Already have '${basename}'"
   476         return 0
   477     fi
   478 
   479     if [ -n "${CT_LOCAL_TARBALLS_DIR}" ]; then
   480         CT_DoLog DEBUG "Trying to retrieve an already downloaded copy of '${basename}'"
   481         # We'd rather have a bzip2'ed tarball, then gzipped tarball, plain tarball,
   482         # or, as a failover, a file without extension.
   483         for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do
   484             CT_DoLog DEBUG "Trying '${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}'"
   485             if [ -r "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" -a \
   486                  "${CT_FORCE_DOWNLOAD}" != "y" ]; then
   487                 CT_DoLog DEBUG "Got '${basename}' from local storage"
   488                 CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" "${CT_TARBALLS_DIR}/${basename}${ext}"
   489                 return 0
   490             fi
   491         done
   492     fi
   493     return 1
   494 }
   495 
   496 # This function saves the specified to local storage if possible,
   497 # and if so, symlinks it for later usage
   498 # Usage: CT_SaveLocal </full/path/file.name>
   499 CT_SaveLocal() {
   500     local file="$1"
   501     local basename="${file##*/}"
   502 
   503     if [ "${CT_SAVE_TARBALLS}" = "y" ]; then
   504         CT_DoLog EXTRA "Saving '${basename}' to local storage"
   505         # The file may already exist if downloads are forced: remove it first
   506         CT_DoExecLog ALL rm -f "${CT_LOCAL_TARBALLS_DIR}/${basename}"
   507         CT_DoExecLog ALL mv -f "${file}" "${CT_LOCAL_TARBALLS_DIR}"
   508         CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}" "${file}"
   509     fi
   510 }
   511 
   512 # Download the file from one of the URLs passed as argument
   513 # Usage: CT_GetFile <basename> [.extension] <url> [url ...]
   514 CT_GetFile() {
   515     local ext
   516     local url URLS LAN_URLS
   517     local file="$1"
   518     local first_ext
   519     shift
   520     # If next argument starts with a dot, then this is not an URL,
   521     # and we can consider that it is a preferred extension.
   522     case "$1" in
   523         .*) first_ext="$1"
   524             shift
   525             ;;
   526     esac
   527 
   528     # Does it exist localy?
   529     CT_GetLocal "${file}" ${first_ext} && return 0 || true
   530     # No, it does not...
   531 
   532     # Try to retrieve the file
   533     CT_DoLog EXTRA "Retrieving '${file}'"
   534 
   535     URLS="$@"
   536 
   537     # Add URLs on the LAN mirror
   538     LAN_URLS=
   539     if [ "${CT_USE_MIRROR}" = "y" ]; then
   540         CT_TestOrAbort "Please set the mirror base URL" -n "${CT_MIRROR_BASE_URL}"
   541         LAN_URLS="${LAN_URLS} ${CT_MIRROR_BASE_URL}/${file%-*}"
   542         LAN_URLS="${LAN_URLS} ${CT_MIRROR_BASE_URL}"
   543 
   544         if [ "${CT_PREFER_MIRROR}" = "y" ]; then
   545             CT_DoLog DEBUG "Pre-pending LAN mirror URLs"
   546             URLS="${LAN_URLS} ${URLS}"
   547         else
   548             CT_DoLog DEBUG "Appending LAN mirror URLs"
   549             URLS="${URLS} ${LAN_URLS}"
   550         fi
   551     fi
   552 
   553     # Scan all URLs in turn, and try to grab a tarball from there
   554     # Do *not* try git trees (ext=/.git), this is handled in a specific
   555     # wrapper, below
   556     for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do
   557         # Try all urls in turn
   558         for url in ${URLS}; do
   559             [ -n "${url}" ] || continue
   560             CT_DoLog DEBUG "Trying '${url}/${file}${ext}'"
   561             CT_DoGetFile "${url}/${file}${ext}"
   562             if [ -f "${CT_TARBALLS_DIR}/${file}${ext}" ]; then
   563                 CT_DoLog DEBUG "Got '${file}' from the Internet"
   564                 CT_SaveLocal "${CT_TARBALLS_DIR}/${file}${ext}"
   565                 return 0
   566             fi
   567         done
   568     done
   569 
   570     # Just warn, someone may want to catch and handle the error
   571     # (eg. glibc/eglibc add-ons can be missing).
   572     CT_DoLog WARN "Could not retrieve '${file}'."
   573     return 1
   574 }
   575 
   576 # Checkout from CVS, and build the associated tarball
   577 # The tarball will be called ${basename}.tar.bz2
   578 # Prerequisite: either the server does not require password,
   579 # or the user must already be logged in.
   580 # 'tag' is the tag to retrieve. Must be specified, but can be empty.
   581 # If dirname is specified, then module will be renamed to dirname
   582 # prior to building the tarball.
   583 # Usage: CT_GetCVS <basename> <url> <module> <tag> [dirname[=subdir]]
   584 # Note: if '=subdir' is given, then it is used instead of 'module'.
   585 CT_GetCVS() {
   586     local basename="$1"
   587     local uri="$2"
   588     local module="$3"
   589     local tag="${4:+-r ${4}}"
   590     local dirname="$5"
   591     local tmp_dir
   592 
   593     # Does it exist localy?
   594     CT_GetLocal "${basename}" && return 0 || true
   595     # No, it does not...
   596 
   597     CT_DoLog EXTRA "Retrieving '${basename}'"
   598 
   599     CT_MktempDir tmp_dir
   600     CT_Pushd "${tmp_dir}"
   601 
   602     CT_DoExecLog ALL cvs -z 9 -d "${uri}" co -P ${tag} "${module}"
   603     if [ -n "${dirname}" ]; then
   604         case "${dirname}" in
   605             *=*)
   606                 CT_DoExecLog DEBUG mv "${dirname#*=}" "${dirname%%=*}"
   607                 CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${dirname%%=*}"
   608                 ;;
   609             *)
   610                 CT_DoExecLog ALL mv "${module}" "${dirname}"
   611                 CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${dirname:-${module}}"
   612                 ;;
   613         esac
   614     fi
   615     CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}.tar.bz2"
   616 
   617     CT_Popd
   618     CT_DoExecLog ALL rm -rf "${tmp_dir}"
   619 }
   620 
   621 # Check out from SVN, and build the associated tarball
   622 # The tarball will be called ${basename}.tar.bz2
   623 # Prerequisite: either the server does not require password,
   624 # or the user must already be logged in.
   625 # 'rev' is the revision to retrieve
   626 # Usage: CT_GetSVN <basename> <url> [rev]
   627 CT_GetSVN() {
   628     local basename="$1"
   629     local uri="$2"
   630     local rev="$3"
   631 
   632     # First try locally, then the mirror
   633     if CT_GetFile "${basename}"; then
   634         # Got it! Return early! :-)
   635         return 0
   636     fi
   637 
   638     CT_MktempDir tmp_dir
   639     CT_Pushd "${tmp_dir}"
   640 
   641     if ! CT_DoExecLog ALL svn export ${rev:+-r ${rev}} "${uri}" "${basename}"; then
   642         CT_DoLog WARN "Could not retrieve '${basename}'"
   643         return 1
   644     fi
   645     CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${basename}"
   646     CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}.tar.bz2"
   647 
   648     CT_Popd
   649     CT_DoExecLog ALL rm -rf "${tmp_dir}"
   650 }
   651 
   652 # Clone a git tree
   653 # Tries the given URLs in turn until one can get cloned. No tarball will be created.
   654 # Prerequisites: either the server does not require password,
   655 # or the user has already taken any action to authenticate to the server.
   656 # The cloned tree will *not* be stored in the local tarballs dir!
   657 # Usage: CT_GetGit <basename> <url [url ...]>
   658 CT_GetGit() {
   659     local basename="$1"; shift
   660     local url
   661     local cloned=0
   662 
   663     # Do we have it in our tarballs dir?
   664     if [ -d "${CT_TARBALLS_DIR}/${basename}/.git" ]; then
   665         CT_DoLog EXTRA "Updating git tree '${basename}'"
   666         CT_Pushd "${CT_TARBALLS_DIR}/${basename}"
   667         CT_DoExecLog ALL git pull
   668         CT_Popd
   669     else
   670         CT_DoLog EXTRA "Retrieving git tree '${basename}'"
   671         for url in "${@}"; do
   672             CT_DoLog ALL "Trying to clone from '${url}'"
   673             CT_DoForceRmdir "${CT_TARBALLS_DIR}/${basename}"
   674             if git clone "${url}" "${CT_TARBALLS_DIR}/${basename}" 2>&1 |CT_DoLog ALL; then
   675                 cloned=1
   676                 break
   677             fi
   678         done
   679         CT_TestOrAbort "Could not clone '${basename}'" ${cloned} -ne 0
   680     fi
   681 }
   682 
   683 # Extract a tarball
   684 # Some tarballs need to be extracted in specific places. Eg.: glibc addons
   685 # must be extracted in the glibc directory; uCLibc locales must be extracted
   686 # in the extra/locale sub-directory of uClibc. This is taken into account
   687 # by the caller, that did a 'cd' into the correct path before calling us
   688 # and sets nochdir to 'nochdir'.
   689 # Note also that this function handles the git trees!
   690 # Usage: CT_Extract <basename> [nochdir] [options]
   691 # where 'options' are dependent on the source (eg. git branch/tag...)
   692 CT_Extract() {
   693     local nochdir="$1"
   694     local basename
   695     local ext
   696 
   697     if [ "${nochdir}" = "nochdir" ]; then
   698         shift
   699         nochdir="$(pwd)"
   700     else
   701         nochdir="${CT_SRC_DIR}"
   702     fi
   703 
   704     basename="$1"
   705     shift
   706 
   707     if ! ext="$(CT_GetFileExtension "${basename}")"; then
   708         CT_DoLog WARN "'${basename}' not found in '${CT_TARBALLS_DIR}'"
   709         return 1
   710     fi
   711     local full_file="${CT_TARBALLS_DIR}/${basename}${ext}"
   712 
   713     # Check if already extracted
   714     if [ -e "${CT_SRC_DIR}/.${basename}.extracted" ]; then
   715         CT_DoLog DEBUG "Already extracted '${basename}'"
   716         return 0
   717     fi
   718 
   719     # Check if previously partially extracted
   720     if [ -e "${CT_SRC_DIR}/.${basename}.extracting" ]; then
   721         CT_DoLog ERROR "The '${basename}' sources were partially extracted."
   722         CT_DoLog ERROR "Please remove first:"
   723         CT_DoLog ERROR " - the source dir for '${basename}', in '${CT_SRC_DIR}'"
   724         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${basename}.extracting'"
   725         CT_Abort "I'll stop now to avoid any carnage..."
   726     fi
   727     CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${basename}.extracting"
   728 
   729     CT_Pushd "${nochdir}"
   730 
   731     CT_DoLog EXTRA "Extracting '${basename}'"
   732     CT_DoExecLog FILE mkdir -p "${basename}"
   733     case "${ext}" in
   734         .tar.bz2)     CT_DoExecLog FILE tar --strip-components=1 -C "${basename}" -xvjf "${full_file}";;
   735         .tar.gz|.tgz) CT_DoExecLog FILE tar --strip-components=1 -C "${basename}" -xvzf "${full_file}";;
   736         .tar)         CT_DoExecLog FILE tar --strip-components=1 -C "${basename}" -xvf "${full_file}";;
   737         /.git)        CT_ExtractGit "${basename}" "${@}";;
   738         *)            CT_DoLog WARN "Don't know how to handle '${basename}${ext}': unknown extension"
   739                       return 1
   740                       ;;
   741     esac
   742 
   743     # Don't mark as being extracted for git
   744     case "${ext}" in
   745         /.git)  ;;
   746         *)      CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${basename}.extracted";;
   747     esac
   748     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/.${basename}.extracting"
   749 
   750     CT_Popd
   751 }
   752 
   753 # Create a working git clone
   754 # Usage: CT_ExtractGit <basename> [ref]
   755 # where 'ref' is the reference to use:
   756 #   the full name of a branch, like "remotes/origin/branch_name"
   757 #   a date as understandable by git, like "YYYY-MM-DD[ hh[:mm[:ss]]]"
   758 #   a tag name
   759 CT_ExtractGit() {
   760     local basename="${1}"
   761     local ref="${2}"
   762     local clone_dir
   763     local ref_type
   764 
   765     # pushd now to be able to get git revlist in case ref is a date
   766     clone_dir="${CT_TARBALLS_DIR}/${basename}"
   767     CT_Pushd "${clone_dir}"
   768 
   769     # What kind of reference is ${ref} ?
   770     if [ -z "${ref}" ]; then
   771         # Don't update the clone, keep as-is
   772         ref_type=none
   773     elif git tag |grep -E "^${ref}$" >/dev/null 2>&1; then
   774         ref_type=tag
   775     elif git branch -a --no-color |grep -E "^. ${ref}$" >/dev/null 2>&1; then
   776         ref_type=branch
   777     elif date -d "${ref}" >/dev/null 2>&1; then
   778         ref_type=date
   779         ref=$(git rev-list -n1 --before="${ref}")
   780     else
   781         CT_Abort "Reference '${ref}' is an incorrect git reference: neither tag, branch nor date"
   782     fi
   783 
   784     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/${basename}"
   785     CT_DoExecLog ALL ln -sf "${clone_dir}" "${CT_SRC_DIR}/${basename}"
   786 
   787     case "${ref_type}" in
   788         none)   ;;
   789         *)      CT_DoExecLog FILE git checkout "${ref}";;
   790     esac
   791 
   792     CT_Popd
   793 }
   794 
   795 # Patches the specified component
   796 # See CT_Extract, above, for explanations on 'nochdir'
   797 # Usage: CT_Patch [nochdir] <packagename> <packageversion>
   798 # If the package directory is *not* packagename-packageversion, then
   799 # the caller must cd into the proper directory first, and call us
   800 # with nochdir
   801 CT_Patch() {
   802     local nochdir="$1"
   803     local pkgname
   804     local version
   805     local pkgdir
   806     local base_file
   807     local ver_file
   808     local d
   809     local -a patch_dirs
   810     local bundled_patch_dir
   811     local local_patch_dir
   812 
   813     if [ "${nochdir}" = "nochdir" ]; then
   814         shift
   815         pkgname="$1"
   816         version="$2"
   817         pkgdir="${pkgname}-${version}"
   818         nochdir="$(pwd)"
   819     else
   820         pkgname="$1"
   821         version="$2"
   822         pkgdir="${pkgname}-${version}"
   823         nochdir="${CT_SRC_DIR}/${pkgdir}"
   824     fi
   825 
   826     # Check if already patched
   827     if [ -e "${CT_SRC_DIR}/.${pkgdir}.patched" ]; then
   828         CT_DoLog DEBUG "Already patched '${pkgdir}'"
   829         return 0
   830     fi
   831 
   832     # Check if already partially patched
   833     if [ -e "${CT_SRC_DIR}/.${pkgdir}.patching" ]; then
   834         CT_DoLog ERROR "The '${pkgdir}' sources were partially patched."
   835         CT_DoLog ERROR "Please remove first:"
   836         CT_DoLog ERROR " - the source dir for '${pkgdir}', in '${CT_SRC_DIR}'"
   837         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${pkgdir}.extracted'"
   838         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${pkgdir}.patching'"
   839         CT_Abort "I'll stop now to avoid any carnage..."
   840     fi
   841     touch "${CT_SRC_DIR}/.${pkgdir}.patching"
   842 
   843     CT_Pushd "${nochdir}"
   844 
   845     CT_DoLog EXTRA "Patching '${pkgdir}'"
   846 
   847     bundled_patch_dir="${CT_LIB_DIR}/patches/${pkgname}/${version}"
   848     local_patch_dir="${CT_LOCAL_PATCH_DIR}/${pkgname}/${version}"
   849 
   850     case "${CT_PATCH_ORDER}" in
   851         bundled)        patch_dirs=("${bundled_patch_dir}");;
   852         local)          patch_dirs=("${local_patch_dir}");;
   853         bundled,local)  patch_dirs=("${bundled_patch_dir}" "${local_patch_dir}");;
   854         local,bundled)  patch_dirs=("${local_patch_dir}" "${bundled_patch_dir}");;
   855         none)           patch_dirs=;;
   856     esac
   857 
   858     for d in "${patch_dirs[@]}"; do
   859         CT_DoLog DEBUG "Looking for patches in '${d}'..."
   860         if [ -n "${d}" -a -d "${d}" ]; then
   861             for p in "${d}"/*.patch; do
   862                 if [ -f "${p}" ]; then
   863                     CT_DoLog DEBUG "Applying patch '${p}'"
   864                     CT_DoExecLog ALL patch --no-backup-if-mismatch -g0 -F1 -p1 -f <"${p}"
   865                 fi
   866             done
   867             if [ "${CT_PATCH_SINGLE}" = "y" ]; then
   868                 break
   869             fi
   870         fi
   871     done
   872 
   873     if [ "${CT_OVERIDE_CONFIG_GUESS_SUB}" = "y" ]; then
   874         CT_DoLog ALL "Overiding config.guess and config.sub"
   875         for cfg in config_guess config_sub; do
   876             eval ${cfg}="${CT_LIB_DIR}/scripts/${cfg/_/.}"
   877             [ -e "${CT_TOP_DIR}/scripts/${cfg/_/.}" ] && eval ${cfg}="${CT_TOP_DIR}/scripts/${cfg/_/.}"
   878             # Can't use CT_DoExecLog because of the '{} \;' to be passed un-mangled to find
   879             find . -type f -name "${cfg/_/.}" -exec cp -v "${!cfg}" {} \; |CT_DoLog ALL
   880         done
   881     fi
   882 
   883     CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${pkgdir}.patched"
   884     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/.${pkgdir}.patching"
   885 
   886     CT_Popd
   887 }
   888 
   889 # Two wrappers to call config.(guess|sub) either from CT_TOP_DIR or CT_LIB_DIR.
   890 # Those from CT_TOP_DIR, if they exist, will be be more recent than those from CT_LIB_DIR.
   891 CT_DoConfigGuess() {
   892     if [ -x "${CT_TOP_DIR}/scripts/config.guess" ]; then
   893         "${CT_TOP_DIR}/scripts/config.guess"
   894     else
   895         "${CT_LIB_DIR}/scripts/config.guess"
   896     fi
   897 }
   898 
   899 CT_DoConfigSub() {
   900     if [ -x "${CT_TOP_DIR}/scripts/config.sub" ]; then
   901         "${CT_TOP_DIR}/scripts/config.sub" "$@"
   902     else
   903         "${CT_LIB_DIR}/scripts/config.sub" "$@"
   904     fi
   905 }
   906 
   907 # Compute the target tuple from what is provided by the user
   908 # Usage: CT_DoBuildTargetTuple
   909 # In fact this function takes the environment variables to build the target
   910 # tuple. It is needed both by the normal build sequence, as well as the
   911 # sample saving sequence.
   912 CT_DoBuildTargetTuple() {
   913     # Set the endianness suffix, and the default endianness gcc option
   914     case "${CT_ARCH_BE},${CT_ARCH_LE}" in
   915         y,) target_endian_eb=eb
   916             target_endian_el=
   917             CT_ARCH_ENDIAN_CFLAG="-mbig-endian"
   918             CT_ARCH_ENDIAN_LDFLAG="-EB"
   919             ;;
   920         ,y) target_endian_eb=
   921             target_endian_el=el
   922             CT_ARCH_ENDIAN_CFLAG="-mlittle-endian"
   923             CT_ARCH_ENDIAN_LDFLAG="-EL"
   924             ;;
   925     esac
   926 
   927     # Build the default architecture tuple part
   928     CT_TARGET_ARCH="${CT_ARCH}"
   929 
   930     # Set defaults for the system part of the tuple. Can be overriden
   931     # by architecture-specific values.
   932     case "${CT_LIBC}" in
   933         *glibc) CT_TARGET_SYS=gnu;;
   934         uClibc) CT_TARGET_SYS=uclibc;;
   935         *)      CT_TARGET_SYS=elf;;
   936     esac
   937 
   938     # Set the default values for ARCH, ABI, CPU, TUNE, FPU and FLOAT
   939     unset CT_ARCH_ARCH_CFLAG CT_ARCH_ABI_CFLAG CT_ARCH_CPU_CFLAG CT_ARCH_TUNE_CFLAG CT_ARCH_FPU_CFLAG CT_ARCH_FLOAT_CFLAG
   940     unset CT_ARCH_WITH_ARCH CT_ARCH_WITH_ABI CT_ARCH_WITH_CPU CT_ARCH_WITH_TUNE CT_ARCH_WITH_FPU CT_ARCH_WITH_FLOAT
   941     [ "${CT_ARCH_ARCH}"     ] && { CT_ARCH_ARCH_CFLAG="-march=${CT_ARCH_ARCH}";  CT_ARCH_WITH_ARCH="--with-arch=${CT_ARCH_ARCH}"; }
   942     [ "${CT_ARCH_ABI}"      ] && { CT_ARCH_ABI_CFLAG="-mabi=${CT_ARCH_ABI}";     CT_ARCH_WITH_ABI="--with-abi=${CT_ARCH_ABI}";    }
   943     [ "${CT_ARCH_CPU}"      ] && { CT_ARCH_CPU_CFLAG="-mcpu=${CT_ARCH_CPU}";     CT_ARCH_WITH_CPU="--with-cpu=${CT_ARCH_CPU}";    }
   944     [ "${CT_ARCH_TUNE}"     ] && { CT_ARCH_TUNE_CFLAG="-mtune=${CT_ARCH_TUNE}";  CT_ARCH_WITH_TUNE="--with-tune=${CT_ARCH_TUNE}"; }
   945     [ "${CT_ARCH_FPU}"      ] && { CT_ARCH_FPU_CFLAG="-mfpu=${CT_ARCH_FPU}";     CT_ARCH_WITH_FPU="--with-fpu=${CT_ARCH_FPU}";    }
   946     [ "${CT_ARCH_FLOAT_SW}" ] && { CT_ARCH_FLOAT_CFLAG="-msoft-float";           CT_ARCH_WITH_FLOAT="--with-float=soft";          }
   947 
   948     # Build the default kernel tuple part
   949     CT_TARGET_KERNEL="${CT_KERNEL}"
   950 
   951     # Overide the default values with the components specific settings
   952     CT_DoArchTupleValues
   953     CT_DoKernelTupleValues
   954 
   955     # Finish the target tuple construction
   956     CT_TARGET="${CT_TARGET_ARCH}"
   957     CT_TARGET="${CT_TARGET}${CT_TARGET_VENDOR:+-${CT_TARGET_VENDOR}}"
   958     CT_TARGET="${CT_TARGET}${CT_TARGET_KERNEL:+-${CT_TARGET_KERNEL}}"
   959     CT_TARGET="${CT_TARGET}${CT_TARGET_SYS:+-${CT_TARGET_SYS}}"
   960 
   961     # Sanity checks
   962     __sed_alias=""
   963     if [ -n "${CT_TARGET_ALIAS_SED_EXPR}" ]; then
   964         __sed_alias=$(echo "${CT_TARGET}" |sed -r -e "${CT_TARGET_ALIAS_SED_EXPR}")
   965     fi
   966     case ":${CT_TARGET_VENDOR}:${CT_TARGET_ALIAS}:${__sed_alias}:" in
   967       :*" "*:*:*:) CT_Abort "Don't use spaces in the vendor string, it breaks things.";;
   968       :*"-"*:*:*:) CT_Abort "Don't use dashes in the vendor string, it breaks things.";;
   969       :*:*" "*:*:) CT_Abort "Don't use spaces in the target alias, it breaks things.";;
   970       :*:*:*" "*:) CT_Abort "Don't use spaces in the target sed transform, it breaks things.";;
   971     esac
   972 
   973     # Canonicalise it
   974     CT_TARGET=$(CT_DoConfigSub "${CT_TARGET}")
   975     # Prepare the target CFLAGS
   976     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ENDIAN_CFLAG}"
   977     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ARCH_CFLAG}"
   978     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ABI_CFLAG}"
   979     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_CPU_CFLAG}"
   980     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_TUNE_CFLAG}"
   981     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FPU_CFLAG}"
   982     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FLOAT_CFLAG}"
   983 
   984     # Now on for the target LDFLAGS
   985     CT_ARCH_TARGET_LDFLAGS="${CT_ARCH_TARGET_LDFLAGS} ${CT_ARCH_ENDIAN_LDFLAG}"
   986 }
   987 
   988 # This function does pause the build until the user strikes "Return"
   989 # Usage: CT_DoPause [optional_message]
   990 CT_DoPause() {
   991     local foo
   992     local message="${1:-Pausing for your pleasure}"
   993     CT_DoLog INFO "${message}"
   994     read -p "Press 'Enter' to continue, or Ctrl-C to stop..." foo >&6
   995     return 0
   996 }
   997 
   998 # This function creates a tarball of the specified directory, but
   999 # only if it exists
  1000 # Usage: CT_DoTarballIfExists <dir> <tarball_basename> [extra_tar_options [...]]
  1001 CT_DoTarballIfExists() {
  1002     local dir="$1"
  1003     local tarball="$2"
  1004     shift 2
  1005     local -a extra_tar_opts=( "$@" )
  1006     local -a compress
  1007 
  1008     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1009         y)  compress=( gzip -c -3 - ); tar_ext=.gz;;
  1010         *)  compress=( cat - );        tar_ext=;;
  1011     esac
  1012 
  1013     if [ -d "${dir}" ]; then
  1014         CT_DoLog DEBUG "  Saving '${dir}'"
  1015         { tar c -C "${dir}" -v -f - "${extra_tar_opts[@]}" .    \
  1016           |"${compress[@]}" >"${tarball}.tar${tar_ext}"         ;
  1017         } 2>&1 |sed -r -e 's/^/    /;' |CT_DoLog STATE
  1018     else
  1019         CT_DoLog STATE "  Not saving '${dir}': does not exist"
  1020     fi
  1021 }
  1022 
  1023 # This function extracts a tarball to the specified directory, but
  1024 # only if the tarball exists
  1025 # Usage: CT_DoExtractTarballIfExists <tarball_basename> <dir> [extra_tar_options [...]]
  1026 CT_DoExtractTarballIfExists() {
  1027     local tarball="$1"
  1028     local dir="$2"
  1029     shift 2
  1030     local -a extra_tar_opts=( "$@" )
  1031     local -a uncompress
  1032 
  1033     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1034         y)  uncompress=( gzip -c -d ); tar_ext=.gz;;
  1035         *)  uncompress=( cat );        tar_ext=;;
  1036     esac
  1037 
  1038     if [ -f "${tarball}.tar${tar_ext}" ]; then
  1039         CT_DoLog DEBUG "  Restoring '${dir}'"
  1040         CT_DoForceRmdir "${dir}"
  1041         CT_DoExecLog DEBUG mkdir -p "${dir}"
  1042         { "${uncompress[@]}" "${tarball}.tar${tar_ext}"     \
  1043           |tar x -C "${dir}" -v -f - "${extra_tar_opts[@]}" ;
  1044         } 2>&1 |sed -r -e 's/^/    /;' |CT_DoLog STATE
  1045     else
  1046         CT_DoLog STATE "  Not restoring '${dir}': does not exist"
  1047     fi
  1048 }
  1049 
  1050 # This function saves the state of the toolchain to be able to restart
  1051 # at any one point
  1052 # Usage: CT_DoSaveState <next_step_name>
  1053 CT_DoSaveState() {
  1054 	[ "${CT_DEBUG_CT_SAVE_STEPS}" = "y" ] || return 0
  1055     local state_name="$1"
  1056     local state_dir="${CT_STATE_DIR}/${state_name}"
  1057 
  1058     # Log this to the log level required by the user
  1059     CT_DoLog ${CT_LOG_LEVEL_MAX} "Saving state to restart at step '${state_name}'..."
  1060 
  1061     rm -rf "${state_dir}"
  1062     mkdir -p "${state_dir}"
  1063 
  1064     CT_DoLog STATE "  Saving environment and aliases"
  1065     # We must omit shell functions, and some specific bash variables
  1066     # that break when restoring the environment, later. We could do
  1067     # all the processing in the awk script, but a sed is easier...
  1068     set |awk '
  1069               BEGIN { _p = 1; }
  1070               $0~/^[^ ]+ \(\)/ { _p = 0; }
  1071               _p == 1
  1072               $0 == "}" { _p = 1; }
  1073               ' |sed -r -e '/^BASH_(ARGC|ARGV|LINENO|SOURCE|VERSINFO)=/d;
  1074                            /^(UID|EUID)=/d;
  1075                            /^(FUNCNAME|GROUPS|PPID|SHELLOPTS)=/d;' >"${state_dir}/env.sh"
  1076 
  1077     CT_DoTarballIfExists "${CT_BUILDTOOLS_PREFIX_DIR}" "${state_dir}/buildtools_dir"
  1078     CT_DoTarballIfExists "${CT_COMPLIBS_DIR}" "${state_dir}/complibs_dir"
  1079     CT_DoTarballIfExists "${CT_CONFIG_DIR}" "${state_dir}/config_dir"
  1080     CT_DoTarballIfExists "${CT_CC_CORE_STATIC_PREFIX_DIR}" "${state_dir}/cc_core_static_prefix_dir"
  1081     CT_DoTarballIfExists "${CT_CC_CORE_SHARED_PREFIX_DIR}" "${state_dir}/cc_core_shared_prefix_dir"
  1082     CT_DoTarballIfExists "${CT_PREFIX_DIR}" "${state_dir}/prefix_dir" --exclude '*.log'
  1083 
  1084     CT_DoLog STATE "  Saving log file"
  1085     exec >/dev/null
  1086     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1087         y)  gzip -3 -c "${tmp_log_file}"  >"${state_dir}/log.gz";;
  1088         *)  cat "${tmp_log_file}" >"${state_dir}/log";;
  1089     esac
  1090     exec >>"${tmp_log_file}"
  1091 }
  1092 
  1093 # This function restores a previously saved state
  1094 # Usage: CT_DoLoadState <state_name>
  1095 CT_DoLoadState(){
  1096     local state_name="$1"
  1097     local state_dir="${CT_STATE_DIR}/${state_name}"
  1098     local old_RESTART="${CT_RESTART}"
  1099     local old_STOP="${CT_STOP}"
  1100 
  1101     CT_TestOrAbort "The previous build did not reach the point where it could be restarted at '${CT_RESTART}'" -d "${state_dir}"
  1102 
  1103     # We need to do something special with the log file!
  1104     if [ "${CT_LOG_TO_FILE}" = "y" ]; then
  1105         exec >"${state_dir}/tail.log"
  1106     fi
  1107 
  1108     # Log this to the log level required by the user
  1109     CT_DoLog ${CT_LOG_LEVEL_MAX} "Restoring state at step '${state_name}', as requested."
  1110 
  1111     CT_DoExtractTarballIfExists "${state_dir}/prefix_dir" "${CT_PREFIX_DIR}"
  1112     CT_DoExtractTarballIfExists "${state_dir}/cc_core_shared_prefix_dir" "${CT_CC_CORE_SHARED_PREFIX_DIR}"
  1113     CT_DoExtractTarballIfExists "${state_dir}/cc_core_static_prefix_dir" "${CT_CC_CORE_STATIC_PREFIX_DIR}"
  1114     CT_DoExtractTarballIfExists "${state_dir}/config_dir" "${CT_CONFIG_DIR}"
  1115     CT_DoExtractTarballIfExists "${state_dir}/complibs_dir" "${CT_COMPLIBS_DIR}"
  1116     CT_DoExtractTarballIfExists "${state_dir}/buildtools_dir" "${CT_BUILDTOOLS_PREFIX_DIR}"
  1117 
  1118     # Restore the environment, discarding any error message
  1119     # (for example, read-only bash internals)
  1120     CT_DoLog STATE "  Restoring environment"
  1121     . "${state_dir}/env.sh" >/dev/null 2>&1 || true
  1122 
  1123     # Restore the new RESTART and STOP steps
  1124     CT_RESTART="${old_RESTART}"
  1125     CT_STOP="${old_STOP}"
  1126     unset old_stop old_restart
  1127 
  1128     CT_DoLog STATE "  Restoring log file"
  1129     exec >/dev/null
  1130     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1131         y)  zcat "${state_dir}/log.gz" >"${tmp_log_file}";;
  1132         *)  cat "${state_dir}/log" >"${tmp_log_file}";;
  1133     esac
  1134     cat "${state_dir}/tail.log" >>"${tmp_log_file}"
  1135     exec >>"${tmp_log_file}"
  1136     rm -f "${state_dir}/tail.log"
  1137 }