scripts/functions
author Johannes Stezenbach <js@sig21.net>
Tue Oct 30 00:22:25 2012 +0000 (2012-10-30)
changeset 3097 5c67476c7342
parent 3096 d1766c2273d1
child 3127 2af20cfd210b
permissions -rw-r--r--
scripts/functions: fix endless loop in debug-shell with IO redirection

CT_DEBUG_INTERACTIVE is disabled when stdin, stdout or
stderr are redirected, but the check is only done at
the start of the build and doesn't catch when individual
build commands use redirection. When stdin is redirected
it will cause the debug shell to exit immediately, causing
and endless loop. Thus, save the stdin/our/err file handles
and restore them before invoking the debug shell.

Signed-off-by: Johannes Stezenbach <js@sig21.net>
Message-Id: <20121030102225.GA8303@sig21.net>
Patchwork-Id: 195409
     1 # This file contains some usefull common functions -*- sh -*-
     2 # Copyright 2007 Yann E. MORIN
     3 # Licensed under the GPL v2. See COPYING in the root of this package
     4 
     5 # Prepare the fault handler
     6 CT_OnError() {
     7     local ret=$?
     8     local result
     9     local old_trap
    10     local intro
    11     local file line func
    12     local step step_depth
    13 
    14     # To avoid printing the backtace for each sub-shell
    15     # up to the top-level, just remember we've dumped it
    16     if [ ! -f "${CT_WORK_DIR}/backtrace" ]; then
    17         touch "${CT_WORK_DIR}/backtrace"
    18 
    19         # Print steps backtrace
    20         step_depth=${CT_STEP_COUNT}
    21         CT_STEP_COUNT=1     # To have a zero-indentation
    22         CT_DoLog ERROR ""
    23         CT_DoLog ERROR ">>"
    24         intro="Build failed"
    25         for((step=step_depth; step>0; step--)); do
    26             CT_DoLog ERROR ">>  ${intro} in step '${CT_STEP_MESSAGE[${step}]}'"
    27             intro="      called"
    28         done
    29 
    30         # Print functions backtrace
    31         intro="Error happened in"
    32         CT_DoLog ERROR ">>"
    33         for((depth=1; ${BASH_LINENO[$((${depth}-1))]}>0; depth++)); do
    34             file="${BASH_SOURCE[${depth}]#${CT_LIB_DIR}/}"
    35             func="${FUNCNAME[${depth}]}"
    36             line="@${BASH_LINENO[${depth}-1]:-?}"
    37             CT_DoLog ERROR ">>  ${intro}: ${func}[${file}${line}]"
    38             intro="      called from"
    39         done
    40 
    41         # If the user asked for interactive debugging, dump him/her to a shell
    42         if [ "${CT_DEBUG_INTERACTIVE}" = "y" ]; then
    43             # We do not want this sub-shell exit status to be caught, because
    44             # it is absolutely legit that it exits with non-zero.
    45             # Save the trap handler to restore it after our debug-shell
    46             old_trap="$(trap -p ERR)"
    47             trap -- ERR
    48             (
    49                 exec >&6 2>&7 <&8
    50                 printf "\r         \n\nCurrent command"
    51                 if [ -n "${cur_cmd}" ]; then
    52                     printf ":\n  %s\n" "${cur_cmd}"
    53                 else
    54                     printf " (unknown), "
    55                 fi
    56                 printf "exited with error code: %d\n" ${ret}
    57                 printf "Please fix it up and finish by exiting the shell with one of these values:\n"
    58                 printf "    1  fixed, continue with next build command\n"
    59                 if [ -n "${cur_cmd}" ]; then
    60                     printf "    2  repeat this build command\n"
    61                 fi
    62                 printf "    3  abort build\n\n"
    63                 while true; do
    64                     ${bash} --rcfile <(printf "PS1='ct-ng:\w> '\nPROMPT_COMMAND=''\n") -i
    65                     result=$?
    66                     case $result in
    67                         1)  printf "\nContinuing past the failed command.\n\n"
    68                             break
    69                             ;;
    70                         2)  if [ -n "${cur_cmd}" ]; then
    71                                 printf "\nRe-trying last command.\n\n"
    72                                 break
    73                             fi
    74                             ;;&
    75                         3)  break;;
    76                         *)  printf "\nPlease exit with one of these values:\n"
    77                             printf "    1  fixed, continue with next build command\n"
    78                             if [ -n "${cur_cmd}" ]; then
    79                                 printf "    2  repeat this build command\n"
    80                             fi
    81                             printf "    3  abort build\n"
    82                             ;;
    83                     esac
    84                 done
    85                 exit $result
    86             )
    87             result=$?
    88             # Restore the trap handler
    89             eval "${old_trap}"
    90             case "${result}" in
    91                 1)  rm -f "${CT_WORK_DIR}/backtrace"; return;;
    92                 2)  rm -f "${CT_WORK_DIR}/backtrace"; touch "${CT_BUILD_DIR}/repeat"; return;;
    93                 # 3 is an abort, continue...
    94             esac
    95         fi
    96     fi
    97 
    98     # And finally, in top-level shell, print some hints
    99     if [ ${BASH_SUBSHELL} -eq 0 ]; then
   100         # Help diagnose the error
   101         CT_STEP_COUNT=1     # To have a zero-indentation
   102         CT_DoLog ERROR ">>"
   103         if [ "${CT_LOG_TO_FILE}" = "y" ]; then
   104             CT_DoLog ERROR ">>  For more info on this error, look at the file: '${tmp_log_file#${CT_TOP_DIR}/}'"
   105         fi
   106         CT_DoLog ERROR ">>  There is a list of known issues, some with workarounds, in:"
   107         CT_DoLog ERROR ">>      '${CT_DOC_DIR#${CT_TOP_DIR}/}/B - Known issues.txt'"
   108 
   109         CT_DoLog ERROR ""
   110         CT_DoEnd ERROR
   111         rm -f "${CT_WORK_DIR}/backtrace"
   112     fi
   113     exit $ret
   114 }
   115 
   116 # Install the fault handler
   117 trap CT_OnError ERR
   118 
   119 # Inherit the fault handler in subshells and functions
   120 set -E
   121 
   122 # Make pipes fail on the _first_ failed command
   123 # Not supported on bash < 3.x, but we need it, so drop the obsoleting bash-2.x
   124 set -o pipefail
   125 
   126 # Don't hash commands' locations, and search every time it is requested.
   127 # This is slow, but needed because of the static/shared core gcc which shall
   128 # always match to shared if it exists, and only fallback to static if the
   129 # shared is not found
   130 set +o hashall
   131 
   132 # Log policy:
   133 #  - first of all, save stdout so we can see the live logs: fd #6
   134 # (also save stdin and stderr for use by CT_DEBUG_INTERACTIVE)
   135 exec 6>&1 7>&2 8<&0
   136 #  - then point stdout to the log file
   137 tmp_log_file="${CT_TOP_DIR}/build.log"
   138 rm -f "${tmp_log_file}"
   139 exec >>"${tmp_log_file}"
   140 
   141 # The different log levels:
   142 CT_LOG_LEVEL_ERROR=0
   143 CT_LOG_LEVEL_WARN=1
   144 CT_LOG_LEVEL_INFO=2
   145 CT_LOG_LEVEL_EXTRA=3
   146 CT_LOG_LEVEL_CFG=4
   147 CT_LOG_LEVEL_FILE=5
   148 CT_LOG_LEVEL_STATE=6
   149 CT_LOG_LEVEL_ALL=7
   150 CT_LOG_LEVEL_DEBUG=8
   151 
   152 # Make it easy to use \n and !
   153 CR=$(printf "\n")
   154 BANG='!'
   155 
   156 # A function to log what is happening
   157 # Different log level are available:
   158 #   - ERROR:   A serious, fatal error occurred
   159 #   - WARN:    A non fatal, non serious error occurred, take your responsbility with the generated build
   160 #   - INFO:    Informational messages
   161 #   - EXTRA:   Extra informational messages
   162 #   - CFG:     Output of various "./configure"-type scripts
   163 #   - FILE:    File / archive unpacking.
   164 #   - STATE:   State save & restore
   165 #   - ALL:     Component's build messages
   166 #   - DEBUG:   Internal debug messages
   167 # Usage: CT_DoLog <level> [message]
   168 # If message is empty, then stdin will be logged.
   169 CT_DoLog() {
   170     local max_level LEVEL level cur_l cur_L
   171     local l
   172     eval max_level="\${CT_LOG_LEVEL_${CT_LOG_LEVEL_MAX}}"
   173     # Set the maximum log level to DEBUG if we have none
   174     [ -z "${max_level}" ] && max_level=${CT_LOG_LEVEL_DEBUG}
   175 
   176     LEVEL="$1"; shift
   177     eval level="\${CT_LOG_LEVEL_${LEVEL}}"
   178 
   179     if [ $# -eq 0 ]; then
   180         cat -
   181     else
   182         printf "%s\n" "${*}"
   183     fi |( IFS="${CR}" # We want the full lines, even leading spaces
   184           _prog_bar_cpt=0
   185           _prog_bar[0]='/'
   186           _prog_bar[1]='-'
   187           _prog_bar[2]='\'
   188           _prog_bar[3]='|'
   189           indent=$((2*CT_STEP_COUNT))
   190           while read line; do
   191               case "${CT_LOG_SEE_TOOLS_WARN},${line}" in
   192                 y,*"warning:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
   193                 y,*"WARNING:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
   194                 *"error:"*)             cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
   195                 *"make["*"]: *** ["*)   cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
   196                 *)                      cur_L="${LEVEL}"; cur_l="${level}";;
   197               esac
   198               # There will always be a log file (stdout, fd #1), be it /dev/null
   199               printf "[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}"
   200               if [ ${cur_l} -le ${max_level} ]; then
   201                   # Only print to console (fd #6) if log level is high enough.
   202                   printf "${CT_LOG_PROGRESS_BAR:+\r}[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}" >&6
   203               fi
   204               if [ "${CT_LOG_PROGRESS_BAR}" = "y" ]; then
   205                   printf "\r[%02d:%02d] %s " $((SECONDS/60)) $((SECONDS%60)) "${_prog_bar[$((_prog_bar_cpt/10))]}" >&6
   206                   _prog_bar_cpt=$(((_prog_bar_cpt+1)%40))
   207               fi
   208           done
   209         )
   210 
   211     return 0
   212 }
   213 
   214 # Execute an action, and log its messages
   215 # It is possible to even log local variable assignments (a-la: var=val ./cmd opts)
   216 # Usage: CT_DoExecLog <level> [VAR=val...] <command> [parameters...]
   217 CT_DoExecLog() {
   218     local level="$1"
   219     local cur_cmd
   220     local ret
   221     shift
   222     (
   223     for i in "$@"; do
   224         cur_cmd+="'${i}' "
   225     done
   226     while true; do
   227         case "${1}" in
   228             *=*)    eval export "'${1}'"; shift;;
   229             *)      break;;
   230         esac
   231     done
   232     # This while-loop goes hand-in-hand with the ERR trap handler:
   233     # - if the command terminates successfully, then we hit the break
   234     #   statement, and we exit the loop
   235     # - if the command terminates in error, then the ERR handler kicks
   236     #   in, then:
   237     #     - if the user did *not* ask for interactive debugging, the ERR
   238     #       handler exits, and we hit the end of the sub-shell
   239     #     - if the user did ask for interactive debugging, the ERR handler
   240     #       spawns a shell. Upon termination of this shell, the ERR handler
   241     #       examines the exit status of the shell:
   242     #         - if 1, the ERR handler returns; then we hit the else statement,
   243     #           then the break, and we exit the 'while' loop, to continue the
   244     #           build;
   245     #         - if 2, the ERR handler touches the repeat file, and returns;
   246     #           then we hit the if statement, and we loop for one more
   247     #           iteration;
   248     #         - if 3, the ERR handler exits with the command's exit status,
   249     #           and we're dead;
   250     #         - for any other exit status of the shell, the ERR handler
   251     #           prints an informational message, and respawns the shell
   252     #
   253     # This allows a user to get an interactive shell that has the same
   254     # environment (PATH and so on) that the failed command was ran with.
   255     while true; do
   256         rm -f "${CT_BUILD_DIR}/repeat"
   257         CT_DoLog DEBUG "==> Executing: ${cur_cmd}"
   258         "${@}" 2>&1 |CT_DoLog "${level}"
   259         ret="${?}"
   260         if [ -f "${CT_BUILD_DIR}/repeat" ]; then
   261             continue
   262         else
   263             break
   264         fi
   265     done
   266     exit ${ret}
   267     )
   268     # Catch failure of the sub-shell
   269     [ $? -eq 0 ]
   270 }
   271 
   272 # Tail message to be logged whatever happens
   273 # Usage: CT_DoEnd <level>
   274 CT_DoEnd()
   275 {
   276     local level="$1"
   277     CT_STOP_DATE=$(CT_DoDate +%s%N)
   278     CT_STOP_DATE_HUMAN=$(CT_DoDate +%Y%m%d.%H%M%S)
   279     if [ "${level}" != "ERROR" ]; then
   280         CT_DoLog "${level:-INFO}" "Build completed at ${CT_STOP_DATE_HUMAN}"
   281     fi
   282     elapsed=$((CT_STOP_DATE-CT_STAR_DATE))
   283     elapsed_min=$((elapsed/(60*1000*1000*1000)))
   284     elapsed_sec=$(printf "%02d" $(((elapsed%(60*1000*1000*1000))/(1000*1000*1000))))
   285     elapsed_csec=$(printf "%02d" $(((elapsed%(1000*1000*1000))/(10*1000*1000))))
   286     CT_DoLog ${level:-INFO} "(elapsed: ${elapsed_min}:${elapsed_sec}.${elapsed_csec})"
   287 }
   288 
   289 # Remove entries referring to . and other relative paths
   290 # Usage: CT_SanitizePath
   291 CT_SanitizePath() {
   292     local new
   293     local p
   294     local IFS=:
   295     for p in $PATH; do
   296         # Only accept absolute paths;
   297         # Note: as a special case the empty string in PATH is equivalent to .
   298         if [ -n "${p}" -a -z "${p%%/*}" ]; then
   299             new="${new}${new:+:}${p}"
   300         fi
   301     done
   302     PATH="${new}"
   303 }
   304 
   305 # Sanitise the directory name contained in the variable passed as argument:
   306 # - remove duplicate /
   307 # Usage: CT_SanitiseVarDir CT_PREFIX_DIR
   308 CT_SanitiseVarDir() {
   309     local var
   310     local old_dir
   311     local new_dir
   312 
   313     for var in "$@"; do
   314         eval "old_dir=\"\${${var}}\""
   315         new_dir="$( printf "${old_dir}"     \
   316                     |sed -r -e 's:/+:/:g;'  \
   317                   )"
   318         eval "${var}=\"${new_dir}\""
   319         CT_DoLog DEBUG "Sanitised '${var}': '${old_dir}' -> '${new_dir}'"
   320     done
   321 }
   322 
   323 # Abort the execution with an error message
   324 # Usage: CT_Abort <message>
   325 CT_Abort() {
   326     CT_DoLog ERROR "$1"
   327     false
   328 }
   329 
   330 # Test a condition, and print a message if satisfied
   331 # Usage: CT_Test <message> <tests>
   332 CT_Test() {
   333     local ret
   334     local m="$1"
   335     shift
   336     CT_DoLog DEBUG "Testing '! ( $* )'"
   337     test "$@" && CT_DoLog WARN "$m"
   338     return 0
   339 }
   340 
   341 # Test a condition, and abort with an error message if satisfied
   342 # Usage: CT_TestAndAbort <message> <tests>
   343 CT_TestAndAbort() {
   344     local m="$1"
   345     shift
   346     CT_DoLog DEBUG "Testing '! ( $* )'"
   347     test "$@" && CT_Abort "$m"
   348     return 0
   349 }
   350 
   351 # Test a condition, and abort with an error message if not satisfied
   352 # Usage: CT_TestAndAbort <message> <tests>
   353 CT_TestOrAbort() {
   354     local m="$1"
   355     shift
   356     CT_DoLog DEBUG "Testing '$*'"
   357     test "$@" || CT_Abort "$m"
   358     return 0
   359 }
   360 
   361 # Test the presence of a tool, or abort if not found
   362 # Usage: CT_HasOrAbort <tool>
   363 CT_HasOrAbort() {
   364     CT_TestAndAbort "'${1}' not found and needed for successful toolchain build." -z "$(CT_Which "${1}")"
   365     return 0
   366 }
   367 
   368 # Search a program: wrap "which" for those system where "which"
   369 # verbosely says there is no match (such as on Mandriva).
   370 # Usage: CT_Which <filename>
   371 CT_Which() {
   372   which "$1" 2>/dev/null || true
   373 }
   374 
   375 # Get current date with nanosecond precision
   376 # On those system not supporting nanosecond precision, faked with rounding down
   377 # to the highest entire second
   378 # Usage: CT_DoDate <fmt>
   379 CT_DoDate() {
   380     date "$1" |sed -r -e 's/%?N$/000000000/;'
   381 }
   382 
   383 CT_STEP_COUNT=1
   384 CT_STEP_MESSAGE[${CT_STEP_COUNT}]="(top-level)"
   385 # Memorise a step being done so that any error is caught
   386 # Usage: CT_DoStep <loglevel> <message>
   387 CT_DoStep() {
   388     local start=$(CT_DoDate +%s%N)
   389     CT_DoLog "$1" "================================================================="
   390     CT_DoLog "$1" "$2"
   391     CT_STEP_COUNT=$((CT_STEP_COUNT+1))
   392     CT_STEP_LEVEL[${CT_STEP_COUNT}]="$1"; shift
   393     CT_STEP_START[${CT_STEP_COUNT}]="${start}"
   394     CT_STEP_MESSAGE[${CT_STEP_COUNT}]="$1"
   395     return 0
   396 }
   397 
   398 # End the step just being done
   399 # Usage: CT_EndStep
   400 CT_EndStep() {
   401     local stop=$(CT_DoDate +%s%N)
   402     local duration=$(printf "%032d" $((stop-${CT_STEP_START[${CT_STEP_COUNT}]})) |sed -r -e 's/([[:digit:]]{2})[[:digit:]]{7}$/\.\1/; s/^0+//; s/^\./0\./;')
   403     local elapsed=$(printf "%02d:%02d" $((SECONDS/60)) $((SECONDS%60)))
   404     local level="${CT_STEP_LEVEL[${CT_STEP_COUNT}]}"
   405     local message="${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}"
   406     CT_STEP_COUNT=$((CT_STEP_COUNT-1))
   407     CT_DoLog "${level}" "${message}: done in ${duration}s (at ${elapsed})"
   408     return 0
   409 }
   410 
   411 # Pushes into a directory, and pops back
   412 CT_Pushd() {
   413     pushd "$1" >/dev/null 2>&1
   414 }
   415 CT_Popd() {
   416     popd >/dev/null 2>&1
   417 }
   418 
   419 # Create a dir and cd or pushd into it
   420 # Usage: CT_mkdir_cd <dir/to/create>
   421 #        CT_mkdir_pushd <dir/to/create>
   422 CT_mkdir_cd() {
   423     local dir="${1}"
   424 
   425     mkdir -p "${dir}"
   426     cd "${dir}"
   427 }
   428 CT_mkdir_pushd() {
   429     local dir="${1}"
   430 
   431     mkdir -p "${dir}"
   432     CT_Pushd "${dir}"
   433 }
   434 
   435 # Creates a temporary directory
   436 # $1: variable to assign to
   437 # Usage: CT_MktempDir foo
   438 CT_MktempDir() {
   439     # Some mktemp do not allow more than 6 Xs
   440     eval "$1"=$(mktemp -q -d "${CT_BUILD_DIR}/tmp.XXXXXX")
   441     CT_TestOrAbort "Could not make temporary directory" -n "${!1}" -a -d "${!1}"
   442     CT_DoLog DEBUG "Made temporary directory '${!1}'"
   443     return 0
   444 }
   445 
   446 # Removes one or more directories, even if it is read-only, or its parent is
   447 # Usage: CT_DoForceRmdir dir [...]
   448 CT_DoForceRmdir() {
   449     local dir
   450     local mode
   451     for dir in "${@}"; do
   452         [ -d "${dir}" ] || continue
   453         case "$CT_SYS_OS" in
   454             Linux|CYGWIN*)
   455                 mode="$(stat -c '%a' "$(dirname "${dir}")")"
   456                 ;;
   457             Darwin|*BSD)
   458                 mode="$(stat -f '%Lp' "$(dirname "${dir}")")"
   459                 ;;
   460             *)
   461                 CT_Abort "Unhandled host OS $CT_SYS_OS"
   462                 ;;
   463         esac
   464         CT_DoExecLog ALL chmod u+w "$(dirname "${dir}")"
   465         CT_DoExecLog ALL chmod -R u+w "${dir}"
   466         CT_DoExecLog ALL rm -rf "${dir}"
   467         CT_DoExecLog ALL chmod ${mode} "$(dirname "${dir}")"
   468     done
   469 }
   470 
   471 # Echoes the specified string on stdout until the pipe breaks.
   472 # Doesn't fail
   473 # $1: string to echo
   474 # Usage: CT_DoYes "" |make oldconfig
   475 CT_DoYes() {
   476     yes "$1" || true
   477 }
   478 
   479 # Add the specified directory to LD_LIBRARY_PATH, and export it
   480 # If the specified patch is already present, just export
   481 # $1: path to add
   482 # $2: add as 'first' or 'last' path, 'first' is assumed if $2 is empty
   483 # Usage CT_SetLibPath /some/where/lib [first|last]
   484 CT_SetLibPath() {
   485     local path="$1"
   486     local pos="$2"
   487 
   488     case ":${LD_LIBRARY_PATH}:" in
   489         *:"${path}":*)  ;;
   490         *)  case "${pos}" in
   491                 last)
   492                     CT_DoLog DEBUG "Adding '${path}' at end of LD_LIBRARY_PATH"
   493                     LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${path}"
   494                     ;;
   495                 first|"")
   496                     CT_DoLog DEBUG "Adding '${path}' at start of LD_LIBRARY_PATH"
   497                     LD_LIBRARY_PATH="${path}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
   498                     ;;
   499                 *)
   500                     CT_Abort "Incorrect position '${pos}' to add '${path}' to LD_LIBRARY_PATH"
   501                     ;;
   502             esac
   503             ;;
   504     esac
   505     CT_DoLog DEBUG "==> LD_LIBRARY_PATH='${LD_LIBRARY_PATH}'"
   506     export LD_LIBRARY_PATH
   507 }
   508 
   509 # Build up the list of allowed tarball extensions
   510 # Add them in the prefered order; most preferred comes first
   511 CT_DoListTarballExt() {
   512     if [ "${CT_CONFIGURE_has_xz}" = "y" ]; then
   513         printf ".tar.xz\n"
   514     fi
   515     if [    "${CT_CONFIGURE_has_lzma}" = "y"    \
   516          -o "${CT_CONFIGURE_has_xz}" = "y" ]; then
   517         printf ".tar.lzma\n"
   518     fi
   519     printf ".tar.bz2\n"
   520     printf ".tar.gz\n.tgz\n"
   521     printf ".tar\n"
   522 }
   523 
   524 # Get the file name extension of a component
   525 # Usage: CT_GetFileExtension <component_name-component_version> [extension]
   526 # If found, echoes the extension to stdout, and return 0
   527 # If not found, echoes nothing on stdout, and return !0.
   528 CT_GetFileExtension() {
   529     local ext
   530     local file="$1"
   531     shift
   532     local first_ext="$1"
   533 
   534     # we need to also check for an empty extension for those very
   535     # peculiar components that don't have one (such as sstrip from
   536     # buildroot).
   537     for ext in ${first_ext} $(CT_DoListTarballExt) /.git ''; do
   538         if [ -e "${CT_TARBALLS_DIR}/${file}${ext}" ]; then
   539             echo "${ext}"
   540             exit 0
   541         fi
   542     done
   543 
   544     exit 1
   545 }
   546 
   547 # Try to retrieve the specified URL (HTTP or FTP)
   548 # Usage: CT_DoGetFile <URL>
   549 # This functions always returns true (0), as it can be legitimate not
   550 # to find the requested URL (think about snapshots, different layouts
   551 # for different gcc versions, etc...).
   552 CT_DoGetFile() {
   553     local url="${1}"
   554     local dest="${CT_TARBALLS_DIR}/${url##*/}"
   555     local tmp="${dest}.tmp-dl"
   556 
   557     # Remove potential left-over from a previous run
   558     rm -f "${tmp}"
   559 
   560     # We also retry a few times, in case there is a transient error (eg. behind
   561     # a dynamic IP that changes during the transfer...)
   562     # With automated download as we are doing, it can be very dangerous to
   563     # continue the downloads. It's far better to simply overwrite the
   564     # destination file.
   565     # Some company networks have firewalls to connect to the internet, but it's
   566     # not easy to detect them, so force a global ${CT_CONNECT_TIMEOUT}-second
   567     # timeout.
   568     # For curl, no good progress indicator is available. So, be silent.
   569     if CT_DoExecLog ALL wget --passive-ftp --tries=3 -nc    \
   570                              --progress=dot:binary          \
   571                              -T ${CT_CONNECT_TIMEOUT}       \
   572                              -O "${tmp}"                    \
   573                              "${url}"
   574     then
   575         # Success, we got it, good!
   576         mv "${tmp}" "${dest}"
   577         CT_DoLog DEBUG "Got it from: \"${url}\""
   578     else
   579         # Woops...
   580         rm -f "${tmp}"
   581         CT_DoLog DEBUG "Not at this location: \"${url}\""
   582     fi
   583 }
   584 
   585 # This function tries to retrieve a tarball form a local directory
   586 # Usage: CT_GetLocal <basename> [.extension]
   587 CT_GetLocal() {
   588     local basename="$1"
   589     local first_ext="$2"
   590     local ext
   591 
   592     # Do we already have it in *our* tarballs dir?
   593     if ext="$( CT_GetFileExtension "${basename}" ${first_ext} )"; then
   594         CT_DoLog DEBUG "Already have '${basename}'"
   595         return 0
   596     fi
   597 
   598     if [ -n "${CT_LOCAL_TARBALLS_DIR}" ]; then
   599         CT_DoLog DEBUG "Trying to retrieve an already downloaded copy of '${basename}'"
   600         # We'd rather have a bzip2'ed tarball, then gzipped tarball, plain tarball,
   601         # or, as a failover, a file without extension.
   602         for ext in ${first_ext} $(CT_DoListTarballExt) ''; do
   603             CT_DoLog DEBUG "Trying '${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}'"
   604             if [ -r "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" -a \
   605                  "${CT_FORCE_DOWNLOAD}" != "y" ]; then
   606                 CT_DoLog DEBUG "Got '${basename}' from local storage"
   607                 CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" "${CT_TARBALLS_DIR}/${basename}${ext}"
   608                 return 0
   609             fi
   610         done
   611     fi
   612     return 1
   613 }
   614 
   615 # This function gets the custom source from either a tarball or directory
   616 # Usage: CT_GetCustom <component> <custom_version> <custom_location>
   617 CT_GetCustom() {
   618     local custom_component="$1"
   619     local custom_version="$2"
   620     local custom_location="$3"
   621     local custom_name="${custom_component}-${custom_version}"
   622 
   623     CT_TestAndAbort "${custom_name}: CT_CUSTOM_LOCATION_ROOT_DIR or ${custom_component}'s CUSTOM_LOCATION must be set." \
   624                     -z "${CT_CUSTOM_LOCATION_ROOT_DIR}" -a -z "${custom_location}"
   625 
   626     if [ -n "${CT_CUSTOM_LOCATION_ROOT_DIR}" \
   627          -a -z "${custom_location}"          ]; then
   628         custom_location="${CT_CUSTOM_LOCATION_ROOT_DIR}/${custom_component}"
   629     fi
   630 
   631     CT_DoLog EXTRA "Using '${custom_name}' from custom location"
   632     if [ ! -d "${custom_location}" ]; then
   633         # We need to know the custom tarball extension,
   634         # so we can create a properly-named symlink, which
   635         # we use later on in 'extract'
   636         case "${custom_location}" in
   637             *.tar.bz2)      custom_name="${custom_name}.tar.bz2";;
   638             *.tar.gz|*.tgz) custom_name="${custom_name}.tar.gz";;
   639             *.tar)          custom_name="${custom_name}.tar";;
   640             *)  CT_Abort "Unknown extension for custom tarball '${custom_location}'";;
   641         esac
   642         CT_DoExecLog DEBUG ln -sf "${custom_location}"  \
   643                                   "${CT_TARBALLS_DIR}/${custom_name}"
   644     else
   645         CT_DoExecLog DEBUG ln -snf "${custom_location}" \
   646                                    "${CT_SRC_DIR}/${custom_name}"
   647     fi
   648 }
   649 
   650 # This function saves the specified to local storage if possible,
   651 # and if so, symlinks it for later usage
   652 # Usage: CT_SaveLocal </full/path/file.name>
   653 CT_SaveLocal() {
   654     local file="$1"
   655     local basename="${file##*/}"
   656 
   657     if [ "${CT_SAVE_TARBALLS}" = "y" ]; then
   658         CT_DoLog EXTRA "Saving '${basename}' to local storage"
   659         # The file may already exist if downloads are forced: remove it first
   660         CT_DoExecLog ALL rm -f "${CT_LOCAL_TARBALLS_DIR}/${basename}"
   661         CT_DoExecLog ALL mv -f "${file}" "${CT_LOCAL_TARBALLS_DIR}"
   662         CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}" "${file}"
   663     fi
   664 }
   665 
   666 # Download the file from one of the URLs passed as argument
   667 # Usage: CT_GetFile <basename> [.extension] <url> [url ...]
   668 CT_GetFile() {
   669     local ext
   670     local -a URLS
   671     local url
   672     local file="$1"
   673     local first_ext
   674     shift
   675     # If next argument starts with a dot, then this is not an URL,
   676     # and we can consider that it is a preferred extension.
   677     case "$1" in
   678         .*) first_ext="$1"
   679             shift
   680             ;;
   681     esac
   682 
   683     # Does it exist localy?
   684     if CT_GetLocal "${file}" ${first_ext}; then
   685         return 0
   686     fi
   687     # No, it does not...
   688 
   689     # If not allowed to download from the Internet, don't
   690     if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then
   691         CT_DoLog DEBUG "Not allowed to download from the Internet, aborting ${file} download"
   692         return 1
   693     fi
   694 
   695     # Try to retrieve the file
   696     CT_DoLog EXTRA "Retrieving '${file}'"
   697 
   698     # Add URLs on the LAN mirror
   699     if [ "${CT_USE_MIRROR}" = "y" ]; then
   700         CT_TestOrAbort "Please set the mirror base URL" -n "${CT_MIRROR_BASE_URL}"
   701         URLS+=( "${CT_MIRROR_BASE_URL}/${file%-*}" )
   702         URLS+=( "${CT_MIRROR_BASE_URL}" )
   703     fi
   704 
   705     if [ "${CT_FORCE_MIRROR}" != "y" ]; then
   706         URLS+=( "${@}" )
   707     fi
   708 
   709     # Scan all URLs in turn, and try to grab a tarball from there
   710     # Do *not* try git trees (ext=/.git), this is handled in a specific
   711     # wrapper, below
   712     for ext in ${first_ext} $(CT_DoListTarballExt) ''; do
   713         # Try all urls in turn
   714         for url in "${URLS[@]}"; do
   715             [ -n "${url}" ] || continue
   716             CT_DoLog DEBUG "Trying '${url}/${file}${ext}'"
   717             CT_DoGetFile "${url}/${file}${ext}"
   718             if [ -f "${CT_TARBALLS_DIR}/${file}${ext}" ]; then
   719                 CT_DoLog DEBUG "Got '${file}' from the Internet"
   720                 CT_SaveLocal "${CT_TARBALLS_DIR}/${file}${ext}"
   721                 return 0
   722             fi
   723         done
   724     done
   725 
   726     # Just return error, someone may want to catch and handle the error
   727     # (eg. glibc/eglibc add-ons can be missing).
   728     return 1
   729 }
   730 
   731 # Checkout from CVS, and build the associated tarball
   732 # The tarball will be called ${basename}.tar.bz2
   733 # Prerequisite: either the server does not require password,
   734 # or the user must already be logged in.
   735 # 'tag' is the tag to retrieve. Must be specified, but can be empty.
   736 # If dirname is specified, then module will be renamed to dirname
   737 # prior to building the tarball.
   738 # Usage: CT_GetCVS <basename> <url> <module> <tag> [dirname[=subdir]]
   739 # Note: if '=subdir' is given, then it is used instead of 'module'.
   740 CT_GetCVS() {
   741     local basename="$1"
   742     local uri="$2"
   743     local module="$3"
   744     local tag="${4:+-r ${4}}"
   745     local dirname="$5"
   746     local tmp_dir
   747 
   748     # First try locally, then the mirror
   749     if CT_GetFile "${basename}"; then
   750         # Got it! Return early! :-)
   751         return 0
   752     fi
   753 
   754     if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then
   755         CT_DoLog WARN "Downloads forbidden, not trying cvs retrieval"
   756         return 1
   757     fi
   758 
   759     CT_MktempDir tmp_dir
   760     CT_Pushd "${tmp_dir}"
   761 
   762     CT_DoExecLog ALL cvs -z 9 -d "${uri}" co -P ${tag} "${module}"
   763     if [ -n "${dirname}" ]; then
   764         case "${dirname}" in
   765             *=*)
   766                 CT_DoExecLog DEBUG mv "${dirname#*=}" "${dirname%%=*}"
   767                 CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${dirname%%=*}"
   768                 ;;
   769             *)
   770                 CT_DoExecLog ALL mv "${module}" "${dirname}"
   771                 CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${dirname:-${module}}"
   772                 ;;
   773         esac
   774     fi
   775     CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}.tar.bz2"
   776 
   777     CT_Popd
   778     CT_DoExecLog ALL rm -rf "${tmp_dir}"
   779 }
   780 
   781 # Check out from SVN, and build the associated tarball
   782 # The tarball will be called ${basename}.tar.bz2
   783 # Prerequisite: either the server does not require password,
   784 # or the user must already be logged in.
   785 # 'rev' is the revision to retrieve
   786 # Usage: CT_GetSVN <basename> <url> [rev]
   787 CT_GetSVN() {
   788     local basename="$1"
   789     local uri="$2"
   790     local rev="$3"
   791 
   792     # First try locally, then the mirror
   793     if CT_GetFile "${basename}"; then
   794         # Got it! Return early! :-)
   795         return 0
   796     fi
   797 
   798     if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then
   799         CT_DoLog WARN "Downloads forbidden, not trying svn retrieval"
   800         return 1
   801     fi
   802 
   803     CT_MktempDir tmp_dir
   804     CT_Pushd "${tmp_dir}"
   805 
   806     if ! CT_DoExecLog ALL svn export ${rev:+-r ${rev}} "${uri}" "${basename}"; then
   807         CT_DoLog WARN "Could not retrieve '${basename}'"
   808         return 1
   809     fi
   810     CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${basename}"
   811     CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}.tar.bz2"
   812 
   813     CT_Popd
   814     CT_DoExecLog ALL rm -rf "${tmp_dir}"
   815 }
   816 
   817 # Clone a git tree
   818 # Tries the given URLs in turn until one can get cloned. No tarball will be created.
   819 # Prerequisites: either the server does not require password,
   820 # or the user has already taken any action to authenticate to the server.
   821 # The cloned tree will *not* be stored in the local tarballs dir!
   822 # Usage: CT_GetGit <basename> <url [url ...]>
   823 CT_GetGit() {
   824     local basename="$1"; shift
   825     local url
   826     local cloned=0
   827 
   828     if [ "${CT_FORBID_DOWNLOAD}" = "y" ]; then
   829         CT_DoLog WARN "Downloads forbidden, not trying git retrieval"
   830         return 1
   831     fi
   832 
   833     # Do we have it in our tarballs dir?
   834     if [ -d "${CT_TARBALLS_DIR}/${basename}/.git" ]; then
   835         CT_DoLog EXTRA "Updating git tree '${basename}'"
   836         CT_Pushd "${CT_TARBALLS_DIR}/${basename}"
   837         CT_DoExecLog ALL git pull
   838         CT_Popd
   839     else
   840         CT_DoLog EXTRA "Retrieving git tree '${basename}'"
   841         for url in "${@}"; do
   842             CT_DoLog ALL "Trying to clone from '${url}'"
   843             CT_DoForceRmdir "${CT_TARBALLS_DIR}/${basename}"
   844             if git clone "${url}" "${CT_TARBALLS_DIR}/${basename}" 2>&1 |CT_DoLog ALL; then
   845                 cloned=1
   846                 break
   847             fi
   848         done
   849         CT_TestOrAbort "Could not clone '${basename}'" ${cloned} -ne 0
   850     fi
   851 }
   852 
   853 # Extract a tarball
   854 # Some tarballs need to be extracted in specific places. Eg.: glibc addons
   855 # must be extracted in the glibc directory; uCLibc locales must be extracted
   856 # in the extra/locale sub-directory of uClibc. This is taken into account
   857 # by the caller, that did a 'cd' into the correct path before calling us
   858 # and sets nochdir to 'nochdir'.
   859 # Note also that this function handles the git trees!
   860 # Usage: CT_Extract <basename> [nochdir] [options]
   861 # where 'options' are dependent on the source (eg. git branch/tag...)
   862 CT_Extract() {
   863     local nochdir="$1"
   864     local basename
   865     local ext
   866     local lzma_prog
   867     local -a tar_opts
   868 
   869     if [ "${nochdir}" = "nochdir" ]; then
   870         shift
   871         nochdir="$(pwd)"
   872     else
   873         nochdir="${CT_SRC_DIR}"
   874     fi
   875 
   876     basename="$1"
   877     shift
   878 
   879     if ! ext="$(CT_GetFileExtension "${basename}")"; then
   880         CT_DoLog WARN "'${basename}' not found in '${CT_TARBALLS_DIR}'"
   881         return 1
   882     fi
   883     local full_file="${CT_TARBALLS_DIR}/${basename}${ext}"
   884 
   885     # Check if already extracted
   886     if [ -e "${CT_SRC_DIR}/.${basename}.extracted" ]; then
   887         CT_DoLog DEBUG "Already extracted '${basename}'"
   888         return 0
   889     fi
   890 
   891     # Check if previously partially extracted
   892     if [ -e "${CT_SRC_DIR}/.${basename}.extracting" ]; then
   893         CT_DoLog ERROR "The '${basename}' sources were partially extracted."
   894         CT_DoLog ERROR "Please remove first:"
   895         CT_DoLog ERROR " - the source dir for '${basename}', in '${CT_SRC_DIR}'"
   896         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${basename}.extracting'"
   897         CT_Abort "I'll stop now to avoid any carnage..."
   898     fi
   899     CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${basename}.extracting"
   900 
   901     CT_Pushd "${nochdir}"
   902 
   903     CT_DoLog EXTRA "Extracting '${basename}'"
   904     CT_DoExecLog FILE mkdir -p "${basename}"
   905     tar_opts=( "--strip-components=1" )
   906     tar_opts+=( "-C" "${basename}" )
   907     tar_opts+=( "-xv" )
   908 
   909     # One note here:
   910     # - lzma can be handled either with 'xz' or 'lzma'
   911     # - we get lzma tarball only if either or both are available
   912     # - so, if we get an lzma tarball, and either 'xz' or 'lzma' is
   913     #   missing, we can assume the other is available
   914     if [ "${CT_CONFIGURE_has_lzma}" = "y" ]; then
   915         lzma_prog="lzma -fdc"
   916     else
   917         lzma_prog="xz -fdc"
   918     fi
   919     case "${ext}" in
   920         .tar.xz)      xz -fdc "${full_file}" | CT_DoExecLog FILE tar "${tar_opts[@]}" -f -;;
   921         .tar.lzma)    ${lzma_prog} "${full_file}" | CT_DoExecLog FILE tar "${tar_opts[@]}" -f -;;
   922         .tar.bz2)     bzip2 -dc "${full_file}" | CT_DoExecLog FILE tar "${tar_opts[@]}" -f -;;
   923         .tar.gz|.tgz) gzip -dc "${full_file}" | CT_DoExecLog FILE tar "${tar_opts[@]}" -f -;;
   924         .tar)         CT_DoExecLog FILE tar "${tar_opts[@]}" -f "${full_file}";;
   925         /.git)        CT_ExtractGit "${basename}" "${@}";;
   926         *)            CT_DoLog WARN "Don't know how to handle '${basename}${ext}': unknown extension"
   927                       return 1
   928                       ;;
   929     esac
   930 
   931     # Don't mark as being extracted for git
   932     case "${ext}" in
   933         /.git)  ;;
   934         *)      CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${basename}.extracted";;
   935     esac
   936     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/.${basename}.extracting"
   937 
   938     CT_Popd
   939 }
   940 
   941 # Create a working git clone of a local git repository
   942 # Usage: CT_ExtractGit <basename> [ref]
   943 # where 'ref' is the reference to use:
   944 #   the full name of a branch, like "remotes/origin/branch_name"
   945 #   a date as understandable by git, like "YYYY-MM-DD[ hh[:mm[:ss]]]"
   946 #   a tag name
   947 # If 'ref' is not given, the current repository HEAD will be used
   948 CT_ExtractGit() {
   949     local basename="${1}"
   950     local ref="${2}"
   951     local repo
   952     local ref_type
   953 
   954     # pushd now to be able to get git revlist in case ref is a date
   955     repo="${CT_TARBALLS_DIR}/${basename}"
   956     CT_Pushd "${repo}"
   957 
   958     # What kind of reference is ${ref} ?
   959     if [ -z "${ref}" ]; then
   960         ref_type=head
   961         ref=$(git rev-list -n1 HEAD)
   962     elif git tag |grep -E "^${ref}$" >/dev/null 2>&1; then
   963         ref_type=tag
   964     elif git branch -a --no-color |grep -E "^. ${ref}$" >/dev/null 2>&1; then
   965         ref_type=branch
   966     elif date -d "${ref}" >/dev/null 2>&1; then
   967         ref_type=date
   968         ref=$(git rev-list -n1 --before="${ref}")
   969     else
   970         CT_Abort "Reference '${ref}' is an incorrect git reference: neither tag, branch nor date"
   971     fi
   972 
   973     CT_Popd
   974 
   975     CT_DoExecLog FILE rmdir "${basename}"
   976     case "${ref_type}" in
   977         branch) CT_DoExecLog FILE git clone -b "${ref}" "${repo}" "${basename}" ;;
   978         *)      CT_DoExecLog FILE git clone "${repo}" "${basename}"
   979                 CT_Pushd "${basename}"
   980                 CT_DoExecLog FILE git checkout "${ref}"
   981                 CT_Popd
   982                 ;;
   983     esac
   984 }
   985 
   986 # Patches the specified component
   987 # See CT_Extract, above, for explanations on 'nochdir'
   988 # Usage: CT_Patch [nochdir] <packagename> <packageversion>
   989 # If the package directory is *not* packagename-packageversion, then
   990 # the caller must cd into the proper directory first, and call us
   991 # with nochdir
   992 CT_Patch() {
   993     local nochdir="$1"
   994     local pkgname
   995     local version
   996     local pkgdir
   997     local base_file
   998     local ver_file
   999     local d
  1000     local -a patch_dirs
  1001     local bundled_patch_dir
  1002     local local_patch_dir
  1003 
  1004     if [ "${nochdir}" = "nochdir" ]; then
  1005         shift
  1006         pkgname="$1"
  1007         version="$2"
  1008         pkgdir="${pkgname}-${version}"
  1009         nochdir="$(pwd)"
  1010     else
  1011         pkgname="$1"
  1012         version="$2"
  1013         pkgdir="${pkgname}-${version}"
  1014         nochdir="${CT_SRC_DIR}/${pkgdir}"
  1015     fi
  1016 
  1017     # Check if already patched
  1018     if [ -e "${CT_SRC_DIR}/.${pkgdir}.patched" ]; then
  1019         CT_DoLog DEBUG "Already patched '${pkgdir}'"
  1020         return 0
  1021     fi
  1022 
  1023     # Check if already partially patched
  1024     if [ -e "${CT_SRC_DIR}/.${pkgdir}.patching" ]; then
  1025         CT_DoLog ERROR "The '${pkgdir}' sources were partially patched."
  1026         CT_DoLog ERROR "Please remove first:"
  1027         CT_DoLog ERROR " - the source dir for '${pkgdir}', in '${CT_SRC_DIR}'"
  1028         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${pkgdir}.extracted'"
  1029         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${pkgdir}.patching'"
  1030         CT_Abort "I'll stop now to avoid any carnage..."
  1031     fi
  1032     touch "${CT_SRC_DIR}/.${pkgdir}.patching"
  1033 
  1034     CT_Pushd "${nochdir}"
  1035 
  1036     CT_DoLog EXTRA "Patching '${pkgdir}'"
  1037 
  1038     bundled_patch_dir="${CT_LIB_DIR}/patches/${pkgname}/${version}"
  1039     local_patch_dir="${CT_LOCAL_PATCH_DIR}/${pkgname}/${version}"
  1040 
  1041     case "${CT_PATCH_ORDER}" in
  1042         bundled)        patch_dirs=("${bundled_patch_dir}");;
  1043         local)          patch_dirs=("${local_patch_dir}");;
  1044         bundled,local)  patch_dirs=("${bundled_patch_dir}" "${local_patch_dir}");;
  1045         local,bundled)  patch_dirs=("${local_patch_dir}" "${bundled_patch_dir}");;
  1046         none)           patch_dirs=;;
  1047     esac
  1048 
  1049     for d in "${patch_dirs[@]}"; do
  1050         CT_DoLog DEBUG "Looking for patches in '${d}'..."
  1051         if [ -n "${d}" -a -d "${d}" ]; then
  1052             for p in "${d}"/*.patch; do
  1053                 if [ -f "${p}" ]; then
  1054                     CT_DoExecLog ALL patch --no-backup-if-mismatch -g0 -F1 -p1 -f -i "${p}"
  1055                 fi
  1056             done
  1057             if [ "${CT_PATCH_SINGLE}" = "y" ]; then
  1058                 break
  1059             fi
  1060         fi
  1061     done
  1062 
  1063     if [ "${CT_OVERIDE_CONFIG_GUESS_SUB}" = "y" ]; then
  1064         CT_DoLog ALL "Overiding config.guess and config.sub"
  1065         for cfg in config_guess config_sub; do
  1066             eval ${cfg}="${CT_LIB_DIR}/scripts/${cfg/_/.}"
  1067             [ -e "${CT_TOP_DIR}/scripts/${cfg/_/.}" ] && eval ${cfg}="${CT_TOP_DIR}/scripts/${cfg/_/.}"
  1068             # Can't use CT_DoExecLog because of the '{} \;' to be passed un-mangled to find
  1069             find . -type f -name "${cfg/_/.}" -exec cp -v "${!cfg}" {} \; |CT_DoLog ALL
  1070         done
  1071     fi
  1072 
  1073     CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${pkgdir}.patched"
  1074     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/.${pkgdir}.patching"
  1075 
  1076     CT_Popd
  1077 }
  1078 
  1079 # Two wrappers to call config.(guess|sub) either from CT_TOP_DIR or CT_LIB_DIR.
  1080 # Those from CT_TOP_DIR, if they exist, will be be more recent than those from CT_LIB_DIR.
  1081 CT_DoConfigGuess() {
  1082     if [ -x "${CT_TOP_DIR}/scripts/config.guess" ]; then
  1083         "${CT_TOP_DIR}/scripts/config.guess"
  1084     else
  1085         "${CT_LIB_DIR}/scripts/config.guess"
  1086     fi
  1087 }
  1088 
  1089 CT_DoConfigSub() {
  1090     if [ -x "${CT_TOP_DIR}/scripts/config.sub" ]; then
  1091         "${CT_TOP_DIR}/scripts/config.sub" "$@"
  1092     else
  1093         "${CT_LIB_DIR}/scripts/config.sub" "$@"
  1094     fi
  1095 }
  1096 
  1097 # Compute the target tuple from what is provided by the user
  1098 # Usage: CT_DoBuildTargetTuple
  1099 # In fact this function takes the environment variables to build the target
  1100 # tuple. It is needed both by the normal build sequence, as well as the
  1101 # sample saving sequence.
  1102 CT_DoBuildTargetTuple() {
  1103     # Set the endianness suffix, and the default endianness gcc option
  1104     case "${CT_ARCH_ENDIAN}" in
  1105         big)
  1106             target_endian_eb=eb
  1107             target_endian_el=
  1108             CT_ARCH_ENDIAN_CFLAG="-mbig-endian"
  1109             CT_ARCH_ENDIAN_LDFLAG="-Wl,-EB"
  1110             ;;
  1111         little)
  1112             target_endian_eb=
  1113             target_endian_el=el
  1114             CT_ARCH_ENDIAN_CFLAG="-mlittle-endian"
  1115             CT_ARCH_ENDIAN_LDFLAG="-Wl,-EL"
  1116             ;;
  1117     esac
  1118 
  1119     # Build the default architecture tuple part
  1120     CT_TARGET_ARCH="${CT_ARCH}"
  1121 
  1122     # Set defaults for the system part of the tuple. Can be overriden
  1123     # by architecture-specific values.
  1124     case "${CT_LIBC}" in
  1125         *glibc) CT_TARGET_SYS=gnu;;
  1126         uClibc) CT_TARGET_SYS=uclibc;;
  1127         *)      CT_TARGET_SYS=elf;;
  1128     esac
  1129 
  1130     # Set the default values for ARCH, ABI, CPU, TUNE, FPU and FLOAT
  1131     unset CT_ARCH_ARCH_CFLAG CT_ARCH_ABI_CFLAG CT_ARCH_CPU_CFLAG CT_ARCH_TUNE_CFLAG CT_ARCH_FPU_CFLAG CT_ARCH_FLOAT_CFLAG
  1132     unset CT_ARCH_WITH_ARCH CT_ARCH_WITH_ABI CT_ARCH_WITH_CPU CT_ARCH_WITH_TUNE CT_ARCH_WITH_FPU CT_ARCH_WITH_FLOAT
  1133     [ "${CT_ARCH_ARCH}"     ] && { CT_ARCH_ARCH_CFLAG="-march=${CT_ARCH_ARCH}";  CT_ARCH_WITH_ARCH="--with-arch=${CT_ARCH_ARCH}"; }
  1134     [ "${CT_ARCH_ABI}"      ] && { CT_ARCH_ABI_CFLAG="-mabi=${CT_ARCH_ABI}";     CT_ARCH_WITH_ABI="--with-abi=${CT_ARCH_ABI}";    }
  1135     [ "${CT_ARCH_CPU}"      ] && { CT_ARCH_CPU_CFLAG="-mcpu=${CT_ARCH_CPU}";     CT_ARCH_WITH_CPU="--with-cpu=${CT_ARCH_CPU}";    }
  1136     [ "${CT_ARCH_TUNE}"     ] && { CT_ARCH_TUNE_CFLAG="-mtune=${CT_ARCH_TUNE}";  CT_ARCH_WITH_TUNE="--with-tune=${CT_ARCH_TUNE}"; }
  1137     [ "${CT_ARCH_FPU}"      ] && { CT_ARCH_FPU_CFLAG="-mfpu=${CT_ARCH_FPU}";     CT_ARCH_WITH_FPU="--with-fpu=${CT_ARCH_FPU}";    }
  1138 
  1139     case "${CT_ARCH_FLOAT}" in
  1140         hard)
  1141             CT_ARCH_FLOAT_CFLAG="-mhard-float"
  1142             CT_ARCH_WITH_FLOAT="--with-float=hard"
  1143             ;;
  1144         soft)
  1145             CT_ARCH_FLOAT_CFLAG="-msoft-float"
  1146             CT_ARCH_WITH_FLOAT="--with-float=soft"
  1147             ;;
  1148         softfp)
  1149             CT_ARCH_FLOAT_CFLAG="-mfloat-abi=softfp"
  1150             CT_ARCH_WITH_FLOAT="--with-float=softfp"
  1151             ;;
  1152     esac
  1153 
  1154     # Build the default kernel tuple part
  1155     CT_TARGET_KERNEL="${CT_KERNEL}"
  1156 
  1157     # Overide the default values with the components specific settings
  1158     CT_DoArchTupleValues
  1159     CT_DoKernelTupleValues
  1160 
  1161     # Finish the target tuple construction
  1162     CT_TARGET="${CT_TARGET_ARCH}"
  1163     CT_TARGET="${CT_TARGET}${CT_TARGET_VENDOR:+-${CT_TARGET_VENDOR}}"
  1164     CT_TARGET="${CT_TARGET}${CT_TARGET_KERNEL:+-${CT_TARGET_KERNEL}}"
  1165     CT_TARGET="${CT_TARGET}${CT_TARGET_SYS:+-${CT_TARGET_SYS}}"
  1166 
  1167     # Sanity checks
  1168     __sed_alias=""
  1169     if [ -n "${CT_TARGET_ALIAS_SED_EXPR}" ]; then
  1170         __sed_alias=$(echo "${CT_TARGET}" |sed -r -e "${CT_TARGET_ALIAS_SED_EXPR}")
  1171     fi
  1172     case ":${CT_TARGET_VENDOR}:${CT_TARGET_ALIAS}:${__sed_alias}:" in
  1173       :*" "*:*:*:) CT_Abort "Don't use spaces in the vendor string, it breaks things.";;
  1174       :*"-"*:*:*:) CT_Abort "Don't use dashes in the vendor string, it breaks things.";;
  1175       :*:*" "*:*:) CT_Abort "Don't use spaces in the target alias, it breaks things.";;
  1176       :*:*:*" "*:) CT_Abort "Don't use spaces in the target sed transform, it breaks things.";;
  1177     esac
  1178 
  1179     # Canonicalise it
  1180     CT_TARGET=$(CT_DoConfigSub "${CT_TARGET}")
  1181     # Prepare the target CFLAGS
  1182     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ENDIAN_CFLAG}"
  1183     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ARCH_CFLAG}"
  1184     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ABI_CFLAG}"
  1185     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_CPU_CFLAG}"
  1186     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_TUNE_CFLAG}"
  1187     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FPU_CFLAG}"
  1188     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FLOAT_CFLAG}"
  1189 
  1190     # Now on for the target LDFLAGS
  1191     CT_ARCH_TARGET_LDFLAGS="${CT_ARCH_TARGET_LDFLAGS} ${CT_ARCH_ENDIAN_LDFLAG}"
  1192 }
  1193 
  1194 # This function does pause the build until the user strikes "Return"
  1195 # Usage: CT_DoPause [optional_message]
  1196 CT_DoPause() {
  1197     local foo
  1198     local message="${1:-Pausing for your pleasure}"
  1199     CT_DoLog INFO "${message}"
  1200     read -p "Press 'Enter' to continue, or Ctrl-C to stop..." foo >&6
  1201     return 0
  1202 }
  1203 
  1204 # This function creates a tarball of the specified directory, but
  1205 # only if it exists
  1206 # Usage: CT_DoTarballIfExists <dir> <tarball_basename> [extra_tar_options [...]]
  1207 CT_DoTarballIfExists() {
  1208     local dir="$1"
  1209     local tarball="$2"
  1210     shift 2
  1211     local -a extra_tar_opts=( "$@" )
  1212     local -a compress
  1213 
  1214     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1215         y)  compress=( gzip -c -3 - ); tar_ext=.gz;;
  1216         *)  compress=( cat - );        tar_ext=;;
  1217     esac
  1218 
  1219     if [ -d "${dir}" ]; then
  1220         CT_DoLog DEBUG "  Saving '${dir}'"
  1221         { tar c -C "${dir}" -v -f - "${extra_tar_opts[@]}" .    \
  1222           |"${compress[@]}" >"${tarball}.tar${tar_ext}"         ;
  1223         } 2>&1 |sed -r -e 's/^/    /;' |CT_DoLog STATE
  1224     else
  1225         CT_DoLog STATE "  Not saving '${dir}': does not exist"
  1226     fi
  1227 }
  1228 
  1229 # This function extracts a tarball to the specified directory, but
  1230 # only if the tarball exists
  1231 # Usage: CT_DoExtractTarballIfExists <tarball_basename> <dir> [extra_tar_options [...]]
  1232 CT_DoExtractTarballIfExists() {
  1233     local tarball="$1"
  1234     local dir="$2"
  1235     shift 2
  1236     local -a extra_tar_opts=( "$@" )
  1237     local -a uncompress
  1238 
  1239     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1240         y)  uncompress=( gzip -c -d ); tar_ext=.gz;;
  1241         *)  uncompress=( cat );        tar_ext=;;
  1242     esac
  1243 
  1244     if [ -f "${tarball}.tar${tar_ext}" ]; then
  1245         CT_DoLog DEBUG "  Restoring '${dir}'"
  1246         CT_DoForceRmdir "${dir}"
  1247         CT_DoExecLog DEBUG mkdir -p "${dir}"
  1248         { "${uncompress[@]}" "${tarball}.tar${tar_ext}"     \
  1249           |tar x -C "${dir}" -v -f - "${extra_tar_opts[@]}" ;
  1250         } 2>&1 |sed -r -e 's/^/    /;' |CT_DoLog STATE
  1251     else
  1252         CT_DoLog STATE "  Not restoring '${dir}': does not exist"
  1253     fi
  1254 }
  1255 
  1256 # This function saves the state of the toolchain to be able to restart
  1257 # at any one point
  1258 # Usage: CT_DoSaveState <next_step_name>
  1259 CT_DoSaveState() {
  1260 	[ "${CT_DEBUG_CT_SAVE_STEPS}" = "y" ] || return 0
  1261     local state_name="$1"
  1262     local state_dir="${CT_STATE_DIR}/${state_name}"
  1263 
  1264     # Log this to the log level required by the user
  1265     CT_DoLog ${CT_LOG_LEVEL_MAX} "Saving state to restart at step '${state_name}'..."
  1266 
  1267     rm -rf "${state_dir}"
  1268     mkdir -p "${state_dir}"
  1269 
  1270     CT_DoLog STATE "  Saving environment and aliases"
  1271     # We must omit shell functions, and some specific bash variables
  1272     # that break when restoring the environment, later. We could do
  1273     # all the processing in the awk script, but a sed is easier...
  1274     set |awk '
  1275               BEGIN { _p = 1; }
  1276               $0~/^[^ ]+ \(\)/ { _p = 0; }
  1277               _p == 1
  1278               $0 == "}" { _p = 1; }
  1279               ' |sed -r -e '/^BASH_(ARGC|ARGV|LINENO|SOURCE|VERSINFO)=/d;
  1280                            /^(UID|EUID)=/d;
  1281                            /^(FUNCNAME|GROUPS|PPID|SHELLOPTS)=/d;' >"${state_dir}/env.sh"
  1282 
  1283     CT_DoTarballIfExists "${CT_BUILDTOOLS_PREFIX_DIR}" "${state_dir}/buildtools_dir"
  1284     CT_DoTarballIfExists "${CT_CONFIG_DIR}" "${state_dir}/config_dir"
  1285     CT_DoTarballIfExists "${CT_PREFIX_DIR}" "${state_dir}/prefix_dir" --exclude '*.log'
  1286 
  1287     CT_DoLog STATE "  Saving log file"
  1288     exec >/dev/null
  1289     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1290         y)  gzip -3 -c "${tmp_log_file}"  >"${state_dir}/log.gz";;
  1291         *)  cat "${tmp_log_file}" >"${state_dir}/log";;
  1292     esac
  1293     exec >>"${tmp_log_file}"
  1294 }
  1295 
  1296 # This function restores a previously saved state
  1297 # Usage: CT_DoLoadState <state_name>
  1298 CT_DoLoadState(){
  1299     local state_name="$1"
  1300     local state_dir="${CT_STATE_DIR}/${state_name}"
  1301     local old_RESTART="${CT_RESTART}"
  1302     local old_STOP="${CT_STOP}"
  1303 
  1304     CT_TestOrAbort "The previous build did not reach the point where it could be restarted at '${CT_RESTART}'" -d "${state_dir}"
  1305 
  1306     # We need to do something special with the log file!
  1307     if [ "${CT_LOG_TO_FILE}" = "y" ]; then
  1308         exec >"${state_dir}/tail.log"
  1309     fi
  1310 
  1311     # Log this to the log level required by the user
  1312     CT_DoLog ${CT_LOG_LEVEL_MAX} "Restoring state at step '${state_name}', as requested."
  1313 
  1314     CT_DoExtractTarballIfExists "${state_dir}/prefix_dir" "${CT_PREFIX_DIR}"
  1315     CT_DoExtractTarballIfExists "${state_dir}/config_dir" "${CT_CONFIG_DIR}"
  1316     CT_DoExtractTarballIfExists "${state_dir}/buildtools_dir" "${CT_BUILDTOOLS_PREFIX_DIR}"
  1317 
  1318     # Restore the environment, discarding any error message
  1319     # (for example, read-only bash internals)
  1320     CT_DoLog STATE "  Restoring environment"
  1321     . "${state_dir}/env.sh" >/dev/null 2>&1 || true
  1322 
  1323     # Restore the new RESTART and STOP steps
  1324     CT_RESTART="${old_RESTART}"
  1325     CT_STOP="${old_STOP}"
  1326     unset old_stop old_restart
  1327 
  1328     CT_DoLog STATE "  Restoring log file"
  1329     exec >/dev/null
  1330     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
  1331         y)  zcat "${state_dir}/log.gz" >"${tmp_log_file}";;
  1332         *)  cat "${state_dir}/log" >"${tmp_log_file}";;
  1333     esac
  1334     cat "${state_dir}/tail.log" >>"${tmp_log_file}"
  1335     exec >>"${tmp_log_file}"
  1336     rm -f "${state_dir}/tail.log"
  1337 }