scripts/functions
author "Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
Tue Nov 10 19:32:11 2009 +0100 (2009-11-10)
changeset 1622 9ad2a3fd1fcc
parent 1592 c08132a93049
child 1630 1409ba438ea1
permissions -rw-r--r--
scripts: output renumbered patches in a new directory

When renumbering patches, the original patches get removed and replaced
with the new ones. This can be annoying to loose the original patches.
Fix this by putting the new patchs in a directory of their own.
     1 # This file contains some usefull common functions
     2 # Copyright 2007 Yann E. MORIN
     3 # Licensed under the GPL v2. See COPYING in the root of this package
     4 
     5 # Prepare the fault handler
     6 CT_OnError() {
     7     ret=$?
     8     # Bail out early in subshell, the upper level shell will act accordingly.
     9     [ ${BASH_SUBSHELL} -eq 0 ] || exit $ret
    10     CT_DoLog ERROR "Build failed in step '${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}'"
    11     for((step=(CT_STEP_COUNT-1); step>1; step--)); do
    12         CT_DoLog ERROR "      called in step '${CT_STEP_MESSAGE[${step}]}'"
    13     done
    14     CT_DoLog ERROR "Error happened in '${BASH_SOURCE[1]}' in function '${FUNCNAME[1]}' (line unknown, sorry)"
    15     for((depth=2; ${BASH_LINENO[$((${depth}-1))]}>0; depth++)); do
    16         CT_DoLog ERROR "      called from '${BASH_SOURCE[${depth}]}' at line # ${BASH_LINENO[${depth}-1]} in function '${FUNCNAME[${depth}]}'"
    17     done
    18     [ "${CT_LOG_TO_FILE}" = "y" ] && CT_DoLog ERROR "Look at '${CT_LOG_FILE}' for more info on this error."
    19     CT_STEP_COUNT=1
    20     CT_DoEnd ERROR
    21     exit $ret
    22 }
    23 
    24 # Install the fault handler
    25 trap CT_OnError ERR
    26 
    27 # Inherit the fault handler in subshells and functions
    28 set -E
    29 
    30 # Make pipes fail on the _first_ failed command
    31 # Not supported on bash < 3.x, but we need it, so drop the obsoleting bash-2.x
    32 set -o pipefail
    33 
    34 # Don't hash commands' locations, and search every time it is requested.
    35 # This is slow, but needed because of the static/shared core gcc which shall
    36 # always match to shared if it exists, and only fallback to static if the
    37 # shared is not found
    38 set +o hashall
    39 
    40 # Log policy:
    41 #  - first of all, save stdout so we can see the live logs: fd #6
    42 exec 6>&1
    43 #  - then point stdout to the log file (temporary for now)
    44 tmp_log_file="${CT_TOP_DIR}/log.$$"
    45 exec >>"${tmp_log_file}"
    46 
    47 # The different log levels:
    48 CT_LOG_LEVEL_ERROR=0
    49 CT_LOG_LEVEL_WARN=1
    50 CT_LOG_LEVEL_INFO=2
    51 CT_LOG_LEVEL_EXTRA=3
    52 CT_LOG_LEVEL_DEBUG=4
    53 CT_LOG_LEVEL_ALL=5
    54 
    55 # Make it easy to use \n and !
    56 CR=$(printf "\n")
    57 BANG='!'
    58 
    59 # A function to log what is happening
    60 # Different log level are available:
    61 #   - ERROR:   A serious, fatal error occurred
    62 #   - WARN:    A non fatal, non serious error occurred, take your responsbility with the generated build
    63 #   - INFO:    Informational messages
    64 #   - EXTRA:   Extra informational messages
    65 #   - DEBUG:   Debug messages
    66 #   - ALL:     Component's build messages
    67 # Usage: CT_DoLog <level> [message]
    68 # If message is empty, then stdin will be logged.
    69 CT_DoLog() {
    70     local max_level LEVEL level cur_l cur_L
    71     local l
    72     eval max_level="\${CT_LOG_LEVEL_${CT_LOG_LEVEL_MAX}}"
    73     # Set the maximum log level to DEBUG if we have none
    74     [ -z "${max_level}" ] && max_level=${CT_LOG_LEVEL_DEBUG}
    75 
    76     LEVEL="$1"; shift
    77     eval level="\${CT_LOG_LEVEL_${LEVEL}}"
    78 
    79     if [ $# -eq 0 ]; then
    80         cat -
    81     else
    82         printf "${*}\n"
    83     fi |( IFS="${CR}" # We want the full lines, even leading spaces
    84           _prog_bar_cpt=0
    85           _prog_bar[0]='/'
    86           _prog_bar[1]='-'
    87           _prog_bar[2]='\'
    88           _prog_bar[3]='|'
    89           indent=$((2*CT_STEP_COUNT))
    90           while read line; do
    91               case "${CT_LOG_SEE_TOOLS_WARN},${line}" in
    92                 y,*"warning:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
    93                 y,*"WARNING:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
    94                 *"error:"*)             cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
    95                 *"make["*"]: *** ["*)   cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
    96                 *)                      cur_L="${LEVEL}"; cur_l="${level}";;
    97               esac
    98               # There will always be a log file (stdout, fd #1), be it /dev/null
    99               printf "[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}"
   100               if [ ${cur_l} -le ${max_level} ]; then
   101                   # Only print to console (fd #6) if log level is high enough.
   102                   printf "\r[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}" >&6
   103               fi
   104               if [ "${CT_LOG_PROGRESS_BAR}" = "y" ]; then
   105                   printf "\r[%02d:%02d] %s " $((SECONDS/60)) $((SECONDS%60)) "${_prog_bar[$((_prog_bar_cpt/10))]}" >&6
   106                   _prog_bar_cpt=$(((_prog_bar_cpt+1)%40))
   107               fi
   108           done
   109         )
   110 
   111     return 0
   112 }
   113 
   114 # Execute an action, and log its messages
   115 # Usage: [VAR=val...] CT_DoExecLog <level> <command [parameters...]>
   116 CT_DoExecLog() {
   117     local level="$1"
   118     shift
   119     CT_DoLog DEBUG "==> Executing: '${*}'"
   120     "${@}" 2>&1 |CT_DoLog "${level}"
   121 }
   122 
   123 # Tail message to be logged whatever happens
   124 # Usage: CT_DoEnd <level>
   125 CT_DoEnd()
   126 {
   127     local level="$1"
   128     CT_STOP_DATE=$(CT_DoDate +%s%N)
   129     CT_STOP_DATE_HUMAN=$(CT_DoDate +%Y%m%d.%H%M%S)
   130     if [ "${level}" != "ERROR" ]; then
   131         CT_DoLog "${level:-INFO}" "Build completed at ${CT_STOP_DATE_HUMAN}"
   132     fi
   133     elapsed=$((CT_STOP_DATE-CT_STAR_DATE))
   134     elapsed_min=$((elapsed/(60*1000*1000*1000)))
   135     elapsed_sec=$(printf "%02d" $(((elapsed%(60*1000*1000*1000))/(1000*1000*1000))))
   136     elapsed_csec=$(printf "%02d" $(((elapsed%(1000*1000*1000))/(10*1000*1000))))
   137     CT_DoLog ${level:-INFO} "(elapsed: ${elapsed_min}:${elapsed_sec}.${elapsed_csec})"
   138 }
   139 
   140 # Abort the execution with an error message
   141 # Usage: CT_Abort <message>
   142 CT_Abort() {
   143     CT_DoLog ERROR "$1"
   144     exit 1
   145 }
   146 
   147 # Test a condition, and print a message if satisfied
   148 # Usage: CT_Test <message> <tests>
   149 CT_Test() {
   150     local ret
   151     local m="$1"
   152     shift
   153     test "$@" && CT_DoLog WARN "$m"
   154     return 0
   155 }
   156 
   157 # Test a condition, and abort with an error message if satisfied
   158 # Usage: CT_TestAndAbort <message> <tests>
   159 CT_TestAndAbort() {
   160     local m="$1"
   161     shift
   162     test "$@" && CT_Abort "$m"
   163     return 0
   164 }
   165 
   166 # Test a condition, and abort with an error message if not satisfied
   167 # Usage: CT_TestAndAbort <message> <tests>
   168 CT_TestOrAbort() {
   169     local m="$1"
   170     shift
   171     test "$@" || CT_Abort "$m"
   172     return 0
   173 }
   174 
   175 # Test the presence of a tool, or abort if not found
   176 # Usage: CT_HasOrAbort <tool>
   177 CT_HasOrAbort() {
   178     CT_TestAndAbort "'${1}' not found and needed for successful toolchain build." -z "$(CT_Which "${1}")"
   179     return 0
   180 }
   181 
   182 # Search a program: wrap "which" for those system where
   183 # "which" verbosely says there is no match (Mandriva is
   184 # such a sucker...)
   185 # Usage: CT_Which <filename>
   186 CT_Which() {
   187   which "$1" 2>/dev/null || true
   188 }
   189 
   190 # Get current date with nanosecond precision
   191 # On those system not supporting nanosecond precision, faked with rounding down
   192 # to the highest entire second
   193 # Usage: CT_DoDate <fmt>
   194 CT_DoDate() {
   195     date "$1" |sed -r -e 's/%N$/000000000/;'
   196 }
   197 
   198 CT_STEP_COUNT=1
   199 CT_STEP_MESSAGE[${CT_STEP_COUNT}]="<none>"
   200 # Memorise a step being done so that any error is caught
   201 # Usage: CT_DoStep <loglevel> <message>
   202 CT_DoStep() {
   203     local start=$(CT_DoDate +%s%N)
   204     CT_DoLog "$1" "================================================================="
   205     CT_DoLog "$1" "$2"
   206     CT_STEP_COUNT=$((CT_STEP_COUNT+1))
   207     CT_STEP_LEVEL[${CT_STEP_COUNT}]="$1"; shift
   208     CT_STEP_START[${CT_STEP_COUNT}]="${start}"
   209     CT_STEP_MESSAGE[${CT_STEP_COUNT}]="$1"
   210     return 0
   211 }
   212 
   213 # End the step just being done
   214 # Usage: CT_EndStep
   215 CT_EndStep() {
   216     local stop=$(CT_DoDate +%s%N)
   217     local duration=$(printf "%032d" $((stop-${CT_STEP_START[${CT_STEP_COUNT}]})) |sed -r -e 's/([[:digit:]]{2})[[:digit:]]{7}$/\.\1/; s/^0+//; s/^\./0\./;')
   218     local elapsed=$(printf "%02d:%02d" $((SECONDS/60)) $((SECONDS%60)))
   219     local level="${CT_STEP_LEVEL[${CT_STEP_COUNT}]}"
   220     local message="${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}"
   221     CT_STEP_COUNT=$((CT_STEP_COUNT-1))
   222     CT_DoLog "${level}" "${message}: done in ${duration}s (at ${elapsed})"
   223     return 0
   224 }
   225 
   226 # Pushes into a directory, and pops back
   227 CT_Pushd() {
   228     pushd "$1" >/dev/null 2>&1
   229 }
   230 CT_Popd() {
   231     popd >/dev/null 2>&1
   232 }
   233 
   234 # Creates a temporary directory
   235 # $1: variable to assign to
   236 # Usage: CT_MktempDir foo
   237 CT_MktempDir() {
   238     # Some mktemp do not allow more than 6 Xs
   239     eval "$1"=$(mktemp -q -d "${CT_BUILD_DIR}/tmp.XXXXXX")
   240     CT_TestOrAbort "Could not make temporary directory" -n "${!1}" -a -d "${!1}"
   241     CT_DoLog DEBUG "Made temporary directory '${!1}'"
   242     return 0
   243 }
   244 
   245 # Removes one or more directories, even if it is read-only, or its parent is
   246 # Usage: CT_DoForceRmdir dir [...]
   247 CT_DoForceRmdir() {
   248     local dir
   249     local mode
   250     for dir in "${@}"; do
   251         [ -d "${dir}" ] || continue
   252         mode="$(stat -c '%a' "$(dirname "${dir}")")"
   253         CT_DoExecLog ALL chmod u+w "$(dirname "${dir}")"
   254         CT_DoExecLog ALL chmod -R u+w "${dir}"
   255         CT_DoExecLog ALL rm -rf "${dir}"
   256         CT_DoExecLog ALL chmod ${mode} "$(dirname "${dir}")"
   257     done
   258 }
   259 
   260 # Echoes the specified string on stdout until the pipe breaks.
   261 # Doesn't fail
   262 # $1: string to echo
   263 # Usage: CT_DoYes "" |make oldconfig
   264 CT_DoYes() {
   265     yes "$1" || true
   266 }
   267 
   268 # Add the specified directory to LD_LIBRARY_PATH, and export it
   269 # If the specified patch is already present, just export
   270 # $1: path to add
   271 # $2: add as 'first' or 'last' path, 'first' is assumed if $2 is empty
   272 # Usage CT_SetLibPath /some/where/lib [first|last]
   273 CT_SetLibPath() {
   274     local path="$1"
   275     local pos="$2"
   276 
   277     case ":${LD_LIBRARY_PATH}:" in
   278         *:"${path}":*)  ;;
   279         *)  case "${pos}" in
   280                 last)
   281                     CT_DoLog DEBUG "Adding '${path}' at end of LD_LIBRARY_PATH"
   282                     LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${path}"
   283                     ;;
   284                 first|"")
   285                     CT_DoLog DEBUG "Adding '${path}' at start of LD_LIBRARY_PATH"
   286                     LD_LIBRARY_PATH="${path}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
   287                     ;;
   288                 *)
   289                     CT_Abort "Incorrect position '${pos}' to add '${path}' to LD_LIBRARY_PATH"
   290                     ;;
   291             esac
   292             ;;
   293     esac
   294     CT_DoLog DEBUG "==> LD_LIBRARY_PATH='${LD_LIBRARY_PATH}'"
   295     export LD_LIBRARY_PATH
   296 }
   297 
   298 # Get the file name extension of a component
   299 # Usage: CT_GetFileExtension <component_name-component_version> [extension]
   300 # If found, echoes the extension to stdout
   301 # If not found, echoes nothing on stdout.
   302 CT_GetFileExtension() {
   303     local ext
   304     local file="$1"
   305     shift
   306     local first_ext="$1"
   307 
   308     # we need to also check for an empty extension for those very
   309     # peculiar components that don't have one (such as sstrip from
   310     # buildroot).
   311     for ext in ${first_ext} .tar.gz .tar.bz2 .tgz .tar ''; do
   312         if [ -f "${CT_TARBALLS_DIR}/${file}${ext}" ]; then
   313             echo "${ext}"
   314             break
   315         fi
   316     done
   317 
   318     return 0
   319 }
   320 
   321 # Download an URL using wget
   322 # Usage: CT_DoGetFileWget <URL>
   323 CT_DoGetFileWget() {
   324     # Need to return true because it is legitimate to not find the tarball at
   325     # some of the provided URLs (think about snapshots, different layouts for
   326     # different gcc versions, etc...)
   327     # Some (very old!) FTP server might not support the passive mode, thus
   328     # retry without
   329     # With automated download as we are doing, it can be very dangerous to use
   330     # -c to continue the downloads. It's far better to simply overwrite the
   331     # destination file
   332     # Some company networks have firewalls to connect to the internet, but it's
   333     # not easy to detect them, and wget does not timeout by default while
   334     # connecting, so force a global ${CT_CONNECT_TIMEOUT}-second timeout.
   335     CT_DoExecLog ALL wget -T ${CT_CONNECT_TIMEOUT} -nc --progress=dot:binary --tries=3 --passive-ftp "$1"    \
   336     || CT_DoExecLog ALL wget -T ${CT_CONNECT_TIMEOUT} -nc --progress=dot:binary --tries=3 "$1"               \
   337     || true
   338 }
   339 
   340 # Download an URL using curl
   341 # Usage: CT_DoGetFileCurl <URL>
   342 CT_DoGetFileCurl() {
   343     # Note: comments about wget method (above) are also valid here
   344     # Plus: no good progress indicator is available with curl,
   345     #       so, be silent.
   346     CT_DoExecLog ALL curl -s --ftp-pasv -O --retry 3 "$1" --connect-timeout ${CT_CONNECT_TIMEOUT}    \
   347     || CT_DoExecLog ALL curl -s -O --retry 3 "$1" --connect-timeout ${CT_CONNECT_TIMEOUT}            \
   348     || true
   349 }
   350 
   351 _wget=$(CT_Which wget)
   352 _curl=$(CT_Which curl)
   353 # Wrapper function to call one of curl or wget
   354 # Usage: CT_DoGetFile <URL>
   355 CT_DoGetFile() {
   356     case "${_wget},${_curl}" in
   357         ,)  CT_Abort "Could find neither wget nor curl";;
   358         ,*) CT_DoGetFileCurl "$1";;
   359         *)  CT_DoGetFileWget "$1";;
   360     esac
   361 }
   362 
   363 # This function tries to retrieve a tarball form a local directory
   364 # Usage: CT_GetLocal <basename> [.extension]
   365 CT_GetLocal() {
   366     local basename="$1"
   367     local first_ext="$2"
   368     local ext
   369 
   370     # Do we already have it in *our* tarballs dir?
   371     ext=$(CT_GetFileExtension "${basename}" ${first_ext})
   372     if [ -n "${ext}" ]; then
   373         CT_DoLog DEBUG "Already have '${basename}'"
   374         return 0
   375     fi
   376 
   377     if [ -n "${CT_LOCAL_TARBALLS_DIR}" ]; then
   378         CT_DoLog DEBUG "Trying to retrieve an already downloaded copy of '${basename}'"
   379         # We'd rather have a bzip2'ed tarball, then gzipped tarball, plain tarball,
   380         # or, as a failover, a file without extension.
   381         for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do
   382             CT_DoLog DEBUG "Trying '${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}'"
   383             if [ -r "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" -a \
   384                  "${CT_FORCE_DOWNLOAD}" != "y" ]; then
   385                 CT_DoLog DEBUG "Got '${basename}' from local storage"
   386                 CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}${ext}" "${CT_TARBALLS_DIR}/${basename}${ext}"
   387                 return 0
   388             fi
   389         done
   390     fi
   391     return 1
   392 }
   393 
   394 # This function saves the specified to local storage if possible,
   395 # and if so, symlinks it for later usage
   396 # Usage: CT_SaveLocal </full/path/file.name>
   397 CT_SaveLocal() {
   398     local file="$1"
   399     local basename="${file##*/}"
   400 
   401     if [ "${CT_SAVE_TARBALLS}" = "y" ]; then
   402         CT_DoLog EXTRA "Saving '${basename}' to local storage"
   403         # The file may already exist if downloads are forced: remove it first
   404         CT_DoExecLog ALL rm -f "${CT_LOCAL_TARBALLS_DIR}/${basename}"
   405         CT_DoExecLog ALL mv -f "${file}" "${CT_LOCAL_TARBALLS_DIR}"
   406         CT_DoExecLog ALL ln -s "${CT_LOCAL_TARBALLS_DIR}/${basename}" "${file}"
   407     fi
   408 }
   409 
   410 # Download the file from one of the URLs passed as argument
   411 # Usage: CT_GetFile <basename> [.extension] <url> [url ...]
   412 CT_GetFile() {
   413     local ext
   414     local url URLS LAN_URLS
   415     local file="$1"
   416     local first_ext
   417     shift
   418     # If next argument starts with a dot, then this is not an URL,
   419     # and we can consider that it is a preferred extension.
   420     case "$1" in
   421         .*) first_ext="$1"
   422             shift
   423             ;;
   424     esac
   425 
   426     # Does it exist localy?
   427     CT_GetLocal "${file}" ${first_ext} && return 0 || true
   428     # No, it does not...
   429 
   430     # Are downloads allowed ?
   431     CT_TestAndAbort "File '${file}' not present locally, and downloads are not allowed" "${CT_FORBID_DOWNLOAD}" = "y"
   432 
   433     # Try to retrieve the file
   434     CT_DoLog EXTRA "Retrieving '${file}'"
   435     CT_Pushd "${CT_TARBALLS_DIR}"
   436 
   437     URLS="$@"
   438 
   439     # Add URLs on the LAN mirror
   440     LAN_URLS=
   441     if [ "${CT_USE_MIRROR}" = "y" ]; then
   442         CT_TestOrAbort "Please set the mirror base URL" -n "${CT_MIRROR_BASE_URL}"
   443         LAN_URLS="${LAN_URLS} ${CT_MIRROR_BASE_URL}/${file%-*}"
   444         LAN_URLS="${LAN_URLS} ${CT_MIRROR_BASE_URL}"
   445 
   446         if [ "${CT_PREFER_MIRROR}" = "y" ]; then
   447             CT_DoLog DEBUG "Pre-pending LAN mirror URLs"
   448             URLS="${LAN_URLS} ${URLS}"
   449         else
   450             CT_DoLog DEBUG "Appending LAN mirror URLs"
   451             URLS="${URLS} ${LAN_URLS}"
   452         fi
   453     fi
   454 
   455     # Scan all URLs in turn, and try to grab a tarball from there
   456     for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do
   457         # Try all urls in turn
   458         for url in ${URLS}; do
   459             CT_DoLog DEBUG "Trying '${url}/${file}${ext}'"
   460             CT_DoGetFile "${url}/${file}${ext}"
   461             if [ -f "${file}${ext}" ]; then
   462                 CT_DoLog DEBUG "Got '${file}' from the Internet"
   463                 CT_SaveLocal "${CT_TARBALLS_DIR}/${file}${ext}"
   464                 return 0
   465             fi
   466         done
   467     done
   468     CT_Popd
   469 
   470     CT_Abort "Could not retrieve '${file}'."
   471 }
   472 
   473 # Checkout from CVS, and build the associated tarball
   474 # The tarball will be called ${basename}.tar.bz2
   475 # Prerequisite: either the server does not require password,
   476 # or the user must already be logged in.
   477 # 'tag' is the tag to retrieve. Must be specified, but can be empty.
   478 # If dirname is specified, then module will be renamed to dirname
   479 # prior to building the tarball.
   480 # Usage: CT_GetCVS <basename> <url> <module> <tag> [dirname[=subdir]]
   481 # Note: if '=subdir' is given, then it is used instead of 'module'.
   482 CT_GetCVS() {
   483     local basename="$1"
   484     local uri="$2"
   485     local module="$3"
   486     local tag="${4:+-r ${4}}"
   487     local dirname="$5"
   488     local tmp_dir
   489 
   490     # Does it exist localy?
   491     CT_GetLocal "${basename}" && return 0 || true
   492     # No, it does not...
   493 
   494     # Are downloads allowed ?
   495     CT_TestAndAbort "File '${basename}' not present locally, and downloads are not allowed" "${CT_FORBID_DOWNLOAD}" = "y"
   496 
   497     CT_DoLog EXTRA "Retrieving '${basename}'"
   498 
   499     CT_MktempDir tmp_dir
   500     CT_Pushd "${tmp_dir}"
   501 
   502     CT_DoExecLog ALL cvs -z 9 -d "${uri}" co -P ${tag} "${module}"
   503     if [ -n "${dirname}" ]; then
   504         case "${dirname}" in
   505             *=*)
   506                 CT_DoExecLog DEBUG mv "${dirname#*=}" "${dirname%%=*}"
   507                 CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${dirname%%=*}"
   508                 ;;
   509             *)
   510                 CT_DoExecLog ALL mv "${module}" "${dirname}"
   511                 CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${dirname:-${module}}"
   512                 ;;
   513         esac
   514     fi
   515     CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}.tar.bz2"
   516 
   517     CT_Popd
   518     CT_DoExecLog ALL rm -rf "${tmp_dir}"
   519 }
   520 
   521 # Check out from SVN, and build the associated tarball
   522 # The tarball will be called ${basename}.tar.bz2
   523 # Prerequisite: either the server does not require password,
   524 # or the user must already be logged in.
   525 # 'rev' is the revision to retrieve
   526 # Usage: CT_GetSVN <basename> <url> [rev]
   527 CT_GetSVN() {
   528     local basename="$1"
   529     local uri="$2"
   530     local rev="$3"
   531 
   532     # Does it exist localy?
   533     CT_GetLocal "${basename}" && return 0 || true
   534     # No, it does not...
   535 
   536     # Are downloads allowed ?
   537     CT_TestAndAbort "File '${basename}' not present locally, and downloads are not allowed" "${CT_FORBID_DOWNLOAD}" = "y"
   538 
   539     CT_DoLog EXTRA "Retrieving '${basename}'"
   540 
   541     CT_MktempDir tmp_dir
   542     CT_Pushd "${tmp_dir}"
   543 
   544     CT_DoExecLog ALL svn export ${rev:+-r ${rev}} "${uri}" "${basename}"
   545     CT_DoExecLog ALL tar cjf "${CT_TARBALLS_DIR}/${basename}.tar.bz2" "${basename}"
   546     CT_SaveLocal "${CT_TARBALLS_DIR}/${basename}.tar.bz2"
   547 
   548     CT_Popd
   549     CT_DoExecLog ALL rm -rf "${tmp_dir}"
   550 }
   551 
   552 # Extract a tarball
   553 # Some tarballs need to be extracted in specific places. Eg.: glibc addons
   554 # must be extracted in the glibc directory; uCLibc locales must be extracted
   555 # in the extra/locale sub-directory of uClibc. This is taken into account
   556 # by the caller, that did a 'cd' into the correct path before calling us
   557 # and sets nochdir to 'nochdir'.
   558 # Usage: CT_Extract <basename> [nochdir]
   559 CT_Extract() {
   560     local basename="$1"
   561     local nochdir="$2"
   562     local ext=$(CT_GetFileExtension "${basename}")
   563     CT_TestAndAbort "'${basename}' not found in '${CT_TARBALLS_DIR}'" -z "${ext}"
   564     local full_file="${CT_TARBALLS_DIR}/${basename}${ext}"
   565 
   566     # Check if already extracted
   567     if [ -e "${CT_SRC_DIR}/.${basename}.extracted" ]; then
   568         CT_DoLog DEBUG "Already extracted '${basename}'"
   569         return 0
   570     fi
   571 
   572     [ "${nochdir}" = "nochdir" ] || CT_Pushd "${CT_SRC_DIR}"
   573 
   574     CT_DoLog EXTRA "Extracting '${basename}'"
   575     case "${ext}" in
   576         .tar.bz2)     CT_DoExecLog ALL tar xvjf "${full_file}";;
   577         .tar.gz|.tgz) CT_DoExecLog ALL tar xvzf "${full_file}";;
   578         .tar)         CT_DoExecLog ALL tar xvf  "${full_file}";;
   579         *)            CT_Abort "Don't know how to handle '${basename}${ext}': unknown extension" ;;
   580     esac
   581 
   582     # Some tarballs have read-only files... :-(
   583     # Because of nochdir, we don't know where we are, so chmod all
   584     # the src tree
   585     CT_DoExecLog DEBUG chmod -R u+w "${CT_SRC_DIR}"
   586 
   587     CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${basename}.extracted"
   588 
   589     [ "${nochdir}" = "nochdir" ] || CT_Popd
   590 }
   591 
   592 # Patches the specified component
   593 # Usage: CT_Patch <basename> [nochdir]
   594 CT_Patch() {
   595     local basename="$1"
   596     local nochdir="$2"
   597     local base_file="${basename%%-*}"
   598     local ver_file="${basename#*-}"
   599     local d
   600     local -a patch_dirs
   601     local bundled_patch_dir
   602     local local_patch_dir
   603 
   604     # Check if already patched
   605     if [ -e "${CT_SRC_DIR}/.${basename}.patched" ]; then
   606         CT_DoLog DEBUG "Already patched '${basename}'"
   607         return 0
   608     fi
   609 
   610     # Check if already partially patched
   611     if [ -e "${CT_SRC_DIR}/.${basename}.patching" ]; then
   612         CT_DoLog ERROR "The '${basename}' sources were partially patched."
   613         CT_DoLog ERROR "Please remove first:"
   614         CT_DoLog ERROR " - the source dir for '${basename}', in '${CT_SRC_DIR}'"
   615         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${basename}.extracted'"
   616         CT_DoLog ERROR " - the file '${CT_SRC_DIR}/.${basename}.patching'"
   617         CT_Abort "I'll stop now to avoid any carnage..."
   618     fi
   619     touch "${CT_SRC_DIR}/.${basename}.patching"
   620 
   621     [ "${nochdir}" = "nochdir" ] || CT_Pushd "${CT_SRC_DIR}/${basename}"
   622 
   623     CT_DoLog EXTRA "Patching '${basename}'"
   624 
   625     bundled_patch_dir="${CT_LIB_DIR}/patches/${base_file}/${ver_file}"
   626     local_patch_dir="${CT_LOCAL_PATCH_DIR}/${base_file}/${ver_file}"
   627 
   628     case "${CT_PATCH_ORDER}" in
   629         bundled)        patch_dirs=("${bundled_patch_dir}");;
   630         local)          patch_dirs=("${local_patch_dir}");;
   631         bundled,local)  patch_dirs=("${bundled_patch_dir}" "${local_patch_dir}");;
   632         local,bundled)  patch_dirs=("${local_patch_dir}" "${bundled_patch_dir}");;
   633     esac
   634 
   635     for d in "${patch_dirs[@]}"; do
   636         CT_DoLog DEBUG "Looking for patches in '${d}'..."
   637         if [ -n "${d}" -a -d "${d}" ]; then
   638             for p in "${d}"/*.patch; do
   639                 if [ -f "${p}" ]; then
   640                     CT_DoLog DEBUG "Applying patch '${p}'"
   641                     CT_DoExecLog ALL patch -g0 -F1 -p1 -f <"${p}"
   642                 fi
   643             done
   644             if [ "${CT_PATCH_SINGLE}" = "y" ]; then
   645                 break
   646             fi
   647         fi
   648     done
   649 
   650     if [ "${CT_OVERIDE_CONFIG_GUESS_SUB}" = "y" ]; then
   651         CT_DoLog ALL "Overiding config.guess and config.sub"
   652         for cfg in config_guess config_sub; do
   653             eval ${cfg}="${CT_LIB_DIR}/scripts/${cfg/_/.}"
   654             [ -e "${CT_TOP_DIR}/scripts/${cfg/_/.}" ] && eval ${cfg}="${CT_TOP_DIR}/scripts/${cfg/_/.}"
   655             # Can't use CT_DoExecLog because of the '{} \;' to be passed un-mangled to find
   656             find . -type f -name "${cfg/_/.}" -exec cp -v "${!cfg}" {} \; |CT_DoLog ALL
   657         done
   658     fi
   659 
   660     CT_DoExecLog DEBUG rm -f "${CT_SRC_DIR}/.${basename}.patching"
   661     CT_DoExecLog DEBUG touch "${CT_SRC_DIR}/.${basename}.patched"
   662 
   663     [ "${nochdir}" = "nochdir" ] || CT_Popd
   664 }
   665 
   666 # Two wrappers to call config.(guess|sub) either from CT_TOP_DIR or CT_LIB_DIR.
   667 # Those from CT_TOP_DIR, if they exist, will be be more recent than those from CT_LIB_DIR.
   668 CT_DoConfigGuess() {
   669     if [ -x "${CT_TOP_DIR}/scripts/config.guess" ]; then
   670         "${CT_TOP_DIR}/scripts/config.guess"
   671     else
   672         "${CT_LIB_DIR}/scripts/config.guess"
   673     fi
   674 }
   675 
   676 CT_DoConfigSub() {
   677     if [ -x "${CT_TOP_DIR}/scripts/config.sub" ]; then
   678         "${CT_TOP_DIR}/scripts/config.sub" "$@"
   679     else
   680         "${CT_LIB_DIR}/scripts/config.sub" "$@"
   681     fi
   682 }
   683 
   684 # Compute the target tuple from what is provided by the user
   685 # Usage: CT_DoBuildTargetTuple
   686 # In fact this function takes the environment variables to build the target
   687 # tuple. It is needed both by the normal build sequence, as well as the
   688 # sample saving sequence.
   689 CT_DoBuildTargetTuple() {
   690     # Set the endianness suffix, and the default endianness gcc option
   691     case "${CT_ARCH_BE},${CT_ARCH_LE}" in
   692         y,) target_endian_eb=eb
   693             target_endian_el=
   694             CT_ARCH_ENDIAN_CFLAG="-mbig-endian"
   695             CT_ARCH_ENDIAN_LDFLAG="-EB"
   696             ;;
   697         ,y) target_endian_eb=
   698             target_endian_el=el
   699             CT_ARCH_ENDIAN_CFLAG="-mlittle-endian"
   700             CT_ARCH_ENDIAN_LDFLAG="-EL"
   701             ;;
   702     esac
   703 
   704     # Build the default architecture tuple part
   705     CT_TARGET_ARCH="${CT_ARCH}"
   706 
   707     # Set defaults for the system part of the tuple. Can be overriden
   708     # by architecture-specific values.
   709     case "${CT_LIBC}" in
   710         *glibc) CT_TARGET_SYS=gnu;;
   711         uClibc) CT_TARGET_SYS=uclibc;;
   712         *)      CT_TARGET_SYS=elf;;
   713     esac
   714 
   715     # Transform the ARCH into a kernel-understandable ARCH
   716     CT_KERNEL_ARCH="${CT_ARCH}"
   717 
   718     # Set the default values for ARCH, ABI, CPU, TUNE, FPU and FLOAT
   719     unset CT_ARCH_ARCH_CFLAG CT_ARCH_ABI_CFLAG CT_ARCH_CPU_CFLAG CT_ARCH_TUNE_CFLAG CT_ARCH_FPU_CFLAG CT_ARCH_FLOAT_CFLAG
   720     unset CT_ARCH_WITH_ARCH CT_ARCH_WITH_ABI CT_ARCH_WITH_CPU CT_ARCH_WITH_TUNE CT_ARCH_WITH_FPU CT_ARCH_WITH_FLOAT
   721     [ "${CT_ARCH_ARCH}"     ] && { CT_ARCH_ARCH_CFLAG="-march=${CT_ARCH_ARCH}";  CT_ARCH_WITH_ARCH="--with-arch=${CT_ARCH_ARCH}"; }
   722     [ "${CT_ARCH_ABI}"      ] && { CT_ARCH_ABI_CFLAG="-mabi=${CT_ARCH_ABI}";     CT_ARCH_WITH_ABI="--with-abi=${CT_ARCH_ABI}";    }
   723     [ "${CT_ARCH_CPU}"      ] && { CT_ARCH_CPU_CFLAG="-mcpu=${CT_ARCH_CPU}";     CT_ARCH_WITH_CPU="--with-cpu=${CT_ARCH_CPU}";    }
   724     [ "${CT_ARCH_TUNE}"     ] && { CT_ARCH_TUNE_CFLAG="-mtune=${CT_ARCH_TUNE}";  CT_ARCH_WITH_TUNE="--with-tune=${CT_ARCH_TUNE}"; }
   725     [ "${CT_ARCH_FPU}"      ] && { CT_ARCH_FPU_CFLAG="-mfpu=${CT_ARCH_FPU}";     CT_ARCH_WITH_FPU="--with-fpu=${CT_ARCH_FPU}";    }
   726     [ "${CT_ARCH_FLOAT_SW}" ] && { CT_ARCH_FLOAT_CFLAG="-msoft-float";           CT_ARCH_WITH_FLOAT="--with-float=soft";          }
   727 
   728     # Build the default kernel tuple part
   729     CT_TARGET_KERNEL="${CT_KERNEL}"
   730 
   731     # Overide the default values with the components specific settings
   732     CT_DoArchTupleValues
   733     CT_DoKernelTupleValues
   734 
   735     # Finish the target tuple construction
   736     CT_TARGET="${CT_TARGET_ARCH}-${CT_TARGET_VENDOR:-unknown}-${CT_TARGET_KERNEL}${CT_TARGET_KERNEL:+-}${CT_TARGET_SYS}"
   737 
   738     # Sanity checks
   739     __sed_alias=""
   740     if [ -n "${CT_TARGET_ALIAS_SED_EXPR}" ]; then
   741         __sed_alias=$(echo "${CT_TARGET}" |sed -r -e "${CT_TARGET_ALIAS_SED_EXPR}")
   742     fi
   743     case ":${CT_TARGET_VENDOR}:${CT_TARGET_ALIAS}:${__sed_alias}:" in
   744       :*" "*:*:*:) CT_Abort "Don't use spaces in the vendor string, it breaks things.";;
   745       :*"-"*:*:*:) CT_Abort "Don't use dashes in the vendor string, it breaks things.";;
   746       :*:*" "*:*:) CT_Abort "Don't use spaces in the target alias, it breaks things.";;
   747       :*:*:*" "*:) CT_Abort "Don't use spaces in the target sed transform, it breaks things.";;
   748     esac
   749 
   750     # Canonicalise it
   751     CT_TARGET=$(CT_DoConfigSub "${CT_TARGET}")
   752 
   753     # Prepare the target CFLAGS
   754     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ENDIAN_CFLAG}"
   755     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ARCH_CFLAG}"
   756     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ABI_CFLAG}"
   757     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_CPU_CFLAG}"
   758     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_TUNE_CFLAG}"
   759     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FPU_CFLAG}"
   760     CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FLOAT_CFLAG}"
   761 
   762     # Now on for the target LDFLAGS
   763     CT_ARCH_TARGET_LDFLAGS="${CT_ARCH_TARGET_LDFLAGS} ${CT_ARCH_ENDIAN_LDFLAG}"
   764 }
   765 
   766 # This function does pause the build until the user strikes "Return"
   767 # Usage: CT_DoPause [optional_message]
   768 CT_DoPause() {
   769     local foo
   770     local message="${1:-Pausing for your pleasure}"
   771     CT_DoLog INFO "${message}"
   772     read -p "Press 'Enter' to continue, or Ctrl-C to stop..." foo >&6
   773     return 0
   774 }
   775 
   776 # This function saves the state of the toolchain to be able to restart
   777 # at any one point
   778 # Usage: CT_DoSaveState <next_step_name>
   779 CT_DoSaveState() {
   780 	[ "${CT_DEBUG_CT_SAVE_STEPS}" = "y" ] || return 0
   781     local state_name="$1"
   782     local state_dir="${CT_STATE_DIR}/${state_name}"
   783 
   784     # Log this to the log level required by the user
   785     CT_DoLog ${CT_LOG_LEVEL_MAX} "Saving state to restart at step '${state_name}'..."
   786 
   787     rm -rf "${state_dir}"
   788     mkdir -p "${state_dir}"
   789 
   790     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
   791         y)  tar_opt=z; tar_ext=.gz;;
   792         *)  tar_opt=;  tar_ext=;;
   793     esac
   794 
   795     CT_DoLog DEBUG "  Saving environment and aliases"
   796     # We must omit shell functions, and some specific bash variables
   797     # that break when restoring the environment, later. We could do
   798     # all the processing in the awk script, but a sed is easier...
   799     set |awk '
   800               BEGIN { _p = 1; }
   801               $0~/^[^ ]+ \(\)/ { _p = 0; }
   802               _p == 1
   803               $0 == "}" { _p = 1; }
   804               ' |sed -r -e '/^BASH_(ARGC|ARGV|LINENO|SOURCE|VERSINFO)=/d;
   805                            /^(UID|EUID)=/d;
   806                            /^(FUNCNAME|GROUPS|PPID|SHELLOPTS)=/d;' >"${state_dir}/env.sh"
   807 
   808     CT_DoLog DEBUG "  Saving CT_CONFIG_DIR='${CT_CONFIG_DIR}'"
   809     CT_Pushd "${CT_CONFIG_DIR}"
   810     CT_DoExecLog DEBUG tar cv${tar_opt}f "${state_dir}/config_dir.tar${tar_ext}" .
   811     CT_Popd
   812 
   813     CT_DoLog DEBUG "  Saving CT_CC_CORE_STATIC_PREFIX_DIR='${CT_CC_CORE_STATIC_PREFIX_DIR}'"
   814     CT_Pushd "${CT_CC_CORE_STATIC_PREFIX_DIR}"
   815     CT_DoExecLog DEBUG tar cv${tar_opt}f "${state_dir}/cc_core_static_prefix_dir.tar${tar_ext}" .
   816     CT_Popd
   817 
   818     CT_DoLog DEBUG "  Saving CT_CC_CORE_SHARED_PREFIX_DIR='${CT_CC_CORE_SHARED_PREFIX_DIR}'"
   819     CT_Pushd "${CT_CC_CORE_SHARED_PREFIX_DIR}"
   820     CT_DoExecLog DEBUG tar cv${tar_opt}f "${state_dir}/cc_core_shared_prefix_dir.tar${tar_ext}" .
   821     CT_Popd
   822 
   823     CT_DoLog DEBUG "  Saving CT_PREFIX_DIR='${CT_PREFIX_DIR}'"
   824     CT_Pushd "${CT_PREFIX_DIR}"
   825     CT_DoExecLog DEBUG tar cv${tar_opt}f "${state_dir}/prefix_dir.tar${tar_ext}" --exclude '*.log' .
   826     CT_Popd
   827 
   828     if [ "${CT_LOG_TO_FILE}" = "y" ]; then
   829         CT_DoLog DEBUG "  Saving log file"
   830         exec >/dev/null
   831         case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
   832             y)  gzip -3 -c "${CT_LOG_FILE}"  >"${state_dir}/log.gz";;
   833             *)  cat "${CT_LOG_FILE}" >"${state_dir}/log";;
   834         esac
   835         exec >>"${CT_LOG_FILE}"
   836     fi
   837 }
   838 
   839 # This function restores a previously saved state
   840 # Usage: CT_DoLoadState <state_name>
   841 CT_DoLoadState(){
   842     local state_name="$1"
   843     local state_dir="${CT_STATE_DIR}/${state_name}"
   844     local old_RESTART="${CT_RESTART}"
   845     local old_STOP="${CT_STOP}"
   846 
   847     CT_TestOrAbort "The previous build did not reach the point where it could be restarted at '${CT_RESTART}'" -d "${state_dir}"
   848 
   849     # We need to do something special with the log file!
   850     if [ "${CT_LOG_TO_FILE}" = "y" ]; then
   851         exec >"${state_dir}/tail.log"
   852     fi
   853 
   854     # Log this to the log level required by the user
   855     CT_DoLog ${CT_LOG_LEVEL_MAX} "Restoring state at step '${state_name}', as requested."
   856 
   857     case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
   858         y)  tar_opt=z; tar_ext=.gz;;
   859         *)  tar_opt=;  tar_ext=;;
   860     esac
   861 
   862     CT_DoLog DEBUG "  Removing previous build directories"
   863     CT_DoForceRmdir             "${CT_PREFIX_DIR}" "${CT_CC_CORE_SHARED_PREFIX_DIR}" "${CT_CC_CORE_STATIC_PREFIX_DIR}" "${CT_CONFIG_DIR}"
   864     CT_DoExecLog DEBUG mkdir -p "${CT_PREFIX_DIR}" "${CT_CC_CORE_SHARED_PREFIX_DIR}" "${CT_CC_CORE_STATIC_PREFIX_DIR}" "${CT_CONFIG_DIR}"
   865 
   866     CT_DoLog DEBUG "  Restoring CT_PREFIX_DIR='${CT_PREFIX_DIR}'"
   867     CT_Pushd "${CT_PREFIX_DIR}"
   868     CT_DoExecLog DEBUG tar xv${tar_opt}f "${state_dir}/prefix_dir.tar${tar_ext}"
   869     CT_Popd
   870 
   871     CT_DoLog DEBUG "  Restoring CT_CC_CORE_SHARED_PREFIX_DIR='${CT_CC_CORE_SHARED_PREFIX_DIR}'"
   872     CT_Pushd "${CT_CC_CORE_SHARED_PREFIX_DIR}"
   873     CT_DoExecLog DEBUG tar xv${tar_opt}f "${state_dir}/cc_core_shared_prefix_dir.tar${tar_ext}"
   874     CT_Popd
   875 
   876     CT_DoLog DEBUG "  Restoring CT_CC_CORE_STATIC_PREFIX_DIR='${CT_CC_CORE_STATIC_PREFIX_DIR}'"
   877     CT_Pushd "${CT_CC_CORE_STATIC_PREFIX_DIR}"
   878     CT_DoExecLog DEBUG tar xv${tar_opt}f "${state_dir}/cc_core_static_prefix_dir.tar${tar_ext}"
   879     CT_Popd
   880 
   881     CT_DoLog DEBUG "  Restoring CT_CONFIG_DIR='${CT_CONFIG_DIR}'"
   882     CT_Pushd "${CT_CONFIG_DIR}"
   883     CT_DoExecLog DEBUG tar xv${tar_opt}f "${state_dir}/config_dir.tar${tar_ext}"
   884     CT_Popd
   885 
   886     # Restore the environment, discarding any error message
   887     # (for example, read-only bash internals)
   888     CT_DoLog DEBUG "  Restoring environment"
   889     . "${state_dir}/env.sh" >/dev/null 2>&1 || true
   890 
   891     # Restore the new RESTART and STOP steps
   892     CT_RESTART="${old_RESTART}"
   893     CT_STOP="${old_STOP}"
   894     unset old_stop old_restart
   895 
   896     if [ "${CT_LOG_TO_FILE}" = "y" ]; then
   897         CT_DoLog DEBUG "  Restoring log file"
   898         exec >/dev/null
   899         case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
   900             y)  zcat "${state_dir}/log.gz" >"${CT_LOG_FILE}";;
   901             *)  cat "${state_dir}/log" >"${CT_LOG_FILE}";;
   902         esac
   903         cat "${state_dir}/tail.log" >>"${CT_LOG_FILE}"
   904         exec >>"${CT_LOG_FILE}"
   905         rm -f "${state_dir}/tail.log"
   906     fi
   907 }