scripts/functions
author "Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
Fri May 18 15:54:42 2007 +0000 (2007-05-18)
changeset 102 ce80474df80e
parent 97 63a30dd47eb8
child 107 06d3636f6611
permissions -rw-r--r--
Really use local copy first in case it does not have the same extension as the downloadable tarball.
     1 # This file contains some usefull common functions
     2 # Copyright 2007 Yann E. MORIN
     3 # Licensed under the GPL v2. See COPYING in the root of this package
     4 
     5 CT_OnError() {
     6     ret=$?
     7     CT_DoLog ERROR "Build failed in step \"${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}\""
     8     for((step=(CT_STEP_COUNT-1); step>1; step--)); do
     9         CT_DoLog ERROR "      called in step \"${CT_STEP_MESSAGE[${step}]}\""
    10     done
    11     CT_DoLog ERROR "Error happened in \"${BASH_SOURCE[1]}\" in function \"${FUNCNAME[1]}\" (line unknown, sorry)"
    12     for((depth=2; ${BASH_LINENO[$((${depth}-1))]}>0; depth++)); do
    13         CT_DoLog ERROR "      called from \"${BASH_SOURCE[${depth}]}\" at line # ${BASH_LINENO[${depth}-1]} in function \"${FUNCNAME[${depth}]}\""
    14     done
    15     CT_DoLog ERROR "Look at \"${CT_ACTUAL_LOG_FILE}\" for more info on this error."
    16     CT_STEP_COUNT=1
    17     CT_DoEnd ERROR
    18     exit $ret
    19 }
    20 trap CT_OnError ERR
    21 
    22 set -E
    23 set -o pipefail
    24 
    25 # The different log levels:
    26 CT_LOG_LEVEL_ERROR=0
    27 CT_LOG_LEVEL_WARN=1
    28 CT_LOG_LEVEL_INFO=2
    29 CT_LOG_LEVEL_EXTRA=3
    30 CT_LOG_LEVEL_DEBUG=4
    31 CT_LOG_LEVEL_ALL=5
    32 
    33 # A function to log what is happening
    34 # Different log level are available:
    35 #   - ERROR:   A serious, fatal error occurred
    36 #   - WARN:    A non fatal, non serious error occurred, take your responsbility with the generated build
    37 #   - INFO:    Informational messages
    38 #   - EXTRA:   Extra informational messages
    39 #   - DEBUG:   Debug messages
    40 #   - ALL:     Component's build messages
    41 # Usage: CT_DoLog <level> [message]
    42 # If message is empty, then stdin will be logged.
    43 CT_DoLog() {
    44     local max_level LEVEL level cur_l cur_L
    45     local l
    46     eval max_level="\${CT_LOG_LEVEL_${CT_LOG_LEVEL_MAX}}"
    47     # Set the maximum log level to DEBUG if we have none
    48     [ -z "${max_level}" ] && max_level=${CT_LOG_LEVEL_DEBUG}
    49 
    50     LEVEL="$1"; shift
    51     eval level="\${CT_LOG_LEVEL_${LEVEL}}"
    52 
    53     if [ $# -eq 0 ]; then
    54         cat -
    55     else
    56         echo "${1}"
    57     fi |( IFS="\n" # We want the full lines, even leading spaces
    58           CT_PROG_BAR_CPT=0
    59           indent=$((2*CT_STEP_COUNT))
    60           while read line; do
    61               case "${CT_LOG_SEE_TOOLS_WARN},${line}" in
    62                 y,*"warning:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
    63                 *"error:"*)             cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
    64                 *"make["?*"]:"*"Stop.") cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
    65                 *)                      cur_L="${LEVEL}"; cur_l="${level}";;
    66               esac
    67               l="`printf \"[%-5s]%*s%s%s\" \"${cur_L}\" \"${indent}\" \" \" \"${line}\"`"
    68               # There will always be a log file, be it /dev/null
    69               echo -e "${l}" >>"${CT_ACTUAL_LOG_FILE}"
    70               if [ ${cur_l} -le ${max_level} ]; then
    71                   echo -e "\r${l}"
    72               fi
    73               if [ "${CT_LOG_PROGRESS_BAR}" = "y" ]; then
    74                   str=`CT_DoDate +%s`
    75                   elapsed=$((str-(CT_STAR_DATE/(1000*1000*1000))))
    76                   [ ${CT_PROG_BAR_CPT} -eq 0  ] && bar="/"
    77                   [ ${CT_PROG_BAR_CPT} -eq 10 ] && bar="-"
    78                   [ ${CT_PROG_BAR_CPT} -eq 20 ] && bar="\\"
    79                   [ ${CT_PROG_BAR_CPT} -eq 30 ] && bar="|"
    80                   printf "\r[%02d:%02d] %s " $((elapsed/60)) $((elapsed%60)) "${bar}"
    81                   CT_PROG_BAR_CPT=$(((CT_PROG_BAR_CPT+1)%40))
    82               fi
    83           done
    84         )
    85 
    86     return 0
    87 }
    88 
    89 # Tail message to be logged whatever happens
    90 # Usage: CT_DoEnd <level>
    91 CT_DoEnd()
    92 {
    93     CT_STOP_DATE=`CT_DoDate +%s%N`
    94     CT_STOP_DATE_HUMAN=`CT_DoDate +%Y%m%d.%H%M%S`
    95     CT_DoLog INFO "Build completed at ${CT_STOP_DATE_HUMAN}"
    96     elapsed=$((CT_STOP_DATE-CT_STAR_DATE))
    97     elapsed_min=$((elapsed/(60*1000*1000*1000)))
    98     elapsed_sec=`printf "%02d" $(((elapsed%(60*1000*1000*1000))/(1000*1000*1000)))`
    99     elapsed_csec=`printf "%02d" $(((elapsed%(1000*1000*1000))/(10*1000*1000)))`
   100     CT_DoLog ${1:-INFO} "(elapsed: ${elapsed_min}:${elapsed_sec}.${elapsed_csec})"
   101 }
   102 
   103 # Abort the execution with an error message
   104 # Usage: CT_Abort <message>
   105 CT_Abort() {
   106     CT_DoLog ERROR "$1" >&2
   107     exit 1
   108 }
   109 
   110 # Test a condition, and print a message if satisfied
   111 # Usage: CT_Test <message> <tests>
   112 CT_Test() {
   113     local ret
   114     local m="$1"
   115     shift
   116     test "$@" && CT_DoLog WARN "$m"
   117     return 0
   118 }
   119 
   120 # Test a condition, and abort with an error message if satisfied
   121 # Usage: CT_TestAndAbort <message> <tests>
   122 CT_TestAndAbort() {
   123     local m="$1"
   124     shift
   125     test "$@" && CT_Abort "$m"
   126     return 0
   127 }
   128 
   129 # Test a condition, and abort with an error message if not satisfied
   130 # Usage: CT_TestAndAbort <message> <tests>
   131 CT_TestOrAbort() {
   132     local m="$1"
   133     shift
   134     test "$@" || CT_Abort "$m"
   135     return 0
   136 }
   137 
   138 # Test the presence of a tool, or abort if not found
   139 # Usage: CT_HasOrAbort <tool>
   140 CT_HasOrAbort() {
   141     CT_TestAndAbort "\"${1}\" not found and needed for successfull toolchain build." -z "`which \"${1}\"`"
   142     return 0
   143 }
   144 
   145 # Get current date with nanosecond precision
   146 # On those system not supporting nanosecond precision, faked with rounding down
   147 # to the highest entire second
   148 # Usage: CT_DoDate <fmt>
   149 CT_DoDate() {
   150     date "$1" |sed -r -e 's/%N$/000000000/;'
   151 }
   152 
   153 CT_STEP_COUNT=1
   154 CT_STEP_MESSAGE[${CT_STEP_COUNT}]="<none>"
   155 # Memorise a step being done so that any error is caught
   156 # Usage: CT_DoStep <loglevel> <message>
   157 CT_DoStep() {
   158     local start=`CT_DoDate +%s%N`
   159     CT_DoLog "$1" "================================================================="
   160     CT_DoLog "$1" "$2"
   161     CT_STEP_COUNT=$((CT_STEP_COUNT+1))
   162     CT_STEP_LEVEL[${CT_STEP_COUNT}]="$1"; shift
   163     CT_STEP_START[${CT_STEP_COUNT}]="${start}"
   164     CT_STEP_MESSAGE[${CT_STEP_COUNT}]="$1"
   165     return 0
   166 }
   167 
   168 # End the step just being done
   169 # Usage: CT_EndStep
   170 CT_EndStep() {
   171     local stop=`CT_DoDate +%s%N`
   172     local duration=`printf "%032d" $((stop-${CT_STEP_START[${CT_STEP_COUNT}]})) |sed -r -e 's/([[:digit:]]{2})[[:digit:]]{7}$/\.\1/; s/^0+//; s/^\./0\./;'`
   173     local level="${CT_STEP_LEVEL[${CT_STEP_COUNT}]}"
   174     local message="${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}"
   175     CT_STEP_COUNT=$((CT_STEP_COUNT-1))
   176     CT_DoLog "${level}" "${message}: done in ${duration}s"
   177     return 0
   178 }
   179 
   180 # Pushes into a directory, and pops back
   181 CT_Pushd() {
   182     pushd "$1" >/dev/null 2>&1
   183 }
   184 CT_Popd() {
   185     popd >/dev/null 2>&1
   186 }
   187 
   188 # Makes a path absolute
   189 # Usage: CT_MakeAbsolutePath path
   190 CT_MakeAbsolutePath() {
   191     # Try to cd in that directory
   192     if [ -d "$1" ]; then
   193         CT_Pushd "$1"
   194         pwd
   195         CT_Popd
   196     else
   197         # No such directory, fail back to guessing
   198         case "$1" in
   199             /*)  echo "$1";;
   200             *)   echo "`pwd`/$1";;
   201         esac
   202     fi
   203     
   204     return 0
   205 }
   206 
   207 # Creates a temporary directory
   208 # $1: variable to assign to
   209 # Usage: CT_MktempDir foo
   210 CT_MktempDir() {
   211     # Some mktemp do not allow more than 6 Xs
   212     eval "$1"="`mktemp -q -d \"${CT_BUILD_DIR}/.XXXXXX\"`"
   213     CT_TestOrAbort "Could not make temporary directory" -n "${!1}" -a -d "${!1}"
   214 }
   215 
   216 # Echoes the specified string on stdout until the pipe breaks.
   217 # Doesn't fail
   218 # $1: string to echo
   219 # Usage: CT_DoYes "" |make oldconfig
   220 CT_DoYes() {
   221     yes "$1" || true
   222 }
   223 
   224 # Get the file name extension of a component
   225 # Usage: CT_GetFileExtension <component_name-component_version>
   226 # If found, echoes the extension to stdout
   227 # If not found, echoes nothing on stdout.
   228 CT_GetFileExtension() {
   229     local ext
   230     local file="$1"
   231     local got_it=1
   232 
   233     CT_Pushd "${CT_TARBALLS_DIR}"
   234     for ext in .tar.gz .tar.bz2 .tgz .tar; do
   235         if [ -f "${file}${ext}" ]; then
   236             echo "${ext}"
   237             got_it=0
   238             break
   239         fi
   240     done
   241     CT_Popd
   242 
   243     return 0
   244 }
   245 
   246 # Download an URL using wget
   247 # Usage: CT_DoGetFileWget <URL>
   248 CT_DoGetFileWget() {
   249     # Need to return true because it is legitimate to not find the tarball at
   250     # some of the provided URLs (think about snapshots, different layouts for
   251     # different gcc versions, etc...)
   252     # Some (very old!) FTP server might not support the passive mode, thus
   253     # retry without
   254     # With automated download as we are doing, it can be very dangerous to use
   255     # -c to continue the downloads. It's far better to simply overwrite the
   256     # destination file
   257     wget -nc --progress=dot:binary --tries=3 --passive-ftp "$1" || wget -nc --progress=dot:binary --tries=3 "$1" || true
   258 }
   259 
   260 # Download an URL using curl
   261 # Usage: CT_DoGetFileCurl <URL>
   262 CT_DoGetFileCurl() {
   263 	# Note: comments about wget method are also valid here
   264 	# Plus: no good progreess indicator is available with curl,
   265 	#       so output is consigned to oblivion
   266 	curl --ftp-pasv -O --retry 3 "$1" >/dev/null || curl -O --retry 3 "$1" >/dev/null || true
   267 }
   268 
   269 # Wrapper function to call one of curl or wget
   270 # Usage: CT_DoGetFile <URL>
   271 CT_DoGetFile() {
   272     local _wget=`which wget`
   273     local _curl=`which curl`
   274     case "${_wget},${_curl}" in
   275         ,)  CT_DoError "Could find neither wget nor curl";;
   276         ,*) CT_DoGetFileCurl "$1" 2>&1 |CT_DoLog DEBUG;;
   277         *)  CT_DoGetFileWget "$1" 2>&1 |CT_DoLog DEBUG;;
   278     esac
   279 }
   280 
   281 # Download the file from one of the URLs passed as argument
   282 # Usage: CT_GetFile <filename> <url> [<url> ...]
   283 CT_GetFile() {
   284     local got_it
   285     local ext
   286     local url
   287     local file="$1"
   288     shift
   289 
   290     # Do we already have it?
   291     ext=`CT_GetFileExtension "${file}"`
   292     if [ -n "${ext}" ]; then
   293         CT_DoLog DEBUG "Already have \"${file}\""
   294         return 0
   295     fi
   296 
   297     CT_DoLog EXTRA "Retrieving \"${file}\""
   298     CT_Pushd "${CT_TARBALLS_DIR}"
   299     # File not yet downloaded, try to get it
   300     got_it=0
   301     # We'd rather have a bzip2'ed tarball, then gzipped, and finally plain tar.
   302     # Try local copy first, if it exists
   303     for ext in .tar.bz2 .tar.gz .tgz .tar; do
   304         if [ -r "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" -a \
   305              "${CT_FORCE_DOWNLOAD}" != "y" ]; then
   306             cp -v "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog DEBUG
   307             return 0
   308         fi
   309     done
   310     # Try to download it
   311     for ext in .tar.bz2 .tar.gz .tgz .tar; do
   312         # Try all urls in turn
   313         for url in "$@"; do
   314             case "${url}" in
   315                 *)  CT_DoLog DEBUG "Trying \"${url}/${file}${ext}\""
   316                     CT_DoGetFile "${url}/${file}${ext}"
   317                     ;;
   318             esac
   319             [ -f "${file}${ext}" ] && return 0 || true
   320         done
   321     done
   322     CT_Popd
   323 
   324     CT_Abort "Could not download \"${file}\", and not present in \"${CT_LOCAL_TARBALLS_DIR}\""
   325 }
   326 
   327 # Extract a tarball and patch the resulting sources if necessary.
   328 # Some tarballs need to be extracted in specific places. Eg.: glibc addons
   329 # must be extracted in the glibc directory; uCLibc locales must be extracted
   330 # in the extra/locale sub-directory of uClibc.
   331 CT_ExtractAndPatch() {
   332     local file="$1"
   333     local base_file=`echo "${file}" |cut -d - -f 1`
   334     local ver_file=`echo "${file}" |cut -d - -f 2-`
   335     local official_patch_dir
   336     local custom_patch_dir
   337     local libc_addon
   338     local ext=`CT_GetFileExtension "${file}"`
   339     CT_TestAndAbort "\"${file}\" not found in \"${CT_TARBALLS_DIR}\"" -z "${ext}"
   340     local full_file="${CT_TARBALLS_DIR}/${file}${ext}"
   341 
   342     CT_Pushd "${CT_SRC_DIR}"
   343 
   344     # Add-ons need a little love, really.
   345     case "${file}" in
   346         glibc-[a-z]*-*)
   347             CT_TestAndAbort "Trying to extract the C-library addon/locales \"${file}\" when C-library not yet extracted" ! -d "${CT_LIBC_FILE}"
   348             cd "${CT_LIBC_FILE}"
   349             libc_addon=y
   350             [ -f ".${file}.extracted" ] && return 0
   351             touch ".${file}.extracted"
   352             ;;
   353         uClibc-locale-*)
   354             CT_TestAndAbort "Trying to extract the C-library addon/locales \"${file}\" when C-library not yet extracted" ! -d "${CT_LIBC_FILE}"
   355             cd "${CT_LIBC_FILE}/extra/locale"
   356             libc_addon=y
   357             [ -f ".${file}.extracted" ] && return 0
   358             touch ".${file}.extracted"
   359             ;;
   360     esac
   361 
   362     # If the directory exists, then consider extraction and patching done
   363     if [ -d "${file}" ]; then
   364         CT_DoLog DEBUG "Already extracted \"${file}\""
   365         return 0
   366     fi
   367 
   368     CT_DoLog EXTRA "Extracting \"${file}\""
   369     case "${ext}" in
   370         .tar.bz2)     tar xvjf "${full_file}" |CT_DoLog ALL;;
   371         .tar.gz|.tgz) tar xvzf "${full_file}" |CT_DoLog ALL;;
   372         .tar)         tar xvf  "${full_file}" |CT_DoLog ALL;;
   373         *)            CT_Abort "Don't know how to handle \"${file}\": unknown extension" ;;
   374     esac
   375 
   376     # Snapshots might not have the version number in the extracted directory
   377     # name. This is also the case for some (old) packages, such as libfloat.
   378     # Overcome this issue by symlink'ing the directory.
   379     if [ ! -d "${file}" -a "${libc_addon}" != "y" ]; then
   380         case "${ext}" in
   381             .tar.bz2)     base=`tar tjf "${full_file}" |head -n 1 |cut -d / -f 1 || true`;;
   382             .tar.gz|.tgz) base=`tar tzf "${full_file}" |head -n 1 |cut -d / -f 1 || true`;;
   383             .tar)         base=`tar tf  "${full_file}" |head -n 1 |cut -d / -f 1 || true`;;
   384         esac
   385         CT_TestOrAbort "There was a problem when extracting \"${file}\"" -d "${base}" -o "${base}" != "${file}"
   386         ln -s "${base}" "${file}"
   387     fi
   388 
   389     # Kludge: outside this function, we wouldn't know if we had just extracted
   390     # a libc addon, or a plain package. Apply patches now.
   391     CT_DoLog EXTRA "Patching \"${file}\""
   392 
   393     # If libc addon, we're already in the correct place.
   394     [ -z "${libc_addon}" ] && cd "${file}"
   395 
   396     [ "${CUSTOM_PATCH_ONLY}" = "y" ] || official_patch_dir="${CT_TOP_DIR}/patches/${base_file}/${ver_file}"
   397     [ "${CT_CUSTOM_PATCH}" = "y" ] && custom_patch_dir="${CT_CUSTOM_PATCH_DIR}/${base_file}/${ver_file}"
   398     for patch_dir in "${official_patch_dir}" "${custom_patch_dir}"; do
   399         if [ -n "${patch_dir}" -a -d "${patch_dir}" ]; then
   400             for p in "${patch_dir}"/*.patch; do
   401                 if [ -f "${p}" ]; then
   402                     CT_DoLog DEBUG "Applying patch \"${p}\""
   403                     patch -g0 -F1 -p1 -f <"${p}" |CT_DoLog ALL
   404                     CT_TestAndAbort "Failed while applying patch file \"${p}\"" ${PIPESTATUS[0]} -ne 0
   405                 fi
   406             done
   407         fi
   408     done
   409 
   410     CT_Popd
   411 }
   412 
   413 # Compute the target triplet from what is provided by the user
   414 # Usage: CT_DoBuildTargetTriplet
   415 # In fact this function takes the environment variables to build the target
   416 # triplet. It is needed both by the normal build sequence, as well as the
   417 # sample saving sequence.
   418 CT_DoBuildTargetTriplet() {
   419     case "${CT_ARCH_BE},${CT_ARCH_LE}" in
   420         y,) target_endian_eb=eb; target_endian_el=;;
   421         ,y) target_endian_eb=; target_endian_el=el;;
   422     esac
   423     case "${CT_ARCH}" in
   424         arm)  CT_TARGET="${CT_ARCH}${target_endian_eb}";;
   425         mips) CT_TARGET="${CT_ARCH}${target_endian_el}";;
   426         x86*) # Much love for this one :-(
   427               # Ultimately, we should use config.sub to output the correct
   428               # procesor name. Work for later...
   429               arch="${CT_ARCH_ARCH}"
   430               [ -z "${arch}" ] && arch="${CT_ARCH_TUNE}"
   431               case "${CT_ARCH}" in
   432                   x86_64)      CT_TARGET=x86_64;;
   433               	  *)  case "${arch}" in
   434                           "")                                       CT_TARGET=i386;;
   435                           i386|i486|i586|i686)                      CT_TARGET="${arch}";;
   436                           winchip*)                                 CT_TARGET=i486;;
   437                           pentium|pentium-mmx|c3*)                  CT_TARGET=i586;;
   438                           nocona|athlon*64|k8|athlon-fx|opteron)    CT_TARGET=x86_64;;
   439                           pentiumpro|pentium*|athlon*)              CT_TARGET=i686;;
   440                           *)                                        CT_TARGET=i586;;
   441                       esac;;
   442               esac;;
   443     esac
   444     case "${CT_TARGET_VENDOR}" in
   445         "") CT_TARGET="${CT_TARGET}-unknown";;
   446         *)  CT_TARGET="${CT_TARGET}-${CT_TARGET_VENDOR}";;
   447     esac
   448     case "${CT_KERNEL}" in
   449         linux*)  CT_TARGET="${CT_TARGET}-linux";;
   450         cygwin*) CT_TARGET="${CT_TARGET}-cygwin";;
   451     esac
   452     case "${CT_LIBC}" in
   453         glibc)  CT_TARGET="${CT_TARGET}-gnu";;
   454         uClibc) CT_TARGET="${CT_TARGET}-uclibc";;
   455     esac
   456     case "${CT_ARCH_ABI}" in
   457         eabi)   CT_TARGET="${CT_TARGET}eabi";;
   458     esac
   459     CT_TARGET="`${CT_TOP_DIR}/tools/config.sub ${CT_TARGET}`"
   460 }