scripts/functions
author "Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
Tue May 08 18:43:26 2007 +0000 (2007-05-08)
changeset 79 1b5fe9bbe694
parent 78 c3868084d81a
child 81 a20be197429a
permissions -rw-r--r--
Fix handling of downloader output: they are now correctly logged using the internal log facility.
     1 # This file contains some usefull common functions
     2 # Copyright 2007 Yann E. MORIN
     3 # Licensed under the GPL v2. See COPYING in the root of this package
     4 
     5 CT_OnError() {
     6     ret=$?
     7     CT_DoLog ERROR "Build failed in step \"${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}\""
     8     for((step=(CT_STEP_COUNT-1); step>1; step--)); do
     9         CT_DoLog ERROR "      called in step \"${CT_STEP_MESSAGE[${step}]}\""
    10     done
    11     CT_DoLog ERROR "Error happened in \"${BASH_SOURCE[1]}\" in function \"${FUNCNAME[1]}\" (line unknown, sorry)"
    12     for((depth=2; ${BASH_LINENO[$((${depth}-1))]}>0; depth++)); do
    13         CT_DoLog ERROR "      called from \"${BASH_SOURCE[${depth}]}\" at line # ${BASH_LINENO[${depth}-1]} in function \"${FUNCNAME[${depth}]}\""
    14     done
    15     CT_DoLog ERROR "Look at \"${CT_ACTUAL_LOG_FILE}\" for more info on this error."
    16     exit $ret
    17 }
    18 trap CT_OnError ERR
    19 
    20 set -E
    21 set -o pipefail
    22 
    23 # This is crosstool-ng-0.0.1
    24 CT_VERSION=ng-0.0.1
    25 
    26 # The different log levels:
    27 CT_LOG_LEVEL_ERROR=0
    28 CT_LOG_LEVEL_WARN=1
    29 CT_LOG_LEVEL_INFO=2
    30 CT_LOG_LEVEL_EXTRA=3
    31 CT_LOG_LEVEL_DEBUG=4
    32 CT_LOG_LEVEL_ALL=5
    33 
    34 # Attributes
    35 _A_NOR="\\033[0m"
    36 _A_BRI="\\033[1m"
    37 _A_DIM="\\033[2m"
    38 _A_UND="\\033[4m"
    39 _A_BRB="\\033[5m"
    40 _A_REV="\\033[7m"
    41 _A_HID="\\033[8m"
    42 
    43 # Fore colors
    44 _F_BLK="\\033[30m"
    45 _F_RED="\\033[31m"
    46 _F_GRN="\\033[32m"
    47 _F_YEL="\\033[33m"
    48 _F_BLU="\\033[34m"
    49 _F_MAG="\\033[35m"
    50 _F_CYA="\\033[36m"
    51 _F_WHI="\\033[37m"
    52 
    53 # A function to log what is happening
    54 # Different log level are available:
    55 #   - ERROR:   A serious, fatal error occurred
    56 #   - WARN:    A non fatal, non serious error occurred, take your responsbility with the generated build
    57 #   - INFO:    Informational messages
    58 #   - EXTRA:   Extra informational messages
    59 #   - DEBUG:   Debug messages
    60 #   - ALL:     Component's build messages
    61 # Usage: CT_DoLog <level> [message]
    62 # If message is empty, then stdin will be logged.
    63 CT_DoLog() {
    64     local max_level LEVEL level cur_l cur_L
    65     local l
    66     eval max_level="\${CT_LOG_LEVEL_${CT_LOG_LEVEL_MAX}}"
    67     # Set the maximum log level to DEBUG if we have none
    68     [ -z "${max_level}" ] && max_level=${CT_LOG_LEVEL_DEBUG}
    69 
    70     LEVEL="$1"; shift
    71     eval level="\${CT_LOG_LEVEL_${LEVEL}}"
    72 
    73     if [ $# -eq 0 ]; then
    74         cat -
    75     else
    76         echo "${1}"
    77     fi |( IFS="\n" # We want the full lines, even leading spaces
    78           cpt=0
    79           indent=$((2*CT_STEP_COUNT))
    80           while read line; do
    81               case "${CT_LOG_SEE_TOOLS_WARN},${line}" in
    82                 y,*"warning:"*)         cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};;
    83                 *"error:"*)             cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
    84                 *"make["?*"]:"*"Stop.") cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};;
    85                 *)                      cur_L="${LEVEL}"; cur_l="${level}";;
    86               esac
    87               l="`printf \"[%-5s]%*s%s%s\" \"${cur_L}\" \"${indent}\" \" \" \"${line}\"`"
    88               # There will always be a log file, be it /dev/null
    89               echo -e "${l}" >>"${CT_ACTUAL_LOG_FILE}"
    90               color="CT_${cur_L}_COLOR"
    91               normal="CT_NORMAL_COLOR"
    92               if [ ${cur_l} -le ${max_level} ]; then
    93                   echo -e "\r${!color}${l}${!normal}"
    94               else
    95                   ${CT_PROG_BAR}
    96               fi
    97           done
    98         )
    99 
   100     return 0
   101 }
   102 
   103 # Abort the execution with a error message
   104 # Usage: CT_Abort <message>
   105 CT_Abort() {
   106     CT_DoLog ERROR "$1" >&2
   107     exit 1
   108 }
   109 
   110 # Test a condition, and print a message if satisfied
   111 # Usage: CT_Test <message> <tests>
   112 CT_Test() {
   113     local ret
   114     local m="$1"
   115     shift
   116     test "$@" && CT_DoLog WARN "$m"
   117     return 0
   118 }
   119 
   120 # Test a condition, and abort with an error message if satisfied
   121 # Usage: CT_TestAndAbort <message> <tests>
   122 CT_TestAndAbort() {
   123     local m="$1"
   124     shift
   125     test "$@" && CT_Abort "$m"
   126     return 0
   127 }
   128 
   129 # Test a condition, and abort with an error message if not satisfied
   130 # Usage: CT_TestAndAbort <message> <tests>
   131 CT_TestOrAbort() {
   132     local m="$1"
   133     shift
   134     test "$@" || CT_Abort "$m"
   135     return 0
   136 }
   137 
   138 # Test the presence of a tool, or abort if not found
   139 # Usage: CT_HasOrAbort <tool>
   140 CT_HasOrAbort() {
   141     CT_TestAndAbort "\"${1}\" not found and needed for successfull toolchain build." -z "`which \"${1}\"`"
   142     return 0
   143 }
   144 
   145 # Get current date with nanosecond precision
   146 # On those system not supporting nanosecond precision, faked with rounding down
   147 # to the highest entire second
   148 # Usage: CT_DoDate <fmt>
   149 CT_DoDate() {
   150     date "$1" |sed -r -e 's/%N$/000000000/;'
   151 }
   152 
   153 CT_STEP_COUNT=1
   154 CT_STEP_MESSAGE[${CT_STEP_COUNT}]="<none>"
   155 # Memorise a step being done so that any error is caught
   156 # Usage: CT_DoStep <loglevel> <message>
   157 CT_DoStep() {
   158     local start=`CT_DoDate +%s%N`
   159     CT_DoLog "$1" "================================================================="
   160     CT_DoLog "$1" "$2"
   161     CT_STEP_COUNT=$((CT_STEP_COUNT+1))
   162     CT_STEP_LEVEL[${CT_STEP_COUNT}]="$1"; shift
   163     CT_STEP_START[${CT_STEP_COUNT}]="${start}"
   164     CT_STEP_MESSAGE[${CT_STEP_COUNT}]="$1"
   165     return 0
   166 }
   167 
   168 # End the step just being done
   169 # Usage: CT_EndStep
   170 CT_EndStep() {
   171     local stop=`CT_DoDate +%s%N`
   172     local duration=`printf "%032d" $((stop-${CT_STEP_START[${CT_STEP_COUNT}]})) |sed -r -e 's/([[:digit:]]{2})[[:digit:]]{7}$/\.\1/; s/^0+//; s/^\./0\./;'`
   173     local level="${CT_STEP_LEVEL[${CT_STEP_COUNT}]}"
   174     local message="${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}"
   175     CT_STEP_COUNT=$((CT_STEP_COUNT-1))
   176     CT_DoLog "${level}" "${message}: done in ${duration}s"
   177     return 0
   178 }
   179 
   180 # Pushes into a directory, and pops back
   181 CT_Pushd() {
   182     pushd "$1" >/dev/null 2>&1
   183 }
   184 CT_Popd() {
   185     popd >/dev/null 2>&1
   186 }
   187 
   188 # Makes a path absolute
   189 # Usage: CT_MakeAbsolutePath path
   190 CT_MakeAbsolutePath() {
   191     # Try to cd in that directory
   192     if [ -d "$1" ]; then
   193         CT_Pushd "$1"
   194         pwd
   195         CT_Popd
   196     else
   197         # No such directory, fail back to guessing
   198         case "$1" in
   199             /*)  echo "$1";;
   200             *)   echo "`pwd`/$1";;
   201         esac
   202     fi
   203     
   204     return 0
   205 }
   206 
   207 # Creates a temporary directory
   208 # $1: variable to assign to
   209 # Usage: CT_MktempDir foo
   210 CT_MktempDir() {
   211     # Some mktemp do not allow more than 6 Xs
   212     eval "$1"="`mktemp -q -d \"${CT_BUILD_DIR}/.XXXXXX\"`"
   213     CT_TestOrAbort "Could not make temporary directory" -n "${!1}" -a -d "${!1}"
   214 }
   215 
   216 # Echoes the specified string on stdout until the pipe breaks.
   217 # Doesn't fail
   218 # $1: string to echo
   219 # Usage: CT_DoYes "" |make oldconfig
   220 CT_DoYes() {
   221     yes "$1" || true
   222 }
   223 
   224 # Download an URL using wget
   225 # Usage: CT_DoGetFileWget <URL>
   226 CT_DoGetFileWget() {
   227     # Need to return true because it is legitimate to not find the tarball at
   228     # some of the provided URLs (think about snapshots, different layouts for
   229     # different gcc versions, etc...)
   230     # Some (very old!) FTP server might not support the passive mode, thus
   231     # retry without
   232     # With automated download as we are doing, it can be very dangerous to use
   233     # -c to continue the downloads. It's far better to simply overwrite the
   234     # destination file
   235     wget -nc --progress=dot:binary --tries=3 --passive-ftp "$1" || wget -nc --progress=dot:binary --tries=3 "$1" || true
   236 }
   237 
   238 # Download an URL using curl
   239 # Usage: CT_DoGetFileCurl <URL>
   240 CT_DoGetFileCurl() {
   241 	# Note: comments about wget method are also valid here
   242 	# Plus: no good progreess indicator is available with curl,
   243 	#       so output is consigned to oblivion
   244 	curl --ftp-pasv -O --retry 3 "$1" >/dev/null || curl -O --retry 3 "$1" >/dev/null || true
   245 }
   246 
   247 # Wrapper function to call one of curl or wget
   248 # Usage: CT_DoGetFile <URL>
   249 CT_DoGetFile() {
   250     local _wget=`which wget`
   251     local _curl=`which curl`
   252     case "${_wget},${_curl}" in
   253         ,)  CT_DoError "Could find neither wget nor curl";;
   254         ,*) CT_DoGetFileCurl "$1" 2>&1 |CT_DoLog DEBUG;;
   255         *)  CT_DoGetFileWget "$1" 2>&1 |CT_DoLog DEBUG;;
   256     esac
   257 }
   258 
   259 # Download the file from one of the URLs passed as argument
   260 # Usage: CT_GetFile <filename> <url> [<url> ...]
   261 CT_GetFile() {
   262     local got_it
   263     local ext
   264     local url
   265     local file="$1"
   266     shift
   267 
   268     # Do we already have it?
   269     ext=`CT_GetFileExtension "${file}"`
   270     if [ -n "${ext}" ]; then
   271         if [ "${CT_FORCE_DOWNLOAD}" = "y" ]; then
   272             CT_DoLog DEBUG "Removing already present \"${file}\""
   273             rm -f "${CT_TARBALLS_DIR}/${file}${ext}"
   274         else
   275             CT_DoLog DEBUG "Already have \"${file}\""
   276             return 0
   277         fi
   278     fi
   279 
   280     CT_DoLog EXTRA "Retrieving \"${file}\""
   281     CT_Pushd "${CT_TARBALLS_DIR}"
   282     # File not yet downloaded, try to get it
   283     got_it=0
   284     if [ "${got_it}" != "y" ]; then
   285         # We'd rather have a bzip2'ed tarball, then gzipped, and finally plain tar.
   286         for ext in .tar.bz2 .tar.gz .tgz .tar; do
   287             # Try all urls in turn
   288             for url in "$@"; do
   289                 case "${url}" in
   290                     *)  CT_DoLog DEBUG "Trying \"${url}/${file}${ext}\""
   291                         CT_DoGetFile "${url}/${file}${ext}"
   292                         ;;
   293                 esac
   294                 [ -f "${file}${ext}" ] && got_it=1 && break 2 || true
   295             done
   296         done
   297     fi
   298     CT_Popd
   299 
   300     CT_TestAndAbort "Could not download \"${file}\", and not present in \"${CT_TARBALLS_DIR}\"" ${got_it} -eq 0
   301 }
   302 
   303 # Get the file name extension of a component
   304 # Usage: CT_GetFileExtension <component_name-component_version>
   305 # If found, echoes the extension to stdout
   306 # If not found, echoes nothing on stdout.
   307 CT_GetFileExtension() {
   308     local ext
   309     local file="$1"
   310     local got_it=1
   311 
   312     CT_Pushd "${CT_TARBALLS_DIR}"
   313     for ext in .tar.gz .tar.bz2 .tgz .tar; do
   314         if [ -f "${file}${ext}" ]; then
   315             echo "${ext}"
   316             got_it=0
   317             break
   318         fi
   319     done
   320     CT_Popd
   321 
   322     return 0
   323 }
   324 
   325 # Extract a tarball and patch the resulting sources if necessary.
   326 # Some tarballs need to be extracted in specific places. Eg.: glibc addons
   327 # must be extracted in the glibc directory; uCLibc locales must be extracted
   328 # in the extra/locale sub-directory of uClibc.
   329 CT_ExtractAndPatch() {
   330     local file="$1"
   331     local base_file=`echo "${file}" |cut -d - -f 1`
   332     local ver_file=`echo "${file}" |cut -d - -f 2-`
   333     local official_patch_dir
   334     local custom_patch_dir
   335     local libc_addon
   336     local ext=`CT_GetFileExtension "${file}"`
   337     CT_TestAndAbort "\"${file}\" not found in \"${CT_TARBALLS_DIR}\"" -z "${ext}"
   338     local full_file="${CT_TARBALLS_DIR}/${file}${ext}"
   339 
   340     CT_Pushd "${CT_SRC_DIR}"
   341 
   342     # Add-ons need a little love, really.
   343     case "${file}" in
   344         glibc-[a-z]*-*)
   345             CT_TestAndAbort "Trying to extract the C-library addon/locales \"${file}\" when C-library not yet extracted" ! -d "${CT_LIBC_FILE}"
   346             cd "${CT_LIBC_FILE}"
   347             libc_addon=y
   348             [ -f ".${file}.extracted" ] && return 0
   349             touch ".${file}.extracted"
   350             ;;
   351         uClibc-locale-*)
   352             CT_TestAndAbort "Trying to extract the C-library addon/locales \"${file}\" when C-library not yet extracted" ! -d "${CT_LIBC_FILE}"
   353             cd "${CT_LIBC_FILE}/extra/locale"
   354             libc_addon=y
   355             [ -f ".${file}.extracted" ] && return 0
   356             touch ".${file}.extracted"
   357             ;;
   358     esac
   359 
   360     # If the directory exists, then consider extraction and patching done
   361     if [ -d "${file}" ]; then
   362         CT_DoLog DEBUG "Already extracted \"${file}\""
   363         return 0
   364     fi
   365 
   366     CT_DoLog EXTRA "Extracting \"${file}\""
   367     case "${ext}" in
   368         .tar.bz2)     tar xvjf "${full_file}" |CT_DoLog ALL;;
   369         .tar.gz|.tgz) tar xvzf "${full_file}" |CT_DoLog ALL;;
   370         .tar)         tar xvf  "${full_file}" |CT_DoLog ALL;;
   371         *)            CT_Abort "Don't know how to handle \"${file}\": unknown extension" ;;
   372     esac
   373 
   374     # Snapshots might not have the version number in the extracted directory
   375     # name. This is also the case for some (old) packages, such as libfloat.
   376     # Overcome this issue by symlink'ing the directory.
   377     if [ ! -d "${file}" -a "${libc_addon}" != "y" ]; then
   378         case "${ext}" in
   379             .tar.bz2)     base=`tar tjf "${full_file}" |head -n 1 |cut -d / -f 1 || true`;;
   380             .tar.gz|.tgz) base=`tar tzf "${full_file}" |head -n 1 |cut -d / -f 1 || true`;;
   381             .tar)         base=`tar tf  "${full_file}" |head -n 1 |cut -d / -f 1 || true`;;
   382         esac
   383         CT_TestOrAbort "There was a problem when extracting \"${file}\"" -d "${base}" -o "${base}" != "${file}"
   384         ln -s "${base}" "${file}"
   385     fi
   386 
   387     # Kludge: outside this function, we wouldn't know if we had just extracted
   388     # a libc addon, or a plain package. Apply patches now.
   389     CT_DoLog EXTRA "Patching \"${file}\""
   390 
   391     # If libc addon, we're already in the correct place.
   392     [ -z "${libc_addon}" ] && cd "${file}"
   393 
   394     [ "${CUSTOM_PATCH_ONLY}" = "y" ] || official_patch_dir="${CT_TOP_DIR}/patches/${base_file}/${ver_file}"
   395     [ "${CT_CUSTOM_PATCH}" = "y" ] && custom_patch_dir="${CT_CUSTOM_PATCH_DIR}/${base_file}/${ver_file}"
   396     for patch_dir in "${official_patch_dir}" "${custom_patch_dir}"; do
   397         if [ -n "${patch_dir}" -a -d "${patch_dir}" ]; then
   398             for p in "${patch_dir}"/*.patch; do
   399                 if [ -f "${p}" ]; then
   400                     CT_DoLog DEBUG "Applying patch \"${p}\""
   401                     patch -g0 -F1 -p1 -f <"${p}" |CT_DoLog ALL
   402                     CT_TestAndAbort "Failed while applying patch file \"${p}\"" ${PIPESTATUS[0]} -ne 0
   403                 fi
   404             done
   405         fi
   406     done
   407 
   408     CT_Popd
   409 }
   410 
   411 # Compute the target triplet from what is provided by the user
   412 # Usage: CT_DoBuildTargetTriplet
   413 # In fact this function takes the environment variables to build the target
   414 # triplet. It is needed both by the normal build sequence, as well as the
   415 # sample saving sequence.
   416 CT_DoBuildTargetTriplet() {
   417     case "${CT_ARCH_BE},${CT_ARCH_LE}" in
   418         y,) target_endian_eb=eb; target_endian_el=;;
   419         ,y) target_endian_eb=; target_endian_el=el;;
   420     esac
   421     case "${CT_ARCH}" in
   422         arm)  CT_TARGET="${CT_ARCH}${target_endian_eb}";;
   423         mips) CT_TARGET="${CT_ARCH}${target_endian_el}";;
   424         x86*) # Much love for this one :-(
   425               # Ultimately, we should use config.sub to output the correct
   426               # procesor name. Work for later...
   427               arch="${CT_ARCH_ARCH}"
   428               [ -z "${arch}" ] && arch="${CT_ARCH_TUNE}"
   429               case "${CT_ARCH}" in
   430                   x86_64)      CT_TARGET=x86_64;;
   431               	  *)  case "${arch}" in
   432                           "")                                       CT_TARGET=i386;;
   433                           i386|i486|i586|i686)                      CT_TARGET="${arch}";;
   434                           winchip*)                                 CT_TARGET=i486;;
   435                           pentium|pentium-mmx|c3*)                  CT_TARGET=i586;;
   436                           nocona|athlon*64|k8|athlon-fx|opteron)    CT_TARGET=x86_64;;
   437                           pentiumpro|pentium*|athlon*)              CT_TARGET=i686;;
   438                           *)                                        CT_TARGET=i586;;
   439                       esac;;
   440               esac;;
   441     esac
   442     case "${CT_TARGET_VENDOR}" in
   443         "") CT_TARGET="${CT_TARGET}-unknown";;
   444         *)  CT_TARGET="${CT_TARGET}-${CT_TARGET_VENDOR}";;
   445     esac
   446     case "${CT_KERNEL}" in
   447         linux*)  CT_TARGET="${CT_TARGET}-linux";;
   448         cygwin*) CT_TARGET="${CT_TARGET}-cygwin";;
   449     esac
   450     case "${CT_LIBC}" in
   451         glibc)  CT_TARGET="${CT_TARGET}-gnu";;
   452         uClibc) CT_TARGET="${CT_TARGET}-uclibc";;
   453     esac
   454     case "${CT_ARCH_ABI}" in
   455         eabi)   CT_TARGET="${CT_TARGET}eabi";;
   456     esac
   457     CT_TARGET="`${CT_TOP_DIR}/tools/config.sub ${CT_TARGET}`"
   458 }