yann@1: # This file contains some usefull common functions yann@1: # Copyright 2007 Yann E. MORIN yann@1: # Licensed under the GPL v2. See COPYING in the root of this package yann@1: yann@136: # Prepare the fault handler yann@1: CT_OnError() { yann@1: ret=$? yann@726: # Bail out early in subshell, the upper level shell will act accordingly. yann@726: [ ${BASH_SUBSHELL} -eq 0 ] || exit $ret yann@523: CT_DoLog ERROR "Build failed in step '${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}'" yann@1: for((step=(CT_STEP_COUNT-1); step>1; step--)); do yann@523: CT_DoLog ERROR " called in step '${CT_STEP_MESSAGE[${step}]}'" yann@1: done yann@523: CT_DoLog ERROR "Error happened in '${BASH_SOURCE[1]}' in function '${FUNCNAME[1]}' (line unknown, sorry)" yann@1: for((depth=2; ${BASH_LINENO[$((${depth}-1))]}>0; depth++)); do yann@523: CT_DoLog ERROR " called from '${BASH_SOURCE[${depth}]}' at line # ${BASH_LINENO[${depth}-1]} in function '${FUNCNAME[${depth}]}'" yann@1: done yann@523: [ "${CT_LOG_TO_FILE}" = "y" ] && CT_DoLog ERROR "Look at '${CT_LOG_FILE}' for more info on this error." yann@96: CT_STEP_COUNT=1 yann@96: CT_DoEnd ERROR yann@1: exit $ret yann@1: } yann@136: yann@136: # Install the fault handler yann@1: trap CT_OnError ERR yann@1: yann@136: # Inherit the fault handler in subshells and functions yann@1: set -E yann@136: yann@136: # Make pipes fail on the _first_ failed command yann@136: # Not supported on bash < 3.x, but we need it, so drop the obsoleting bash-2.x yann@1: set -o pipefail yann@1: yann@136: # Don't hash commands' locations, and search every time it is requested. yann@136: # This is slow, but needed because of the static/shared core gcc which shall yann@136: # always match to shared if it exists, and only fallback to static if the yann@136: # shared is not found yann@136: set +o hashall yann@136: yann@165: # Log policy: yann@165: # - first of all, save stdout so we can see the live logs: fd #6 yann@165: exec 6>&1 yann@165: # - then point stdout to the log file (temporary for now) yann@165: tmp_log_file="${CT_TOP_DIR}/log.$$" yann@165: exec >>"${tmp_log_file}" yann@165: yann@1: # The different log levels: yann@1: CT_LOG_LEVEL_ERROR=0 yann@1: CT_LOG_LEVEL_WARN=1 yann@1: CT_LOG_LEVEL_INFO=2 yann@1: CT_LOG_LEVEL_EXTRA=3 yann@1: CT_LOG_LEVEL_DEBUG=4 yann@78: CT_LOG_LEVEL_ALL=5 yann@1: yann@1: # A function to log what is happening yann@1: # Different log level are available: yann@1: # - ERROR: A serious, fatal error occurred yann@1: # - WARN: A non fatal, non serious error occurred, take your responsbility with the generated build yann@1: # - INFO: Informational messages yann@1: # - EXTRA: Extra informational messages yann@1: # - DEBUG: Debug messages yann@78: # - ALL: Component's build messages yann@1: # Usage: CT_DoLog [message] yann@1: # If message is empty, then stdin will be logged. yann@1: CT_DoLog() { yann@47: local max_level LEVEL level cur_l cur_L yann@47: local l yann@1: eval max_level="\${CT_LOG_LEVEL_${CT_LOG_LEVEL_MAX}}" yann@1: # Set the maximum log level to DEBUG if we have none yann@78: [ -z "${max_level}" ] && max_level=${CT_LOG_LEVEL_DEBUG} yann@1: yann@47: LEVEL="$1"; shift yann@1: eval level="\${CT_LOG_LEVEL_${LEVEL}}" yann@1: yann@1: if [ $# -eq 0 ]; then yann@1: cat - yann@1: else yann@646: echo "${@}" yann@141: fi |( IFS="\n" # We want the full lines, even leading spaces yann@523: _prog_bar_cpt=0 yann@523: _prog_bar[0]='/' yann@523: _prog_bar[1]='-' yann@523: _prog_bar[2]='\' yann@523: _prog_bar[3]='|' yann@1: indent=$((2*CT_STEP_COUNT)) yann@1: while read line; do yann@47: case "${CT_LOG_SEE_TOOLS_WARN},${line}" in yann@47: y,*"warning:"*) cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};; yann@112: y,*"WARNING:"*) cur_L=WARN; cur_l=${CT_LOG_LEVEL_WARN};; yann@47: *"error:"*) cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};; yann@78: *"make["?*"]:"*"Stop.") cur_L=ERROR; cur_l=${CT_LOG_LEVEL_ERROR};; yann@47: *) cur_L="${LEVEL}"; cur_l="${level}";; yann@47: esac yann@523: # There will always be a log file (stdout, fd #1), be it /dev/null yann@523: printf "[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}" yann@47: if [ ${cur_l} -le ${max_level} ]; then yann@523: # Only print to console (fd #6) if log level is high enough. yann@523: printf "\r[%-5s]%*s%s%s\n" "${cur_L}" "${indent}" " " "${line}" >&6 yann@82: fi yann@82: if [ "${CT_LOG_PROGRESS_BAR}" = "y" ]; then yann@523: printf "\r[%02d:%02d] %s " $((SECONDS/60)) $((SECONDS%60)) "${_prog_bar[$((_prog_bar_cpt/10))]}" >&6 yann@523: _prog_bar_cpt=$(((_prog_bar_cpt+1)%40)) yann@1: fi yann@1: done yann@1: ) yann@1: yann@1: return 0 yann@1: } yann@1: yann@535: # Execute an action, and log its messages yann@535: # Usage: CT_DoExecLog <[VAR=val...] command [parameters...]> yann@535: CT_DoExecLog() { yann@535: local level="$1" yann@535: shift yann@657: CT_DoLog DEBUG "==> Executing: '${@}'" yann@668: "${@}" 2>&1 |CT_DoLog "${level}" yann@535: } yann@535: yann@96: # Tail message to be logged whatever happens yann@96: # Usage: CT_DoEnd yann@96: CT_DoEnd() yann@96: { yann@145: local level="$1" yann@523: CT_STOP_DATE=$(CT_DoDate +%s%N) yann@523: CT_STOP_DATE_HUMAN=$(CT_DoDate +%Y%m%d.%H%M%S) yann@599: if [ "${level}" != "ERROR" ]; then yann@582: CT_DoLog "${level:-INFO}" "Build completed at ${CT_STOP_DATE_HUMAN}" yann@582: fi yann@96: elapsed=$((CT_STOP_DATE-CT_STAR_DATE)) yann@96: elapsed_min=$((elapsed/(60*1000*1000*1000))) yann@523: elapsed_sec=$(printf "%02d" $(((elapsed%(60*1000*1000*1000))/(1000*1000*1000)))) yann@523: elapsed_csec=$(printf "%02d" $(((elapsed%(1000*1000*1000))/(10*1000*1000)))) yann@145: CT_DoLog ${level:-INFO} "(elapsed: ${elapsed_min}:${elapsed_sec}.${elapsed_csec})" yann@96: } yann@96: yann@96: # Abort the execution with an error message yann@1: # Usage: CT_Abort yann@1: CT_Abort() { yann@128: CT_DoLog ERROR "$1" yann@1: exit 1 yann@1: } yann@1: yann@1: # Test a condition, and print a message if satisfied yann@1: # Usage: CT_Test yann@1: CT_Test() { yann@1: local ret yann@1: local m="$1" yann@1: shift yann@1: test "$@" && CT_DoLog WARN "$m" yann@1: return 0 yann@1: } yann@1: yann@1: # Test a condition, and abort with an error message if satisfied yann@1: # Usage: CT_TestAndAbort yann@1: CT_TestAndAbort() { yann@1: local m="$1" yann@1: shift yann@1: test "$@" && CT_Abort "$m" yann@1: return 0 yann@1: } yann@1: yann@1: # Test a condition, and abort with an error message if not satisfied yann@1: # Usage: CT_TestAndAbort yann@1: CT_TestOrAbort() { yann@1: local m="$1" yann@1: shift yann@1: test "$@" || CT_Abort "$m" yann@1: return 0 yann@1: } yann@1: yann@1: # Test the presence of a tool, or abort if not found yann@1: # Usage: CT_HasOrAbort yann@1: CT_HasOrAbort() { yann@523: CT_TestAndAbort "'${1}' not found and needed for successful toolchain build." -z ""$(CT_Which "${1}") yann@1: return 0 yann@1: } yann@1: yann@210: # Search a program: wrap "which" for those system where yann@210: # "which" verbosely says there is no match (Mdk are such yann@210: # suckers...) yann@210: # Usage: CT_Which yann@210: CT_Which() { yann@210: which "$1" 2>/dev/null || true yann@210: } yann@210: yann@1: # Get current date with nanosecond precision yann@1: # On those system not supporting nanosecond precision, faked with rounding down yann@1: # to the highest entire second yann@1: # Usage: CT_DoDate yann@1: CT_DoDate() { yann@1: date "$1" |sed -r -e 's/%N$/000000000/;' yann@1: } yann@1: yann@1: CT_STEP_COUNT=1 yann@1: CT_STEP_MESSAGE[${CT_STEP_COUNT}]="" yann@1: # Memorise a step being done so that any error is caught yann@1: # Usage: CT_DoStep yann@1: CT_DoStep() { yann@523: local start=$(CT_DoDate +%s%N) yann@1: CT_DoLog "$1" "=================================================================" yann@1: CT_DoLog "$1" "$2" yann@1: CT_STEP_COUNT=$((CT_STEP_COUNT+1)) yann@1: CT_STEP_LEVEL[${CT_STEP_COUNT}]="$1"; shift yann@1: CT_STEP_START[${CT_STEP_COUNT}]="${start}" yann@1: CT_STEP_MESSAGE[${CT_STEP_COUNT}]="$1" yann@1: return 0 yann@1: } yann@1: yann@1: # End the step just being done yann@1: # Usage: CT_EndStep yann@1: CT_EndStep() { yann@523: local stop=$(CT_DoDate +%s%N) yann@523: local duration=$(printf "%032d" $((stop-${CT_STEP_START[${CT_STEP_COUNT}]})) |sed -r -e 's/([[:digit:]]{2})[[:digit:]]{7}$/\.\1/; s/^0+//; s/^\./0\./;') yann@582: local elapsed=$(printf "%02d:%02d" $((SECONDS/60)) $((SECONDS%60))) yann@1: local level="${CT_STEP_LEVEL[${CT_STEP_COUNT}]}" yann@1: local message="${CT_STEP_MESSAGE[${CT_STEP_COUNT}]}" yann@1: CT_STEP_COUNT=$((CT_STEP_COUNT-1)) yann@582: CT_DoLog "${level}" "${message}: done in ${duration}s (at ${elapsed})" yann@1: return 0 yann@1: } yann@1: yann@1: # Pushes into a directory, and pops back yann@1: CT_Pushd() { yann@1: pushd "$1" >/dev/null 2>&1 yann@1: } yann@1: CT_Popd() { yann@1: popd >/dev/null 2>&1 yann@1: } yann@1: yann@1: # Makes a path absolute yann@1: # Usage: CT_MakeAbsolutePath path yann@1: CT_MakeAbsolutePath() { yann@1: # Try to cd in that directory yann@1: if [ -d "$1" ]; then yann@1: CT_Pushd "$1" yann@1: pwd yann@1: CT_Popd yann@1: else yann@1: # No such directory, fail back to guessing yann@1: case "$1" in yann@1: /*) echo "$1";; yann@523: *) echo "$(pwd)/$1";; yann@1: esac yann@1: fi yann@1: yann@1: return 0 yann@1: } yann@1: yann@1: # Creates a temporary directory yann@1: # $1: variable to assign to yann@1: # Usage: CT_MktempDir foo yann@1: CT_MktempDir() { yann@1: # Some mktemp do not allow more than 6 Xs yann@523: eval "$1"=$(mktemp -q -d "${CT_BUILD_DIR}/.XXXXXX") yann@1: CT_TestOrAbort "Could not make temporary directory" -n "${!1}" -a -d "${!1}" yann@1: } yann@1: yann@1: # Echoes the specified string on stdout until the pipe breaks. yann@1: # Doesn't fail yann@1: # $1: string to echo yann@1: # Usage: CT_DoYes "" |make oldconfig yann@1: CT_DoYes() { yann@1: yes "$1" || true yann@1: } yann@63: yann@85: # Get the file name extension of a component yann@719: # Usage: CT_GetFileExtension [extension] yann@85: # If found, echoes the extension to stdout yann@85: # If not found, echoes nothing on stdout. yann@85: CT_GetFileExtension() { yann@85: local ext yann@85: local file="$1" yann@719: shift yann@719: local first_ext="$1" yann@85: yann@85: CT_Pushd "${CT_TARBALLS_DIR}" yann@160: # we need to also check for an empty extension for those very yann@160: # peculiar components that don't have one (such as sstrip from yann@160: # buildroot). yann@719: for ext in ${first_ext} .tar.gz .tar.bz2 .tgz .tar ''; do yann@85: if [ -f "${file}${ext}" ]; then yann@85: echo "${ext}" yann@85: break yann@85: fi yann@85: done yann@85: CT_Popd yann@85: yann@85: return 0 yann@85: } yann@85: yann@754: # Set environment for proxy access yann@754: # Usage: CT_DoSetProxy yann@754: # where proxy_type is one of 'none', 'http', 'sockssys', 'socks4' or 'socks5' yann@754: CT_DoSetProxy() { yann@754: case "${CT_PROXY_TYPE}" in yann@754: http) yann@754: http_proxy="http://" yann@754: case "${CT_PROXY_USER}:${CT_PROXY_PASS}" in yann@754: :) ;; yann@754: :*) http_proxy="${http_proxy}:${CT_PROXY_PASS}@";; yann@754: *:) http_proxy="${http_proxy}${CT_PROXY_USER}@";; yann@754: *:*) http_proxy="${http_proxy}${CT_PROXY_USER}:${CT_PROXY_PASS}@";; yann@754: esac yann@754: export http_proxy="${http_proxy}${CT_PROXY_HOST}:${CT_PROXY_PORT}/" yann@754: export https_proxy="${http_proxy}" yann@754: export ftp_proxy="${http_proxy}" yann@754: CT_DoLog DEBUG "http_proxy='${http_proxy}'" yann@754: ;; yann@754: sockssys) yann@754: # Force not using HTTP proxy yann@754: unset http_proxy ftp_proxy https_proxy yann@754: CT_HasOrAbort tsocks yann@754: . tsocks -on yann@754: ;; yann@754: socks*) yann@754: # Force not using HTTP proxy yann@754: unset http_proxy ftp_proxy https_proxy yann@754: # Remove any lingering config file from any previous run yann@754: rm -f "${CT_BUILD_DIR}/tsocks.conf" yann@754: # Find all interfaces and build locally accessible networks yann@754: server_ip=$(ping -c 1 -W 2 "${CT_PROXY_HOST}" |head -n 1 |sed -r -e 's/^[^\(]+\(([^\)]+)\).*$/\1/;' || true) yann@754: CT_TestOrAbort "SOCKS proxy '${CT_PROXY_HOST}' has no IP." -n "${server_ip}" yann@754: /sbin/ifconfig |awk -v server_ip="${server_ip}" ' yann@754: BEGIN { yann@754: split( server_ip, tmp, "\\." ); yann@754: server_ip_num = tmp[1] * 2^24 + tmp[2] * 2^16 + tmp[3] * 2^8 + tmp[4] * 2^0; yann@754: pairs = 0; yann@754: } yann@754: yann@754: $0 ~ /^[[:space:]]*inet addr:/ { yann@754: split( $2, tmp, ":|\\." ); yann@754: if( ( tmp[2] == 127 ) && ( tmp[3] == 0 ) && ( tmp[4] == 0 ) && ( tmp[5] == 1 ) ) { yann@754: /* Skip 127.0.0.1, it'\''s taken care of by tsocks itself */ yann@754: next; yann@754: } yann@754: ip_num = tmp[2] * 2^24 + tmp[3] * 2^16 + tmp[4] * 2 ^8 + tmp[5] * 2^0; yann@754: i = 32; yann@754: do { yann@754: i--; yann@754: mask = 2^32 - 2^i; yann@754: } while( (i!=0) && ( and( server_ip_num, mask ) == and( ip_num, mask ) ) ); yann@754: mask = and( 0xFFFFFFFF, lshift( mask, 1 ) ); yann@754: if( (i!=0) && (mask!=0) ) { yann@754: masked_ip = and( ip_num, mask ); yann@754: for( i=0; i"${CT_BUILD_DIR}/tsocks.conf" yann@754: ( echo "server = ${server_ip}"; yann@754: echo "server_port = ${CT_PROXY_PORT}"; yann@754: [ -n "${CT_PROXY_USER}" ] && echo "default_user=${CT_PROXY_USER}"; yann@754: [ -n "${CT_PROXY_PASS}" ] && echo "default_pass=${CT_PROXY_PASS}"; yann@754: ) >>"${CT_BUILD_DIR}/tsocks.conf" yann@754: case "${CT_PROXY_TYPE/socks}" in yann@754: 4|5) proxy_type="${CT_PROXY_TYPE/socks}";; yann@754: auto) yann@754: reply=$(inspectsocks "${server_ip}" "${CT_PROXY_PORT}" 2>&1 || true) yann@754: case "${reply}" in yann@754: *"server is a version 4 socks server") proxy_type=4;; yann@754: *"server is a version 5 socks server") proxy_type=5;; yann@754: *) CT_Abort "Unable to determine SOCKS proxy type for '${CT_PROXY_HOST}:${CT_PROXY_PORT}'" yann@754: esac yann@754: ;; yann@754: esac yann@754: echo "server_type = ${proxy_type}" >> "${CT_BUILD_DIR}/tsocks.conf" yann@754: CT_HasOrAbort tsocks yann@754: # If tsocks was found, then validateconf is present (distributed with tsocks). yann@754: CT_DoExecLog DEBUG validateconf -f "${CT_BUILD_DIR}/tsocks.conf" yann@754: export TSOCKS_CONF_FILE="${CT_BUILD_DIR}/tsocks.conf" yann@754: . tsocks -on yann@754: ;; yann@754: esac yann@754: } yann@754: yann@63: # Download an URL using wget yann@63: # Usage: CT_DoGetFileWget yann@63: CT_DoGetFileWget() { yann@63: # Need to return true because it is legitimate to not find the tarball at yann@63: # some of the provided URLs (think about snapshots, different layouts for yann@63: # different gcc versions, etc...) yann@63: # Some (very old!) FTP server might not support the passive mode, thus yann@63: # retry without yann@63: # With automated download as we are doing, it can be very dangerous to use yann@63: # -c to continue the downloads. It's far better to simply overwrite the yann@63: # destination file yann@492: # Some company networks have firewalls to connect to the internet, but it's yann@492: # not easy to detect them, and wget does not timeout by default while yann@492: # connecting, so force a global ${CT_CONNECT_TIMEOUT}-second timeout. yann@492: wget -T ${CT_CONNECT_TIMEOUT} -nc --progress=dot:binary --tries=3 --passive-ftp "$1" \ yann@492: || wget -T ${CT_CONNECT_TIMEOUT} -nc --progress=dot:binary --tries=3 "$1" \ yann@439: || true yann@63: } yann@63: yann@63: # Download an URL using curl yann@63: # Usage: CT_DoGetFileCurl yann@63: CT_DoGetFileCurl() { yann@492: # Note: comments about wget method (above) are also valid here yann@439: # Plus: no good progress indicator is available with curl, yann@439: # so output is consigned to oblivion yann@492: curl --ftp-pasv -O --retry 3 "$1" --connect-timeout ${CT_CONNECT_TIMEOUT} >/dev/null \ yann@492: || curl -O --retry 3 "$1" --connect-timeout ${CT_CONNECT_TIMEOUT} >/dev/null \ yann@439: || true yann@63: } yann@63: yann@523: _wget=$(CT_Which wget) yann@523: _curl=$(CT_Which curl) yann@63: # Wrapper function to call one of curl or wget yann@63: # Usage: CT_DoGetFile yann@63: CT_DoGetFile() { yann@63: case "${_wget},${_curl}" in yann@63: ,) CT_DoError "Could find neither wget nor curl";; yann@146: ,*) CT_DoGetFileCurl "$1" 2>&1 |CT_DoLog ALL;; yann@146: *) CT_DoGetFileWget "$1" 2>&1 |CT_DoLog ALL;; yann@63: esac yann@63: } yann@63: yann@63: # Download the file from one of the URLs passed as argument yann@242: # Usage: CT_GetFile [extension] [url ...] yann@63: CT_GetFile() { yann@63: local ext yann@63: local url yann@63: local file="$1" yann@242: local first_ext="" yann@63: shift yann@712: # If next argument starts with a dot, then this is not an URL, yann@712: # and we can consider that it is a preferred extension. yann@242: case "$1" in yann@712: .*) first_ext="$1" yann@242: shift yann@242: ;; yann@242: esac yann@63: yann@63: # Do we already have it? yann@719: ext=$(CT_GetFileExtension "${file}" ${first_ext}) yann@63: if [ -n "${ext}" ]; then yann@523: CT_DoLog DEBUG "Already have '${file}'" yann@85: return 0 yann@63: fi yann@63: yann@754: # Try to retrieve the file yann@63: CT_Pushd "${CT_TARBALLS_DIR}" yann@754: yann@754: if [ -n "${CT_LOCAL_TARBALLS_DIR}" ]; then yann@754: CT_DoLog DEBUG "Trying to retrieve an already downloaded copy of '${file}'" yann@754: # We'd rather have a bzip2'ed tarball, then gzipped tarball, plain tarball, yann@754: # or, as a failover, a file without extension. yann@754: # Try local copy first, if it exists yann@754: for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do yann@754: CT_DoLog DEBUG "Trying '${CT_LOCAL_TARBALLS_DIR}/${file}${ext}'" yann@754: if [ -r "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" -a \ yann@754: "${CT_FORCE_DOWNLOAD}" != "y" ]; then yann@754: CT_DoLog EXTRA "Got '${file}' from local storage" yann@754: ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL yann@754: return 0 yann@754: fi yann@754: done yann@754: fi yann@695: yann@695: # Not found locally, try from the network yann@695: yann@695: # Start with LAN mirror yann@695: if [ "${CT_USE_LAN_MIRROR}" = "y" ]; then yann@754: CT_DoSetProxy ${CT_LAN_MIRROR_USE_PROXY:+${CT_PROXY_TYPE}} yann@754: CT_DoLog DEBUG "Trying to retrieve a copy of '${file}' from LAN mirror '${CT_LAN_MIRROR_HOSTNAME}'" yann@754: CT_TestOrAbort "Please set the LAN mirror hostname" -n "${CT_LAN_MIRROR_HOSTNAME}" yann@754: CT_TestOrAbort "Please tell me where to find tarballs on the LAN mirror '${CT_LAN_MIRROR_HOSTNAME}'" -n "${CT_LAN_MIRROR_BASE}" yann@754: for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do yann@695: # Please note: we just have the file's basename in a single piece. yann@695: # So we have to just try and split it back into name and version... :-( yann@754: for url in "${CT_LAN_MIRROR_SCHEME}://${CT_LAN_MIRROR_HOSTNAME}/${CT_LAN_MIRROR_BASE}/${file%-*}" \ yann@754: "${CT_LAN_MIRROR_SCHEME}://${CT_LAN_MIRROR_HOSTNAME}/${CT_LAN_MIRROR_BASE}"; \ yann@754: do yann@695: CT_DoLog DEBUG "Trying '${url}/${file}${ext}'" yann@695: CT_DoGetFile "${url}/${file}${ext}" yann@695: if [ -f "${file}${ext}" ]; then yann@695: if [ "${CT_SAVE_TARBALLS}" = "y" ]; then yann@695: # No need to test if the file already exists because yann@695: # it does NOT. If it did exist, we'd have been stopped yann@695: # above, when looking for local copies. yann@695: CT_DoLog EXTRA "Saving '${file}' to local storage" yann@695: mv "${file}${ext}" "${CT_LOCAL_TARBALLS_DIR}" |CT_DoLog ALL yann@695: ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL yann@695: fi yann@754: CT_DoLog EXTRA "Got '${file}' from the LAN mirror" yann@695: return 0 yann@695: fi yann@695: done yann@695: done yann@695: fi yann@695: yann@695: # OK, available neither localy, nor from the LAN mirror (if any). yann@754: CT_DoSetProxy ${CT_PROXY_TYPE} yann@242: for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do yann@102: # Try all urls in turn yann@102: for url in "$@"; do yann@523: CT_DoLog DEBUG "Trying '${url}/${file}${ext}'" yann@265: CT_DoGetFile "${url}/${file}${ext}" yann@265: if [ -f "${file}${ext}" ]; then yann@265: if [ "${CT_SAVE_TARBALLS}" = "y" ]; then yann@695: # No need to test if the file already exists because yann@695: # it does NOT. If it did exist, we'd have been stopped yann@695: # above, when looking for local copies. yann@523: CT_DoLog EXTRA "Saving '${file}' to local storage" yann@371: mv "${file}${ext}" "${CT_LOCAL_TARBALLS_DIR}" |CT_DoLog ALL yann@371: ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL yann@265: fi yann@754: CT_DoLog EXTRA "Got '${file}' from the Internet" yann@265: return 0 yann@265: fi yann@102: done yann@102: done yann@63: CT_Popd yann@63: yann@754: CT_Abort "Could not retrieve '${file}'." yann@63: } yann@63: yann@63: # Extract a tarball and patch the resulting sources if necessary. yann@63: # Some tarballs need to be extracted in specific places. Eg.: glibc addons yann@63: # must be extracted in the glibc directory; uCLibc locales must be extracted yann@63: # in the extra/locale sub-directory of uClibc. yann@63: CT_ExtractAndPatch() { yann@63: local file="$1" yann@523: local base_file=$(echo "${file}" |cut -d - -f 1) yann@523: local ver_file=$(echo "${file}" |cut -d - -f 2-) yann@63: local official_patch_dir yann@63: local custom_patch_dir yann@63: local libc_addon yann@523: local ext=$(CT_GetFileExtension "${file}") yann@523: CT_TestAndAbort "'${file}' not found in '${CT_TARBALLS_DIR}'" -z "${ext}" yann@63: local full_file="${CT_TARBALLS_DIR}/${file}${ext}" yann@63: yann@63: CT_Pushd "${CT_SRC_DIR}" yann@63: yann@63: # Add-ons need a little love, really. yann@63: case "${file}" in yann@63: glibc-[a-z]*-*) yann@523: CT_TestAndAbort "Trying to extract the C-library addon/locales '${file}' when C-library not yet extracted" ! -d "${CT_LIBC_FILE}" yann@63: cd "${CT_LIBC_FILE}" yann@63: libc_addon=y yann@63: [ -f ".${file}.extracted" ] && return 0 yann@63: touch ".${file}.extracted" yann@63: ;; yann@63: uClibc-locale-*) yann@523: CT_TestAndAbort "Trying to extract the C-library addon/locales '${file}' when C-library not yet extracted" ! -d "${CT_LIBC_FILE}" yann@63: cd "${CT_LIBC_FILE}/extra/locale" yann@63: libc_addon=y yann@63: [ -f ".${file}.extracted" ] && return 0 yann@63: touch ".${file}.extracted" yann@63: ;; yann@63: esac yann@63: yann@63: # If the directory exists, then consider extraction and patching done yann@77: if [ -d "${file}" ]; then yann@523: CT_DoLog DEBUG "Already extracted '${file}'" yann@77: return 0 yann@77: fi yann@63: yann@523: CT_DoLog EXTRA "Extracting '${file}'" yann@63: case "${ext}" in yann@78: .tar.bz2) tar xvjf "${full_file}" |CT_DoLog ALL;; yann@78: .tar.gz|.tgz) tar xvzf "${full_file}" |CT_DoLog ALL;; yann@78: .tar) tar xvf "${full_file}" |CT_DoLog ALL;; yann@523: *) CT_Abort "Don't know how to handle '${file}': unknown extension" ;; yann@63: esac yann@63: yann@63: # Snapshots might not have the version number in the extracted directory yann@305: # name. This is also the case for some (odd) packages, such as D.U.M.A. yann@63: # Overcome this issue by symlink'ing the directory. yann@63: if [ ! -d "${file}" -a "${libc_addon}" != "y" ]; then yann@63: case "${ext}" in yann@523: .tar.bz2) base=$(tar tjf "${full_file}" |head -n 1 |cut -d / -f 1 || true);; yann@523: .tar.gz|.tgz) base=$(tar tzf "${full_file}" |head -n 1 |cut -d / -f 1 || true);; yann@523: .tar) base=$(tar tf "${full_file}" |head -n 1 |cut -d / -f 1 || true);; yann@63: esac yann@523: CT_TestOrAbort "There was a problem when extracting '${file}'" -d "${base}" -o "${base}" != "${file}" yann@63: ln -s "${base}" "${file}" yann@63: fi yann@63: yann@63: # Kludge: outside this function, we wouldn't know if we had just extracted yann@63: # a libc addon, or a plain package. Apply patches now. yann@523: CT_DoLog EXTRA "Patching '${file}'" yann@63: yann@136: if [ "${libc_addon}" = "y" ]; then yann@508: # Some addon tarballs directly contain the correct addon directory, yann@508: # while others have the addon directory named after the tarball. yann@365: # Fix that by always using the short name (eg: linuxthreads, ports, etc...) yann@523: addon_short_name=$(echo "${file}" |sed -r -e 's/^[^-]+-//; s/-[^-]+$//;') yann@136: [ -d "${addon_short_name}" ] || ln -s "${file}" "${addon_short_name}" yann@136: # If libc addon, we're already in the correct place yann@136: else yann@136: cd "${file}" yann@136: fi yann@63: yann@182: official_patch_dir= yann@182: custom_patch_dir= yann@182: [ "${CUSTOM_PATCH_ONLY}" = "y" ] || official_patch_dir="${CT_LIB_DIR}/patches/${base_file}/${ver_file}" yann@63: [ "${CT_CUSTOM_PATCH}" = "y" ] && custom_patch_dir="${CT_CUSTOM_PATCH_DIR}/${base_file}/${ver_file}" yann@63: for patch_dir in "${official_patch_dir}" "${custom_patch_dir}"; do yann@63: if [ -n "${patch_dir}" -a -d "${patch_dir}" ]; then yann@63: for p in "${patch_dir}"/*.patch; do yann@63: if [ -f "${p}" ]; then yann@523: CT_DoLog DEBUG "Applying patch '${p}'" yann@78: patch -g0 -F1 -p1 -f <"${p}" |CT_DoLog ALL yann@523: CT_TestAndAbort "Failed while applying patch file '${p}'" ${PIPESTATUS[0]} -ne 0 yann@63: fi yann@63: done yann@63: fi yann@63: done yann@63: yann@508: if [ "${CT_OVERIDE_CONFIG_GUESS_SUB}" = "y" ]; then yann@508: CT_DoLog ALL "Overiding config.guess and config.sub" yann@508: for cfg in config_guess config_sub; do yann@508: eval ${cfg}="${CT_LIB_DIR}/tools/${cfg/_/.}" yann@508: [ -e "${CT_TOP_DIR}/tools/${cfg/_/.}" ] && eval ${cfg}="${CT_TOP_DIR}/tools/${cfg/_/.}" yann@508: find . -type f -name "${cfg/_/.}" -exec cp -v "${!cfg}" {} \; |CT_DoLog ALL yann@508: done yann@508: fi yann@508: yann@63: CT_Popd yann@63: } yann@63: yann@182: # Two wrappers to call config.(guess|sub) either from CT_TOP_DIR or CT_LIB_DIR. yann@182: # Those from CT_TOP_DIR, if they exist, will be be more recent than those from CT_LIB_DIR. yann@182: CT_DoConfigGuess() { yann@182: if [ -x "${CT_TOP_DIR}/tools/config.guess" ]; then yann@182: "${CT_TOP_DIR}/tools/config.guess" yann@182: else yann@182: "${CT_LIB_DIR}/tools/config.guess" yann@182: fi yann@182: } yann@182: yann@182: CT_DoConfigSub() { yann@182: if [ -x "${CT_TOP_DIR}/tools/config.sub" ]; then yann@182: "${CT_TOP_DIR}/tools/config.sub" "$@" yann@182: else yann@182: "${CT_LIB_DIR}/tools/config.sub" "$@" yann@182: fi yann@182: } yann@182: yann@335: # Compute the target tuple from what is provided by the user yann@335: # Usage: CT_DoBuildTargetTuple yann@63: # In fact this function takes the environment variables to build the target yann@335: # tuple. It is needed both by the normal build sequence, as well as the yann@63: # sample saving sequence. yann@335: CT_DoBuildTargetTuple() { yann@383: # Set the endianness suffix, and the default endianness gcc option yann@63: case "${CT_ARCH_BE},${CT_ARCH_LE}" in yann@383: y,) target_endian_eb=eb yann@383: target_endian_el= yann@391: CT_ARCH_ENDIAN_CFLAG="-mbig-endian" yann@527: CT_ARCH_ENDIAN_LDFLAG="-EB" yann@383: ;; yann@383: ,y) target_endian_eb= yann@383: target_endian_el=el yann@391: CT_ARCH_ENDIAN_CFLAG="-mlittle-endian" yann@527: CT_ARCH_ENDIAN_LDFLAG="-EL" yann@383: ;; yann@63: esac yann@383: yann@383: # Set defaults for the system part of the tuple. Can be overriden yann@383: # by architecture-specific values. yann@383: case "${CT_LIBC}" in yann@383: glibc) CT_TARGET_SYS=gnu;; yann@383: uClibc) CT_TARGET_SYS=uclibc;; yann@63: esac yann@383: yann@387: # Transform the ARCH into a kernel-understandable ARCH yann@387: CT_KERNEL_ARCH="${CT_ARCH}" yann@387: yann@391: # Set the default values for ARCH, ABI, CPU, TUNE, FPU and FLOAT yann@497: unset CT_ARCH_ARCH_CFLAG CT_ARCH_ABI_CFLAG CT_ARCH_CPU_CFLAG CT_ARCH_TUNE_CFLAG CT_ARCH_FPU_CFLAG CT_ARCH_FLOAT_CFLAG yann@497: unset CT_ARCH_WITH_ARCH CT_ARCH_WITH_ABI CT_ARCH_WITH_CPU CT_ARCH_WITH_TUNE CT_ARCH_WITH_FPU CT_ARCH_WITH_FLOAT yann@391: [ "${CT_ARCH_ARCH}" ] && { CT_ARCH_ARCH_CFLAG="-march=${CT_ARCH_ARCH}"; CT_ARCH_WITH_ARCH="--with-arch=${CT_ARCH_ARCH}"; } yann@391: [ "${CT_ARCH_ABI}" ] && { CT_ARCH_ABI_CFLAG="-mabi=${CT_ARCH_ABI}"; CT_ARCH_WITH_ABI="--with-abi=${CT_ARCH_ABI}"; } yann@391: [ "${CT_ARCH_CPU}" ] && { CT_ARCH_CPU_CFLAG="-mcpu=${CT_ARCH_CPU}"; CT_ARCH_WITH_CPU="--with-cpu=${CT_ARCH_CPU}"; } yann@405: [ "${CT_ARCH_TUNE}" ] && { CT_ARCH_TUNE_CFLAG="-mtune=${CT_ARCH_TUNE}"; CT_ARCH_WITH_TUNE="--with-tune=${CT_ARCH_TUNE}"; } yann@391: [ "${CT_ARCH_FPU}" ] && { CT_ARCH_FPU_CFLAG="-mfpu=${CT_ARCH_FPU}"; CT_ARCH_WITH_FPU="--with-fpu=${CT_ARCH_FPU}"; } yann@412: [ "${CT_ARCH_FLOAT_SW}" ] && { CT_ARCH_FLOAT_CFLAG="-msoft-float"; CT_ARCH_WITH_FLOAT="--with-float=soft"; } yann@391: yann@383: # Call the architecture specific settings yann@383: CT_DoArchValues yann@383: yann@391: # Finish the target tuple construction yann@383: case "${CT_KERNEL}" in yann@383: linux*) CT_TARGET_KERNEL=linux;; yann@63: esac yann@523: CT_TARGET=$(CT_DoConfigSub "${CT_TARGET_ARCH}-${CT_TARGET_VENDOR:-unknown}-${CT_TARGET_KERNEL}-${CT_TARGET_SYS}") yann@391: yann@391: # Prepare the target CFLAGS yann@527: CT_ARCH_TARGET_CFLAGS="${CT_ARCH_ENDIAN_CFLAG}" yann@527: CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ARCH_CFLAG}" yann@397: CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_ABI_CFLAG}" yann@397: CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_CPU_CFLAG}" yann@397: CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_TUNE_CFLAG}" yann@397: CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FPU_CFLAG}" yann@397: CT_ARCH_TARGET_CFLAGS="${CT_ARCH_TARGET_CFLAGS} ${CT_ARCH_FLOAT_CFLAG}" yann@527: yann@527: # Now on for the target LDFLAGS yann@527: CT_ARCH_TARGET_LDFLAGS="${CT_ARCH_ENDIAN_LDFLAG}" yann@63: } yann@121: yann@121: # This function does pause the build until the user strikes "Return" yann@121: # Usage: CT_DoPause [optional_message] yann@121: CT_DoPause() { yann@121: local foo yann@121: local message="${1:-Pausing for your pleasure}" yann@121: CT_DoLog INFO "${message}" yann@523: read -p "Press 'Enter' to continue, or Ctrl-C to stop..." foo >&6 yann@121: return 0 yann@121: } yann@121: yann@121: # This function saves the state of the toolchain to be able to restart yann@121: # at any one point yann@121: # Usage: CT_DoSaveState yann@121: CT_DoSaveState() { yann@121: [ "${CT_DEBUG_CT_SAVE_STEPS}" = "y" ] || return 0 yann@121: local state_name="$1" yann@121: local state_dir="${CT_STATE_DIR}/${state_name}" yann@121: yann@523: CT_DoLog DEBUG "Saving state to restart at step '${state_name}'..." yann@121: rm -rf "${state_dir}" yann@121: mkdir -p "${state_dir}" yann@121: yann@121: case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in yann@327: y) tar_opt=z; tar_ext=.gz;; yann@327: *) tar_opt=; tar_ext=;; yann@121: esac yann@121: yann@121: CT_DoLog DEBUG " Saving environment and aliases" yann@738: # We must omit shell functions, and some specific bash variables yann@738: # that break when restoring the environment, later. We could do yann@738: # all the processing in the awk script, but a sed is easier... yann@121: set |awk ' yann@738: BEGIN { _p = 1; } yann@738: $0~/^[^ ]+ \(\)/ { _p = 0; } yann@738: _p == 1 yann@738: $0 == "}" { _p = 1; } yann@738: ' |sed -r -e '/^BASH_(ARGC|ARGV|LINENO|SOURCE|VERSINFO)=/d; yann@738: /^(UID|EUID)=/d; yann@738: /^(FUNCNAME|GROUPS|PPID|SHELLOPTS)=/d;' >"${state_dir}/env.sh" yann@121: yann@523: CT_DoLog DEBUG " Saving CT_CC_CORE_STATIC_PREFIX_DIR='${CT_CC_CORE_STATIC_PREFIX_DIR}'" yann@136: CT_Pushd "${CT_CC_CORE_STATIC_PREFIX_DIR}" yann@327: tar cv${tar_opt}f "${state_dir}/cc_core_static_prefix_dir.tar${tar_ext}" . |CT_DoLog DEBUG yann@136: CT_Popd yann@136: yann@523: CT_DoLog DEBUG " Saving CT_CC_CORE_SHARED_PREFIX_DIR='${CT_CC_CORE_SHARED_PREFIX_DIR}'" yann@136: CT_Pushd "${CT_CC_CORE_SHARED_PREFIX_DIR}" yann@327: tar cv${tar_opt}f "${state_dir}/cc_core_shared_prefix_dir.tar${tar_ext}" . |CT_DoLog DEBUG yann@121: CT_Popd yann@121: yann@523: CT_DoLog DEBUG " Saving CT_PREFIX_DIR='${CT_PREFIX_DIR}'" yann@121: CT_Pushd "${CT_PREFIX_DIR}" yann@327: tar cv${tar_opt}f "${state_dir}/prefix_dir.tar${tar_ext}" --exclude '*.log' . |CT_DoLog DEBUG yann@121: CT_Popd yann@121: yann@121: if [ "${CT_LOG_TO_FILE}" = "y" ]; then yann@121: CT_DoLog DEBUG " Saving log file" yann@121: exec >/dev/null yann@121: case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in yann@121: y) gzip -3 -c "${CT_LOG_FILE}" >"${state_dir}/log.gz";; yann@121: *) cat "${CT_LOG_FILE}" >"${state_dir}/log";; yann@121: esac yann@121: exec >>"${CT_LOG_FILE}" yann@121: fi yann@121: } yann@121: yann@136: # This function restores a previously saved state yann@121: # Usage: CT_DoLoadState yann@121: CT_DoLoadState(){ yann@121: local state_name="$1" yann@121: local state_dir="${CT_STATE_DIR}/${state_name}" yann@135: local old_RESTART="${CT_RESTART}" yann@135: local old_STOP="${CT_STOP}" yann@121: yann@523: CT_TestOrAbort "The previous build did not reach the point where it could be restarted at '${CT_RESTART}'" -d "${state_dir}" yann@141: yann@121: # We need to do something special with the log file! yann@121: if [ "${CT_LOG_TO_FILE}" = "y" ]; then yann@121: exec >"${state_dir}/tail.log" yann@121: fi yann@523: CT_DoLog INFO "Restoring state at step '${state_name}', as requested." yann@121: yann@121: case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in yann@327: y) tar_opt=z; tar_ext=.gz;; yann@327: *) tar_opt=; tar_ext=;; yann@121: esac yann@121: yann@121: CT_DoLog DEBUG " Removing previous build directories" yann@136: chmod -R u+rwX "${CT_PREFIX_DIR}" "${CT_CC_CORE_SHARED_PREFIX_DIR}" "${CT_CC_CORE_STATIC_PREFIX_DIR}" yann@136: rm -rf "${CT_PREFIX_DIR}" "${CT_CC_CORE_SHARED_PREFIX_DIR}" "${CT_CC_CORE_STATIC_PREFIX_DIR}" yann@136: mkdir -p "${CT_PREFIX_DIR}" "${CT_CC_CORE_SHARED_PREFIX_DIR}" "${CT_CC_CORE_STATIC_PREFIX_DIR}" yann@121: yann@523: CT_DoLog DEBUG " Restoring CT_PREFIX_DIR='${CT_PREFIX_DIR}'" yann@121: CT_Pushd "${CT_PREFIX_DIR}" yann@343: tar xv${tar_opt}f "${state_dir}/prefix_dir.tar${tar_ext}" |CT_DoLog DEBUG yann@121: CT_Popd yann@121: yann@523: CT_DoLog DEBUG " Restoring CT_CC_CORE_SHARED_PREFIX_DIR='${CT_CC_CORE_SHARED_PREFIX_DIR}'" yann@136: CT_Pushd "${CT_CC_CORE_SHARED_PREFIX_DIR}" yann@327: tar xv${tar_opt}f "${state_dir}/cc_core_shared_prefix_dir.tar${tar_ext}" |CT_DoLog DEBUG yann@136: CT_Popd yann@136: yann@523: CT_DoLog DEBUG " Restoring CT_CC_CORE_STATIC_PREFIX_DIR='${CT_CC_CORE_STATIC_PREFIX_DIR}'" yann@136: CT_Pushd "${CT_CC_CORE_STATIC_PREFIX_DIR}" yann@327: tar xv${tar_opt}f "${state_dir}/cc_core_static_prefix_dir.tar${tar_ext}" |CT_DoLog DEBUG yann@121: CT_Popd yann@121: yann@121: # Restore the environment, discarding any error message yann@121: # (for example, read-only bash internals) yann@121: CT_DoLog DEBUG " Restoring environment" yann@121: . "${state_dir}/env.sh" >/dev/null 2>&1 || true yann@121: yann@135: # Restore the new RESTART and STOP steps yann@135: CT_RESTART="${old_RESTART}" yann@135: CT_STOP="${old_STOP}" yann@135: unset old_stop old_restart yann@135: yann@121: if [ "${CT_LOG_TO_FILE}" = "y" ]; then yann@121: CT_DoLog DEBUG " Restoring log file" yann@121: exec >/dev/null yann@121: case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in yann@121: y) zcat "${state_dir}/log.gz" >"${CT_LOG_FILE}";; yann@121: *) cat "${state_dir}/log" >"${CT_LOG_FILE}";; yann@121: esac yann@121: cat "${state_dir}/tail.log" >>"${CT_LOG_FILE}" yann@121: exec >>"${CT_LOG_FILE}" yann@121: rm -f "${state_dir}/tail.log" yann@121: fi yann@121: }