From b75f03146218a09a2882d5ecbb8130dac1fc7ab9 Mon Sep 17 00:00:00 2001 From: Alexey Neyman Date: Sat, 19 Aug 2017 15:33:23 -0700 Subject: Check for config file changes when restarting Also, save/restore per-target source directory. Fixes #800. Signed-off-by: Alexey Neyman diff --git a/scripts/crosstool-NG.sh.in b/scripts/crosstool-NG.sh.in index bd07c6b..3d9a34e 100644 --- a/scripts/crosstool-NG.sh.in +++ b/scripts/crosstool-NG.sh.in @@ -651,7 +651,7 @@ if [ "${CT_ONLY_DOWNLOAD}" != "y" -a "${CT_ONLY_EXTRACT}" != "y" ]; then else CT_DoSaveState ${step} if [ ${do_stop} -eq 1 ]; then - CT_DoLog ERROR "Stopping just after step '${prev_step}', as requested." + CT_DoLog INFO "Stopping just after step '${prev_step}', as requested." exit 0 fi fi diff --git a/scripts/functions b/scripts/functions index 147de34..ce99a17 100644 --- a/scripts/functions +++ b/scripts/functions @@ -1052,7 +1052,10 @@ CT_DoSaveState() { /^(UID|EUID)=/d; /^(FUNCNAME|GROUPS|PPID|SHELLOPTS)=/d;' >"${state_dir}/env.sh" + CT_DoExecLog STATE cp ".config" "${state_dir}/config" + CT_DoTarballIfExists "${CT_BUILDTOOLS_PREFIX_DIR}" "${state_dir}/buildtools_dir" + CT_DoTarballIfExists "${CT_SRC_DIR}" "${state_dir}/src_dir" CT_DoTarballIfExists "${CT_PREFIX_DIR}" "${state_dir}/prefix_dir" --exclude '*.log' CT_DoLog STATE " Saving log file" @@ -1073,10 +1076,14 @@ CT_DoLoadState(){ local old_STOP="${CT_STOP}" CT_TestOrAbort "The previous build did not reach the point where it could be restarted at '${CT_RESTART}'" -d "${state_dir}" + if ! cmp ".config" "${state_dir}/config" >/dev/null 2>&1; then + CT_Abort "The configuration file has changed between two runs" + fi CT_DoLog INFO "Restoring state at step '${state_name}', as requested." CT_DoExtractTarballIfExists "${state_dir}/prefix_dir" "${CT_PREFIX_DIR}" + CT_DoExtractTarballIfExists "${state_dir}/src_dir" "${CT_SRC_DIR}" CT_DoExtractTarballIfExists "${state_dir}/buildtools_dir" "${CT_BUILDTOOLS_PREFIX_DIR}" # Restore the environment, discarding any error message -- cgit v0.10.2-6-g49f6 From 2bdb632495e5e9e34ee3e8856cda134bef7d218e Mon Sep 17 00:00:00 2001 From: Alexey Neyman Date: Sat, 19 Aug 2017 18:40:42 -0700 Subject: Reject set-but-empty environment variables ... which also lead to cryptic failures. Fixed #743. Signed-off-by: Alexey Neyman diff --git a/scripts/crosstool-NG.sh.in b/scripts/crosstool-NG.sh.in index 3d9a34e..1d872df 100644 --- a/scripts/crosstool-NG.sh.in +++ b/scripts/crosstool-NG.sh.in @@ -51,17 +51,17 @@ CT_SanitizePath # Some sanity checks in the environment and needed tools CT_DoLog INFO "Performing some trivial sanity checks" -CT_TestAndAbort "Don't set LD_LIBRARY_PATH. It screws up the build." -n "${LD_LIBRARY_PATH}" -CT_TestAndAbort "Don't set LIBRARY_PATH. It screws up the build." -n "${LIBRARY_PATH}" -CT_TestAndAbort "Don't set LPATH. It screws up the build." -n "${LPATH}" -CT_TestAndAbort "Don't set CPATH. It screws up the build." -n "${CPATH}" -CT_TestAndAbort "Don't set C_INCLUDE_PATH. It screws up the build." -n "${C_INCLUDE_PATH}" -CT_TestAndAbort "Don't set CPLUS_INCLUDE_PATH. It screws up the build." -n "${CPLUS_INCLUDE_PATH}" -CT_TestAndAbort "Don't set OBJC_INCLUDE_PATH. It screws up the build." -n "${OBJC_INCLUDE_PATH}" -CT_TestAndAbort "Don't set CFLAGS. It screws up the build." -n "${CFLAGS}" -CT_TestAndAbort "Don't set CXXFLAGS. It screws up the build." -n "${CXXFLAGS}" -CT_TestAndAbort "Don't set CC. It screws up the build." -n "${CC}" -CT_TestAndAbort "Don't set CXX. It screws up the build." -n "${CXX}" +CT_TestAndAbort "Don't set LD_LIBRARY_PATH. It screws up the build." -n "${LD_LIBRARY_PATH+set}" +CT_TestAndAbort "Don't set LIBRARY_PATH. It screws up the build." -n "${LIBRARY_PATH+set}" +CT_TestAndAbort "Don't set LPATH. It screws up the build." -n "${LPATH+set}" +CT_TestAndAbort "Don't set CPATH. It screws up the build." -n "${CPATH+set}" +CT_TestAndAbort "Don't set C_INCLUDE_PATH. It screws up the build." -n "${C_INCLUDE_PATH+set}" +CT_TestAndAbort "Don't set CPLUS_INCLUDE_PATH. It screws up the build." -n "${CPLUS_INCLUDE_PATH+set}" +CT_TestAndAbort "Don't set OBJC_INCLUDE_PATH. It screws up the build." -n "${OBJC_INCLUDE_PATH+set}" +CT_TestAndAbort "Don't set CFLAGS. It screws up the build." -n "${CFLAGS+set}" +CT_TestAndAbort "Don't set CXXFLAGS. It screws up the build." -n "${CXXFLAGS+set}" +CT_TestAndAbort "Don't set CC. It screws up the build." -n "${CC+set}" +CT_TestAndAbort "Don't set CXX. It screws up the build." -n "${CXX+set}" CT_Test "GREP_OPTIONS screws up the build. Resetting." -n "${GREP_OPTIONS}" export GREP_OPTIONS= # Workaround against openSUSE 12.1 that breaks ./configure for cross-compilation: -- cgit v0.10.2-6-g49f6 From 0346153bdbbcbccfb51469135bf019551df93471 Mon Sep 17 00:00:00 2001 From: Alexey Neyman Date: Sat, 19 Aug 2017 23:11:27 -0700 Subject: Save export status for environment vars ... and limit our modifications to our variables. Fixes #762. Signed-off-by: Alexey Neyman diff --git a/ct-ng.in b/ct-ng.in index cf2edd1..0fc5499 100644 --- a/ct-ng.in +++ b/ct-ng.in @@ -32,10 +32,6 @@ include $(CT_LIB_DIR)/paths.mk # bash. export SHELL=$(bash) -# GREP_OPTIONS=--color=always will break the generated .in files -# We do not need any GREP_OPTIONS anyway, so set it to empty. -export GREP_OPTIONS= - # Make the restart/stop steps available to scripts/crostool-NG.sh export CT_STOP:=$(STOP) export CT_RESTART:=$(RESTART) diff --git a/packages/gmp/package.desc b/packages/gmp/package.desc index f7dd3da..bb97cb5 100644 --- a/packages/gmp/package.desc +++ b/packages/gmp/package.desc @@ -1,5 +1,5 @@ repository='hg https://gmplib.org/repo/gmp/' bootstrap='./.bootstrap' -mirrors='https://gmplib.org/download/gmp https:////gmplib.org/download/gmp/archive $(CT_Mirrors GNU gmp)' +mirrors='https://gmplib.org/download/gmp https://gmplib.org/download/gmp/archive $(CT_Mirrors GNU gmp)' milestones='5.1' archive_formats='.tar.xz .tar.lz .tar.bz2' diff --git a/scripts/build/binutils/binutils.sh b/scripts/build/binutils/binutils.sh index 8fb51b8..e0fe7f4 100644 --- a/scripts/build/binutils/binutils.sh +++ b/scripts/build/binutils/binutils.sh @@ -188,7 +188,7 @@ do_binutils_backend() { --disable-werror \ "${extra_config[@]}" \ ${CT_ARCH_WITH_FLOAT} \ - ${BINUTILS_SYSROOT_ARG} \ + ${CT_BINUTILS_SYSROOT_ARG} \ "${CT_BINUTILS_EXTRA_CONFIG_ARRAY[@]}" if [ "${static_build}" = "y" ]; then diff --git a/scripts/build/cc/gcc.sh b/scripts/build/cc/gcc.sh index 163a6a2..d4c1b43 100644 --- a/scripts/build/cc/gcc.sh +++ b/scripts/build/cc/gcc.sh @@ -560,7 +560,7 @@ do_gcc_core_backend() { --target=${CT_TARGET} \ --prefix="${prefix}" \ --with-local-prefix="${CT_SYSROOT_DIR}" \ - ${CC_CORE_SYSROOT_ARG} \ + ${CT_CC_CORE_SYSROOT_ARG} \ "${extra_config[@]}" \ --enable-languages="${lang_list}" \ "${extra_user_config[@]}" @@ -1105,7 +1105,7 @@ do_gcc_backend() { --host=${host} \ --target=${CT_TARGET} \ --prefix="${prefix}" \ - ${CC_SYSROOT_ARG} \ + ${CT_CC_SYSROOT_ARG} \ "${extra_config[@]}" \ --with-local-prefix="${CT_SYSROOT_DIR}" \ --enable-long-long \ diff --git a/scripts/crosstool-NG.sh.in b/scripts/crosstool-NG.sh.in index 1d872df..0ea0625 100644 --- a/scripts/crosstool-NG.sh.in +++ b/scripts/crosstool-NG.sh.in @@ -40,6 +40,8 @@ if [ "${CT_DEBUG_INTERACTIVE}" = "y" -a ! \( -t 0 -a -t 6 -a -t 2 \) ]; then exit 1 fi +CT_TrapEnvExport + # Override the locale early, in case we ever translate crosstool-NG messages if [ -z "${CT_NO_OVERRIDE_LC_MESSAGES}" ]; then export LC_ALL=C @@ -62,8 +64,8 @@ CT_TestAndAbort "Don't set CFLAGS. It screws up the build." -n "${CFLAGS+set}" CT_TestAndAbort "Don't set CXXFLAGS. It screws up the build." -n "${CXXFLAGS+set}" CT_TestAndAbort "Don't set CC. It screws up the build." -n "${CC+set}" CT_TestAndAbort "Don't set CXX. It screws up the build." -n "${CXX+set}" -CT_Test "GREP_OPTIONS screws up the build. Resetting." -n "${GREP_OPTIONS}" -export GREP_OPTIONS= +CT_Test "GREP_OPTIONS screws up the build. Unsetting." -n "${GREP_OPTIONS+set}" +unset GREP_OPTIONS # Workaround against openSUSE 12.1 that breaks ./configure for cross-compilation: export CONFIG_SITE= @@ -141,9 +143,13 @@ CT_DoStep DEBUG "Dumping user-supplied crosstool-NG configuration" CT_DoExecLog DEBUG ${grep} -E '^(# )?CT_' .config CT_EndStep -CT_DoLog DEBUG "Unsetting and unexporting MAKEFLAGS" +CT_DoLog DEBUG "Unsetting MAKEFLAGS" unset MAKEFLAGS -export MAKEFLAGS + +# Set the shell to be used by ./configure scripts and by Makefiles (those +# that support it!). +export CONFIG_SHELL="${CT_CONFIG_SHELL}" # for ./configure +export SHELL="${CT_CONFIG_SHELL}" # for Makefiles CT_DoLog INFO "Building environment variables" @@ -304,9 +310,9 @@ if [ -z "${CT_RESTART}" ]; then CT_DEBUGROOT_DIR="${CT_PREFIX_DIR}/${CT_TARGET}/${CT_SYSROOT_DIR_PREFIX}/debug-root" CT_HEADERS_DIR="${CT_SYSROOT_DIR}/usr/include" CT_SanitizeVarDir CT_SYSROOT_DIR CT_DEBUGROOT_DIR CT_HEADERS_DIR - BINUTILS_SYSROOT_ARG="--with-sysroot=${CT_SYSROOT_DIR}" - CC_CORE_SYSROOT_ARG="--with-sysroot=${CT_SYSROOT_DIR}" - CC_SYSROOT_ARG="--with-sysroot=${CT_SYSROOT_DIR}" + CT_BINUTILS_SYSROOT_ARG="--with-sysroot=${CT_SYSROOT_DIR}" + CT_CC_CORE_SYSROOT_ARG="--with-sysroot=${CT_SYSROOT_DIR}" + CT_CC_SYSROOT_ARG="--with-sysroot=${CT_SYSROOT_DIR}" # glibc's prefix must be exactly /usr, else --with-sysroot'd gcc will get # confused when $sysroot/usr/include is not present. # Note: --prefix=/usr is magic! @@ -320,11 +326,11 @@ if [ -z "${CT_RESTART}" ]; then # hack! Always use --with-sysroot for binutils. # binutils 2.14 and later obey it, older binutils ignore it. # Lets you build a working 32->64 bit cross gcc - BINUTILS_SYSROOT_ARG="--with-sysroot=${CT_SYSROOT_DIR}" + CT_BINUTILS_SYSROOT_ARG="--with-sysroot=${CT_SYSROOT_DIR}" # Use --with-headers, else final gcc will define disable_glibc while # building libgcc, and you'll have no profiling - CC_CORE_SYSROOT_ARG="--without-headers" - CC_SYSROOT_ARG="--with-headers=${CT_HEADERS_DIR}" + CT_CC_CORE_SYSROOT_ARG="--without-headers" + CT_CC_SYSROOT_ARG="--with-headers=${CT_HEADERS_DIR}" fi CT_DoExecLog ALL mkdir -p "${CT_SYSROOT_DIR}" CT_DoExecLog ALL mkdir -p "${CT_DEBUGROOT_DIR}" @@ -529,11 +535,6 @@ if [ -z "${CT_RESTART}" ]; then CT_DoLog DEBUG "CFLAGS for host compiler: '${CT_CFLAGS_FOR_HOST}'" CT_DoLog DEBUG "LDFLAGS for host compiler: '${CT_LDFLAGS_FOR_HOST}'" - # Set the shell to be used by ./configure scripts and by Makefiles (those - # that support it!). - export CONFIG_SHELL="${CT_CONFIG_SHELL}" # for ./configure - export SHELL="${CT_CONFIG_SHELL}" # for Makefiles - # And help make go faster JOBSFLAGS= # Override the configured jobs with what's been given on the command line @@ -601,9 +602,7 @@ if [ -z "${CT_RESTART}" ]; then CT_DoLog DEBUG "Other environment:" printenv |${grep} -v -E '^CT_.+=' |CT_DoLog DEBUG CT_EndStep -fi -if [ -z "${CT_RESTART}" ]; then CT_DoStep INFO "Retrieving needed toolchain components' tarballs" do_companion_tools_get do_kernel_get diff --git a/scripts/functions b/scripts/functions index ce99a17..cc7e131 100644 --- a/scripts/functions +++ b/scripts/functions @@ -974,6 +974,30 @@ CT_DoPause() { return 0 } +# This function sets up trapping export/unset operations so that saving/restoring +# the state can restore status of environment exactly. +CT_TrapEnvExport() +{ + unset() + { + eval "builtin unset $*" + CT_ENVVAR_UNSET="${CT_ENVVAR_UNSET} $*" + } + + export() + { + local v + + for v in "$@"; do + eval "builtin export \"${v}\"" + case "${CT_ENVVAR_EXPORTED} " in + *" ${v%%=*} "*) continue;; + esac + CT_ENVVAR_EXPORTED="${CT_ENVVAR_EXPORTED} ${v%%=*}" + done + } +} + # This function creates a tarball of the specified directory, but # only if it exists # Usage: CT_DoTarballIfExists [extra_tar_options [...]] @@ -1033,25 +1057,31 @@ CT_DoSaveState() { [ "${CT_DEBUG_CT_SAVE_STEPS}" = "y" ] || return 0 local state_name="$1" local state_dir="${CT_STATE_DIR}/${state_name}" + local v CT_DoLog INFO "Saving state to restart at step '${state_name}'..." rm -rf "${state_dir}" mkdir -p "${state_dir}" + # Save only environment variables, not functions. + # Limit saving to our variables (CT_*) and exported variables. + # Also unset variables that have been removed from the environment. + # This generated script will be sourced from a function, so make + # all the definitions global by adding -g. Hope we don't have + # a multi-line variable that has a line starting with "declare" + # (or we'll need to run sed on each variable separately, only on + # the first line of it). CT_DoLog STATE " Saving environment and aliases" - # We must omit shell functions, and some specific bash variables - # that break when restoring the environment, later. We could do - # all the processing in the awk script, but a sed is easier... - set |${awk} ' - BEGIN { _p = 1; } - $0~/^[^ ]+ \(\)/ { _p = 0; } - _p == 1 - $0 == "}" { _p = 1; } - ' |${sed} -r -e '/^BASH_(ARGC|ARGV|LINENO|SOURCE|VERSINFO)=/d; - /^(UID|EUID)=/d; - /^(FUNCNAME|GROUPS|PPID|SHELLOPTS)=/d;' >"${state_dir}/env.sh" - + { + for v in "${!CT_@}" ${CT_ENVVAR_EXPORTED}; do + # Check if it is still set + [ -n "${!v+set}" ] && declare -p "${v}" + done | ${sed} 's/^declare /declare -g /' + echo "builtin unset ${CT_ENVVAR_UNSET}" + } >"${state_dir}/env.sh" + + # Save .config to check it hasn't changed when resuming. CT_DoExecLog STATE cp ".config" "${state_dir}/config" CT_DoTarballIfExists "${CT_BUILDTOOLS_PREFIX_DIR}" "${state_dir}/buildtools_dir" @@ -1094,7 +1124,6 @@ CT_DoLoadState(){ # Restore the new RESTART and STOP steps CT_RESTART="${old_RESTART}" CT_STOP="${old_STOP}" - unset old_stop old_restart CT_DoLog STATE " Restoring log file" CT_LogDisable -- cgit v0.10.2-6-g49f6 From 2aaa25d2e97460ab940225e9d10611996f541442 Mon Sep 17 00:00:00 2001 From: Alexey Neyman Date: Sat, 19 Aug 2017 23:28:40 -0700 Subject: Update TODO Signed-off-by: Alexey Neyman diff --git a/TODO b/TODO index 3957934..e75103c 100644 --- a/TODO +++ b/TODO @@ -10,8 +10,6 @@ A (slightly) ordered set of tasks for crosstool-NG. Written in a cryptic languag [ ] use to retrieve ax_pthread.m4 (gettext?) [ ] retire wiki-samples [ ] Fix displaying the versions in case devel is used (custom location/repo) - display "devel" or "custom" in those cases -[ ] clean up GDB versions - no X.Y if X.Y.1 is present - [ ] Check other packages, leave only the most recent on each branch [ ] arm_neon.h - offer as a companion "library" for the target [ ] gdbinit (installed if CT_GDB_INSTALL_GDBINIT is set) is not relocatable, contains absolute paths [ ] FreeBSD @@ -28,11 +26,14 @@ A (slightly) ordered set of tasks for crosstool-NG. Written in a cryptic languag [ ] mingw.sh: create a hook for "pre-checks" for all libcs [ ] move CT_DoMultilibList to this new hook? [ ] go over all config options and evaluate their applicability to cross/canadian/cross-native (i.e. WANTS_STATIC_LINK does not have to test build's support for static link) -[ ] Remove per-arch patching (this makes .build/src non-sharable between builds!) [ ] Update "Internals" chapter in the docs to match current state [ ] Integrate openrisc support [ ] Re-enable shared libraries - can it be done without wrapper scripts, e.g. via rpath? -[ ] 'ct-ng' source to disregard FORBID_DOWNLOADS and to avoid clobbering .build and build.log +[ ] 'ct-ng source' + [ ] disregard FORBID_DOWNLOADS and to avoid clobbering .build and build.log + [ ] should not check/prepare environment (#776) + [ ] move download/patching/prepare into separate steps + [ ] beware of TBD - want to use autotools for all packages after they're built, so need to defer preparation of the sources from a repo [ ] Restore blackfin/fdpic support? [ ] Install bash completion script - are there any default locations for it? [ ] Do we need "HACK: we need to override SHLIB_LC from gcc/config/t-slibgcc-elf-ver" block in 100-gcc.sh? We do create a dummy libc.so @@ -94,8 +95,11 @@ A (slightly) ordered set of tasks for crosstool-NG. Written in a cryptic languag [ ] #534 Merge gcc backends in 100-gcc.sh [ ] Currently some options (e.g. plugins) are not supported in core backend, hence aren't available on baremetal configurations [ ] Support clang -[ ] Generate per-package version Kconfig fragments, and include them +[X] Generate per-package version Kconfig fragments, and include them [ ] Add a script to poll for new releases + [ ] Add a script to verify + [ ] URL accessibility - note that some URLs are mutually exclusive, e.g. foo.com/bar and foo.com/bar/archive + [ ] Patch applicability [ ] Install a single lib/ directory with all the stuff needed - scripts, makefile fragments, etc [ ] Separate maintainer's scripts from the scripts used by crosstool-ng itself [ ] Commit testing.py to the new maintainer's dir -- cgit v0.10.2-6-g49f6