Merge pull request #594 from stilor/wget-vs-curl

Select wget vs curl, BSD stat vs GNU stat

9 files changed, 194 insertions, 40 deletions

diff --git a/Makefile.in b/Makefile.in
index 92759ca..4e8db5c 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -62,6 +62,8 @@ export awk             := @AWK@
 export grep            := @GREP@
 export make            := @MAKE@
 export sed             := @SED@
+export wget            := @wget@
+export curl            := @curl@
 export libtool         := @LIBTOOL@
 export libtoolize      := @LIBTOOLIZE@
 export objcopy         := @OBJCOPY@
@@ -155,6 +157,7 @@ uninstall: real-uninstall
 # Build rules
 
 build-bin: $(PROG_NAME)             \
+           scripts/scripts.mk       \
            scripts/crosstool-NG.sh  \
            scripts/saveSample.sh    \
            scripts/showConfig.sh
@@ -187,6 +190,8 @@ define sed_it
 	           -e 's,@@CT_make@@,$(make),g;'            \
 	           -e 's,@@CT_bash@@,$(bash),g;'            \
 	           -e 's,@@CT_awk@@,$(awk),g;'              \
+	           -e 's,@@CT_wget@@,$(wget),g;'            \
+	           -e 's,@@CT_curl@@,$(curl),g;'            \
 	           $< >$@
 endef
 
@@ -258,6 +263,8 @@ FORCE:
 clean-bin:
 	@echo "  RM     '$(PROG_NAME)'"
 	@rm -f $(PROG_NAME)
+	@echo "  RM     'scripts/scripts.mk'"
+	@rm -f scripts/scripts.mk
 	@echo "  RM     'scripts/crosstool-NG.sh'"
 	@rm -f scripts/crosstool-NG.sh
 	@echo "  RM     'scripts/saveSample.sh'"
diff --git a/config/global/download.in b/config/global/download.in
index 377905e..fc79400 100644
--- a/config/global/download.in
+++ b/config/global/download.in
@@ -2,6 +2,23 @@
 
 comment "Downloading"
 
+choice
+    prompt "Download agent"
+
+config DOWNLOAD_AGENT_WGET
+    bool "wget"
+    depends on CONFIGURE_has_wget
+
+config DOWNLOAD_AGENT_CURL
+    bool "curl"
+    depends on CONFIGURE_has_curl
+
+config DOWNLOAD_AGENT_NONE
+    bool "none (forbid download)"
+    select FORBID_DOWNLOAD
+
+endchoice
+
 config FORBID_DOWNLOAD
     bool
     prompt "Forbid downloads"
@@ -11,7 +28,7 @@ config FORBID_DOWNLOAD
       If you do not have network connectivity when you run crosstool-NG,
       and some files are missing, it can be a long time before crosstool-NG
       fails.
-      
+
       Saying 'y' here will prevent crosstool-NG from downloading missing
       files, thus failing early so that you don't get stuck.
 
@@ -22,7 +39,7 @@ config FORCE_DOWNLOAD
     prompt "Force downloads"
     help
       Force downloading tarballs, even if one already exists.
-      
+
       Useful if you suspect a tarball to be damaged.
 
 config CONNECT_TIMEOUT
@@ -31,7 +48,7 @@ config CONNECT_TIMEOUT
     default 10
     help
       Maximum time in seconds that you allow the connection to the server to take.
-      
+
       The scenario is as follows;
         - some enterprise networks have firewalls that prohibit FTP traffic, while
           still allowing HTTP
@@ -39,13 +56,29 @@ config CONNECT_TIMEOUT
         - after this number of seconds, it is considered that the connection could
           not be established, and the next URL in the list is tried, until we reach
           an URL that will go through the firewall, most probably an http:// URL.
-      
+
       If you have a slow network, you'd better set this value higher than the default
       10s. If you know a firewall is blocking connections, but your network is globally
       fast, you can try to lower this value to jump more quickly to allowed URLs. YMMV.
-      
+
       If '-1' is specified, no timeout reconfiguration options are passed to wget.
 
+if DOWNLOAD_AGENT_WGET
+
+config DOWNLOAD_WGET_OPTIONS
+    string "Extra options to wget"
+    default "--passive-ftp --tries=3 -nc --progress=dot:binary"
+
+endif
+
+if DOWNLOAD_AGENT_CURL
+
+config DOWNLOAD_CURL_OPTIONS
+    string "Extra options to curl"
+    default "--location --ftp-pasv --retry 3 --fail --silent"
+
+endif
+
 config ONLY_DOWNLOAD
     bool
     prompt "Stop after downloading tarballs"
diff --git a/configure.ac b/configure.ac
index ba24711..11c5a8e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -216,14 +216,43 @@ ACX_CHECK_PROGS_REQ([bison], [bison])
 ACX_CHECK_PROGS_REQ([flex], [flex])
 ACX_CHECK_PROGS_REQ([makeinfo], [makeinfo])
 ACX_CHECK_PROGS_REQ([cut], [cut])
-ACX_CHECK_PROGS_REQ([stat], [stat])
 ACX_CHECK_PROGS_REQ([readlink], [readlink])
-ACX_CHECK_PROGS_REQ([wget], [wget])
 ACX_CHECK_PROGS_REQ([tar], [tar])
 ACX_CHECK_PROGS_REQ([gzip], [gzip])
 ACX_CHECK_PROGS_REQ([bzip2], [bzip2])
 ACX_CHECK_PROGS_REQ([help2man], [help2man])
 
+# Not a fatal failure even if we have neither - the tarballs may
+# be provided in a local directory.
+AC_CHECK_PROGS([wget], [wget])
+ACX_SET_KCONFIG_OPTION([wget])
+AC_SUBST([wget])
+
+AC_CHECK_PROGS([curl], [curl])
+ACX_SET_KCONFIG_OPTION([curl])
+AC_SUBST([curl])
+
+ACX_CHECK_PROGS_REQ([stat], [stat])
+AC_CACHE_CHECK([whether stat takes GNU or BSD format],
+    [acx_cv_stat_flavor],
+    [touch conftest
+     chmod 642 conftest
+     attr_bsd=`stat -f '%Lp' conftest 2>/dev/null`
+     attr_gnu=`stat -c '%a' conftest 2>/dev/null`
+     rm -f conftest
+     AS_IF([test "$attr_bsd" = "642"],
+       [acx_cv_stat_flavor=BSD],
+       [test "$attr_gnu" = "642"],
+       [acx_cv_stat_flavor=GNU],
+       [AC_MSG_ERROR([cannot determine stat(1) format option])])])
+
+# FIXME: support SET_KCONFIG_OPTION with string values? But then
+# again, these checks may be moved into ct-ng script.
+test "$acx_cv_stat_flavor" = "BSD" && stat_flavor_BSD=y
+ACX_SET_KCONFIG_OPTION([stat_flavor_BSD])
+test "$acx_cv_stat_flavor" = "GNU" && stat_flavor_GNU=y
+ACX_SET_KCONFIG_OPTION([stat_flavor_GNU])
+
 #--------------------------------------------------------------------
 # Still boring, but remember the path, now...
 #--------------------------------------------------------------------
diff --git a/ct-ng.in b/ct-ng.in
index d856828..1204908 100644
--- a/ct-ng.in
+++ b/ct-ng.in
@@ -28,7 +28,7 @@ include $(CT_LIB_DIR)/paths.mk
 # Some distributions (eg. Ubuntu) thought it wise to point /bin/sh to
 # a truly POSIX-conforming shell, ash in this case. This is not so good
 # as we, smart (haha!) developers (as smart we ourselves think we are),
-# got used to bashisms, and are enclined to easiness... So force use of
+# got used to bashisms, and are inclined to easiness... So force use of
 # bash.
 export SHELL=$(bash)
 
@@ -36,7 +36,7 @@ export SHELL=$(bash)
 # We do not need any GREP_OPTIONS anyway, so set it to empty.
 export GREP_OPTIONS=
 
-# Make the restart/stop steps availabe to scripts/crostool-NG.sh
+# Make the restart/stop steps available to scripts/crostool-NG.sh
 export CT_STOP:=$(STOP)
 export CT_RESTART:=$(RESTART)
 
diff --git a/scripts/config.guess b/scripts/config.guess
index bbd48b6..1000e2b 100755
--- a/scripts/config.guess
+++ b/scripts/config.guess
@@ -2,7 +2,7 @@
 # Attempt to guess a canonical system name.
 #   Copyright 1992-2017 Free Software Foundation, Inc.
 
-timestamp='2017-01-01'
+timestamp='2017-02-07'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -1343,6 +1343,9 @@ EOF
     NSR-?:NONSTOP_KERNEL:*:*)
 	echo nsr-tandem-nsk${UNAME_RELEASE}
 	exit ;;
+    NSX-?:NONSTOP_KERNEL:*:*)
+	echo nsx-tandem-nsk${UNAME_RELEASE}
+	exit ;;
     *:NonStop-UX:*:*)
 	echo mips-compaq-nonstopux
 	exit ;;
diff --git a/scripts/config.sub b/scripts/config.sub
index 7e792b4..87abeab 100755
--- a/scripts/config.sub
+++ b/scripts/config.sub
@@ -2,7 +2,7 @@
 # Configuration validation subroutine script.
 #   Copyright 1992-2017 Free Software Foundation, Inc.
 
-timestamp='2017-01-01'
+timestamp='2017-02-07'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -948,6 +948,9 @@ case $basic_machine in
 	nsr-tandem)
 		basic_machine=nsr-tandem
 		;;
+	nsx-tandem)
+		basic_machine=nsx-tandem
+		;;
 	op50n-* | op60c-*)
 		basic_machine=hppa1.1-oki
 		os=-proelf
diff --git a/scripts/functions b/scripts/functions
index 9f852b3..84054d5 100644
--- a/scripts/functions
+++ b/scripts/functions
@@ -567,15 +567,15 @@ CT_DoForceRmdir() {
     local mode
     for dir in "${@}"; do
         [ -d "${dir}" ] || continue
-        case "$CT_SYS_OS" in
-            Linux|CYGWIN*)
+        case "${CT_CONFIGURE_has_stat_flavor_GNU},${CT_CONFIGURE_has_stat_flavor_BSD}" in
+            y,*)
                 mode="$(stat -c '%a' "$(dirname "${dir}")")"
                 ;;
-            Darwin|*BSD)
+            *,y)
                 mode="$(stat -f '%Lp' "$(dirname "${dir}")")"
                 ;;
             *)
-                CT_Abort "Unhandled host OS $CT_SYS_OS"
+                CT_Abort "Unknown stat format options"
                 ;;
         esac
         CT_DoExecLog ALL chmod u+w "$(dirname "${dir}")"
@@ -663,29 +663,34 @@ CT_DoGetFile() {
     local url="${1}"
     local dest="${CT_TARBALLS_DIR}/${url##*/}"
     local tmp="${dest}.tmp-dl"
+    local ok
+    local T
 
     # Remove potential left-over from a previous run
     rm -f "${tmp}"
 
-    # We also retry a few times, in case there is a transient error (eg. behind
-    # a dynamic IP that changes during the transfer...)
-    # With automated download as we are doing, it can be very dangerous to
-    # continue the downloads. It's far better to simply overwrite the
-    # destination file.
-    # Some company networks have firewalls to connect to the internet, but it's
-    # not easy to detect them, so force a global ${CT_CONNECT_TIMEOUT}-second
-    # timeout.
-    if [ ${CT_CONNECT_TIMEOUT} = -1 ]; then
-        T=
-    else
-        T="-T ${CT_CONNECT_TIMEOUT}"
+    # Replace a special value of '-1' with empty string
+    if [ ${CT_CONNECT_TIMEOUT} != -1 ]; then
+        T="${CT_CONNECT_TIMEOUT}"
+    fi
+
+    if [ "${CT_DOWNLOAD_AGENT_WGET}" = "y" ]; then
+        if CT_DoExecLog ALL wget ${CT_DOWNLOAD_WGET_OPTIONS} \
+                                 ${T:+-T ${T}}               \
+                                 -O "${tmp}"                 \
+                                 "${url}"; then
+            ok=y
+        fi
+    elif [ "${CT_DOWNLOAD_AGENT_CURL}" = "y" ]; then
+        if CT_DoExecLog ALL curl ${CT_DOWNLOAD_CURL_OPTIONS}  \
+                                 ${T:+--connect-timeout ${T}} \
+                                 -o "${tmp}"                  \
+                                 "${url}"; then
+            ok=y
+        fi
     fi
-    if CT_DoExecLog ALL wget --passive-ftp --tries=3 -nc    \
-                             --progress=dot:binary          \
-                             ${T}                           \
-                             -O "${tmp}"                    \
-                             "${url}"
-    then
+
+    if [ "${ok}" = "y" ]; then
         # Success, we got it, good!
         mv "${tmp}" "${dest}"
         CT_DoLog DEBUG "Got it from: \"${url}\""
diff --git a/scripts/scripts.mk b/scripts/scripts.mk
index 77368a7..42aacab 100644..100755
--- a/scripts/scripts.mk
+++ b/scripts/scripts.mk
@@ -26,9 +26,19 @@ updatetools: $(CONFIG_SUB_DEST) $(CONFIG_GUESS_DEST)
 # ----------------------------------------------------------
 # How to retrieve the tools
 
-wget_opt=-o /dev/null
-ifeq ($(strip $(V)),2)
-  wget_opt=
+ifneq ($(strip $(V)),2)
+  wget_silent_opt = -o /dev/null
+  curl_silent_opt = --silent
+endif
+
+ifneq (wget,)
+download_cmd = wget --passive-ftp $(wget_silent_opt) -O $@
+else
+ifneq (curl,)
+download_cmd = curl --ftp-pasv $(curl_silent_opt) -o $@
+else
+download_cmd = $(error wget or curl needed for downloads)
+endif
 endif
 
 PHONY += scripts
@@ -37,13 +47,13 @@ scripts:
 	$(SILENT)mkdir -p $@
 
 $(CONFIG_SUB_DEST): scripts FORCE
-	@$(CT_ECHO) '  WGET  $@'
-	$(SILENT)wget $(wget_opt) -O $@ $(CONFIG_SUB_SRC)
+	@$(CT_ECHO) '  DOWNLOAD  $@'
+	$(SILENT)$(download_cmd) $(CONFIG_SUB_SRC)
 	$(SILENT)chmod u+rwx,go+rx-w $@
 
 $(CONFIG_GUESS_DEST): scripts FORCE
-	@$(CT_ECHO) '  WGET  $@'
-	$(SILENT)wget $(wget_opt) -O $@ $(CONFIG_GUESS_SRC)
+	@$(CT_ECHO) '  DOWNLOAD  $@'
+	$(SILENT)$(download_cmd) $(CONFIG_GUESS_SRC)
 	$(SILENT)chmod u+rwx,go+rx-w $@
 
 # ----------------------------------------------------------
diff --git a/scripts/scripts.mk.in b/scripts/scripts.mk.in
new file mode 100644
index 0000000..99b9d76
--- /dev/null
+++ b/scripts/scripts.mk.in
@@ -0,0 +1,64 @@
+# Makefile for the scripts/ sub-directory
+
+# Here, we can update the config.* scripts.
+# If we're in CT_LIB_DIR, then CT_LIB_DIR == CT_TOP_DIR, and we can update those
+# scripts for later inclusion mainline. If CT_LIB_DIR != CT_TOP_DIR, then those
+# scripts are downloaded only for use in CT_TOP_DIR.
+
+# ----------------------------------------------------------
+# The tools help entry
+
+help-distrib::
+	@echo  '  updatetools        - Update the config tools'
+
+# ----------------------------------------------------------
+# Where to get tools from, and where to store them into
+# The tools are: config.guess and config.sub
+
+CONFIG_SUB_SRC="http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD"
+CONFIG_SUB_DEST=scripts/config.sub
+CONFIG_GUESS_SRC="http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD"
+CONFIG_GUESS_DEST=scripts/config.guess
+
+PHONY += updatetools
+updatetools: $(CONFIG_SUB_DEST) $(CONFIG_GUESS_DEST)
+
+# ----------------------------------------------------------
+# How to retrieve the tools
+
+ifneq ($(strip $(V)),2)
+  wget_silent_opt = -o /dev/null
+  curl_silent_opt = --silent
+endif
+
+ifneq (@@CT_wget@@,)
+download_cmd = wget --passive-ftp $(wget_silent_opt) -O $@
+else
+ifneq (@@CT_curl@@,)
+download_cmd = curl --ftp-pasv $(curl_silent_opt) -o $@
+else
+download_cmd = $(error wget or curl needed for downloads)
+endif
+endif
+
+PHONY += scripts
+scripts:
+	@$(CT_ECHO) '  MKDIR $@'
+	$(SILENT)mkdir -p $@
+
+$(CONFIG_SUB_DEST): scripts FORCE
+	@$(CT_ECHO) '  DOWNLOAD  $@'
+	$(SILENT)$(download_cmd) $(CONFIG_SUB_SRC)
+	$(SILENT)chmod u+rwx,go+rx-w $@
+
+$(CONFIG_GUESS_DEST): scripts FORCE
+	@$(CT_ECHO) '  DOWNLOAD  $@'
+	$(SILENT)$(download_cmd) $(CONFIG_GUESS_SRC)
+	$(SILENT)chmod u+rwx,go+rx-w $@
+
+# ----------------------------------------------------------
+# Clean up the mess
+
+distclean::
+	@$(CT_ECHO) "  CLEAN scripts"
+	$(SILENT)[ $(CT_TOP_DIR) = $(CT_LIB_DIR) ] || rm -rf $(CT_TOP_DIR)/scripts