Introduce using a LAN mirror for retrieving tarballs.
/trunk/scripts/functions | 40 36 4 0 ++++++++++++-
/trunk/config/global/download.in | 146 115 31 0 ++++++++++++++++++++++++++++++++++++----------
2 files changed, 151 insertions(+), 35 deletions(-)
1.1 --- a/config/global/download.in Sat Jul 19 22:39:15 2008 +0000
1.2 +++ b/config/global/download.in Sat Jul 19 22:45:17 2008 +0000
1.3 @@ -11,51 +11,99 @@
1.4
1.5 Usefull if you suspect a tarball to be damaged.
1.6
1.7 -config ONLY_DOWNLOAD
1.8 +menuconfig USE_LAN_MIRROR
1.9 bool
1.10 - prompt "Stop after downloading tarballs"
1.11 + prompt "Use LAN mirror"
1.12 default n
1.13 help
1.14 - Only download the tarballs. Exit once it done.
1.15 + If you have a machine on your LAN that mirrors some of the needed
1.16 + tarballs, you can say 'Y' here, and configure adequate values in
1.17 + the following options.
1.18
1.19 - Usefull to pre-retrieve the tarballs before going off-line.
1.20 + Tarballs will be be preferably fetched from the LAN mirror, and if
1.21 + not found there, standard places will be searched for.
1.22
1.23 -config CONNECT_TIMEOUT
1.24 - int
1.25 - prompt "connection timeout"
1.26 - default 10
1.27 +if USE_LAN_MIRROR
1.28 +
1.29 +choice
1.30 + bool
1.31 + prompt "Server type:"
1.32 +
1.33 +config LAN_MIRROR_HTTP
1.34 + bool
1.35 + prompt "http"
1.36 +
1.37 +config LAN_MIRROR_FTP
1.38 + bool
1.39 + prompt "ftp"
1.40 +
1.41 +endchoice # Server type
1.42 +
1.43 +config LAN_MIRROR_SCHEME
1.44 + string
1.45 + default "http" if LAN_MIRROR_HTTP
1.46 + default "ftp" if LAN_MIRROR_FTP
1.47 +
1.48 +config LAN_MIRROR_HOSTNAME
1.49 + string
1.50 + prompt "hostname"
1.51 + depends on EXPERIMENTAL
1.52 + default "localhost"
1.53 help
1.54 - From the curl manual:
1.55 - Maximum time in seconds that you allow the connection to the server to take.
1.56 + Enter here the hostname on your LAN mirror.
1.57
1.58 - The scenario is as follows;
1.59 - - some enterprise networks have firewalls that prohibit FTP traffic, while
1.60 - still allowing HTTP
1.61 - - most download sites have http:// equivalent for the ftp:// URL
1.62 - - after this number of seconds, it is considered that the connection could
1.63 - not be established, and the next URL in the list is tried, until we reach
1.64 - an URL that will go through the firewall, most probably an http:// URL.
1.65 +config LAN_MIRROR_PATTERNS
1.66 + string
1.67 + prompt "locations patterns (READ HELP!)"
1.68 + default "/downloads /downloads/%pkg /downloads/%pkg/%ver /downloads/%pkg-%ver"
1.69 + help
1.70 + A space-separated list of patterns to find the tarballs on the
1.71 + LAN mirror.
1.72
1.73 - If you have a slow network, you'd better set this value higher than the default
1.74 - 10s. If you know a firewall is blocking connections, but your network is globally
1.75 - fast, you can try to lower this value to jump more quickly to allowed URLs. YMMV.
1.76 + You can use the following 'variables', and they'll get replaced by
1.77 + adequate values:
1.78 + %pkg : name of the package
1.79 + %ver : version of the package
1.80
1.81 - Note that this value applies equally to wget if you have that installed.
1.82 + Thus for gcc-4.2.4, %pkg will be replaced with 'gcc', and %ver with
1.83 + '4.2.4' (both without quotes). Thus if you entered the default
1.84 + pattern: "/downloads/%pkg/ /downloads/%pkg/%ver /downloads/%pkg-%ver"
1.85 + the tarball for gcc-4.2.4 would be searched for in:
1.86 + /downloads/gcc/
1.87 + /downloads/gcc/4.2.4/
1.88 + /downloads/gcc-4.2.4/
1.89
1.90 - Of course, you'd be better off to use a proxy, as offered by the following
1.91 - choice of options.
1.92 + Of course, if the files on the LAN mirror are really arranged in weird
1.93 + ways, entering a list of patterns will be tedious... But see below.
1.94 +
1.95 +config LAN_MIRROR_LS_R
1.96 + bool
1.97 + prompt "Use ls-lr.gz et al."
1.98 + default n
1.99 + help
1.100 + If the tarball was not found at any of the above locations, see if the
1.101 + server has a ls-lr.gz (or similar) file, and use that file to see if
1.102 + the tarball is listed somewhere in that file.
1.103 +
1.104 + Common file names looked for are:
1.105 + ls-lrRt.txt (used at ftp.gnu.org)
1.106 + find.txt (ditto)
1.107 + ls-lR
1.108 +
1.109 +endif # USE_LAN_MIRROR
1.110 +
1.111 +menuconfig USE_PROXY
1.112 + bool
1.113 + prompt "Proxy settings"
1.114 + default n
1.115 + help
1.116 + Say 'Y' here if you need to use a proxy to connect to the internet.
1.117 +
1.118 +if USE_PROXY
1.119
1.120 choice
1.121 bool
1.122 prompt "Proxy type"
1.123 - default PROXY_TYPE_NONE
1.124 -
1.125 -config PROXY_TYPE_NONE
1.126 - bool
1.127 - prompt "No proxy"
1.128 - help
1.129 - Select this option if you have a direct connection to the internet,
1.130 - or if you already set the environment adequately.
1.131
1.132 config PROXY_TYPE_HTTP
1.133 bool
1.134 @@ -173,11 +221,47 @@
1.135
1.136 endchoice
1.137
1.138 +endif # Proxy settings
1.139 +
1.140 config PROXY_TYPE
1.141 string
1.142 - default "none" if PROXY_TYPE_NONE
1.143 + default "none" if ! USE_PROXY
1.144 default "http" if PROXY_TYPE_HTTP
1.145 default "sockssys" if PROXY_TYPE_SOCKS_SYS
1.146 default "socksauto" if PROXY_TYPE_SOCKS_AUTO
1.147 default "socks4" if PROXY_TYPE_SOCKS_4
1.148 default "socks5" if PROXY_TYPE_SOCKS_5
1.149 +
1.150 +config CONNECT_TIMEOUT
1.151 + int
1.152 + prompt "connection timeout"
1.153 + default 10
1.154 + help
1.155 + From the curl manual:
1.156 + Maximum time in seconds that you allow the connection to the server to take.
1.157 +
1.158 + The scenario is as follows;
1.159 + - some enterprise networks have firewalls that prohibit FTP traffic, while
1.160 + still allowing HTTP
1.161 + - most download sites have http:// equivalent for the ftp:// URL
1.162 + - after this number of seconds, it is considered that the connection could
1.163 + not be established, and the next URL in the list is tried, until we reach
1.164 + an URL that will go through the firewall, most probably an http:// URL.
1.165 +
1.166 + If you have a slow network, you'd better set this value higher than the default
1.167 + 10s. If you know a firewall is blocking connections, but your network is globally
1.168 + fast, you can try to lower this value to jump more quickly to allowed URLs. YMMV.
1.169 +
1.170 + Note that this value applies equally to wget if you have that installed.
1.171 +
1.172 + Of course, you'd be better off to use a proxy, as offered by the previous
1.173 + option.
1.174 +
1.175 +config ONLY_DOWNLOAD
1.176 + bool
1.177 + prompt "Stop after downloading tarballs"
1.178 + default n
1.179 + help
1.180 + Only download the tarballs. Exit once it done.
1.181 +
1.182 + Usefull to pre-retrieve the tarballs before going off-line.
2.1 --- a/scripts/functions Sat Jul 19 22:39:15 2008 +0000
2.2 +++ b/scripts/functions Sat Jul 19 22:45:17 2008 +0000
2.3 @@ -361,18 +361,50 @@
2.4 return 0
2.5 fi
2.6 done
2.7 - # Try to download it
2.8 +
2.9 + # Not found locally, try from the network
2.10 CT_DoLog EXTRA "Retrieving '${file}' from network"
2.11 +
2.12 + # Start with LAN mirror
2.13 + if [ "${CT_USE_LAN_MIRROR}" = "y" ]; then
2.14 + LAN_URLs=
2.15 + for pat in ${CT_LAN_MIRROR_PATTERNS}; do
2.16 + # Please note: we just have the file's basename in a single piece.
2.17 + # So we have to just try and split it back into name and version... :-(
2.18 + pat="${pat//\%pkg/${file%-*}}"
2.19 + pat="${pat//\%ver/${file##*-}}"
2.20 + LAN_URLs="${LAN_URLs} ${CT_LAN_MIRROR_SCHEME}://${CT_LAN_MIRROR_HOSTNAME}/${pat}"
2.21 + done
2.22 + for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do
2.23 + for url in ${LAN_URLs}; do
2.24 + CT_DoLog DEBUG "Trying '${url}/${file}${ext}'"
2.25 + CT_DoGetFile "${url}/${file}${ext}"
2.26 + if [ -f "${file}${ext}" ]; then
2.27 + if [ "${CT_SAVE_TARBALLS}" = "y" ]; then
2.28 + # No need to test if the file already exists because
2.29 + # it does NOT. If it did exist, we'd have been stopped
2.30 + # above, when looking for local copies.
2.31 + CT_DoLog EXTRA "Saving '${file}' to local storage"
2.32 + mv "${file}${ext}" "${CT_LOCAL_TARBALLS_DIR}" |CT_DoLog ALL
2.33 + ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL
2.34 + fi
2.35 + return 0
2.36 + fi
2.37 + done
2.38 + done
2.39 + fi
2.40 +
2.41 + # OK, available neither localy, nor from the LAN mirror (if any).
2.42 for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do
2.43 # Try all urls in turn
2.44 for url in "$@"; do
2.45 CT_DoLog DEBUG "Trying '${url}/${file}${ext}'"
2.46 CT_DoGetFile "${url}/${file}${ext}"
2.47 if [ -f "${file}${ext}" ]; then
2.48 - # No need to test if the file already exists because
2.49 - # it does NOT. If it did exist, we'd have been stopped
2.50 - # above, when looking for local copies.
2.51 if [ "${CT_SAVE_TARBALLS}" = "y" ]; then
2.52 + # No need to test if the file already exists because
2.53 + # it does NOT. If it did exist, we'd have been stopped
2.54 + # above, when looking for local copies.
2.55 CT_DoLog EXTRA "Saving '${file}' to local storage"
2.56 mv "${file}${ext}" "${CT_LOCAL_TARBALLS_DIR}" |CT_DoLog ALL
2.57 ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL