scripts/functions: add aria2, a powerfull downloader
author"Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
Wed Dec 30 15:36:22 2009 +0100 (2009-12-30)
changeset 166961edd9d19e3c
parent 1668 383c37e754df
child 1670 08b0982e4da9
scripts/functions: add aria2, a powerfull downloader

aria2 is a powerfull downloader that is capable of chunking and
parallel retrieval.

Due to li;itations in crosstool-NG retrieval facilities, it's not possible
to take fully advantage of aria2. It might happen that, in the future,
those limitations get lifted away, so we can take use features such as
parallel downloading from more than one server at the same time. For now,
it should still speed up downloads thanks to parallel downloading of chunks.
config/global/download.in
configure
scripts/functions
     1.1 --- a/config/global/download.in	Tue Dec 29 22:11:09 2009 +0100
     1.2 +++ b/config/global/download.in	Wed Dec 30 15:36:22 2009 +0100
     1.3 @@ -89,7 +89,7 @@
     1.4  
     1.5  config CONNECT_TIMEOUT
     1.6      int
     1.7 -    prompt "connection timeout"
     1.8 +    prompt "Connection timeout"
     1.9      default 10
    1.10      help
    1.11        From the curl manual:
    1.12 @@ -109,6 +109,21 @@
    1.13        
    1.14        Note that this value applies equally to wget if you have that installed.
    1.15  
    1.16 +config DOWNLOAD_MAX_CHUNKS
    1.17 +    int
    1.18 +    prompt "Maximum number of // chunks"
    1.19 +    default 5
    1.20 +    range 1 10
    1.21 +    help
    1.22 +      If you have aria2 installed, then it will be used to download files.
    1.23 +      Aria2 can split the download in chunks, and download those chunks in //
    1.24 +      which can be interesting to speed up the download.
    1.25 +      
    1.26 +      On the other hand, using many chunks, or even chunking in general, may
    1.27 +      be seen by some site admins as being kind of unfair, or even as a DoS.
    1.28 +      That's why the range of acceptable values is [1..10], and the default
    1.29 +      is 5 (aria2's default).
    1.30 +
    1.31  config ONLY_DOWNLOAD
    1.32      bool
    1.33      prompt "Stop after downloading tarballs"
     2.1 --- a/configure	Tue Dec 29 22:11:09 2009 +0100
     2.2 +++ b/configure	Wed Dec 30 15:36:22 2009 +0100
     2.3 @@ -354,7 +354,7 @@
     2.4               ver='\(GNU libtool.*\) (2[[:digit:]]*\.|1\.6[[:digit:]]*\.|1\.5\.[2-9][[:digit:]]+)'   \
     2.5               err="'libtool' 1.5.26 or above was not found"
     2.6  has_or_abort prog=stat ver='GNU coreutils'
     2.7 -has_or_abort prog="curl wget"
     2.8 +has_or_abort prog="aria2c curl wget"
     2.9  has_or_abort prog=cvs
    2.10  has_or_abort prog=patch
    2.11  has_or_abort prog=tar
     3.1 --- a/scripts/functions	Tue Dec 29 22:11:09 2009 +0100
     3.2 +++ b/scripts/functions	Wed Dec 30 15:36:22 2009 +0100
     3.3 @@ -348,12 +348,31 @@
     3.4      || true
     3.5  }
     3.6  
     3.7 +# Download using aria2
     3.8 +# Usage: CT_DoGetFileAria2 <URL>
     3.9 +CT_DoGetFileAria2() {
    3.10 +    # Note: comments about curl method (above) are also valid here
    3.11 +    # Plus: default progress indicator is a single line, so use verbose log
    3.12 +    #       so that the CT-NG's ouput is 'live'.
    3.13 +    CT_DoExecLog ALL aria2c -l - -s ${CT_DOWNLOAD_MAX_CHUNKS} -m 3 --retry-wait 5 -t ${CT_CONNECT_TIMEOUT} -p "$1" \
    3.14 +    || CT_DoExecLog ALL aria2c -l - -s ${CT_DOWNLOAD_MAX_CHUNKS} -m 3 --retry-wait 5 -t ${CT_CONNECT_TIMEOUT} "$1" \
    3.15 +    || true
    3.16 +}
    3.17 +
    3.18 +# OK, just look if we have them...
    3.19 +_aria2c=$(CT_Which aria2c)
    3.20  _wget=$(CT_Which wget)
    3.21  _curl=$(CT_Which curl)
    3.22 -# Wrapper function to call one of curl or wget
    3.23 +
    3.24 +# Wrapper function to call one of, in order of preference:
    3.25 +#   aria2
    3.26 +#   curl
    3.27 +#   wget
    3.28  # Usage: CT_DoGetFile <URL>
    3.29  CT_DoGetFile() {
    3.30 -    if   [ -n "${_curl}" ]; then
    3.31 +    if   [ -n "${_aria2c}" ]; then
    3.32 +        CT_DoGetFileAria2 "$1"
    3.33 +    elif [ -n "${_curl}" ]; then
    3.34          CT_DoGetFileCurl "$1"
    3.35      elif [ -n "${_wget}" ]; then
    3.36          CT_DoGetFileWget "$1"