scripts/populate.in
author "Yann E. MORIN" <yann.morin.1998@anciens.enib.fr>
Mon Mar 29 00:15:32 2010 +0200 (2010-03-29)
changeset 1864 758d5137fe87
parent 1863 11cc7a5102fa
child 2279 a559d9890c02
permissions -rw-r--r--
scripts/populate: optimise search loop

Curently, populate will iterate over all ELF (shared objects|executables)
to look for missing NEEDED DSOs, adding to the list at every iterations
of the search loop.

Instead of looking again at previously handled ELF files, recursively
resolve every ELf files.

Also, in case there are a whole lot of files (more than the shell can
accept as arguments list, or creating a command line longer than the
shell can cope with), use a temporary file with the list of files
to search for missing dependencies.
yann@1863
     1
#!/bin/sh
yann@217
     2
# This script will populate the root directory with libs from the sysroot.
yann@217
     3
# (C) 2007 Yann E. MORIN
yann@217
     4
# Licensed under the GPL v2
yann@1177
     5
set -e
yann@217
     6
yann@1400
     7
# Use the tools discovered by crosstool-NG's ./configure:
yann@1190
     8
install="@@CT_install@@"
yann@1190
     9
grep="@@CT_grep@@"
yann@1190
    10
sed="@@CT_sed@@"
yann@1177
    11
yann@1861
    12
# Detect where the toolchain is:
yann@1861
    13
CT_PREFIX_DIR="$(cd "$(dirname "$0")/.."; pwd)"
yann@1861
    14
CT_GCC="${0%-populate}-gcc"
yann@1861
    15
CT_READELF="${0%-populate}-readelf"
yann@1861
    16
CT_CFG_PREFIX_DIR="$("${CT_GCC}" -v 2>&1            \
yann@1861
    17
                     |tr ' ' '\n'                   \
yann@1861
    18
                     |"${grep}" -E -- '--prefix='   \
yann@1861
    19
                     |cut -d = -f 2-
yann@1861
    20
                    )"
yann@1861
    21
CT_CFG_SYSROOT_DIR="$("${CT_GCC}" -v 2>&1                   \
yann@1861
    22
                      |tr ' ' '\n'                          \
yann@1861
    23
                      |"${grep}" -E -- '--with-sysroot='    \
yann@1861
    24
                      |cut -d = -f 2-
yann@1861
    25
                     )"
yann@1863
    26
CT_SYSROOT_DIR="$(printf "${CT_CFG_SYSROOT_DIR}\n"                              \
yann@1861
    27
                  |"${sed}" -r -e "s:^${CT_CFG_PREFIX_DIR}:${CT_PREFIX_DIR}:;"  \
yann@1861
    28
                  |"${sed}" -r -e 's,/+,/,g;'                                   \
yann@1861
    29
                 )"
yann@1861
    30
yann@1861
    31
myname=$(basename "$0")
yann@1861
    32
yann@217
    33
doHelp() {
yann@217
    34
    cat <<_EOF_
yann@910
    35
NAME
yann@910
    36
    $myname - populate the target root file system
yann@217
    37
yann@910
    38
SYNOPSIS
yann@910
    39
    $myname OPTIONS -s source_root -d destination_root
yann@217
    40
yann@910
    41
DESCRIPTION
yann@1863
    42
    This script will 'populate' your target root file system 'source_root'
yann@1863
    43
    with libraries from the toolchain (eg. libc.so...), storing the result
yann@1863
    44
    into 'dst_dir'.
yann@217
    45
yann@910
    46
OPTIONS
yann@910
    47
    -s src_dir
yann@1863
    48
        Use 'src_dir' as the un-populated (source) root directory.
yann@910
    49
yann@910
    50
    -d dst_dir
yann@1863
    51
        Use 'dst_dir' as the place to put the populated root directory.
yann@1863
    52
        See the -f and -m options, below, on the required (non-)existence
yann@1863
    53
        of this directory.
yann@910
    54
ncase@1859
    55
    -r sysroot_dir
yann@1863
    56
        Use 'sysroot_dir' as the sysroot instead of the toolchain default.
ncase@1859
    57
yann@910
    58
    -l name1[:name2[...]]
yann@910
    59
        Always add the specified shared library/ies name1, name2... from the
yann@910
    60
        toolchain (in the sys-root). Actual library names are searched as
yann@910
    61
        follows (where 'name' is replaced with the given name) in the
yann@910
    62
        sys-root directory:
yann@910
    63
          - libname.so
yann@910
    64
          - name.so
yann@910
    65
          - name
yann@910
    66
        If the file is found, then the SONAME of the library is used, and the
yann@910
    67
        library is copied with that name. If the library was not found, this
yann@910
    68
        yields an error (unless -f was given).
yann@910
    69
yann@910
    70
    -L file
yann@910
    71
        Read 'file' for a list of shared libraries to always add from the
yann@910
    72
        toolchain. The file should contain one library name per line; text
yann@910
    73
        after a # is ignored until the end of the line; spaces are ignored;
yann@910
    74
        empty lines are ignored. Libraries are searched for as with -l.
yann@910
    75
yann@1863
    76
    -f  Force execution: if destination directory already exists, it will be
yann@910
    77
        removed first; if a specified library (above) was not found, continue.
yann@1860
    78
        Note: if using -m and the destination directory already exists, it
yann@1860
    79
        is *not* removed, see below.
yann@1860
    80
yann@1860
    81
    -m  Merge the source root directory with the destination root directory.
yann@1860
    82
        If the latter does not exist, it is created, and -m is ignored.
yann@1863
    83
        If the destination root directory exists, then the content of the
yann@1860
    84
        source root directory is copied in there, and the result is populated
yann@1860
    85
        as usual.
yann@1860
    86
        It can be usefull if constructing a rootfs incrementally from many
yann@1860
    87
        smaller source root directories, or if your destination root directory
yann@1860
    88
        is an NFS export that your target mounts as / (and you don't want to
yann@1860
    89
        re-run exportfs -av everytime). USE WITH CARE!
yann@258
    90
yann@1352
    91
    -v  Be verbose. By default, populate is absolutely silent.
yann@910
    92
yann@217
    93
_EOF_
yann@217
    94
}
yann@217
    95
yann@217
    96
CT_ROOT_SRC_DIR=
yann@217
    97
CT_ROOT_DST_DIR=
yann@910
    98
CT_LIB_LIST=
yann@910
    99
CT_LIB_FILE=
yann@1860
   100
CT_MERGE=
yann@1860
   101
CT_FORCE=
yann@1352
   102
CT_PRINTF=:
yann@1177
   103
OPTIND=1
yann@1860
   104
while getopts ":s:d:r:l:L:fmvh" CT_OPT; do
yann@217
   105
    case "${CT_OPT}" in
yann@217
   106
        s)  CT_ROOT_SRC_DIR="${OPTARG}";;
yann@217
   107
        d)  CT_ROOT_DST_DIR="${OPTARG}";;
ncase@1859
   108
        r)  CT_SYSROOT_DIR="${OPTARG}";;
yann@910
   109
        l)  CT_LIB_LIST="${CT_LIB_LIST}:${OPTARG}";;
yann@910
   110
        L)  CT_LIB_FILE="${OPTARG}";;
yann@259
   111
        f)  CT_FORCE=y;;
yann@1860
   112
        m)  CT_MERGE=y;;
yann@1352
   113
        v)  CT_PRINTF=printf;;
yann@217
   114
        h)  doHelp
yann@217
   115
            exit 0
yann@217
   116
            ;;
yann@1863
   117
        :)  printf "$myname: '-${OPTARG}' takes exactly one argument.\n"
yann@217
   118
            exit 1
yann@217
   119
            ;;
yann@1863
   120
        ?)  printf "$myname: unknown option '-${OPTARG}'.\n"
yann@217
   121
            exit 1
yann@217
   122
            ;;
yann@217
   123
    esac
yann@217
   124
done
yann@217
   125
yann@217
   126
# Sanity checks
yann@217
   127
if [ -z "${CT_ROOT_SRC_DIR}" -o -z "${CT_ROOT_DST_DIR}" ]; then
yann@217
   128
    doHelp
yann@217
   129
    exit 1
yann@217
   130
fi
yann@217
   131
if [ ! -d "${CT_ROOT_SRC_DIR}" ]; then
yann@1863
   132
    printf "$myname: '${CT_ROOT_SRC_DIR}': no such file or directory\n"
yann@217
   133
    exit 1
yann@217
   134
fi
ncase@1859
   135
if [ ! -d "${CT_SYSROOT_DIR}" ]; then
yann@1863
   136
    printf "$myname: '${CT_SYSROOT_DIR}': no such file or directory\n"
ncase@1859
   137
    exit 1
ncase@1859
   138
fi
yann@1860
   139
# If the dest dir does not exist, all is well
yann@1860
   140
# If merging, we accept an existing dest directory
yann@1860
   141
# If forcing and not merging, we remove an exiting dest directory
yann@1860
   142
# If not forcing and not merging, we do not accept an exiting dest directory
yann@1860
   143
if [ -d "${CT_ROOT_DST_DIR}" ]; then
yann@1860
   144
    case "${CT_FORCE}:${CT_MERGE}" in
yann@1860
   145
        *:y)    ;;
yann@1860
   146
        y:)     rm -rf "${CT_ROOT_DST_DIR}";;
yann@1863
   147
        :)      printf "$myname: '${CT_ROOT_DST_DIR}': already exists\n"
yann@1863
   148
                exit 1
yann@1863
   149
                ;;
yann@1860
   150
    esac
yann@217
   151
fi
yann@1863
   152
src_inode=$(ls -1id "${CT_ROOT_SRC_DIR}/." |awk '{ print $1 }')
yann@1863
   153
dst_inode=$(ls -1id "${CT_ROOT_DST_DIR}/." 2>/dev/null |awk '{ print $1 }')
yann@1352
   154
if [ "${src_inode}" -eq "$((dst_inode+0))" ]; then
yann@1863
   155
    printf "$myname: source and destination are the same!\n"
yann@217
   156
    exit 1
yann@217
   157
fi
yann@217
   158
yann@910
   159
# Check existence of the forced libraries file
yann@910
   160
if [ -n "${CT_LIB_FILE}" -a ! \( -f "${CT_LIB_FILE}" -a -r "${CT_LIB_FILE}" \) ]; then
yann@1863
   161
    printf "$myname: forced libraries file '${CT_LIB_FILE}' not found!\n"
yann@910
   162
    exit 1
yann@910
   163
fi
yann@910
   164
yann@1860
   165
# Create the working copy, no issue if already existing
yann@217
   166
mkdir -p "${CT_ROOT_DST_DIR}"
yann@217
   167
yann@217
   168
# Make all path absolute
yann@253
   169
CT_ROOT_SRC_DIR=$(cd "${CT_ROOT_SRC_DIR}"; pwd)
yann@253
   170
CT_ROOT_DST_DIR=$(cd "${CT_ROOT_DST_DIR}"; pwd)
yann@1863
   171
CT_SYSROOT_DIR=$(cd "${CT_SYSROOT_DIR}"; pwd)
yann@217
   172
yann@1678
   173
# Populate the destination directory with files from the source directory
yann@1863
   174
cd "${CT_ROOT_SRC_DIR}"
yann@1678
   175
cp -a . "${CT_ROOT_DST_DIR}"
yann@1863
   176
cd - >/dev/null
yann@910
   177
yann@910
   178
# A function do search for a library
yann@910
   179
# Usage: do_add_lib libname
yann@910
   180
# returns: 0 if library was found and added, !0 otherwise
yann@910
   181
do_add_lib() {
yann@910
   182
    local libname="$1"
yann@910
   183
    local true_libname
yann@1177
   184
    local dir
yann@1352
   185
    local mode
yann@1352
   186
yann@1177
   187
    for dir in lib usr/lib; do
yann@1864
   188
        if [ -e "${dir}/${libname}" ]; then
yann@1864
   189
            ${CT_PRINTF} "    already present\n"
yann@1864
   190
            return 0
yann@1864
   191
        fi
yann@1864
   192
    done
yann@1864
   193
    for dir in lib usr/lib; do
yann@1352
   194
        ${CT_PRINTF} "    trying in '%s'" "${dir}"
yann@1177
   195
        libfile="${CT_SYSROOT_DIR}/${dir}/${libname}"
yann@1352
   196
        ${CT_PRINTF} ": '%s'\n" "${libfile}"
yann@910
   197
        if [ -e "${libfile}" ]; then
yann@1177
   198
            mkdir -p "${dir}"
yann@1177
   199
            true_libname=$("${CT_READELF}" -d "${libfile}"          \
yann@1177
   200
                           |"${grep}" "Library soname:"             \
yann@1177
   201
                           |"${sed}" -r -e 's,.+\[(.+)\] *$,\1,;'   \
yann@1177
   202
                          )
yann@1352
   203
            case "${libfile}" in
yann@1352
   204
                */ld*)  mode=0755;;
yann@1352
   205
                *)      mode=0644;;
yann@1352
   206
            esac
yann@1352
   207
            ${CT_PRINTF} "      installing as '%s/%s', mode='%s'\n" "${dir}" "${true_libname}" "${mode}"
yann@1352
   208
            "${install}" -m "${mode}" "${libfile}" "${dir}/${true_libname}"
yann@1864
   209
            do_resolve_deps "${dir}/${true_libname}"
yann@1177
   210
            return 0
yann@910
   211
        fi
yann@910
   212
    done
yann@1177
   213
    return 1
yann@910
   214
}
yann@910
   215
yann@1864
   216
# A function to resolve all NEEDED entries for the given file, relative
yann@1864
   217
# to the working directory (eg. dst_dir)
yann@1864
   218
# Usage: do_resolve_deps some/where/some/file
yann@1864
   219
# Returns: 0, meaning all dependencies were found
yann@1864
   220
# If not all dependencies could be found, exists with error code 1
yann@1864
   221
# (unless forced)
yann@1864
   222
do_resolve_deps() {
yann@1864
   223
    local file="${1}"
yann@1864
   224
    local libname
yann@1864
   225
    
yann@1864
   226
    for libname in $("${CT_READELF}" -d "${file}"                           \
yann@1864
   227
                     |"${grep}" -E '\(NEEDED\)[[:space:]]+Shared library:'  \
yann@1864
   228
                     |"${sed}" -r -e 's,.+\[(.+)\] *$,\1,;'                 \
yann@1864
   229
                    ); do
yann@1864
   230
        [ -n "${libname}" ] || continue
yann@1864
   231
        ${CT_PRINTF} "Searching for '%s' needed by '%s'\n" "${libname}" "${file}"
yann@1864
   232
        if ! do_add_lib "${libname}"; then
yann@1864
   233
            printf "$myname: library '${libname}' not found!\n"
yann@1864
   234
            [ "${CT_FORCE}" = "y" ] || exit 1
yann@1864
   235
        fi
yann@1864
   236
    done
yann@1864
   237
}
yann@1864
   238
yann@1352
   239
# We'll work in the copied rootfs
yann@1863
   240
cd "${CT_ROOT_DST_DIR}"
yann@1352
   241
yann@910
   242
# First of, copy the forced libraries into the working copy
yann@1862
   243
lib_list=
yann@910
   244
if [ -n "${CT_LIB_FILE}" ]; then
yann@1177
   245
    lib_list=$("${sed}" -r -e ':loop; s/#.*//;'         \
yann@1177
   246
                           -e 's/[[:space:]]+//g;'      \
yann@1177
   247
                           -e 's/([^:])$/\1:/;'         \
yann@1177
   248
                           -e '/$/N; s/\n//; tloop;'    \
yann@1177
   249
                        "${CT_LIB_FILE}"
yann@1177
   250
              )
yann@910
   251
fi
yann@1863
   252
CT_LIB_LIST=$(printf "${CT_LIB_LIST}:${lib_list}\n"         \
yann@1177
   253
              |"${sed}" -r -e 's/^:+//; s/:+$//; s/:+/ /g;' \
yann@1177
   254
             )
yann@1352
   255
if [ -n "${CT_LIB_LIST}" ]; then
yann@1352
   256
    for name in ${CT_LIB_LIST}; do
yann@1352
   257
        [ -z "${name}" ] && continue
yann@1352
   258
        found=0
yann@1352
   259
        for libname in "lib${name}.so" "${name}.so" "${name}"; do
yann@1864
   260
            ${CT_PRINTF} "Searching for forced library '%s'\n" "${libname}"
yann@1352
   261
            if do_add_lib "${libname}"; then
yann@1352
   262
                found=1
yann@1352
   263
                break
yann@1352
   264
            fi
yann@1352
   265
        done
yann@1352
   266
        if [ ${found} -eq 0 ]; then
yann@1863
   267
            printf "$myname: library '${libname}' not found!\n"
yann@1863
   268
            [ "${CT_FORCE}" = "y" ] || exit 1
yann@910
   269
        fi
yann@910
   270
    done
yann@1352
   271
fi
yann@217
   272
yann@1864
   273
# Create a temporary place where to store... temp files.
yann@1864
   274
rand="$( dd if=/dev/urandom bs=1024 count=1 2>/dev/null \
yann@1864
   275
         |md5sum                                        \
yann@1864
   276
         |awk '{ print $1; }'
yann@1864
   277
       )"
yann@1864
   278
CT_TMP_DIR="${TMPDIR:-/tmp}/populate-${rand}-${$}"
yann@1864
   279
( umask 0077; mkdir "${CT_TMP_DIR}" ) || { printf "Could not create temporary directory\n"; exit 1; }
yann@1864
   280
trap "rm -rf ${CT_TMP_DIR}" EXIT
yann@1864
   281
yann@1864
   282
# List all ELF (executables|shared objects)...
yann@1864
   283
find . -type f -exec file {} \;                                             \
yann@1864
   284
|"${grep}" -E ': ELF [[:digit:]]+-bit (L|M)SB (executable|shared object),'  \
yann@1864
   285
|cut -d ":" -f 1                                                            \
yann@1864
   286
>"${CT_TMP_DIR}/files.list"
yann@1864
   287
yann@1864
   288
# ... and use that list to find missing dependencies
yann@1864
   289
while read file; do
yann@1864
   290
    do_resolve_deps "${file}"
yann@1864
   291
done <"${CT_TMP_DIR}/files.list"
yann@1864
   292
yann@1864
   293
rm -rf "${CT_TMP_DIR}"
yann@1864
   294
trap - EXIT
yann@1352
   295
yann@1352
   296
# OK, we're done. Back off.
yann@1863
   297
cd - >/dev/null