Re: [Qemu-devel] [PATCH v3 01/14] scripts: use git archive in archive-source

[Philippe Mathieu-Daudé]

On 5/20/19 2:47 PM, Gerd Hoffmann wrote:
> Use git archive to create tarballs of qemu and submodules instead of
> cloning the repository and the submodules.  This is a order of magnitude
> faster because it doesn't fetch the submodules from the internet each
> time the script runs.
> 
> Signed-off-by: Gerd Hoffmann <address@hidden>
> Tested-by: Thomas Huth <address@hidden>
> ---
>  scripts/archive-source.sh | 72 +++++++++++++++++----------------------
>  1 file changed, 31 insertions(+), 41 deletions(-)
> 
> diff --git a/scripts/archive-source.sh b/scripts/archive-source.sh
> index 8b8994826000..93a48d2063f9 100755
> --- a/scripts/archive-source.sh
> +++ b/scripts/archive-source.sh
> @@ -19,16 +19,25 @@ if test $# -lt 1; then
>  fi
>  
>  tar_file=$(realpath "$1")
> -list_file="${tar_file}.list"
> -vroot_dir="${tar_file}.vroot"
> +sub_tdir=$(mktemp -d "${tar_file%.tar}.sub.XXXXXXXX")
> +sub_file="${sub_tdir}/submodule.tar"
>  
>  # We want a predictable list of submodules for builds, that is
>  # independent of what the developer currently has initialized
>  # in their checkout, because the build environment is completely
>  # different to the host OS.
>  submodules="dtc slirp ui/keycodemapdb tests/fp/berkeley-softfloat-3 
> tests/fp/berkeley-testfloat-3"
> +sub_deinit=""
>  
> -trap "status=$?; rm -rf \"$list_file\" \"$vroot_dir\"; exit \$status" 0 1 2 
> 3 15
> +function cleanup() {
> +    local status=$?
> +    rm -rf "$sub_tdir"
> +    if test "$sub_deinit" != ""; then
> +     git submodule deinit $sub_deinit
> +    fi
> +    exit $status
> +}
> +trap "cleanup" 0 1 2 3 15
>  
>  if git diff-index --quiet HEAD -- &>/dev/null
>  then
> @@ -36,45 +45,26 @@ then
>  else
>      HEAD=$(git stash create)
>  fi
> -git clone --shared . "$vroot_dir"
> -test $? -ne 0 && error "failed to clone into '$vroot_dir'"
> -for sm in $submodules; do
> -    if test -d "$sm/.git"
> -    then
> -       git clone --shared "$sm" "$vroot_dir/$sm"
> -       test $? -ne 0 && error "failed to clone submodule $sm"
> -    fi
> -done
> -
> -cd "$vroot_dir"
> -test $? -ne 0 && error "failed to change into '$vroot_dir'"
> -
> -git checkout $HEAD
> -test $? -ne 0 && error "failed to checkout $HEAD revision"
>  
> +git archive --format tar $HEAD > "$tar_file"
> +test $? -ne 0 && error "failed to archive qemu"
>  for sm in $submodules; do
> -    git submodule update --init $sm
> -    test $? -ne 0 && error "failed to init submodule $sm"
> +    status="$(git submodule status "$sm")"
> +    smhash="${status#[ +-]}"
> +    smhash="${smhash%% *}"
> +    case "$status" in
> +     -*)
> +         sub_deinit="$sub_deinit $sm"
> +         git submodule update --init "$sm"
> +         test $? -ne 0 && error "failed to update submodule $sm"
> +         ;;
> +     +*)
> +         echo "WARNING: submodule $sm is out of sync"
> +         ;;
> +    esac
> +    (cd $sm; git archive --format tar --prefix "$sm/" $smhash) > "$sub_file"
> +    test $? -ne 0 && error "failed to archive submodule $sm ($smhash)"
> +    tar --concatenate --file "$tar_file" "$sub_file"
> +    test $? -ne 0 && error "failed append submodule $sm to $tar_file"
>  done
> -
> -if test -n "$submodules"; then
> -    {
> -        git ls-files || error "git ls-files failed"
> -        for sm in $submodules; do
> -            (cd $sm; git ls-files) | sed "s:^:$sm/:"
> -            if test "${PIPESTATUS[*]}" != "0 0"; then
> -                error "git ls-files in submodule $sm failed"
> -            fi
> -        done
> -    } | grep -x -v $(for sm in $submodules; do echo "-e $sm"; done) > 
> "$list_file"
> -else
> -    git ls-files > "$list_file"
> -fi
> -
> -if test $? -ne 0; then
> -    error "failed to generate list file"
> -fi
> -
> -tar -cf "$tar_file" -T "$list_file" || error "failed to create tar file"
> -
>  exit 0
> 

Tested-by: Philippe Mathieu-Daudé <address@hidden>