From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp33.i.mail.ru (smtp33.i.mail.ru [94.100.177.93]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 9BEE94696C3 for ; Thu, 9 Apr 2020 08:34:00 +0300 (MSK) From: "Alexander V. Tikhonov" Date: Thu, 9 Apr 2020 08:33:58 +0300 Message-Id: Subject: [Tarantool-patches] [PATCH v1] Added ability to remove packages from S3 List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Oleg Piskunov , Sergey Bronnikov Cc: tarantool-patches@dev.tarantool.org Added ability to remove given in options package from S3. TO remove the needed package need to set '-r=' option, like: ./tools/update_repo.sh -o= -d= -b= \ -r=tarantool-2.2.2.0 it will remove all found appropriate source and binaries packages from the given S3 repository, also the meta files will be corrected there. Close #4839 --- Github: https://github.com/tarantool/tarantool/tree/avtikhon/gh-4839-s3-script-full-ci Issue: https://github.com/tarantool/tarantool/issues/4839 tools/update_repo.sh | 389 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 309 insertions(+), 80 deletions(-) diff --git a/tools/update_repo.sh b/tools/update_repo.sh index df94cd1ee..9459c2997 100755 --- a/tools/update_repo.sh +++ b/tools/update_repo.sh @@ -8,6 +8,7 @@ ws_prefix=/tmp/tarantool_repo_s3 alloss='ubuntu debian el fedora' product=tarantool +remove= force= skip_errors= # the path with binaries either repository @@ -62,6 +63,9 @@ Usage for store package binaries from the given path: Usage for mirroring Debian|Ubuntu OS repositories: $0 -o= -d= -b= [-p=] +Usage for removing specific package from S3 repository: + $0 -o= -d= -b= [-p=] -r + Arguments: Path points to the directory with deb/prm packages to be used. @@ -101,6 +105,24 @@ EOF - for RPM packages: # prepare local path with needed "tarantool-queue-*" packages only ./$0 -b=s3:// -o=fedora -d=30 + -r|--remove + Remove package specified by version from S3 repository. It will remove + all found appropriate source and binaries packages from the given S3 + repository, also the meta files will be corrected there. + Example of usage on 'tarantool-2.2.2.0' version: + ./tools/update_repo.sh -o= -d= -b= \\ + -r=tarantool-2.2.2.0 + It will search and try to remove packages: + - for DEB repositories: + tarantool-2.2.2.0-1_all.deb + tarantool-2.2.2.0-1_amd64.deb + tarantool-2.2.2.0-1.dsc + tarantool-2.2.2.0-1.debian.tar.xz + tarantool-2.2.2.0.orig.tar.xz + - for RPM repositories: + x86_64/tarantool-2.2.2.0-1.*.x86_64.rpm + x86_64/tarantool-2.2.2.0-1.*.noarch.rpm + SRPMS/tarantool-2.2.2.0--1.*.src.rpm -f|--force Force updating the remote package with the local one despite the checksum difference -s|--skip_errors @@ -134,6 +156,10 @@ case $i in product="${i#*=}" shift # past argument=value ;; + -r=*|--remove=*) + remove="${i#*=}" + shift # past argument=value + ;; -f|--force) force=1 ;; @@ -146,12 +172,15 @@ case $i in ;; *) repo="${i#*=}" - pushd $repo >/dev/null ; repo=$PWD ; popd >/dev/null shift # past argument=value ;; esac done +if [ "$remove" == "" ]; then + pushd $repo >/dev/null ; repo=$PWD ; popd >/dev/null +fi + # check that all needed options were set and correct if [ "$bucket" == "" ]; then echo "ERROR: need to set -b|--bucket bucket option, check usage" @@ -195,7 +224,7 @@ function update_deb_packfile { # WORKAROUND: unknown why, but reprepro doesn`t save the Sources file, # let`s recreate it manualy from it's zipped version - [ ! -f $psources ] || gunzip -c $psources.gz >$psources + gunzip -c $psources.gz >$psources # WORKAROUND: unknown why, but reprepro creates paths w/o distribution in # it and no solution using configuration setup neither options @@ -226,6 +255,8 @@ function update_deb_metadata { touch $packpath.saved fi + [ "$remove" == "" ] || cp $packpath.saved $packpath + if [ "$packtype" == "dsc" ]; then # check if the DSC hash already exists in old Sources file from S3 # find the hash from the new Sources file @@ -328,6 +359,135 @@ function update_deb_metadata { cat $packpath >>$packpath.saved } +function update_deb_dists { + # 2(binaries). update Packages file archives + for packpath in dists/$loop_dist/$component/binary-* ; do + pushd $packpath + if [ -f Packages ]; then + sed -i '/./,$!d' Packages + bzip2 -c Packages >Packages.bz2 + gzip -c Packages >Packages.gz + fi + popd + done + + # 2(sources). update Sources file archives + pushd dists/$loop_dist/$component/source + if [ -f Sources ]; then + sed -i '/./,$!d' Sources + bzip2 -c Sources >Sources.bz2 + gzip -c Sources >Sources.gz + fi + popd + + # 3. update checksums entries of the Packages* files in *Release files + # NOTE: it is stable structure of the *Release files when the checksum + # entries in it in the following way: + # MD5Sum: + # + # + # SHA1: + # + # + # SHA256: + # + # + # The script bellow puts 'md5' value at the 1st found file entry, + # 'sha1' - at the 2nd and 'sha256' at the 3rd + pushd dists/$loop_dist + for file in $(grep " $component/" Release | awk '{print $3}' | sort -u) ; do + [ -f $file ] || continue + sz=$(stat -c "%s" $file) + md5=$(md5sum $file | awk '{print $1}') + sha1=$(sha1sum $file | awk '{print $1}') + sha256=$(sha256sum $file | awk '{print $1}') + awk 'BEGIN{c = 0} ; { + if ($3 == p) { + c = c + 1 + if (c == 1) {print " " md " " s " " p} + if (c == 2) {print " " sh1 " " s " " p} + if (c == 3) {print " " sh2 " " s " " p} + } else {print $0} + }' p="$file" s="$sz" md="$md5" sh1="$sha1" sh2="$sha256" \ + Release >Release.new + mv Release.new Release + done + # resign the selfsigned InRelease file + $rm_file InRelease + gpg --clearsign -o InRelease Release + # resign the Release file + $rm_file Release.gpg + gpg -u $GPG_SIGN_KEY -abs -o Release.gpg Release + popd + + # 4. sync the latest distribution path changes to S3 + $aws_sync_public dists/$loop_dist "$bucket_path/dists/$loop_dist" +} + +function remove_deb { + pushd $ws + + # get Release file + relpath=dists/$option_dist + $mk_dir $relpath + $aws cp $bucket_path/$relpath/Release $relpath/ + + # get Packages files + for bindirs in $($aws ls $bucket_path/dists/$option_dist/$component/ \ + | awk '{print $2}' | grep -v 'source/' | sed 's#/##g') ; do + packpath=$relpath/$component/$bindirs + $mk_dir $packpath + pushd $packpath + $aws cp $bucket_path/$packpath/Packages . + + hashes_old=$(grep -e "^Filename: " -e "^SHA256: " Packages | \ + grep -A1 "$remove" | grep "^SHA256: " | awk '{print $2}') + # NOTE: for the single file name may exists more than one + # entry in damaged file, to fix it all found entries + # of this file need to be removed + # find and remove all package blocks for the bad hashes + for hash_rm in $hashes_old ; do + echo "Removing from Packages file old hash: $hash_rm" + sed -i '1s/^/\n/' Packages + pcregrep -Mi -v "(?s)Package: (\N+\n)+(?=SHA256: ${hash_rm}).*?^$" \ + Packages >Packages.new + mv Packages.new Packages + done + popd + done + + # get Sources file + packpath=$relpath/$component/source + $mk_dir $packpath + pushd $packpath + $aws cp $bucket_path/$packpath/Sources . + + hashes_old=$(grep '^Checksums-Sha256:' -A3 Sources | \ + grep " .* .* $remove" | awk '{print $1}') + # NOTE: for the single file name may exists more than one + # entry in damaged file, to fix it all found entries + # of this file need to be removed + # find and remove all package blocks for the bad hashes + for hash_rm in $hashes_old ; do + echo "Removing from Sources file old hash: $hash_rm" + sed -i '1s/^/\n/' Sources + pcregrep -Mi -v "(?s)Package: (\N+\n)+(?=^ ${hash_rm}).*?^$" \ + Sources >Sources.new + mv Sources.new Sources + done + popd + + # call DEB dists path updater + loop_dist=$option_dist + update_deb_dists + + # remove all found file by the given pattern in options + for suffix in '-1_all.deb' '-1_amd64.deb' '-1.dsc' '-1.debian.tar.xz' '.orig.tar.xz' ; do + $aws ls "$bucket_path/$poolpath/${remove}$suffix" || continue + $aws rm "$bucket_path/$poolpath/${remove}$suffix" + done +} + # The 'pack_deb' function especialy created for DEB packages. It works # with DEB packing OS like Ubuntu, Debian. It is based on globally known # tool 'reprepro' from: @@ -339,18 +499,7 @@ function update_deb_metadata { # https://www.tarantool.io/en/download/os-installation/debian/ # https://www.tarantool.io/en/download/os-installation/ubuntu/ function pack_deb { - # we need to push packages into 'main' repository only - component=main - - # debian has special directory 'pool' for packages - debdir=pool - - # set the subpath with binaries based on literal character of the product name - proddir=$(echo $product | head -c 1) - # copy single distribution with binaries packages - repopath=$ws/pool/${option_dist}/$component/$proddir/$product - $mk_dir ${repopath} file_found=0 for file in $repo/*.deb $repo/*.dsc $repo/*.tar.*z ; do [ -f $file ] || continue @@ -450,68 +599,8 @@ EOF mv $sources.saved $sources fi - # 2(binaries). update Packages file archives - for packpath in dists/$loop_dist/$component/binary-* ; do - pushd $packpath - if [ -f Packages ]; then - sed -i '/./,$!d' Packages - bzip2 -c Packages >Packages.bz2 - gzip -c Packages >Packages.gz - fi - popd - done - - # 2(sources). update Sources file archives - pushd dists/$loop_dist/$component/source - if [ -f Sources ]; then - sed -i '/./,$!d' Sources - bzip2 -c Sources >Sources.bz2 - gzip -c Sources >Sources.gz - fi - popd - - # 3. update checksums entries of the Packages* files in *Release files - # NOTE: it is stable structure of the *Release files when the checksum - # entries in it in the following way: - # MD5Sum: - # - # - # SHA1: - # - # - # SHA256: - # - # - # The script bellow puts 'md5' value at the 1st found file entry, - # 'sha1' - at the 2nd and 'sha256' at the 3rd - pushd dists/$loop_dist - for file in $(grep " $component/" Release | awk '{print $3}' | sort -u) ; do - [ -f $file ] || continue - sz=$(stat -c "%s" $file) - md5=$(md5sum $file | awk '{print $1}') - sha1=$(sha1sum $file | awk '{print $1}') - sha256=$(sha256sum $file | awk '{print $1}') - awk 'BEGIN{c = 0} ; { - if ($3 == p) { - c = c + 1 - if (c == 1) {print " " md " " s " " p} - if (c == 2) {print " " sh1 " " s " " p} - if (c == 3) {print " " sh2 " " s " " p} - } else {print $0} - }' p="$file" s="$sz" md="$md5" sh1="$sha1" sh2="$sha256" \ - Release >Release.new - mv Release.new Release - done - # resign the selfsigned InRelease file - $rm_file InRelease - gpg --clearsign -o InRelease Release - # resign the Release file - $rm_file Release.gpg - gpg -u $GPG_SIGN_KEY -abs -o Release.gpg Release - popd - - # 4. sync the latest distribution path changes to S3 - $aws_sync_public dists/$loop_dist "$bucket_path/dists/$loop_dist" + # call DEB dists path updater + update_deb_dists done } @@ -525,6 +614,7 @@ EOF # at the Tarantool web site: # https://www.tarantool.io/en/download/os-installation/rhel-centos/ # https://www.tarantool.io/en/download/os-installation/fedora/ + function pack_rpm { pack_subdir=$1 pack_patterns=$2 @@ -631,12 +721,12 @@ function pack_rpm { packs_rm=$(($packs_rm+1)) done # reduce number of packages in metafile counter - gunzip ${metafile}.xml.gz + gunzip -f ${metafile}.xml.gz packs=$(($(grep " packages=" ${metafile}.xml | \ sed 's#.* packages="\([0-9]*\)".*#\1#g')-${packs_rm})) sed "s# packages=\"[0-9]*\"# packages=\"${packs}\"#g" \ -i ${metafile}.xml - gzip ${metafile}.xml + gzip -f ${metafile}.xml done fi done @@ -711,10 +801,141 @@ EOF $aws_sync_public repodata "$bucket_path/$repopath/repodata" } +function remove_rpm { + pack_subdir=$1 + pack_patterns=$2 + + pushd $ws + + # set the paths + repopath=$option_dist/$pack_subdir + rpmpath=Packages + packpath=$repopath/$rpmpath + + # prepare local repository with packages + $mk_dir $packpath + cd $repopath + + # copy the current metadata files from S3 + mkdir repodata + pushd repodata + for file in $($aws ls $bucket_path/$repopath/repodata/ | awk '{print $NF}') ; do + $aws ls $bucket_path/$repopath/repodata/$file || continue + $aws cp $bucket_path/$repopath/repodata/$file $file + [[ ! $file =~ .*.gz ]] || gunzip $file + done + + updated_rpms=0 + # loop by the new hashes from the new meta file + for hash in $(grep " tags for the bad hashes + for hash_rm in $hashes_old ; do + echo "Removing from ${metafile}.xml file old hash: $hash_rm" + sed -i '1s/^/\n/' ${metafile}.xml + pcregrep -Mi -v "(?s)" \ + ${metafile}.xml >${metafile}_new.xml + mv ${metafile}_new.xml ${metafile}.xml + packs_rm=$(($packs_rm+1)) + done + # reduce number of packages in metafile counter + packs=$(($(grep " packages=" ${metafile}.xml | \ + sed 's#.* packages="\([0-9]*\)".*#\1#g')-${packs_rm})) + sed "s# packages=\"[0-9]*\"# packages=\"${packs}\"#g" \ + -i ${metafile}.xml + done + done + done + + # check if any RPM files were newly registered + [ "$updated_rpms" == "0" ] && \ + return || echo "Updating dists" + + # merge metadata files + mv repomd.xml repomd_saved.xml + head -n 2 repomd_saved.xml >repomd.xml + for file in filelists.xml other.xml primary.xml ; do + # get the new data + chsnew=$(sha256sum $file | awk '{print $1}') + sz=$(stat --printf="%s" $file) + gzip $file + chsgznew=$(sha256sum $file.gz | awk '{print $1}') + szgz=$(stat --printf="%s" $file.gz) + timestamp=$(date +%s -r $file.gz) + + # add info to repomd.xml file + name=$(echo $file | sed 's#\.xml$##g') + cat <>repomd.xml + + $chsgznew + $chsnew + + $timestamp + $szgz + $sz +" +EOF + done + tail -n 1 repomd_saved.xml >>repomd.xml + rm -f repomd_saved.xml repomd.xml.asc + popd + gpg --detach-sign --armor repodata/repomd.xml + + # update the metadata at the S3 + $aws_sync_public repodata "$bucket_path/$repopath/repodata" +} + if [ "$os" == "ubuntu" -o "$os" == "debian" ]; then # prepare the workspace prepare_ws ${os} - pack_deb + + # we need to push packages into 'main' repository only + component=main + + # debian has special directory 'pool' for packages + debdir=pool + + # set the subpath with binaries based on literal character of the product name + proddir=$(echo $product | head -c 1) + + # copy single distribution with binaries packages + poolpath=pool/${option_dist}/$component/$proddir/$product + repopath=$ws/$poolpath + $mk_dir ${repopath} + + if [ "$remove" != "" ]; then + remove_deb + else + pack_deb + fi + # unlock the publishing $rm_file $ws_lockfile popd @@ -726,14 +947,22 @@ elif [ "$os" == "el" -o "$os" == "fedora" ]; then # prepare the workspace prepare_ws ${os}_${option_dist} - pack_rpm x86_64 "*.x86_64.rpm *.noarch.rpm" + if [ "$remove" != "" ]; then + remove_rpm x86_64 "-1.*.x86_64.rpm -1.*.noarch.rpm" + else + pack_rpm x86_64 "*.x86_64.rpm *.noarch.rpm" + fi # unlock the publishing $rm_file $ws_lockfile popd # prepare the workspace prepare_ws ${os}_${option_dist} - pack_rpm SRPMS "*.src.rpm" + if [ "$remove" != "" ]; then + remove_rpm SRPMS "-1.*.src.rpm" + else + pack_rpm SRPMS "*.src.rpm" + fi # unlock the publishing $rm_file $ws_lockfile popd -- 2.17.1