* [Tarantool-patches] [PATCH v3] Testing changes after 2nd review on S3
@ 2019-12-16 4:46 Alexander V. Tikhonov
2020-01-13 10:29 ` Igor Munkin
0 siblings, 1 reply; 3+ messages in thread
From: Alexander V. Tikhonov @ 2019-12-16 4:46 UTC (permalink / raw)
To: Igor Munkin; +Cc: tarantool-patches
---
Github: https://github.com/tarantool/tarantool/tree/avtikhon/gh-3380-push-packages-s3-full-ci
Issue: https://github.com/tarantool/tarantool/issues/3380
v1: https://lists.tarantool.org/pipermail/tarantool-patches/2019-October/012021.html
v2: https://lists.tarantool.org/pipermail/tarantool-patches/2019-November/012352.html
Changes v2:
- made changes in script from draft to pre-release stages
Changes v3:
- corrected code style, minor updates
- script is ready for release
tools/add_pack_s3_repo.sh | 381 +++++++++++++++++++++-----------------
1 file changed, 209 insertions(+), 172 deletions(-)
diff --git a/tools/add_pack_s3_repo.sh b/tools/add_pack_s3_repo.sh
index 2316b9015..280a07ab0 100755
--- a/tools/add_pack_s3_repo.sh
+++ b/tools/add_pack_s3_repo.sh
@@ -4,8 +4,12 @@ set -e
rm_file='rm -f'
rm_dir='rm -rf'
mk_dir='mkdir -p'
+ws_prefix=/tmp/tarantool_repo_s3
-alloss='ubuntu debian centos el fedora'
+alloss='ubuntu debian el fedora'
+product=tarantool
+# the path with binaries either repository
+repo=.
get_os_dists()
{
@@ -16,7 +20,7 @@ get_os_dists()
alldists='trusty xenial cosmic disco bionic eoan'
elif [ "$os" == "debian" ]; then
alldists='jessie stretch buster'
- elif [ "$os" == "centos" -o "$os" == "el" ]; then
+ elif [ "$os" == "el" ]; then
alldists='6 7 8'
elif [ "$os" == "fedora" ]; then
alldists='27 28 29 30 31'
@@ -25,16 +29,45 @@ get_os_dists()
echo "$alldists"
}
-ws_prefix=/tmp/tarantool_repo_s3
-create_lockfile()
+prepare_ws()
{
- lockfile -l 1000 $1
+ # temporary lock the publication to the repository
+ ws=${ws_prefix}_${branch}_${os}
+ if [ "$os" != "ubuntu" -a "$os" != "debian" ]; then
+ ws=${ws}_${option_dist}
+ fi
+ ws_lockfile=${ws}.lock
+ if [ -f $ws_lockfile ]; then
+ old_proc=$(cat $ws_lockfile)
+ fi
+ lockfile -l 60 $ws_lockfile
+ chmod u+w $ws_lockfile && echo $$ >$ws_lockfile && chmod u-w $ws_lockfile
+ if [ "$old_proc" != "" -a "$old_proc" != "0" ]; then
+ kill -9 $old_proc >/dev/null 2>&1 || true
+ fi
+
+ # create temporary workspace with repository copy
+ $rm_dir $ws
+ $mk_dir $ws
}
usage()
{
cat <<EOF
-Usage: $0 -b <S3 bucket> -o <OS name> -d <OS distribuition> [-p <product>] <path>
+Usage for store package binaries from the given path:
+ $0 -b=<S3 bucket> -o=<OS name> -d=<OS distribuition> [-p=<product>] <path to package binaries>
+
+Usage for mirroring Debian|Ubuntu OS repositories:
+ $0 -b=<S3 bucket> -o=<OS name> [-p=<product>] <path to 'pool' subdirectory with packages repository>
+
+Arguments:
+ <path>
+ Path points to the directory with deb/prm packages to be used.
+ Script can be used in one of 2 modes:
+ - path with binaries packages for a single distribution
+ - path with 'pool' subdirectory with APT repository (only: debian|ubuntu), like:
+ /var/spool/apt-mirror/mirror/packagecloud.io/tarantool/$branch/$os
+
Options:
-b|--bucket
MCS S3 bucket which will be used for storing the packages.
@@ -44,24 +77,19 @@ Options:
2.2: 2_2
1.10: 1_10
-o|--os
- OS to be checked, one of the list (NOTE: centos == el):
+ OS to be checked, one of the list:
$alloss
-d|--distribution
Distribution appropriate to the given OS:
EOF
for os in $alloss ; do
- echo " $os: <"`get_os_dists $os`">"
+ echo " $os: <"$(get_os_dists $os)">"
done
cat <<EOF
-p|--product
Product name to be packed with, default name is 'tarantool'
-h|--help
Usage help message
- <path>
- Path points to the directory with deb/prm packages to be used.
- Script can be used in one of 2 modes:
- - path with binaries packages for a single distribution
- - path with 'pool' directory with APT repository (only: debian|ubuntu)
EOF
}
@@ -70,7 +98,7 @@ do
case $i in
-b=*|--bucket=*)
branch="${i#*=}"
- if [ "$branch" != "2x" -a "$branch" != "2_3" -a "$branch" != "2_2" -a "$branch" != "1_10" ]; then
+ if echo "$branch" | grep -qvP '^(1_10|2(x|_[2-4]))$' ; then
echo "ERROR: bucket '$branch' is not supported"
usage
exit 1
@@ -79,9 +107,6 @@ case $i in
;;
-o=*|--os=*)
os="${i#*=}"
- if [ "$os" == "el" ]; then
- os=centos
- fi
if ! echo $alloss | grep -F -q -w $os ; then
echo "ERROR: OS '$os' is not supported"
usage
@@ -90,7 +115,7 @@ case $i in
shift # past argument=value
;;
-d=*|--distribution=*)
- DIST="${i#*=}"
+ option_dist="${i#*=}"
shift # past argument=value
;;
-p=*|--product=*)
@@ -120,27 +145,68 @@ if [ "$os" == "" ]; then
usage
exit 1
fi
-alldists=`get_os_dists $os`
-if [ -n "$DIST" ] && ! echo $alldists | grep -F -q -w $DIST ; then
- echo "ERROR: set distribution at options '$DIST' not found at supported list '$alldists'"
+alldists=$(get_os_dists $os)
+if [ -n "$option_dist" ] && ! echo $alldists | grep -F -q -w $option_dist ; then
+ echo "ERROR: set distribution at options '$option_dist' not found at supported list '$alldists'"
usage
exit 1
fi
-# set the path with binaries
-product=$product
-if [ "$product" == "" ]; then
- product=tarantool
-fi
-proddir=`echo $product | head -c 1`
+# set the subpath with binaries based on literal character of the product name
+proddir=$(echo $product | head -c 1)
-# set the path with binaries
-if [ "$repo" == "" ]; then
- repo=.
-fi
-
-aws='aws --endpoint-url https://hb.bizmrg.com'
+# AWS defines
+aws='aws --endpoint-url https://hb.bizmrg.com s3'
s3="s3://tarantool_repo/$branch/$os"
+aws_cp_to_s3="$aws cp --acl public-read"
+aws_sync_to_s3="$aws sync --acl public-read"
+
+update_packfile()
+{
+ packfile=$1
+ packtype=$2
+
+ locpackfile=$(echo $packfile | sed "s#^$ws/##g")
+ # register DEB/DSC pack file to Packages/Sources file
+ reprepro -Vb . include$packtype $loop_dist $packfile
+ # reprepro copied DEB/DSC file to component which is not needed
+ $rm_dir $debdir/$component
+ # to have all sources avoid reprepro set DEB/DSC file to its own registry
+ $rm_dir db
+}
+
+update_metadata()
+{
+ packpath=$1
+ packtype=$2
+
+ if [ ! -f $packpath.saved ] ; then
+ # get the latest Sources file from S3
+ $aws ls "$s3/$packpath" 2>/dev/null && \
+ $aws cp "$s3/$packpath" $packpath.saved || \
+ touch $packpath.saved
+ fi
+
+ if [ "$packtype" == "dsc" ]; then
+ # WORKAROUND: unknown why, but reprepro doesn`t save the Sources file
+ gunzip -c $packpath.gz >$packpath
+ # check if the DSC file already exists in Sources from S3
+ hash=$(grep '^Checksums-Sha256:' -A3 $packpath | \
+ tail -n 1 | awk '{print $1}')
+ if grep " $hash .*\.dsc$" $packpath.saved ; then
+ echo "WARNING: DSC file already registered in S3!"
+ return 1
+ fi
+ elif [ "$packtype" == "deb" ]; then
+ # check if the DEB file already exists in Packages from S3
+ if grep "^$(grep '^SHA256: ' $packages)$" $packages.saved ; then
+ echo "WARNING: DEB file already registered in S3!"
+ return 1
+ fi
+ fi
+ # store the new DEB entry
+ cat $packpath >>$packpath.saved
+}
# The 'pack_deb' function especialy created for DEB packages. It works
# with DEB packing OS like Ubuntu, Debian. It is based on globaly known
@@ -158,32 +224,27 @@ function pack_deb {
# debian has special directory 'pool' for packages
debdir=pool
- # get packages from pointed location either mirror path
- if [ "$repo" == "" ] ; then
- repo=/var/spool/apt-mirror/mirror/packagecloud.io/tarantool/$branch/$os
- fi
- if [ ! -d $repo/$debdir ] && ( [ "$DIST" == "" ] || ! ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ) ; then
+ # get packages from pointed location
+ if [ ! -d $repo/$debdir ] && \
+ ( [ "$option_dist" == "" ] || \
+ ! ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ) ; then
echo "ERROR: Current '$repo' path doesn't have any of the following:"
- echo "Usage with set distribuition with option '-d' and packages: $0 [path with *.deb *.dsc *.tar.*z files]"
- echo "Usage with repositories: $0 [path to repository with '$debdir' subdirectory]"
+ echo " - $0 run option '-d' and DEB packages in path"
+ echo " - 'pool' subdirectory with APT repositories"
+ usage
exit 1
fi
- # temporary lock the publication to the repository
- ws=${ws_prefix}_${branch}_${os}
- ws_lockfile=${ws}.lock
- create_lockfile $ws_lockfile
-
- # create temporary workspace with repository copy
- $rm_dir $ws
- $mk_dir $ws
+ # prepare the workspace
+ prepare_ws
# script works in one of 2 modes:
# - path with binaries packages for a single distribution
# - path with 'pool' directory with APT repository
- if [ "$DIST" != "" ] && ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ; then
+ if [ "$option_dist" != "" ] && \
+ ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ; then
# copy single distribution with binaries packages
- repopath=$ws/pool/${DIST}/main/$proddir/$product
+ repopath=$ws/pool/${option_dist}/$component/$proddir/$product
$mk_dir ${repopath}
cp $repo/*.deb $repo/*.dsc $repo/*.tar.*z $repopath/.
elif [ -d $repo/$debdir ]; then
@@ -191,10 +252,12 @@ function pack_deb {
cp -rf $repo/$debdir $ws/.
else
echo "ERROR: neither distribution option '-d' with files $repo/*.deb $repo/*.dsc $repo/*.tar.*z set nor '$repo/$debdir' path found"
- usage
- $rm_file $wslock
- exit 1
+ usage
+ $rm_file $wslock
+ exit 1
fi
+
+ swd=$PWD
cd $ws
# create the configuration file for 'reprepro' tool
@@ -202,14 +265,14 @@ function pack_deb {
$rm_dir $confpath
$mk_dir $confpath
- for dist in $alldists ; do
+ for loop_dist in $alldists ; do
cat <<EOF >>$confpath/distributions
Origin: Tarantool
Label: tarantool.org
Suite: stable
-Codename: $dist
+Codename: $loop_dist
Architectures: amd64 source
-Components: main
+Components: $component
Description: Unofficial Ubuntu Packages maintained by Tarantool
SignWith: 91B625E5
DebIndices: Packages Release . .gz .bz2
@@ -217,84 +280,49 @@ UDebIndices: Packages . .gz .bz2
DscIndices: Sources Release .gz .bz2
EOF
-done
+ done
# create standalone repository with separate components
- for dist in $alldists ; do
- echo =================== DISTRIBUTION: $dist =========================
+ for loop_dist in $alldists ; do
+ echo ================ DISTRIBUTION: $loop_dist ====================
updated_deb=0
updated_dsc=0
# 1(binaries). use reprepro tool to generate Packages file
- for deb in $ws/$debdir/$dist/$component/*/*/*.deb ; do
+ for deb in $ws/$debdir/$loop_dist/$component/*/*/*.deb ; do
[ -f $deb ] || continue
- locdeb=`echo $deb | sed "s#^$ws\/##g"`
- echo "DEB: $deb"
- # register DEB file to Packages file
- reprepro -Vb . includedeb $dist $deb
- # reprepro copied DEB file to local component which is not needed
- $rm_dir $debdir/$component
- # to have all packages avoid reprepro set DEB file to its own registry
- $rm_dir db
- # copy Packages file to avoid of removing by the new DEB version
- for packages in dists/$dist/$component/binary-*/Packages ; do
- if [ ! -f $packages.saved ] ; then
- # get the latest Packages file from S3
- $aws s3 ls "$s3/$packages" 2>/dev/null && \
- $aws s3 cp --acl public-read \
- "$s3/$packages" $packages.saved || \
- touch $packages.saved
- fi
- # check if the DEB file already exists in Packages from S3
- if grep "^`grep "^SHA256: " $packages`$" $packages.saved ; then
- echo "WARNING: DEB file already registered in S3!"
- continue
- fi
- # store the new DEB entry
- cat $packages >>$packages.saved
- # save the registered DEB file to S3
- $aws s3 cp --acl public-read $deb $s3/$locdeb
+ # regenerate DEB pack
+ update_packfile $deb deb
+ echo "Regenerated DEB file: $locpackfile"
+ for packages in dists/$loop_dist/$component/binary-*/Packages ; do
+ # copy Packages file to avoid of removing by the new DEB version
+ # update metadata 'Packages' files
+ if ! update_metadata $packages deb ; then continue ; fi
updated_deb=1
done
+ # save the registered DEB file to S3
+ if [ "$updated_deb" == 1 ]; then
+ $aws_cp_to_s3 $deb $s3/$locpackfile
+ fi
done
# 1(sources). use reprepro tool to generate Sources file
- for dsc in $ws/$debdir/$dist/$component/*/*/*.dsc ; do
+ for dsc in $ws/$debdir/$loop_dist/$component/*/*/*.dsc ; do
[ -f $dsc ] || continue
- locdsc=`echo $dsc | sed "s#^$ws\/##g"`
- echo "DSC: $dsc"
- # register DSC file to Sources file
- reprepro -Vb . includedsc $dist $dsc
- # reprepro copied DSC file to component which is not needed
- $rm_dir $debdir/$component
- # to have all sources avoid reprepro set DSC file to its own registry
- $rm_dir db
+ # regenerate DSC pack
+ update_packfile $dsc dsc
+ echo "Regenerated DSC file: $locpackfile"
# copy Sources file to avoid of removing by the new DSC version
- sources=dists/$dist/$component/source/Sources
- if [ ! -f $sources.saved ] ; then
- # get the latest Sources file from S3
- $aws s3 ls "$s3/$sources" && \
- $aws s3 cp --acl public-read "$s3/$sources" $sources.saved || \
- touch $sources.saved
- fi
- # WORKAROUND: unknown why, but reprepro doesn`t save the Sources file
- gunzip -c $sources.gz >$sources
- # check if the DSC file already exists in Sources from S3
- hash=`grep '^Checksums-Sha256:' -A3 $sources | \
- tail -n 1 | awk '{print $1}'`
- if grep " $hash .*\.dsc$" $sources.saved ; then
- echo "WARNING: DSC file already registered in S3!"
- continue
- fi
- # store the new DSC entry
- cat $sources >>$sources.saved
- # save the registered DSC file to S3
- $aws s3 cp --acl public-read $dsc $s3/$locdsc
- tarxz=`echo $locdsc | sed 's#\.dsc$#.debian.tar.xz#g'`
- $aws s3 cp --acl public-read $ws/$tarxz "$s3/$tarxz"
- orig=`echo $locdsc | sed 's#-1\.dsc$#.orig.tar.xz#g'`
- $aws s3 cp --acl public-read $ws/$orig "$s3/$orig"
+ # update metadata 'Sources' file
+ if ! update_metadata dists/$loop_dist/$component/source/Sources dsc \
+ ; then continue ; fi
updated_dsc=1
+ # save the registered DSC file to S3
+ $aws_cp_to_s3 $dsc $s3/$locpackfile
+ tarxz=$(echo $locpackfile | sed 's#\.dsc$#.debian.tar.xz#g')
+ $aws_cp_to_s3 $ws/$tarxz "$s3/$tarxz"
+ orig=$(echo $locpackfile | sed 's#-1\.dsc$#.orig.tar.xz#g')
+ $aws_cp_to_s3 $ws/$orig "$s3/$orig"
done
# check if any DEB/DSC files were newly registered
@@ -302,30 +330,30 @@ done
continue || echo "Updating dists"
# finalize the Packages file
- for packages in dists/$dist/$component/binary-*/Packages ; do
+ for packages in dists/$loop_dist/$component/binary-*/Packages ; do
mv $packages.saved $packages
done
# 2(binaries). update Packages file archives
- for packpath in dists/$dist/$component/binary-* ; do
+ for packpath in dists/$loop_dist/$component/binary-* ; do
pushd $packpath
- sed "s#Filename: $debdir/$component/#Filename: $debdir/$dist/$component/#g" -i Packages
+ sed "s#Filename: $debdir/$component/#Filename: $debdir/$loop_dist/$component/#g" -i Packages
bzip2 -c Packages >Packages.bz2
gzip -c Packages >Packages.gz
popd
done
# 2(sources). update Sources file archives
- pushd dists/$dist/$component/source
- sed "s#Directory: $debdir/$component/#Directory: $debdir/$dist/$component/#g" -i Sources
+ pushd dists/$loop_dist/$component/source
+ sed "s#Directory: $debdir/$component/#Directory: $debdir/$loop_dist/$component/#g" -i Sources
bzip2 -c Sources >Sources.bz2
gzip -c Sources >Sources.gz
popd
# 3. update checksums entries of the Packages* files in *Release files
- # NOTE: it is stable structure of the *Release files when the checksum
- # entries in it in the following way:
- # MD5Sum:
+ # NOTE: it is stable structure of the *Release files when the checksum
+ # entries in it in the following way:
+ # MD5Sum:
# <checksum> <size> <file orig>
# <checksum> <size> <file debian>
# SHA1:
@@ -334,14 +362,14 @@ done
# SHA256:
# <checksum> <size> <file orig>
# <checksum> <size> <file debian>
- # The script bellow puts 'md5' value at the 1st found file entry,
- # 'sha1' - at the 2nd and 'sha256' at the 3rd
- pushd dists/$dist
- for file in `grep " $component/" Release | awk '{print $3}' | sort -u` ; do
- sz=`stat -c "%s" $file`
- md5=`md5sum $file | awk '{print $1}'`
- sha1=`sha1sum $file | awk '{print $1}'`
- sha256=`sha256sum $file | awk '{print $1}'`
+ # The script bellow puts 'md5' value at the 1st found file entry,
+ # 'sha1' - at the 2nd and 'sha256' at the 3rd
+ pushd dists/$loop_dist
+ for file in $(grep " $component/" Release | awk '{print $3}' | sort -u) ; do
+ sz=$(stat -c "%s" $file)
+ md5=$(md5sum $file | awk '{print $1}')
+ sha1=$(sha1sum $file | awk '{print $1}')
+ sha256=$(sha256sum $file | awk '{print $1}')
awk 'BEGIN{c = 0} ; {
if ($3 == p) {
c = c + 1
@@ -350,7 +378,7 @@ done
if (c == 3) {print " " sh2 " " s " " p}
} else {print $0}
}' p="$file" s="$sz" md="$md5" sh1="$sha1" sh2="$sha256" \
- Release >Release.new
+ Release >Release.new
mv Release.new Release
done
# resign the selfsigned InRelease file
@@ -362,11 +390,13 @@ done
popd
# 4. sync the latest distribution path changes to S3
- $aws s3 sync --acl public-read dists/$dist "$s3/dists/$dist"
+ $aws_sync_to_s3 dists/$loop_dist "$s3/dists/$loop_dist"
done
# unlock the publishing
$rm_file $ws_lockfile
+
+ cd $swd
}
# The 'pack_rpm' function especialy created for RPM packages. It works
@@ -380,29 +410,26 @@ done
# http://mirrors.kernel.org/fedora/releases/30/Everything/x86_64/os/Packages/t/
function pack_rpm {
if ! ls $repo/*.rpm >/dev/null 2>&1 ; then
- echo "ERROR: Current '$repo' has:"
- ls -al $repo
- echo "Usage: $0 [path with *.rpm files]"
+ echo "ERROR: Current '$repo' path doesn't have RPM packages in path"
+ usage
exit 1
fi
- # temporary lock the publication to the repository
- ws=${prefix_lockfile}_${branch}_${os}_${DIST}
- ws_lockfile=${ws}.lock
- create_lockfile $ws_lockfile
+ # prepare the workspace
+ prepare_ws
- # create temporary workspace with packages copies
- $rm_dir $ws
- $mk_dir $ws
+ # copy the needed package binaries to the workspace
cp $repo/*.rpm $ws/.
+
+ swd=$PWD
cd $ws
# set the paths
- if [ "$os" == "centos" ]; then
- repopath=$DIST/os/x86_64
+ if [ "$os" == "el" ]; then
+ repopath=$option_dist/os/x86_64
rpmpath=Packages
elif [ "$os" == "fedora" ]; then
- repopath=releases/$DIST/Everything/x86_64/os
+ repopath=releases/$option_dist/Everything/x86_64/os
rpmpath=Packages/$proddir
fi
packpath=$repopath/$rpmpath
@@ -410,53 +437,61 @@ function pack_rpm {
# prepare local repository with packages
$mk_dir $packpath
mv *.rpm $packpath/.
- cd $repopath
+ cd $ws/$repopath
# copy the current metadata files from S3
mkdir repodata.base
- for file in `$aws s3 ls $s3/$repopath/repodata/ | awk '{print $NF}'` ; do
- $aws s3 ls $s3/$repopath/repodata/$file || continue
- $aws s3 cp $s3/$repopath/repodata/$file repodata.base/$file
+ for file in $($aws ls $s3/$repopath/repodata/ | awk '{print $NF}') ; do
+ $aws ls $s3/$repopath/repodata/$file || continue
+ $aws cp $s3/$repopath/repodata/$file repodata.base/$file
done
# create the new repository metadata files
- createrepo --no-database --update --workers=2 --compress-type=gz --simple-md-filenames .
+ createrepo --no-database --update --workers=2 \
+ --compress-type=gz --simple-md-filenames .
mv repodata repodata.adding
# merge metadata files
mkdir repodata
head -n 2 repodata.adding/repomd.xml >repodata/repomd.xml
for file in filelists.xml other.xml primary.xml ; do
- # 1. take the 1st line only - to skip the line with number of packages which is not needed
+ # 1. take the 1st line only - to skip the line with
+ # number of packages which is not needed
zcat repodata.adding/$file.gz | head -n 1 >repodata/$file
- # 2. take 2nd line with metadata tag and update the packages number in it
+ # 2. take 2nd line with metadata tag and update
+ # the packages number in it
packsold=0
if [ -f repodata.base/$file.gz ] ; then
- packsold=`zcat repodata.base/$file.gz | head -n 2 | tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g'`
+ packsold=$(zcat repodata.base/$file.gz | head -n 2 | \
+ tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g')
fi
- packsnew=`zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g'`
+ packsnew=$(zcat repodata.adding/$file.gz | head -n 2 | \
+ tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g')
packs=$(($packsold+$packsnew))
- zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | sed "s#packages=\".*\"#packages=\"$packs\"#g" >>repodata/$file
+ zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | \
+ sed "s#packages=\".*\"#packages=\"$packs\"#g" >>repodata/$file
# 3. take only 'package' tags from new file
- zcat repodata.adding/$file.gz | tail -n +3 | head -n -1 >>repodata/$file
+ zcat repodata.adding/$file.gz | tail -n +3 | head -n -1 \
+ >>repodata/$file
# 4. take only 'package' tags from old file if exists
if [ -f repodata.base/$file.gz ] ; then
- zcat repodata.base/$file.gz | tail -n +3 | head -n -1 >>repodata/$file
+ zcat repodata.base/$file.gz | tail -n +3 | head -n -1 \
+ >>repodata/$file
fi
# 5. take the last closing line with metadata tag
zcat repodata.adding/$file.gz | tail -n 1 >>repodata/$file
# get the new data
- chsnew=`sha256sum repodata/$file | awk '{print $1}'`
- sz=`stat --printf="%s" repodata/$file`
+ chsnew=$(sha256sum repodata/$file | awk '{print $1}')
+ sz=$(stat --printf="%s" repodata/$file)
gzip repodata/$file
- chsgznew=`sha256sum repodata/$file.gz | awk '{print $1}'`
- szgz=`stat --printf="%s" repodata/$file.gz`
- timestamp=`date +%s -r repodata/$file.gz`
+ chsgznew=$(sha256sum repodata/$file.gz | awk '{print $1}')
+ szgz=$(stat --printf="%s" repodata/$file.gz)
+ timestamp=$(date +%s -r repodata/$file.gz)
# add info to repomd.xml file
- name=`echo $file | sed 's#\.xml$##g'`
- cat <<EOF >>repodata/repomd.xml
+ name=$(echo $file | sed 's#\.xml$##g')
+ cat <<EOF >>repodata/repomd.xml
<data type="$name">
<checksum type="sha256">$chsgznew</checksum>
<open-checksum type="sha256">$chsnew</open-checksum>
@@ -472,19 +507,21 @@ EOF
# copy the packages to S3
for file in $rpmpath/*.rpm ; do
- $aws s3 cp --acl public-read $file "$s3/$repopath/$file"
+ $aws_cp_to_s3 $file "$s3/$repopath/$file"
done
# update the metadata at the S3
- $aws s3 sync --acl public-read repodata "$s3/$repopath/repodata"
+ $aws_sync_to_s3 repodata "$s3/$repopath/repodata"
# unlock the publishing
$rm_file $ws_lockfile
+
+ cd $swd
}
if [ "$os" == "ubuntu" -o "$os" == "debian" ]; then
pack_deb
-elif [ "$os" == "centos" -o "$os" == "fedora" ]; then
+elif [ "$os" == "el" -o "$os" == "fedora" ]; then
pack_rpm
else
echo "USAGE: given OS '$os' is not supported, use any single from the list: $alloss"
--
2.17.1
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [Tarantool-patches] [PATCH v3] Testing changes after 2nd review on S3
2019-12-16 4:46 [Tarantool-patches] [PATCH v3] Testing changes after 2nd review on S3 Alexander V. Tikhonov
@ 2020-01-13 10:29 ` Igor Munkin
2020-01-13 11:36 ` Alexander Tikhonov
0 siblings, 1 reply; 3+ messages in thread
From: Igor Munkin @ 2020-01-13 10:29 UTC (permalink / raw)
To: Alexander V. Tikhonov; +Cc: tarantool-patches
Sasha,
Thanks for the patch! It LGTM to me, but I left some polish-aimed
comments below, please consider them.
Furthermore, I Cced Sasha Tu. to take a look on it since the patch is a
large one an I can overlooked something vital.
On 16.12.19, Alexander V. Tikhonov wrote:
> ---
>
> Github: https://github.com/tarantool/tarantool/tree/avtikhon/gh-3380-push-packages-s3-full-ci
> Issue: https://github.com/tarantool/tarantool/issues/3380
>
> v1: https://lists.tarantool.org/pipermail/tarantool-patches/2019-October/012021.html
> v2: https://lists.tarantool.org/pipermail/tarantool-patches/2019-November/012352.html
>
> Changes v2:
> - made changes in script from draft to pre-release stages
>
> Changes v3:
> - corrected code style, minor updates
> - script is ready for release
Please consider adding the changelog in the reverse order.
>
> tools/add_pack_s3_repo.sh | 381 +++++++++++++++++++++-----------------
> 1 file changed, 209 insertions(+), 172 deletions(-)
>
General comments:
* Please choose a single code style for function definition. Currently
you mix two following approaches:
| usage()
| {
and
| function pack_rpm {
> diff --git a/tools/add_pack_s3_repo.sh b/tools/add_pack_s3_repo.sh
> index 2316b9015..280a07ab0 100755
> --- a/tools/add_pack_s3_repo.sh
> +++ b/tools/add_pack_s3_repo.sh
> @@ -4,8 +4,12 @@ set -e
> rm_file='rm -f'
> rm_dir='rm -rf'
> mk_dir='mkdir -p'
> +ws_prefix=/tmp/tarantool_repo_s3
>
> -alloss='ubuntu debian centos el fedora'
> +alloss='ubuntu debian el fedora'
> +product=tarantool
> +# the path with binaries either repository
> +repo=.
>
> get_os_dists()
> {
> @@ -16,7 +20,7 @@ get_os_dists()
> alldists='trusty xenial cosmic disco bionic eoan'
Minor: IMHO it's more convenient to sort distros in the release date
order.
> elif [ "$os" == "debian" ]; then
> alldists='jessie stretch buster'
> - elif [ "$os" == "centos" -o "$os" == "el" ]; then
> + elif [ "$os" == "el" ]; then
> alldists='6 7 8'
> elif [ "$os" == "fedora" ]; then
> alldists='27 28 29 30 31'
> @@ -25,16 +29,45 @@ get_os_dists()
> echo "$alldists"
> }
>
> -ws_prefix=/tmp/tarantool_repo_s3
> -create_lockfile()
> +prepare_ws()
> {
> - lockfile -l 1000 $1
> + # temporary lock the publication to the repository
> + ws=${ws_prefix}_${branch}_${os}
> + if [ "$os" != "ubuntu" -a "$os" != "debian" ]; then
> + ws=${ws}_${option_dist}
> + fi
I guess the suffix can be constructed within pack_{rpm,deb} subroutine
(i.e. in prepare_ws caller) and passed as an argument to prepare_ws.
Consider the following:
| prepare_ws ${os}_${option_dist} # for deb based distros
| prepare_ws ${os} # for rpm based distros
> + ws_lockfile=${ws}.lock
> + if [ -f $ws_lockfile ]; then
> + old_proc=$(cat $ws_lockfile)
> + fi
> + lockfile -l 60 $ws_lockfile
> + chmod u+w $ws_lockfile && echo $$ >$ws_lockfile && chmod u-w $ws_lockfile
> + if [ "$old_proc" != "" -a "$old_proc" != "0" ]; then
> + kill -9 $old_proc >/dev/null 2>&1 || true
> + fi
> +
> + # create temporary workspace with repository copy
> + $rm_dir $ws
> + $mk_dir $ws
> }
>
> usage()
> {
> cat <<EOF
> -Usage: $0 -b <S3 bucket> -o <OS name> -d <OS distribuition> [-p <product>] <path>
> +Usage for store package binaries from the given path:
> + $0 -b=<S3 bucket> -o=<OS name> -d=<OS distribuition> [-p=<product>] <path to package binaries>
> +
> +Usage for mirroring Debian|Ubuntu OS repositories:
> + $0 -b=<S3 bucket> -o=<OS name> [-p=<product>] <path to 'pool' subdirectory with packages repository>
> +
> +Arguments:
> + <path>
> + Path points to the directory with deb/prm packages to be used.
> + Script can be used in one of 2 modes:
> + - path with binaries packages for a single distribution
> + - path with 'pool' subdirectory with APT repository (only: debian|ubuntu), like:
> + /var/spool/apt-mirror/mirror/packagecloud.io/tarantool/$branch/$os
> +
> Options:
> -b|--bucket
> MCS S3 bucket which will be used for storing the packages.
> @@ -44,24 +77,19 @@ Options:
> 2.2: 2_2
> 1.10: 1_10
> -o|--os
> - OS to be checked, one of the list (NOTE: centos == el):
> + OS to be checked, one of the list:
> $alloss
> -d|--distribution
> Distribution appropriate to the given OS:
> EOF
> for os in $alloss ; do
> - echo " $os: <"`get_os_dists $os`">"
> + echo " $os: <"$(get_os_dists $os)">"
> done
> cat <<EOF
> -p|--product
> Product name to be packed with, default name is 'tarantool'
> -h|--help
> Usage help message
> - <path>
> - Path points to the directory with deb/prm packages to be used.
> - Script can be used in one of 2 modes:
> - - path with binaries packages for a single distribution
> - - path with 'pool' directory with APT repository (only: debian|ubuntu)
> EOF
> }
>
> @@ -70,7 +98,7 @@ do
> case $i in
> -b=*|--bucket=*)
> branch="${i#*=}"
> - if [ "$branch" != "2x" -a "$branch" != "2_3" -a "$branch" != "2_2" -a "$branch" != "1_10" ]; then
> + if echo "$branch" | grep -qvP '^(1_10|2(x|_[2-4]))$' ; then
> echo "ERROR: bucket '$branch' is not supported"
> usage
> exit 1
> @@ -79,9 +107,6 @@ case $i in
> ;;
> -o=*|--os=*)
> os="${i#*=}"
> - if [ "$os" == "el" ]; then
> - os=centos
> - fi
> if ! echo $alloss | grep -F -q -w $os ; then
> echo "ERROR: OS '$os' is not supported"
> usage
> @@ -90,7 +115,7 @@ case $i in
> shift # past argument=value
> ;;
> -d=*|--distribution=*)
> - DIST="${i#*=}"
> + option_dist="${i#*=}"
> shift # past argument=value
> ;;
> -p=*|--product=*)
> @@ -120,27 +145,68 @@ if [ "$os" == "" ]; then
> usage
> exit 1
> fi
> -alldists=`get_os_dists $os`
> -if [ -n "$DIST" ] && ! echo $alldists | grep -F -q -w $DIST ; then
> - echo "ERROR: set distribution at options '$DIST' not found at supported list '$alldists'"
> +alldists=$(get_os_dists $os)
> +if [ -n "$option_dist" ] && ! echo $alldists | grep -F -q -w $option_dist ; then
> + echo "ERROR: set distribution at options '$option_dist' not found at supported list '$alldists'"
> usage
> exit 1
> fi
>
> -# set the path with binaries
> -product=$product
> -if [ "$product" == "" ]; then
> - product=tarantool
> -fi
> -proddir=`echo $product | head -c 1`
> +# set the subpath with binaries based on literal character of the product name
> +proddir=$(echo $product | head -c 1)
>
> -# set the path with binaries
> -if [ "$repo" == "" ]; then
> - repo=.
> -fi
> -
> -aws='aws --endpoint-url https://hb.bizmrg.com'
> +# AWS defines
> +aws='aws --endpoint-url https://hb.bizmrg.com s3'
> s3="s3://tarantool_repo/$branch/$os"
Minor: s3 variable name seems to be an ambiguous one here. IMHO, bucket
looks the way better.
> +aws_cp_to_s3="$aws cp --acl public-read"
> +aws_sync_to_s3="$aws sync --acl public-read"
Nice change, but other commands also interact with S3. I propose to
change the naming to the following one:
| <aws_<action>_public
> +
> +update_packfile()
> +{
> + packfile=$1
> + packtype=$2
> +
> + locpackfile=$(echo $packfile | sed "s#^$ws/##g")
> + # register DEB/DSC pack file to Packages/Sources file
> + reprepro -Vb . include$packtype $loop_dist $packfile
Well, it's really bad idea to use external loop variable within
function. You can pass it as an explicit argument, please do it.
> + # reprepro copied DEB/DSC file to component which is not needed
> + $rm_dir $debdir/$component
> + # to have all sources avoid reprepro set DEB/DSC file to its own registry
> + $rm_dir db
> +}
> +
Minor: Since update_{packfile,metadata} are used only for deb packages,
it can be showed in naming: update_deb_{packfile,metadata}.
> +update_metadata()
> +{
> + packpath=$1
> + packtype=$2
> +
> + if [ ! -f $packpath.saved ] ; then
> + # get the latest Sources file from S3
> + $aws ls "$s3/$packpath" 2>/dev/null && \
> + $aws cp "$s3/$packpath" $packpath.saved || \
> + touch $packpath.saved
> + fi
> +
> + if [ "$packtype" == "dsc" ]; then
> + # WORKAROUND: unknown why, but reprepro doesn`t save the Sources file
> + gunzip -c $packpath.gz >$packpath
> + # check if the DSC file already exists in Sources from S3
> + hash=$(grep '^Checksums-Sha256:' -A3 $packpath | \
> + tail -n 1 | awk '{print $1}')
> + if grep " $hash .*\.dsc$" $packpath.saved ; then
> + echo "WARNING: DSC file already registered in S3!"
> + return 1
> + fi
> + elif [ "$packtype" == "deb" ]; then
> + # check if the DEB file already exists in Packages from S3
> + if grep "^$(grep '^SHA256: ' $packages)$" $packages.saved ; then
> + echo "WARNING: DEB file already registered in S3!"
> + return 1
> + fi
> + fi
> + # store the new DEB entry
> + cat $packpath >>$packpath.saved
> +}
>
> # The 'pack_deb' function especialy created for DEB packages. It works
> # with DEB packing OS like Ubuntu, Debian. It is based on globaly known
> @@ -158,32 +224,27 @@ function pack_deb {
> # debian has special directory 'pool' for packages
> debdir=pool
>
> - # get packages from pointed location either mirror path
> - if [ "$repo" == "" ] ; then
> - repo=/var/spool/apt-mirror/mirror/packagecloud.io/tarantool/$branch/$os
> - fi
> - if [ ! -d $repo/$debdir ] && ( [ "$DIST" == "" ] || ! ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ) ; then
> + # get packages from pointed location
> + if [ ! -d $repo/$debdir ] && \
> + ( [ "$option_dist" == "" ] || \
> + ! ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ) ; then
> echo "ERROR: Current '$repo' path doesn't have any of the following:"
> - echo "Usage with set distribuition with option '-d' and packages: $0 [path with *.deb *.dsc *.tar.*z files]"
> - echo "Usage with repositories: $0 [path to repository with '$debdir' subdirectory]"
> + echo " - $0 run option '-d' and DEB packages in path"
> + echo " - 'pool' subdirectory with APT repositories"
> + usage
> exit 1
> fi
>
> - # temporary lock the publication to the repository
> - ws=${ws_prefix}_${branch}_${os}
> - ws_lockfile=${ws}.lock
> - create_lockfile $ws_lockfile
> -
> - # create temporary workspace with repository copy
> - $rm_dir $ws
> - $mk_dir $ws
> + # prepare the workspace
> + prepare_ws
>
> # script works in one of 2 modes:
> # - path with binaries packages for a single distribution
> # - path with 'pool' directory with APT repository
> - if [ "$DIST" != "" ] && ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ; then
> + if [ "$option_dist" != "" ] && \
> + ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ; then
> # copy single distribution with binaries packages
> - repopath=$ws/pool/${DIST}/main/$proddir/$product
> + repopath=$ws/pool/${option_dist}/$component/$proddir/$product
> $mk_dir ${repopath}
> cp $repo/*.deb $repo/*.dsc $repo/*.tar.*z $repopath/.
> elif [ -d $repo/$debdir ]; then
> @@ -191,10 +252,12 @@ function pack_deb {
> cp -rf $repo/$debdir $ws/.
> else
> echo "ERROR: neither distribution option '-d' with files $repo/*.deb $repo/*.dsc $repo/*.tar.*z set nor '$repo/$debdir' path found"
> - usage
> - $rm_file $wslock
> - exit 1
> + usage
> + $rm_file $wslock
> + exit 1
> fi
> +
> + swd=$PWD
> cd $ws
Minor: I see no reason to avoid using more fitting pushd/popd here.
>
> # create the configuration file for 'reprepro' tool
> @@ -202,14 +265,14 @@ function pack_deb {
> $rm_dir $confpath
> $mk_dir $confpath
>
> - for dist in $alldists ; do
> + for loop_dist in $alldists ; do
> cat <<EOF >>$confpath/distributions
> Origin: Tarantool
> Label: tarantool.org
> Suite: stable
> -Codename: $dist
> +Codename: $loop_dist
> Architectures: amd64 source
> -Components: main
> +Components: $component
> Description: Unofficial Ubuntu Packages maintained by Tarantool
> SignWith: 91B625E5
> DebIndices: Packages Release . .gz .bz2
> @@ -217,84 +280,49 @@ UDebIndices: Packages . .gz .bz2
> DscIndices: Sources Release .gz .bz2
>
> EOF
> -done
> + done
>
> # create standalone repository with separate components
> - for dist in $alldists ; do
> - echo =================== DISTRIBUTION: $dist =========================
> + for loop_dist in $alldists ; do
> + echo ================ DISTRIBUTION: $loop_dist ====================
> updated_deb=0
> updated_dsc=0
>
> # 1(binaries). use reprepro tool to generate Packages file
> - for deb in $ws/$debdir/$dist/$component/*/*/*.deb ; do
> + for deb in $ws/$debdir/$loop_dist/$component/*/*/*.deb ; do
> [ -f $deb ] || continue
> - locdeb=`echo $deb | sed "s#^$ws\/##g"`
> - echo "DEB: $deb"
> - # register DEB file to Packages file
> - reprepro -Vb . includedeb $dist $deb
> - # reprepro copied DEB file to local component which is not needed
> - $rm_dir $debdir/$component
> - # to have all packages avoid reprepro set DEB file to its own registry
> - $rm_dir db
> - # copy Packages file to avoid of removing by the new DEB version
> - for packages in dists/$dist/$component/binary-*/Packages ; do
> - if [ ! -f $packages.saved ] ; then
> - # get the latest Packages file from S3
> - $aws s3 ls "$s3/$packages" 2>/dev/null && \
> - $aws s3 cp --acl public-read \
> - "$s3/$packages" $packages.saved || \
> - touch $packages.saved
> - fi
> - # check if the DEB file already exists in Packages from S3
> - if grep "^`grep "^SHA256: " $packages`$" $packages.saved ; then
> - echo "WARNING: DEB file already registered in S3!"
> - continue
> - fi
> - # store the new DEB entry
> - cat $packages >>$packages.saved
> - # save the registered DEB file to S3
> - $aws s3 cp --acl public-read $deb $s3/$locdeb
> + # regenerate DEB pack
> + update_packfile $deb deb
> + echo "Regenerated DEB file: $locpackfile"
> + for packages in dists/$loop_dist/$component/binary-*/Packages ; do
> + # copy Packages file to avoid of removing by the new DEB version
> + # update metadata 'Packages' files
> + if ! update_metadata $packages deb ; then continue ; fi
I see the following approach to be more convenient for oneline if:
| update_metadata $packages deb || continue
> updated_deb=1
> done
> + # save the registered DEB file to S3
> + if [ "$updated_deb" == 1 ]; then
> + $aws_cp_to_s3 $deb $s3/$locpackfile
> + fi
> done
>
> # 1(sources). use reprepro tool to generate Sources file
> - for dsc in $ws/$debdir/$dist/$component/*/*/*.dsc ; do
> + for dsc in $ws/$debdir/$loop_dist/$component/*/*/*.dsc ; do
> [ -f $dsc ] || continue
> - locdsc=`echo $dsc | sed "s#^$ws\/##g"`
> - echo "DSC: $dsc"
> - # register DSC file to Sources file
> - reprepro -Vb . includedsc $dist $dsc
> - # reprepro copied DSC file to component which is not needed
> - $rm_dir $debdir/$component
> - # to have all sources avoid reprepro set DSC file to its own registry
> - $rm_dir db
> + # regenerate DSC pack
> + update_packfile $dsc dsc
> + echo "Regenerated DSC file: $locpackfile"
> # copy Sources file to avoid of removing by the new DSC version
> - sources=dists/$dist/$component/source/Sources
> - if [ ! -f $sources.saved ] ; then
> - # get the latest Sources file from S3
> - $aws s3 ls "$s3/$sources" && \
> - $aws s3 cp --acl public-read "$s3/$sources" $sources.saved || \
> - touch $sources.saved
> - fi
> - # WORKAROUND: unknown why, but reprepro doesn`t save the Sources file
> - gunzip -c $sources.gz >$sources
> - # check if the DSC file already exists in Sources from S3
> - hash=`grep '^Checksums-Sha256:' -A3 $sources | \
> - tail -n 1 | awk '{print $1}'`
> - if grep " $hash .*\.dsc$" $sources.saved ; then
> - echo "WARNING: DSC file already registered in S3!"
> - continue
> - fi
> - # store the new DSC entry
> - cat $sources >>$sources.saved
> - # save the registered DSC file to S3
> - $aws s3 cp --acl public-read $dsc $s3/$locdsc
> - tarxz=`echo $locdsc | sed 's#\.dsc$#.debian.tar.xz#g'`
> - $aws s3 cp --acl public-read $ws/$tarxz "$s3/$tarxz"
> - orig=`echo $locdsc | sed 's#-1\.dsc$#.orig.tar.xz#g'`
> - $aws s3 cp --acl public-read $ws/$orig "$s3/$orig"
> + # update metadata 'Sources' file
> + if ! update_metadata dists/$loop_dist/$component/source/Sources dsc \
> + ; then continue ; fi
I see the following approach to be more convenient for oneline if:
| update_metadata $dist/$loop_dist/$component/source/Sources dsc || continue
> updated_dsc=1
> + # save the registered DSC file to S3
> + $aws_cp_to_s3 $dsc $s3/$locpackfile
> + tarxz=$(echo $locpackfile | sed 's#\.dsc$#.debian.tar.xz#g')
> + $aws_cp_to_s3 $ws/$tarxz "$s3/$tarxz"
> + orig=$(echo $locpackfile | sed 's#-1\.dsc$#.orig.tar.xz#g')
> + $aws_cp_to_s3 $ws/$orig "$s3/$orig"
> done
>
> # check if any DEB/DSC files were newly registered
> @@ -302,30 +330,30 @@ done
> continue || echo "Updating dists"
>
> # finalize the Packages file
> - for packages in dists/$dist/$component/binary-*/Packages ; do
> + for packages in dists/$loop_dist/$component/binary-*/Packages ; do
> mv $packages.saved $packages
> done
>
> # 2(binaries). update Packages file archives
> - for packpath in dists/$dist/$component/binary-* ; do
> + for packpath in dists/$loop_dist/$component/binary-* ; do
> pushd $packpath
> - sed "s#Filename: $debdir/$component/#Filename: $debdir/$dist/$component/#g" -i Packages
> + sed "s#Filename: $debdir/$component/#Filename: $debdir/$loop_dist/$component/#g" -i Packages
> bzip2 -c Packages >Packages.bz2
> gzip -c Packages >Packages.gz
> popd
> done
>
> # 2(sources). update Sources file archives
> - pushd dists/$dist/$component/source
> - sed "s#Directory: $debdir/$component/#Directory: $debdir/$dist/$component/#g" -i Sources
> + pushd dists/$loop_dist/$component/source
> + sed "s#Directory: $debdir/$component/#Directory: $debdir/$loop_dist/$component/#g" -i Sources
> bzip2 -c Sources >Sources.bz2
> gzip -c Sources >Sources.gz
> popd
>
> # 3. update checksums entries of the Packages* files in *Release files
> - # NOTE: it is stable structure of the *Release files when the checksum
> - # entries in it in the following way:
> - # MD5Sum:
> + # NOTE: it is stable structure of the *Release files when the checksum
> + # entries in it in the following way:
> + # MD5Sum:
> # <checksum> <size> <file orig>
> # <checksum> <size> <file debian>
> # SHA1:
> @@ -334,14 +362,14 @@ done
> # SHA256:
> # <checksum> <size> <file orig>
> # <checksum> <size> <file debian>
> - # The script bellow puts 'md5' value at the 1st found file entry,
> - # 'sha1' - at the 2nd and 'sha256' at the 3rd
> - pushd dists/$dist
> - for file in `grep " $component/" Release | awk '{print $3}' | sort -u` ; do
> - sz=`stat -c "%s" $file`
> - md5=`md5sum $file | awk '{print $1}'`
> - sha1=`sha1sum $file | awk '{print $1}'`
> - sha256=`sha256sum $file | awk '{print $1}'`
> + # The script bellow puts 'md5' value at the 1st found file entry,
> + # 'sha1' - at the 2nd and 'sha256' at the 3rd
> + pushd dists/$loop_dist
> + for file in $(grep " $component/" Release | awk '{print $3}' | sort -u) ; do
> + sz=$(stat -c "%s" $file)
> + md5=$(md5sum $file | awk '{print $1}')
> + sha1=$(sha1sum $file | awk '{print $1}')
> + sha256=$(sha256sum $file | awk '{print $1}')
> awk 'BEGIN{c = 0} ; {
> if ($3 == p) {
> c = c + 1
> @@ -350,7 +378,7 @@ done
> if (c == 3) {print " " sh2 " " s " " p}
> } else {print $0}
> }' p="$file" s="$sz" md="$md5" sh1="$sha1" sh2="$sha256" \
> - Release >Release.new
> + Release >Release.new
> mv Release.new Release
> done
> # resign the selfsigned InRelease file
> @@ -362,11 +390,13 @@ done
> popd
>
> # 4. sync the latest distribution path changes to S3
> - $aws s3 sync --acl public-read dists/$dist "$s3/dists/$dist"
> + $aws_sync_to_s3 dists/$loop_dist "$s3/dists/$loop_dist"
> done
>
> # unlock the publishing
> $rm_file $ws_lockfile
> +
> + cd $swd
> }
>
> # The 'pack_rpm' function especialy created for RPM packages. It works
> @@ -380,29 +410,26 @@ done
> # http://mirrors.kernel.org/fedora/releases/30/Everything/x86_64/os/Packages/t/
> function pack_rpm {
> if ! ls $repo/*.rpm >/dev/null 2>&1 ; then
> - echo "ERROR: Current '$repo' has:"
> - ls -al $repo
> - echo "Usage: $0 [path with *.rpm files]"
> + echo "ERROR: Current '$repo' path doesn't have RPM packages in path"
> + usage
> exit 1
> fi
>
> - # temporary lock the publication to the repository
> - ws=${prefix_lockfile}_${branch}_${os}_${DIST}
> - ws_lockfile=${ws}.lock
> - create_lockfile $ws_lockfile
> + # prepare the workspace
> + prepare_ws
>
> - # create temporary workspace with packages copies
> - $rm_dir $ws
> - $mk_dir $ws
> + # copy the needed package binaries to the workspace
> cp $repo/*.rpm $ws/.
> +
> + swd=$PWD
> cd $ws
Minor: I see no reason to avoid using more fitting pushd/popd here.
>
> # set the paths
> - if [ "$os" == "centos" ]; then
> - repopath=$DIST/os/x86_64
> + if [ "$os" == "el" ]; then
> + repopath=$option_dist/os/x86_64
> rpmpath=Packages
> elif [ "$os" == "fedora" ]; then
> - repopath=releases/$DIST/Everything/x86_64/os
> + repopath=releases/$option_dist/Everything/x86_64/os
> rpmpath=Packages/$proddir
> fi
> packpath=$repopath/$rpmpath
> @@ -410,53 +437,61 @@ function pack_rpm {
> # prepare local repository with packages
> $mk_dir $packpath
> mv *.rpm $packpath/.
> - cd $repopath
> + cd $ws/$repopath
Minor: As discussed offline $ws is always an absolute path, but the
command above looks confusing to me considering the prior cd command.
Feel free to ignore.
>
> # copy the current metadata files from S3
> mkdir repodata.base
> - for file in `$aws s3 ls $s3/$repopath/repodata/ | awk '{print $NF}'` ; do
> - $aws s3 ls $s3/$repopath/repodata/$file || continue
> - $aws s3 cp $s3/$repopath/repodata/$file repodata.base/$file
> + for file in $($aws ls $s3/$repopath/repodata/ | awk '{print $NF}') ; do
> + $aws ls $s3/$repopath/repodata/$file || continue
> + $aws cp $s3/$repopath/repodata/$file repodata.base/$file
> done
>
> # create the new repository metadata files
> - createrepo --no-database --update --workers=2 --compress-type=gz --simple-md-filenames .
> + createrepo --no-database --update --workers=2 \
> + --compress-type=gz --simple-md-filenames .
> mv repodata repodata.adding
>
> # merge metadata files
> mkdir repodata
> head -n 2 repodata.adding/repomd.xml >repodata/repomd.xml
> for file in filelists.xml other.xml primary.xml ; do
> - # 1. take the 1st line only - to skip the line with number of packages which is not needed
> + # 1. take the 1st line only - to skip the line with
> + # number of packages which is not needed
> zcat repodata.adding/$file.gz | head -n 1 >repodata/$file
> - # 2. take 2nd line with metadata tag and update the packages number in it
> + # 2. take 2nd line with metadata tag and update
> + # the packages number in it
> packsold=0
> if [ -f repodata.base/$file.gz ] ; then
> - packsold=`zcat repodata.base/$file.gz | head -n 2 | tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g'`
> + packsold=$(zcat repodata.base/$file.gz | head -n 2 | \
> + tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g')
> fi
> - packsnew=`zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g'`
> + packsnew=$(zcat repodata.adding/$file.gz | head -n 2 | \
> + tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g')
> packs=$(($packsold+$packsnew))
> - zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | sed "s#packages=\".*\"#packages=\"$packs\"#g" >>repodata/$file
> + zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | \
> + sed "s#packages=\".*\"#packages=\"$packs\"#g" >>repodata/$file
> # 3. take only 'package' tags from new file
> - zcat repodata.adding/$file.gz | tail -n +3 | head -n -1 >>repodata/$file
> + zcat repodata.adding/$file.gz | tail -n +3 | head -n -1 \
> + >>repodata/$file
> # 4. take only 'package' tags from old file if exists
> if [ -f repodata.base/$file.gz ] ; then
> - zcat repodata.base/$file.gz | tail -n +3 | head -n -1 >>repodata/$file
> + zcat repodata.base/$file.gz | tail -n +3 | head -n -1 \
> + >>repodata/$file
> fi
> # 5. take the last closing line with metadata tag
> zcat repodata.adding/$file.gz | tail -n 1 >>repodata/$file
>
> # get the new data
> - chsnew=`sha256sum repodata/$file | awk '{print $1}'`
> - sz=`stat --printf="%s" repodata/$file`
> + chsnew=$(sha256sum repodata/$file | awk '{print $1}')
> + sz=$(stat --printf="%s" repodata/$file)
> gzip repodata/$file
> - chsgznew=`sha256sum repodata/$file.gz | awk '{print $1}'`
> - szgz=`stat --printf="%s" repodata/$file.gz`
> - timestamp=`date +%s -r repodata/$file.gz`
> + chsgznew=$(sha256sum repodata/$file.gz | awk '{print $1}')
> + szgz=$(stat --printf="%s" repodata/$file.gz)
> + timestamp=$(date +%s -r repodata/$file.gz)
>
> # add info to repomd.xml file
> - name=`echo $file | sed 's#\.xml$##g'`
> - cat <<EOF >>repodata/repomd.xml
> + name=$(echo $file | sed 's#\.xml$##g')
> + cat <<EOF >>repodata/repomd.xml
> <data type="$name">
> <checksum type="sha256">$chsgznew</checksum>
> <open-checksum type="sha256">$chsnew</open-checksum>
> @@ -472,19 +507,21 @@ EOF
>
> # copy the packages to S3
> for file in $rpmpath/*.rpm ; do
> - $aws s3 cp --acl public-read $file "$s3/$repopath/$file"
> + $aws_cp_to_s3 $file "$s3/$repopath/$file"
> done
>
> # update the metadata at the S3
> - $aws s3 sync --acl public-read repodata "$s3/$repopath/repodata"
> + $aws_sync_to_s3 repodata "$s3/$repopath/repodata"
>
> # unlock the publishing
> $rm_file $ws_lockfile
> +
> + cd $swd
> }
>
> if [ "$os" == "ubuntu" -o "$os" == "debian" ]; then
> pack_deb
> -elif [ "$os" == "centos" -o "$os" == "fedora" ]; then
> +elif [ "$os" == "el" -o "$os" == "fedora" ]; then
> pack_rpm
> else
> echo "USAGE: given OS '$os' is not supported, use any single from the list: $alloss"
> --
> 2.17.1
>
--
Best regards,
IM
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [Tarantool-patches] [PATCH v3] Testing changes after 2nd review on S3
2020-01-13 10:29 ` Igor Munkin
@ 2020-01-13 11:36 ` Alexander Tikhonov
0 siblings, 0 replies; 3+ messages in thread
From: Alexander Tikhonov @ 2020-01-13 11:36 UTC (permalink / raw)
To: Igor Munkin; +Cc: tarantool-patches
[-- Attachment #1: Type: text/plain, Size: 30330 bytes --]
Igor,
Thanks a lot for the review, I've made all the changes as you suggested. Please recheck the changes.
>Понедельник, 13 января 2020, 13:31 +03:00 от Igor Munkin <imun@tarantool.org>:
>
>Sasha,
>
>Thanks for the patch! It LGTM to me, but I left some polish-aimed
>comments below, please consider them.
>
>Furthermore, I Cced Sasha Tu. to take a look on it since the patch is a
>large one an I can overlooked something vital.
>
>On 16.12.19, Alexander V. Tikhonov wrote:
>> ---
>>
>> Github: https://github.com/tarantool/tarantool/tree/avtikhon/gh-3380-push-packages-s3-full-ci
>> Issue: https://github.com/tarantool/tarantool/issues/3380
>>
>> v1: https://lists.tarantool.org/pipermail/tarantool-patches/2019-October/012021.html
>> v2: https://lists.tarantool.org/pipermail/tarantool-patches/2019-November/012352.html
>>
>> Changes v2:
>> - made changes in script from draft to pre-release stages
>>
>> Changes v3:
>> - corrected code style, minor updates
>> - script is ready for release
>
>Please consider adding the changelog in the reverse order.
>
>>
>> tools/add_pack_s3_repo.sh | 381 +++++++++++++++++++++-----------------
>> 1 file changed, 209 insertions(+), 172 deletions(-)
>>
>
>General comments:
>* Please choose a single code style for function definition. Currently
> you mix two following approaches:
>| usage()
>| {
> and
>| function pack_rpm {
Corrected.
>
>> diff --git a/tools/add_pack_s3_repo.sh b/tools/add_pack_s3_repo.sh
>> index 2316b9015..280a07ab0 100755
>> --- a/tools/add_pack_s3_repo.sh
>> +++ b/tools/add_pack_s3_repo.sh
>> @@ -4,8 +4,12 @@ set -e
>> rm_file='rm -f'
>> rm_dir='rm -rf'
>> mk_dir='mkdir -p'
>> +ws_prefix=/tmp/tarantool_repo_s3
>>
>> -alloss='ubuntu debian centos el fedora'
>> +alloss='ubuntu debian el fedora'
>> +product=tarantool
>> +# the path with binaries either repository
>> +repo=.
>>
>> get_os_dists()
>> {
>> @@ -16,7 +20,7 @@ get_os_dists()
>> alldists='trusty xenial cosmic disco bionic eoan'
>
>Minor: IMHO it's more convenient to sort distros in the release date
>order.
Corrected.
>
>
>> elif [ "$os" == "debian" ]; then
>> alldists='jessie stretch buster'
>> - elif [ "$os" == "centos" -o "$os" == "el" ]; then
>> + elif [ "$os" == "el" ]; then
>> alldists='6 7 8'
>> elif [ "$os" == "fedora" ]; then
>> alldists='27 28 29 30 31'
>> @@ -25,16 +29,45 @@ get_os_dists()
>> echo "$alldists"
>> }
>>
>> -ws_prefix=/tmp/tarantool_repo_s3
>> -create_lockfile()
>> +prepare_ws()
>> {
>> - lockfile -l 1000 $1
>> + # temporary lock the publication to the repository
>> + ws=${ws_prefix}_${branch}_${os}
>> + if [ "$os" != "ubuntu" -a "$os" != "debian" ]; then
>> + ws=${ws}_${option_dist}
>> + fi
>
>I guess the suffix can be constructed within pack_{rpm,deb} subroutine
>(i.e. in prepare_ws caller) and passed as an argument to prepare_ws.
>Consider the following:
>| prepare_ws ${os}_${option_dist} # for deb based distros
>| prepare_ws ${os} # for rpm based distros
Changed as you suggested, just:
prepare_ws ${os} # for deb based distros
prepare_ws ${os}_${option_dist} # for rpm based distros
>
>> + ws_lockfile=${ws}.lock
>> + if [ -f $ws_lockfile ]; then
>> + old_proc=$(cat $ws_lockfile)
>> + fi
>> + lockfile -l 60 $ws_lockfile
>> + chmod u+w $ws_lockfile && echo $$ >$ws_lockfile && chmod u-w $ws_lockfile
>> + if [ "$old_proc" != "" -a "$old_proc" != "0" ]; then
>> + kill -9 $old_proc >/dev/null 2>&1 || true
>> + fi
>> +
>> + # create temporary workspace with repository copy
>> + $rm_dir $ws
>> + $mk_dir $ws
>> }
>>
>> usage()
>> {
>> cat <<EOF
>> -Usage: $0 -b <S3 bucket> -o <OS name> -d <OS distribuition> [-p <product>] <path>
>> +Usage for store package binaries from the given path:
>> + $0 -b=<S3 bucket> -o=<OS name> -d=<OS distribuition> [-p=<product>] <path to package binaries>
>> +
>> +Usage for mirroring Debian|Ubuntu OS repositories:
>> + $0 -b=<S3 bucket> -o=<OS name> [-p=<product>] <path to 'pool' subdirectory with packages repository>
>> +
>> +Arguments:
>> + <path>
>> + Path points to the directory with deb/prm packages to be used.
>> + Script can be used in one of 2 modes:
>> + - path with binaries packages for a single distribution
>> + - path with 'pool' subdirectory with APT repository (only: debian|ubuntu), like:
>> + /var/spool/apt-mirror/mirror/packagecloud.io/tarantool/$branch/$os
>> +
>> Options:
>> -b|--bucket
>> MCS S3 bucket which will be used for storing the packages.
>> @@ -44,24 +77,19 @@ Options:
>> 2.2: 2_2
>> 1.10: 1_10
>> -o|--os
>> - OS to be checked, one of the list (NOTE: centos == el):
>> + OS to be checked, one of the list:
>> $alloss
>> -d|--distribution
>> Distribution appropriate to the given OS:
>> EOF
>> for os in $alloss ; do
>> - echo " $os: <"`get_os_dists $os`">"
>> + echo " $os: <"$(get_os_dists $os)">"
>> done
>> cat <<EOF
>> -p|--product
>> Product name to be packed with, default name is 'tarantool'
>> -h|--help
>> Usage help message
>> - <path>
>> - Path points to the directory with deb/prm packages to be used.
>> - Script can be used in one of 2 modes:
>> - - path with binaries packages for a single distribution
>> - - path with 'pool' directory with APT repository (only: debian|ubuntu)
>> EOF
>> }
>>
>> @@ -70,7 +98,7 @@ do
>> case $i in
>> -b=*|--bucket=*)
>> branch="${i#*=}"
>> - if [ "$branch" != "2x" -a "$branch" != "2_3" -a "$branch" != "2_2" -a "$branch" != "1_10" ]; then
>> + if echo "$branch" | grep -qvP '^(1_10|2(x|_[2-4]))$' ; then
>> echo "ERROR: bucket '$branch' is not supported"
>> usage
>> exit 1
>> @@ -79,9 +107,6 @@ case $i in
>> ;;
>> -o=*|--os=*)
>> os="${i#*=}"
>> - if [ "$os" == "el" ]; then
>> - os=centos
>> - fi
>> if ! echo $alloss | grep -F -q -w $os ; then
>> echo "ERROR: OS '$os' is not supported"
>> usage
>> @@ -90,7 +115,7 @@ case $i in
>> shift # past argument=value
>> ;;
>> -d=*|--distribution=*)
>> - DIST="${i#*=}"
>> + option_dist="${i#*=}"
>> shift # past argument=value
>> ;;
>> -p=*|--product=*)
>> @@ -120,27 +145,68 @@ if [ "$os" == "" ]; then
>> usage
>> exit 1
>> fi
>> -alldists=`get_os_dists $os`
>> -if [ -n "$DIST" ] && ! echo $alldists | grep -F -q -w $DIST ; then
>> - echo "ERROR: set distribution at options '$DIST' not found at supported list '$alldists'"
>> +alldists=$(get_os_dists $os)
>> +if [ -n "$option_dist" ] && ! echo $alldists | grep -F -q -w $option_dist ; then
>> + echo "ERROR: set distribution at options '$option_dist' not found at supported list '$alldists'"
>> usage
>> exit 1
>> fi
>>
>> -# set the path with binaries
>> -product=$product
>> -if [ "$product" == "" ]; then
>> - product=tarantool
>> -fi
>> -proddir=`echo $product | head -c 1`
>> +# set the subpath with binaries based on literal character of the product name
>> +proddir=$(echo $product | head -c 1)
>>
>> -# set the path with binaries
>> -if [ "$repo" == "" ]; then
>> - repo=.
>> -fi
>> -
>> -aws='aws --endpoint-url https://hb.bizmrg.com '
>> +# AWS defines
>> +aws='aws --endpoint-url https://hb.bizmrg.com s3'
>> s3="s3://tarantool_repo/$branch/$os"
>
>Minor: s3 variable name seems to be an ambiguous one here. IMHO, bucket
>looks the way better.
Renamed 's3' variable to 'bucket_path'
>
>> +aws_cp_to_s3="$aws cp --acl public-read"
>> +aws_sync_to_s3="$aws sync --acl public-read"
>
>Nice change, but other commands also interact with S3. I propose to
>change the naming to the following one:
>| <aws_<action>_public
Changed.
>
>> +
>> +update_packfile()
>> +{
>> + packfile=$1
>> + packtype=$2
>> +
>> + locpackfile=$(echo $packfile | sed "s#^$ws/##g")
>> + # register DEB/DSC pack file to Packages/Sources file
>> + reprepro -Vb . include$packtype $loop_dist $packfile
>
>Well, it's really bad idea to use external loop variable within
>function. You can pass it as an explicit argument, please do it.
Corrected.
>
>> + # reprepro copied DEB/DSC file to component which is not needed
>> + $rm_dir $debdir/$component
>> + # to have all sources avoid reprepro set DEB/DSC file to its own registry
>> + $rm_dir db
>> +}
>> +
>
>Minor: Since update_{packfile,metadata} are used only for deb packages,
>it can be showed in naming: update_deb_{packfile,metadata}.
Changed.
>
>
>> +update_metadata()
>> +{
>> + packpath=$1
>> + packtype=$2
>> +
>> + if [ ! -f $packpath.saved ] ; then
>> + # get the latest Sources file from S3
>> + $aws ls "$s3/$packpath" 2>/dev/null && \
>> + $aws cp "$s3/$packpath" $packpath.saved || \
>> + touch $packpath.saved
>> + fi
>> +
>> + if [ "$packtype" == "dsc" ]; then
>> + # WORKAROUND: unknown why, but reprepro doesn`t save the Sources file
>> + gunzip -c $packpath.gz >$packpath
>> + # check if the DSC file already exists in Sources from S3
>> + hash=$(grep '^Checksums-Sha256:' -A3 $packpath | \
>> + tail -n 1 | awk '{print $1}')
>> + if grep " $hash .*\.dsc$" $packpath.saved ; then
>> + echo "WARNING: DSC file already registered in S3!"
>> + return 1
>> + fi
>> + elif [ "$packtype" == "deb" ]; then
>> + # check if the DEB file already exists in Packages from S3
>> + if grep "^$(grep '^SHA256: ' $packages)$" $packages.saved ; then
>> + echo "WARNING: DEB file already registered in S3!"
>> + return 1
>> + fi
>> + fi
>> + # store the new DEB entry
>> + cat $packpath >>$packpath.saved
>> +}
>>
>> # The 'pack_deb' function especialy created for DEB packages. It works
>> # with DEB packing OS like Ubuntu, Debian. It is based on globaly known
>> @@ -158,32 +224,27 @@ function pack_deb {
>> # debian has special directory 'pool' for packages
>> debdir=pool
>>
>> - # get packages from pointed location either mirror path
>> - if [ "$repo" == "" ] ; then
>> - repo=/var/spool/apt-mirror/mirror/packagecloud.io/tarantool/$branch/$os
>> - fi
>> - if [ ! -d $repo/$debdir ] && ( [ "$DIST" == "" ] || ! ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ) ; then
>> + # get packages from pointed location
>> + if [ ! -d $repo/$debdir ] && \
>> + ( [ "$option_dist" == "" ] || \
>> + ! ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ) ; then
>> echo "ERROR: Current '$repo' path doesn't have any of the following:"
>> - echo "Usage with set distribuition with option '-d' and packages: $0 [path with *.deb *.dsc *.tar.*z files]"
>> - echo "Usage with repositories: $0 [path to repository with '$debdir' subdirectory]"
>> + echo " - $0 run option '-d' and DEB packages in path"
>> + echo " - 'pool' subdirectory with APT repositories"
>> + usage
>> exit 1
>> fi
>>
>> - # temporary lock the publication to the repository
>> - ws=${ws_prefix}_${branch}_${os}
>> - ws_lockfile=${ws}.lock
>> - create_lockfile $ws_lockfile
>> -
>> - # create temporary workspace with repository copy
>> - $rm_dir $ws
>> - $mk_dir $ws
>> + # prepare the workspace
>> + prepare_ws
>>
>> # script works in one of 2 modes:
>> # - path with binaries packages for a single distribution
>> # - path with 'pool' directory with APT repository
>> - if [ "$DIST" != "" ] && ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ; then
>> + if [ "$option_dist" != "" ] && \
>> + ls $repo/*.deb $repo/*.dsc $repo/*.tar.*z >/dev/null 2>&1 ; then
>> # copy single distribution with binaries packages
>> - repopath=$ws/pool/${DIST}/main/$proddir/$product
>> + repopath=$ws/pool/${option_dist}/$component/$proddir/$product
>> $mk_dir ${repopath}
>> cp $repo/*.deb $repo/*.dsc $repo/*.tar.*z $repopath/.
>> elif [ -d $repo/$debdir ]; then
>> @@ -191,10 +252,12 @@ function pack_deb {
>> cp -rf $repo/$debdir $ws/.
>> else
>> echo "ERROR: neither distribution option '-d' with files $repo/*.deb $repo/*.dsc $repo/*.tar.*z set nor '$repo/$debdir' path found"
>> - usage
>> - $rm_file $wslock
>> - exit 1
>> + usage
>> + $rm_file $wslock
>> + exit 1
>> fi
>> +
>> + swd=$PWD
>> cd $ws
>
>Minor: I see no reason to avoid using more fitting pushd/popd here.
Changed.
>
>
>>
>> # create the configuration file for 'reprepro' tool
>> @@ -202,14 +265,14 @@ function pack_deb {
>> $rm_dir $confpath
>> $mk_dir $confpath
>>
>> - for dist in $alldists ; do
>> + for loop_dist in $alldists ; do
>> cat <<EOF >>$confpath/distributions
>> Origin: Tarantool
>> Label: tarantool.org
>> Suite: stable
>> -Codename: $dist
>> +Codename: $loop_dist
>> Architectures: amd64 source
>> -Components: main
>> +Components: $component
>> Description: Unofficial Ubuntu Packages maintained by Tarantool
>> SignWith: 91B625E5
>> DebIndices: Packages Release . .gz .bz2
>> @@ -217,84 +280,49 @@ UDebIndices: Packages . .gz .bz2
>> DscIndices: Sources Release .gz .bz2
>>
>> EOF
>> -done
>> + done
>>
>> # create standalone repository with separate components
>> - for dist in $alldists ; do
>> - echo =================== DISTRIBUTION: $dist =========================
>> + for loop_dist in $alldists ; do
>> + echo ================ DISTRIBUTION: $loop_dist ====================
>> updated_deb=0
>> updated_dsc=0
>>
>> # 1(binaries). use reprepro tool to generate Packages file
>> - for deb in $ws/$debdir/$dist/$component/*/*/*.deb ; do
>> + for deb in $ws/$debdir/$loop_dist/$component/*/*/*.deb ; do
>> [ -f $deb ] || continue
>> - locdeb=`echo $deb | sed "s#^$ws\/##g"`
>> - echo "DEB: $deb"
>> - # register DEB file to Packages file
>> - reprepro -Vb . includedeb $dist $deb
>> - # reprepro copied DEB file to local component which is not needed
>> - $rm_dir $debdir/$component
>> - # to have all packages avoid reprepro set DEB file to its own registry
>> - $rm_dir db
>> - # copy Packages file to avoid of removing by the new DEB version
>> - for packages in dists/$dist/$component/binary-*/Packages ; do
>> - if [ ! -f $packages.saved ] ; then
>> - # get the latest Packages file from S3
>> - $aws s3 ls "$s3/$packages" 2>/dev/null && \
>> - $aws s3 cp --acl public-read \
>> - "$s3/$packages" $packages.saved || \
>> - touch $packages.saved
>> - fi
>> - # check if the DEB file already exists in Packages from S3
>> - if grep "^`grep "^SHA256: " $packages`$" $packages.saved ; then
>> - echo "WARNING: DEB file already registered in S3!"
>> - continue
>> - fi
>> - # store the new DEB entry
>> - cat $packages >>$packages.saved
>> - # save the registered DEB file to S3
>> - $aws s3 cp --acl public-read $deb $s3/$locdeb
>> + # regenerate DEB pack
>> + update_packfile $deb deb
>> + echo "Regenerated DEB file: $locpackfile"
>> + for packages in dists/$loop_dist/$component/binary-*/Packages ; do
>> + # copy Packages file to avoid of removing by the new DEB version
>> + # update metadata 'Packages' files
>> + if ! update_metadata $packages deb ; then continue ; fi
>
>I see the following approach to be more convenient for oneline if:
>| update_metadata $packages deb || continue
Changed.
>
>
>> updated_deb=1
>> done
>> + # save the registered DEB file to S3
>> + if [ "$updated_deb" == 1 ]; then
>> + $aws_cp_to_s3 $deb $s3/$locpackfile
>> + fi
>> done
>>
>> # 1(sources). use reprepro tool to generate Sources file
>> - for dsc in $ws/$debdir/$dist/$component/*/*/*.dsc ; do
>> + for dsc in $ws/$debdir/$loop_dist/$component/*/*/*.dsc ; do
>> [ -f $dsc ] || continue
>> - locdsc=`echo $dsc | sed "s#^$ws\/##g"`
>> - echo "DSC: $dsc"
>> - # register DSC file to Sources file
>> - reprepro -Vb . includedsc $dist $dsc
>> - # reprepro copied DSC file to component which is not needed
>> - $rm_dir $debdir/$component
>> - # to have all sources avoid reprepro set DSC file to its own registry
>> - $rm_dir db
>> + # regenerate DSC pack
>> + update_packfile $dsc dsc
>> + echo "Regenerated DSC file: $locpackfile"
>> # copy Sources file to avoid of removing by the new DSC version
>> - sources=dists/$dist/$component/source/Sources
>> - if [ ! -f $sources.saved ] ; then
>> - # get the latest Sources file from S3
>> - $aws s3 ls "$s3/$sources" && \
>> - $aws s3 cp --acl public-read "$s3/$sources" $sources.saved || \
>> - touch $sources.saved
>> - fi
>> - # WORKAROUND: unknown why, but reprepro doesn`t save the Sources file
>> - gunzip -c $sources.gz >$sources
>> - # check if the DSC file already exists in Sources from S3
>> - hash=`grep '^Checksums-Sha256:' -A3 $sources | \
>> - tail -n 1 | awk '{print $1}'`
>> - if grep " $hash .*\.dsc$" $sources.saved ; then
>> - echo "WARNING: DSC file already registered in S3!"
>> - continue
>> - fi
>> - # store the new DSC entry
>> - cat $sources >>$sources.saved
>> - # save the registered DSC file to S3
>> - $aws s3 cp --acl public-read $dsc $s3/$locdsc
>> - tarxz=`echo $locdsc | sed 's#\.dsc$#.debian.tar.xz#g'`
>> - $aws s3 cp --acl public-read $ws/$tarxz "$s3/$tarxz"
>> - orig=`echo $locdsc | sed 's#-1\.dsc$#.orig.tar.xz#g'`
>> - $aws s3 cp --acl public-read $ws/$orig "$s3/$orig"
>> + # update metadata 'Sources' file
>> + if ! update_metadata dists/$loop_dist/$component/source/Sources dsc \
>> + ; then continue ; fi
>
>I see the following approach to be more convenient for oneline if:
>| update_metadata $dist/$loop_dist/$component/source/Sources dsc || continue
Changed.
>
>
>> updated_dsc=1
>> + # save the registered DSC file to S3
>> + $aws_cp_to_s3 $dsc $s3/$locpackfile
>> + tarxz=$(echo $locpackfile | sed 's#\.dsc$#.debian.tar.xz#g')
>> + $aws_cp_to_s3 $ws/$tarxz "$s3/$tarxz"
>> + orig=$(echo $locpackfile | sed 's#-1\.dsc$#.orig.tar.xz#g')
>> + $aws_cp_to_s3 $ws/$orig "$s3/$orig"
>> done
>>
>> # check if any DEB/DSC files were newly registered
>> @@ -302,30 +330,30 @@ done
>> continue || echo "Updating dists"
>>
>> # finalize the Packages file
>> - for packages in dists/$dist/$component/binary-*/Packages ; do
>> + for packages in dists/$loop_dist/$component/binary-*/Packages ; do
>> mv $packages.saved $packages
>> done
>>
>> # 2(binaries). update Packages file archives
>> - for packpath in dists/$dist/$component/binary-* ; do
>> + for packpath in dists/$loop_dist/$component/binary-* ; do
>> pushd $packpath
>> - sed "s#Filename: $debdir/$component/#Filename: $debdir/$dist/$component/#g" -i Packages
>> + sed "s#Filename: $debdir/$component/#Filename: $debdir/$loop_dist/$component/#g" -i Packages
>> bzip2 -c Packages >Packages.bz2
>> gzip -c Packages >Packages.gz
>> popd
>> done
>>
>> # 2(sources). update Sources file archives
>> - pushd dists/$dist/$component/source
>> - sed "s#Directory: $debdir/$component/#Directory: $debdir/$dist/$component/#g" -i Sources
>> + pushd dists/$loop_dist/$component/source
>> + sed "s#Directory: $debdir/$component/#Directory: $debdir/$loop_dist/$component/#g" -i Sources
>> bzip2 -c Sources >Sources.bz2
>> gzip -c Sources >Sources.gz
>> popd
>>
>> # 3. update checksums entries of the Packages* files in *Release files
>> - # NOTE: it is stable structure of the *Release files when the checksum
>> - # entries in it in the following way:
>> - # MD5Sum:
>> + # NOTE: it is stable structure of the *Release files when the checksum
>> + # entries in it in the following way:
>> + # MD5Sum:
>> # <checksum> <size> <file orig>
>> # <checksum> <size> <file debian>
>> # SHA1:
>> @@ -334,14 +362,14 @@ done
>> # SHA256:
>> # <checksum> <size> <file orig>
>> # <checksum> <size> <file debian>
>> - # The script bellow puts 'md5' value at the 1st found file entry,
>> - # 'sha1' - at the 2nd and 'sha256' at the 3rd
>> - pushd dists/$dist
>> - for file in `grep " $component/" Release | awk '{print $3}' | sort -u` ; do
>> - sz=`stat -c "%s" $file`
>> - md5=`md5sum $file | awk '{print $1}'`
>> - sha1=`sha1sum $file | awk '{print $1}'`
>> - sha256=`sha256sum $file | awk '{print $1}'`
>> + # The script bellow puts 'md5' value at the 1st found file entry,
>> + # 'sha1' - at the 2nd and 'sha256' at the 3rd
>> + pushd dists/$loop_dist
>> + for file in $(grep " $component/" Release | awk '{print $3}' | sort -u) ; do
>> + sz=$(stat -c "%s" $file)
>> + md5=$(md5sum $file | awk '{print $1}')
>> + sha1=$(sha1sum $file | awk '{print $1}')
>> + sha256=$(sha256sum $file | awk '{print $1}')
>> awk 'BEGIN{c = 0} ; {
>> if ($3 == p) {
>> c = c + 1
>> @@ -350,7 +378,7 @@ done
>> if (c == 3) {print " " sh2 " " s " " p}
>> } else {print $0}
>> }' p="$file" s="$sz" md="$md5" sh1="$sha1" sh2="$sha256" \
>> - Release >Release.new
>> + Release >Release.new
>> mv Release.new Release
>> done
>> # resign the selfsigned InRelease file
>> @@ -362,11 +390,13 @@ done
>> popd
>>
>> # 4. sync the latest distribution path changes to S3
>> - $aws s3 sync --acl public-read dists/$dist "$s3/dists/$dist"
>> + $aws_sync_to_s3 dists/$loop_dist "$s3/dists/$loop_dist"
>> done
>>
>> # unlock the publishing
>> $rm_file $ws_lockfile
>> +
>> + cd $swd
>> }
>>
>> # The 'pack_rpm' function especialy created for RPM packages. It works
>> @@ -380,29 +410,26 @@ done
>> # http://mirrors.kernel.org/fedora/releases/30/Everything/x86_64/os/Packages/t/
>> function pack_rpm {
>> if ! ls $repo/*.rpm >/dev/null 2>&1 ; then
>> - echo "ERROR: Current '$repo' has:"
>> - ls -al $repo
>> - echo "Usage: $0 [path with *.rpm files]"
>> + echo "ERROR: Current '$repo' path doesn't have RPM packages in path"
>> + usage
>> exit 1
>> fi
>>
>> - # temporary lock the publication to the repository
>> - ws=${prefix_lockfile}_${branch}_${os}_${DIST}
>> - ws_lockfile=${ws}.lock
>> - create_lockfile $ws_lockfile
>> + # prepare the workspace
>> + prepare_ws
>>
>> - # create temporary workspace with packages copies
>> - $rm_dir $ws
>> - $mk_dir $ws
>> + # copy the needed package binaries to the workspace
>> cp $repo/*.rpm $ws/.
>> +
>> + swd=$PWD
>> cd $ws
>
>Minor: I see no reason to avoid using more fitting pushd/popd here.
Changed.
>
>
>>
>> # set the paths
>> - if [ "$os" == "centos" ]; then
>> - repopath=$DIST/os/x86_64
>> + if [ "$os" == "el" ]; then
>> + repopath=$option_dist/os/x86_64
>> rpmpath=Packages
>> elif [ "$os" == "fedora" ]; then
>> - repopath=releases/$DIST/Everything/x86_64/os
>> + repopath=releases/$option_dist/Everything/x86_64/os
>> rpmpath=Packages/$proddir
>> fi
>> packpath=$repopath/$rpmpath
>> @@ -410,53 +437,61 @@ function pack_rpm {
>> # prepare local repository with packages
>> $mk_dir $packpath
>> mv *.rpm $packpath/.
>> - cd $repopath
>> + cd $ws/$repopath
>
>Minor: As discussed offline $ws is always an absolute path, but the
>command above looks confusing to me considering the prior cd command.
>Feel free to ignore.
Removed '$ws' as suggested.
>
>
>>
>> # copy the current metadata files from S3
>> mkdir repodata.base
>> - for file in `$aws s3 ls $s3/$repopath/repodata/ | awk '{print $NF}'` ; do
>> - $aws s3 ls $s3/$repopath/repodata/$file || continue
>> - $aws s3 cp $s3/$repopath/repodata/$file repodata.base/$file
>> + for file in $($aws ls $s3/$repopath/repodata/ | awk '{print $NF}') ; do
>> + $aws ls $s3/$repopath/repodata/$file || continue
>> + $aws cp $s3/$repopath/repodata/$file repodata.base/$file
>> done
>>
>> # create the new repository metadata files
>> - createrepo --no-database --update --workers=2 --compress-type=gz --simple-md-filenames .
>> + createrepo --no-database --update --workers=2 \
>> + --compress-type=gz --simple-md-filenames .
>> mv repodata repodata.adding
>>
>> # merge metadata files
>> mkdir repodata
>> head -n 2 repodata.adding/repomd.xml >repodata/repomd.xml
>> for file in filelists.xml other.xml primary.xml ; do
>> - # 1. take the 1st line only - to skip the line with number of packages which is not needed
>> + # 1. take the 1st line only - to skip the line with
>> + # number of packages which is not needed
>> zcat repodata.adding/$file.gz | head -n 1 >repodata/$file
>> - # 2. take 2nd line with metadata tag and update the packages number in it
>> + # 2. take 2nd line with metadata tag and update
>> + # the packages number in it
>> packsold=0
>> if [ -f repodata.base/$file.gz ] ; then
>> - packsold=`zcat repodata.base/$file.gz | head -n 2 | tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g'`
>> + packsold=$(zcat repodata.base/$file.gz | head -n 2 | \
>> + tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g')
>> fi
>> - packsnew=`zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g'`
>> + packsnew=$(zcat repodata.adding/$file.gz | head -n 2 | \
>> + tail -n 1 | sed 's#.*packages="\(.*\)".*#\1#g')
>> packs=$(($packsold+$packsnew))
>> - zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | sed "s#packages=\".*\"#packages=\"$packs\"#g" >>repodata/$file
>> + zcat repodata.adding/$file.gz | head -n 2 | tail -n 1 | \
>> + sed "s#packages=\".*\"#packages=\"$packs\"#g" >>repodata/$file
>> # 3. take only 'package' tags from new file
>> - zcat repodata.adding/$file.gz | tail -n +3 | head -n -1 >>repodata/$file
>> + zcat repodata.adding/$file.gz | tail -n +3 | head -n -1 \
>> + >>repodata/$file
>> # 4. take only 'package' tags from old file if exists
>> if [ -f repodata.base/$file.gz ] ; then
>> - zcat repodata.base/$file.gz | tail -n +3 | head -n -1 >>repodata/$file
>> + zcat repodata.base/$file.gz | tail -n +3 | head -n -1 \
>> + >>repodata/$file
>> fi
>> # 5. take the last closing line with metadata tag
>> zcat repodata.adding/$file.gz | tail -n 1 >>repodata/$file
>>
>> # get the new data
>> - chsnew=`sha256sum repodata/$file | awk '{print $1}'`
>> - sz=`stat --printf="%s" repodata/$file`
>> + chsnew=$(sha256sum repodata/$file | awk '{print $1}')
>> + sz=$(stat --printf="%s" repodata/$file)
>> gzip repodata/$file
>> - chsgznew=`sha256sum repodata/$file.gz | awk '{print $1}'`
>> - szgz=`stat --printf="%s" repodata/$file.gz`
>> - timestamp=`date +%s -r repodata/$file.gz`
>> + chsgznew=$(sha256sum repodata/$file.gz | awk '{print $1}')
>> + szgz=$(stat --printf="%s" repodata/$file.gz)
>> + timestamp=$(date +%s -r repodata/$file.gz)
>>
>> # add info to repomd.xml file
>> - name=`echo $file | sed 's#\.xml$##g'`
>> - cat <<EOF >>repodata/repomd.xml
>> + name=$(echo $file | sed 's#\.xml$##g')
>> + cat <<EOF >>repodata/repomd.xml
>> <data type="$name">
>> <checksum type="sha256">$chsgznew</checksum>
>> <open-checksum type="sha256">$chsnew</open-checksum>
>> @@ -472,19 +507,21 @@ EOF
>>
>> # copy the packages to S3
>> for file in $rpmpath/*.rpm ; do
>> - $aws s3 cp --acl public-read $file "$s3/$repopath/$file"
>> + $aws_cp_to_s3 $file "$s3/$repopath/$file"
>> done
>>
>> # update the metadata at the S3
>> - $aws s3 sync --acl public-read repodata "$s3/$repopath/repodata"
>> + $aws_sync_to_s3 repodata "$s3/$repopath/repodata"
>>
>> # unlock the publishing
>> $rm_file $ws_lockfile
>> +
>> + cd $swd
>> }
>>
>> if [ "$os" == "ubuntu" -o "$os" == "debian" ]; then
>> pack_deb
>> -elif [ "$os" == "centos" -o "$os" == "fedora" ]; then
>> +elif [ "$os" == "el" -o "$os" == "fedora" ]; then
>> pack_rpm
>> else
>> echo "USAGE: given OS '$os' is not supported, use any single from the list: $alloss"
>> --
>> 2.17.1
>>
>
>--
>Best regards,
>IM
--
Alexander Tikhonov
[-- Attachment #2: Type: text/html, Size: 38623 bytes --]
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2020-01-13 11:36 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-12-16 4:46 [Tarantool-patches] [PATCH v3] Testing changes after 2nd review on S3 Alexander V. Tikhonov
2020-01-13 10:29 ` Igor Munkin
2020-01-13 11:36 ` Alexander Tikhonov
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox