From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtpng3.m.smailru.net (smtpng3.m.smailru.net [94.100.177.149]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 638944696C5 for ; Mon, 13 Jan 2020 15:04:43 +0300 (MSK) From: "Alexander V. Tikhonov" Date: Mon, 13 Jan 2020 15:04:37 +0300 Message-Id: <2c726e7f73906789f4d863d7bd3f3d194a24a610.1578916717.git.avtikhon@tarantool.org> In-Reply-To: References: In-Reply-To: References: Subject: [Tarantool-patches] [PATCH v4 3/3] Testing changes after 3rd review on S3 List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Igor Munkin Cc: tarantool-patches@dev.tarantool.org --- tools/add_pack_s3_repo.sh | 77 +++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/tools/add_pack_s3_repo.sh b/tools/add_pack_s3_repo.sh index 280a07ab0..148ffd0d9 100755 --- a/tools/add_pack_s3_repo.sh +++ b/tools/add_pack_s3_repo.sh @@ -11,13 +11,13 @@ product=tarantool # the path with binaries either repository repo=. -get_os_dists() +function get_os_dists() { os=$1 alldists= if [ "$os" == "ubuntu" ]; then - alldists='trusty xenial cosmic disco bionic eoan' + alldists='trusty xenial bionic cosmic disco eoan' elif [ "$os" == "debian" ]; then alldists='jessie stretch buster' elif [ "$os" == "el" ]; then @@ -29,13 +29,11 @@ get_os_dists() echo "$alldists" } -prepare_ws() +function prepare_ws() { # temporary lock the publication to the repository - ws=${ws_prefix}_${branch}_${os} - if [ "$os" != "ubuntu" -a "$os" != "debian" ]; then - ws=${ws}_${option_dist} - fi + ws_suffix=$1 + ws=${ws_prefix}_${branch}_${ws_suffix} ws_lockfile=${ws}.lock if [ -f $ws_lockfile ]; then old_proc=$(cat $ws_lockfile) @@ -51,7 +49,7 @@ prepare_ws() $mk_dir $ws } -usage() +function usage() { cat </dev/null && \ - $aws cp "$s3/$packpath" $packpath.saved || \ + $aws ls "$bucket_path/$packpath" 2>/dev/null && \ + $aws cp "$bucket_path/$packpath" $packpath.saved || \ touch $packpath.saved fi @@ -236,7 +235,7 @@ function pack_deb { fi # prepare the workspace - prepare_ws + prepare_ws ${os} # script works in one of 2 modes: # - path with binaries packages for a single distribution @@ -257,8 +256,7 @@ function pack_deb { exit 1 fi - swd=$PWD - cd $ws + pushd $ws # create the configuration file for 'reprepro' tool confpath=$ws/conf @@ -292,17 +290,17 @@ EOF for deb in $ws/$debdir/$loop_dist/$component/*/*/*.deb ; do [ -f $deb ] || continue # regenerate DEB pack - update_packfile $deb deb + update_deb_packfile $deb deb $loop_dist echo "Regenerated DEB file: $locpackfile" for packages in dists/$loop_dist/$component/binary-*/Packages ; do # copy Packages file to avoid of removing by the new DEB version # update metadata 'Packages' files - if ! update_metadata $packages deb ; then continue ; fi + ! update_deb_metadata $packages deb || continue updated_deb=1 done # save the registered DEB file to S3 if [ "$updated_deb" == 1 ]; then - $aws_cp_to_s3 $deb $s3/$locpackfile + $aws_cp_to_s3_public $deb $bucket_path/$locpackfile fi done @@ -310,19 +308,19 @@ EOF for dsc in $ws/$debdir/$loop_dist/$component/*/*/*.dsc ; do [ -f $dsc ] || continue # regenerate DSC pack - update_packfile $dsc dsc + update_deb_packfile $dsc dsc $loop_dist echo "Regenerated DSC file: $locpackfile" # copy Sources file to avoid of removing by the new DSC version # update metadata 'Sources' file - if ! update_metadata dists/$loop_dist/$component/source/Sources dsc \ - ; then continue ; fi + ! update_deb_metadata dists/$loop_dist/$component/source/Sources dsc \ + || continue updated_dsc=1 # save the registered DSC file to S3 - $aws_cp_to_s3 $dsc $s3/$locpackfile + $aws_cp_to_s3_public $dsc $bucket_path/$locpackfile tarxz=$(echo $locpackfile | sed 's#\.dsc$#.debian.tar.xz#g') - $aws_cp_to_s3 $ws/$tarxz "$s3/$tarxz" + $aws_cp_to_s3_public $ws/$tarxz "$bucket_path/$tarxz" orig=$(echo $locpackfile | sed 's#-1\.dsc$#.orig.tar.xz#g') - $aws_cp_to_s3 $ws/$orig "$s3/$orig" + $aws_cp_to_s3_public $ws/$orig "$bucket_path/$orig" done # check if any DEB/DSC files were newly registered @@ -390,13 +388,13 @@ EOF popd # 4. sync the latest distribution path changes to S3 - $aws_sync_to_s3 dists/$loop_dist "$s3/dists/$loop_dist" + $aws_sync_to_s3_public dists/$loop_dist "$bucket_path/dists/$loop_dist" done # unlock the publishing $rm_file $ws_lockfile - cd $swd + popd } # The 'pack_rpm' function especialy created for RPM packages. It works @@ -416,13 +414,12 @@ function pack_rpm { fi # prepare the workspace - prepare_ws + prepare_ws ${os}_${option_dist} # copy the needed package binaries to the workspace cp $repo/*.rpm $ws/. - swd=$PWD - cd $ws + pushd $ws # set the paths if [ "$os" == "el" ]; then @@ -437,13 +434,13 @@ function pack_rpm { # prepare local repository with packages $mk_dir $packpath mv *.rpm $packpath/. - cd $ws/$repopath + cd $repopath # copy the current metadata files from S3 mkdir repodata.base - for file in $($aws ls $s3/$repopath/repodata/ | awk '{print $NF}') ; do - $aws ls $s3/$repopath/repodata/$file || continue - $aws cp $s3/$repopath/repodata/$file repodata.base/$file + for file in $($aws ls $bucket_path/$repopath/repodata/ | awk '{print $NF}') ; do + $aws ls $bucket_path/$repopath/repodata/$file || continue + $aws cp $bucket_path/$repopath/repodata/$file repodata.base/$file done # create the new repository metadata files @@ -507,16 +504,16 @@ EOF # copy the packages to S3 for file in $rpmpath/*.rpm ; do - $aws_cp_to_s3 $file "$s3/$repopath/$file" + $aws_cp_to_s3_public $file "$bucket_path/$repopath/$file" done # update the metadata at the S3 - $aws_sync_to_s3 repodata "$s3/$repopath/repodata" + $aws_sync_to_s3_public repodata "$bucket_path/$repopath/repodata" # unlock the publishing $rm_file $ws_lockfile - cd $swd + popd } if [ "$os" == "ubuntu" -o "$os" == "debian" ]; then -- 2.17.1