From: "Alexander V. Tikhonov" <avtikhon@tarantool.org>
To: Kirill Yukhin <kyukhin@tarantool.org>
Cc: tarantool-patches@dev.tarantool.org
Subject: [Tarantool-patches] [PATCH v1] Enable test reruns on failed fragiled tests
Date: Tue, 22 Sep 2020 22:18:27 +0300 [thread overview]
Message-ID: <795f8ba8ccda19103e0357d0edc920fc4765a97b.1600801955.git.avtikhon@tarantool.org> (raw)
In test-run implemented the new format of the fragile lists based on
JSON format set as fragile option in 'suite.ini' files per each suite:
fragile = {
"retries": 10,
"tests": {
"bitset.test.lua": {
"issues": [ "gh-4095" ],
"checksums": [ "050af3a99561a724013995668a4bc71c", "f34be60193cfe9221d3fe50df657e9d3" ]
}
}}
Added ability to check results file checksum on tests fail and
compare with the checksums of the known issues mentioned in the
fragile list.
Also added ability to set 'retries' option, which sets the number
of accepted reruns of the tests failed from 'fragile' list that
have checksums on its fails.
Closes #5050
---
Github: https://github.com/tarantool/tarantool/tree/avtikhon/gh-5050-retries
Issue: https://github.com/tarantool/tarantool/issues/5050
test/app-tap/suite.ini | 9 ++++-
test/app/suite.ini | 17 +++++++--
test/box-py/suite.ini | 10 +++++-
test/box-tap/suite.ini | 13 +++++--
test/box/suite.ini | 67 +++++++++++++++++++++++++++--------
test/engine/suite.ini | 15 ++++++--
test/replication-py/suite.ini | 17 +++++++--
test/replication/suite.ini | 58 +++++++++++++++++++++++-------
test/sql/suite.ini | 9 ++++-
test/vinyl/suite.ini | 49 +++++++++++++++++++------
test/wal_off/suite.ini | 9 ++++-
test/xlog/suite.ini | 13 +++++--
12 files changed, 232 insertions(+), 54 deletions(-)
diff --git a/test/app-tap/suite.ini b/test/app-tap/suite.ini
index f06f89ed5..ec665c397 100644
--- a/test/app-tap/suite.ini
+++ b/test/app-tap/suite.ini
@@ -5,4 +5,11 @@ lua_libs = lua/require_mod.lua lua/serializer_test.lua
is_parallel = True
pretest_clean = True
use_unix_sockets_iproto = True
-fragile = tarantoolctl.test.lua ; gh-5059
+fragile = {
+ "retries": 10,
+ "tests": {
+ "tarantoolctl.test.lua": {
+ "issues": [ "gh-5059" ]
+ }
+ }
+ }
diff --git a/test/app/suite.ini b/test/app/suite.ini
index b87afd2b8..21ca21d58 100644
--- a/test/app/suite.ini
+++ b/test/app/suite.ini
@@ -7,6 +7,17 @@ use_unix_sockets = True
use_unix_sockets_iproto = True
is_parallel = True
pretest_clean = True
-fragile = fiber_channel.test.lua ; gh-4961
- socket.test.lua ; gh-4978
- fiber.test.lua ; gh-4987
+fragile = {
+ "retries": 10,
+ "tests": {
+ "fiber_channel.test.lua": {
+ "issues": [ "gh-4961" ]
+ },
+ "socket.test.lua": {
+ "issues": [ "gh-4978" ]
+ },
+ "fiber.test.lua": {
+ "issues": [ "gh-4987" ]
+ }
+ }
+ }
diff --git a/test/box-py/suite.ini b/test/box-py/suite.ini
index d1ff0fbfd..346f9829c 100644
--- a/test/box-py/suite.ini
+++ b/test/box-py/suite.ini
@@ -5,4 +5,12 @@ script = box.lua
lua_libs = lua/fiber.lua lua/fifo.lua
use_unix_sockets = True
is_parallel = True
-fragile = snapshot.test.py ; gh-4514
+fragile = {
+ "retries": 10,
+ "tests": {
+ "snapshot.test.py": {
+ "issues": [ "gh-4514" ],
+ "snapshots": [ "0e9db9dd391ea124bc5181089350a36e" ]
+ }
+ }
+ }
diff --git a/test/box-tap/suite.ini b/test/box-tap/suite.ini
index 0f4f4a566..67b1e7d09 100644
--- a/test/box-tap/suite.ini
+++ b/test/box-tap/suite.ini
@@ -5,5 +5,14 @@ is_parallel = True
pretest_clean = True
use_unix_sockets_iproto = True
config = suite.cfg
-fragile = cfg.test.lua ; gh-4344
- key_def.test.lua ; gh-5049
+fragile = {
+ "retries": 10,
+ "tests": {
+ "cfg.test.lua": {
+ "issues": [ "gh-4344" ]
+ },
+ "key_def.test.lua": {
+ "issues": [ "gh-5049" ]
+ }
+ }
+ }
diff --git a/test/box/suite.ini b/test/box/suite.ini
index a9ed671db..5c163a169 100644
--- a/test/box/suite.ini
+++ b/test/box/suite.ini
@@ -11,18 +11,55 @@ use_unix_sockets = True
use_unix_sockets_iproto = True
is_parallel = True
pretest_clean = True
-fragile = bitset.test.lua ; gh-4095
- func_reload.test.lua ; gh-4425
- function1.test.lua ; gh-4199
- push.test.lua ; gh-4882
- tree_pk.test.lua ; gh-4882
- alter_limits.test.lua ; gh-4926
- misc.test.lua ; gh-4982
- tuple.test.lua ; gh-4988
- transaction.test.lua ; gh-4990
- rtree_rect.test.lua ; gh-4994
- sequence.test.lua ; gh-4996
- on_replace.test.lua ; gh-4997
- role.test.lua ; gh-4998
- net.box_reconnect_after_gh-3164.test.lua ; gh-5081
- net.box_disconnect_gh-3859.test.lua ; gh-5056
+fragile = {
+ "retries": 10,
+ "tests": {
+ "bitset.test.lua": {
+ "issues": [ "gh-4095" ]
+ },
+ "func_reload.test.lua": {
+ "issues": [ "gh-4425" ],
+ "checksums": [ "2e8206fbddc62d12332d71b82bbcce6d" ]
+ },
+ "function1.test.lua": {
+ "issues": [ "gh-4199" ]
+ },
+ "push.test.lua": {
+ "issues": [ "gh-4882" ]
+ },
+ "tree_pk.test.lua": {
+ "issues": [ "gh-4882" ]
+ },
+ "alter_limits.test.lua": {
+ "issues": [ "gh-4926" ]
+ },
+ "misc.test.lua": {
+ "issues": [ "gh-4982" ]
+ },
+ "tuple.test.lua": {
+ "issues": [ "gh-4988" ]
+ },
+ "transaction.test.lua": {
+ "issues": [ "gh-4990" ]
+ },
+ "rtree_rect.test.lua": {
+ "issues": [ "gh-4994" ]
+ },
+ "sequence.test.lua": {
+ "issues": [ "gh-4996" ]
+ },
+ "on_replace.test.lua": {
+ "issues": [ "gh-4997" ]
+ },
+ "role.test.lua": {
+ "issues": [ "gh-4998" ]
+ },
+ "net.box_reconnect_after_gh-3164.test.lua": {
+ "issues": [ "gh-5081" ]
+ },
+ "net.box_disconnect_gh-3859.test.lua": {
+ "issues": [ "gh-5156" ],
+ "checksums": [ "345f55562f152e52d79f78e2920b28a2" ]
+ }
+ }
+ }
diff --git a/test/engine/suite.ini b/test/engine/suite.ini
index 4ee5e94da..01899f088 100644
--- a/test/engine/suite.ini
+++ b/test/engine/suite.ini
@@ -10,5 +10,16 @@ config = engine.cfg
lua_libs = conflict.lua ../box/lua/utils.lua ../box/lua/push.lua
is_parallel = True
pretest_clean = True
-fragile = ddl.test.lua ; gh-4353
- gh-4973-concurrent-alter-fails.test.lua ; gh-5157
+fragile = {
+ "retries": 10,
+ "tests": {
+ "ddl.test.lua": {
+ "issues": [ "gh-4353" ],
+ "checksums": [ "dd8851d80183cc75052119ba646e295d" ]
+ },
+ "gh-4973-concurrent-alter-fails.test.lua": {
+ "issues": [ "gh-5157" ],
+ "checksums": [ "4e797e63335cebe24dab15eae4aa8044" ]
+ }
+ }
+ }
diff --git a/test/replication-py/suite.ini b/test/replication-py/suite.ini
index 45add028c..78d69f4d9 100644
--- a/test/replication-py/suite.ini
+++ b/test/replication-py/suite.ini
@@ -3,6 +3,17 @@ core = tarantool
script = master.lua
description = tarantool/box, replication
is_parallel = True
-fragile = init_storage.test.py ; gh-4949
- conflict.test.py ; gh-4980
- cluster.test.py ; gh-5109
+fragile = {
+ "retries": 10,
+ "tests": {
+ "init_storage.test.py": {
+ "issues": [ "gh-4949" ]
+ },
+ "conflict.test.py": {
+ "issues": [ "gh-4980" ]
+ },
+ "cluster.test.py": {
+ "issues": [ "gh-5109" ]
+ }
+ }
+ }
diff --git a/test/replication/suite.ini b/test/replication/suite.ini
index 109850f6b..fe6b2c398 100644
--- a/test/replication/suite.ini
+++ b/test/replication/suite.ini
@@ -11,15 +11,49 @@ use_unix_sockets_iproto = True
long_run = prune.test.lua
is_parallel = True
pretest_clean = True
-fragile = errinj.test.lua ; gh-3870
- long_row_timeout.test.lua ; gh-4351
- gh-3160-misc-heartbeats-on-master-changes.test.lua ; gh-4940
- skip_conflict_row.test.lua ; gh-4958
- sync.test.lua ; gh-3835
- transaction.test.lua ; gh-4312
- autobootstrap.test.lua ; gh-4533
- replica_rejoin.test.lua ; gh-4985
- recover_missing_xlog.test.lua ; gh-4989
- box_set_replication_stress.test.lua ; gh-4992 gh-4986
- gh-4605-empty-password.test.lua ; gh-5030
- anon.test.lua ; gh-5058
+fragile = {
+ "retries": 10,
+ "tests": {
+ "errinj.test.lua": {
+ "issues": [ "gh-3870" ]
+ },
+ "long_row_timeout.test.lua": {
+ "issues": [ "gh-4351" ]
+ },
+ "gh-3160-misc-heartbeats-on-master-changes.test.lua": {
+ "issues": [ "gh-4940" ]
+ },
+ "skip_conflict_row.test.lua": {
+ "issues": [ "gh-4958" ]
+ },
+ "sync.test.lua": {
+ "issues": [ "gh-3835" ]
+ },
+ "transaction.test.lua": {
+ "issues": [ "gh-4312" ]
+ },
+ "autobootstrap.test.lua": {
+ "issues": [ "gh-4533" ]
+ },
+ "replica_rejoin.test.lua": {
+ "issues": [ "gh-4985" ]
+ },
+ "recover_missing_xlog.test.lua": {
+ "issues": [ "gh-4989" ]
+ },
+ "box_set_replication_stress.test.lua": {
+ "issues": [ "gh-4992", "gh-4986" ],
+ "checksums": [ "58cd2e36248c230e96c02397af5d7dbd" ]
+ },
+ "gh-4605-empty-password.test.lua": {
+ "issues": [ "gh-5030" ]
+ },
+ "anon.test.lua": {
+ "issues": [ "gh-5058" ]
+ },
+ "on_schema_init.test.lua": {
+ "issues": [ "gh-5291" ],
+ "checksums": [ "1cb01c103258e26c8a80458f6c40fd44" ]
+ }
+ }
+ }
diff --git a/test/sql/suite.ini b/test/sql/suite.ini
index 57733deec..de5ab1537 100644
--- a/test/sql/suite.ini
+++ b/test/sql/suite.ini
@@ -10,4 +10,11 @@ lua_libs = lua/sql_tokenizer.lua
release_disabled = errinj.test.lua view_delayed_wal.test.lua
disabled = sql-statN-index-drop.test.lua
pretest_clean = True
-fragile = func-recreate.test.lua ; gh-4384
+fragile = {
+ "retries": 10,
+ "tests": {
+ "func-recreate.test.lua": {
+ "issues": [ "gh-4384" ]
+ }
+ }
+ }
diff --git a/test/vinyl/suite.ini b/test/vinyl/suite.ini
index 3e11b087b..475e8b475 100644
--- a/test/vinyl/suite.ini
+++ b/test/vinyl/suite.ini
@@ -12,14 +12,41 @@ is_parallel = True
# throttle.test.lua temporary disabled for gh-4168
disabled = upgrade.test.lua throttle.test.lua
pretest_clean = True
-fragile = errinj.test.lua ; gh-4346
- select_consistency.test.lua ; gh-4385
- throttle.test.lua ; gh-4168
- stat.test.lua ; gh-4951
- misc.test.lua ; gh-4979
- snapshot.test.lua ; gh-4984
- write_iterator.test.lua ; gh-4572
- errinj_ddl.test.lua ; gh-4993
- gh-4810-dump-during-index-build.test.lua ; gh-5031
- deferred_delete.test.lua ; gh-5089
- gh.test.lua ; gh-5141
+fragile = {
+ "retries": 10,
+ "tests": {
+ "errinj.test.lua": {
+ "issues": [ "gh-4346" ]
+ },
+ "select_consistency.test.lua": {
+ "issues": [ "gh-4385" ]
+ },
+ "throttle.test.lua": {
+ "issues": [ "gh-4168" ]
+ },
+ "stat.test.lua": {
+ "issues": [ "gh-4951" ]
+ },
+ "misc.test.lua": {
+ "issues": [ "gh-4979" ]
+ },
+ "snapshot.test.lua": {
+ "issues": [ "gh-4984" ]
+ },
+ "write_iterator.test.lua": {
+ "issues": [ "gh-4572" ]
+ },
+ "errinj_ddl.test.lua": {
+ "issues": [ "gh-4993" ]
+ },
+ "gh-4810-dump-during-index-build.test.lua": {
+ "issues": [ "gh-5031" ]
+ },
+ "deferred_delete.test.lua": {
+ "issues": [ "gh-5089" ]
+ },
+ "gh.test.lua": {
+ "issues": [ "gh-5141" ]
+ }
+ }
+ }
diff --git a/test/wal_off/suite.ini b/test/wal_off/suite.ini
index ed77be157..303584ed7 100644
--- a/test/wal_off/suite.ini
+++ b/test/wal_off/suite.ini
@@ -7,4 +7,11 @@ use_unix_sockets_iproto = True
is_parallel = True
pretest_clean = True
disabled = iterator_lt_gt.test.lua
-fragile = iterator_lt_gt.test.lua ; gh-3925
+fragile = {
+ "retries": 10,
+ "tests": {
+ "iterator_lt_gt.test.lua": {
+ "issues": [ "gh-3925" ]
+ }
+ }
+ }
diff --git a/test/xlog/suite.ini b/test/xlog/suite.ini
index 5143650e4..e99220242 100644
--- a/test/xlog/suite.ini
+++ b/test/xlog/suite.ini
@@ -10,5 +10,14 @@ use_unix_sockets_iproto = True
long_run = snap_io_rate.test.lua
is_parallel = True
pretest_clean = True
-fragile = checkpoint_daemon.test.lua ; gh-4952
- panic_on_broken_lsn.test.lua ; gh-4991
+fragile = {
+ "retries": 10,
+ "tests": {
+ "checkpoint_daemon.test.lua": {
+ "issues": [ "gh-4952" ]
+ },
+ "panic_on_broken_lsn.test.lua": {
+ "issues": [ "gh-4991" ]
+ }
+ }
+ }
--
2.17.1
next reply other threads:[~2020-09-22 19:18 UTC|newest]
Thread overview: 2+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-09-22 19:18 Alexander V. Tikhonov [this message]
2020-09-25 15:58 ` Kirill Yukhin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=795f8ba8ccda19103e0357d0edc920fc4765a97b.1600801955.git.avtikhon@tarantool.org \
--to=avtikhon@tarantool.org \
--cc=kyukhin@tarantool.org \
--cc=tarantool-patches@dev.tarantool.org \
--subject='Re: [Tarantool-patches] [PATCH v1] Enable test reruns on failed fragiled tests' \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox