From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp41.i.mail.ru (smtp41.i.mail.ru [94.100.177.101]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id BD643469719 for ; Tue, 22 Sep 2020 22:18:31 +0300 (MSK) From: "Alexander V. Tikhonov" Date: Tue, 22 Sep 2020 22:18:27 +0300 Message-Id: <795f8ba8ccda19103e0357d0edc920fc4765a97b.1600801955.git.avtikhon@tarantool.org> Subject: [Tarantool-patches] [PATCH v1] Enable test reruns on failed fragiled tests List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Kirill Yukhin Cc: tarantool-patches@dev.tarantool.org In test-run implemented the new format of the fragile lists based on JSON format set as fragile option in 'suite.ini' files per each suite: fragile = { "retries": 10, "tests": { "bitset.test.lua": { "issues": [ "gh-4095" ], "checksums": [ "050af3a99561a724013995668a4bc71c", "f34be60193cfe9221d3fe50df657e9d3" ] } }} Added ability to check results file checksum on tests fail and compare with the checksums of the known issues mentioned in the fragile list. Also added ability to set 'retries' option, which sets the number of accepted reruns of the tests failed from 'fragile' list that have checksums on its fails. Closes #5050 --- Github: https://github.com/tarantool/tarantool/tree/avtikhon/gh-5050-retries Issue: https://github.com/tarantool/tarantool/issues/5050 test/app-tap/suite.ini | 9 ++++- test/app/suite.ini | 17 +++++++-- test/box-py/suite.ini | 10 +++++- test/box-tap/suite.ini | 13 +++++-- test/box/suite.ini | 67 +++++++++++++++++++++++++++-------- test/engine/suite.ini | 15 ++++++-- test/replication-py/suite.ini | 17 +++++++-- test/replication/suite.ini | 58 +++++++++++++++++++++++------- test/sql/suite.ini | 9 ++++- test/vinyl/suite.ini | 49 +++++++++++++++++++------ test/wal_off/suite.ini | 9 ++++- test/xlog/suite.ini | 13 +++++-- 12 files changed, 232 insertions(+), 54 deletions(-) diff --git a/test/app-tap/suite.ini b/test/app-tap/suite.ini index f06f89ed5..ec665c397 100644 --- a/test/app-tap/suite.ini +++ b/test/app-tap/suite.ini @@ -5,4 +5,11 @@ lua_libs = lua/require_mod.lua lua/serializer_test.lua is_parallel = True pretest_clean = True use_unix_sockets_iproto = True -fragile = tarantoolctl.test.lua ; gh-5059 +fragile = { + "retries": 10, + "tests": { + "tarantoolctl.test.lua": { + "issues": [ "gh-5059" ] + } + } + } diff --git a/test/app/suite.ini b/test/app/suite.ini index b87afd2b8..21ca21d58 100644 --- a/test/app/suite.ini +++ b/test/app/suite.ini @@ -7,6 +7,17 @@ use_unix_sockets = True use_unix_sockets_iproto = True is_parallel = True pretest_clean = True -fragile = fiber_channel.test.lua ; gh-4961 - socket.test.lua ; gh-4978 - fiber.test.lua ; gh-4987 +fragile = { + "retries": 10, + "tests": { + "fiber_channel.test.lua": { + "issues": [ "gh-4961" ] + }, + "socket.test.lua": { + "issues": [ "gh-4978" ] + }, + "fiber.test.lua": { + "issues": [ "gh-4987" ] + } + } + } diff --git a/test/box-py/suite.ini b/test/box-py/suite.ini index d1ff0fbfd..346f9829c 100644 --- a/test/box-py/suite.ini +++ b/test/box-py/suite.ini @@ -5,4 +5,12 @@ script = box.lua lua_libs = lua/fiber.lua lua/fifo.lua use_unix_sockets = True is_parallel = True -fragile = snapshot.test.py ; gh-4514 +fragile = { + "retries": 10, + "tests": { + "snapshot.test.py": { + "issues": [ "gh-4514" ], + "snapshots": [ "0e9db9dd391ea124bc5181089350a36e" ] + } + } + } diff --git a/test/box-tap/suite.ini b/test/box-tap/suite.ini index 0f4f4a566..67b1e7d09 100644 --- a/test/box-tap/suite.ini +++ b/test/box-tap/suite.ini @@ -5,5 +5,14 @@ is_parallel = True pretest_clean = True use_unix_sockets_iproto = True config = suite.cfg -fragile = cfg.test.lua ; gh-4344 - key_def.test.lua ; gh-5049 +fragile = { + "retries": 10, + "tests": { + "cfg.test.lua": { + "issues": [ "gh-4344" ] + }, + "key_def.test.lua": { + "issues": [ "gh-5049" ] + } + } + } diff --git a/test/box/suite.ini b/test/box/suite.ini index a9ed671db..5c163a169 100644 --- a/test/box/suite.ini +++ b/test/box/suite.ini @@ -11,18 +11,55 @@ use_unix_sockets = True use_unix_sockets_iproto = True is_parallel = True pretest_clean = True -fragile = bitset.test.lua ; gh-4095 - func_reload.test.lua ; gh-4425 - function1.test.lua ; gh-4199 - push.test.lua ; gh-4882 - tree_pk.test.lua ; gh-4882 - alter_limits.test.lua ; gh-4926 - misc.test.lua ; gh-4982 - tuple.test.lua ; gh-4988 - transaction.test.lua ; gh-4990 - rtree_rect.test.lua ; gh-4994 - sequence.test.lua ; gh-4996 - on_replace.test.lua ; gh-4997 - role.test.lua ; gh-4998 - net.box_reconnect_after_gh-3164.test.lua ; gh-5081 - net.box_disconnect_gh-3859.test.lua ; gh-5056 +fragile = { + "retries": 10, + "tests": { + "bitset.test.lua": { + "issues": [ "gh-4095" ] + }, + "func_reload.test.lua": { + "issues": [ "gh-4425" ], + "checksums": [ "2e8206fbddc62d12332d71b82bbcce6d" ] + }, + "function1.test.lua": { + "issues": [ "gh-4199" ] + }, + "push.test.lua": { + "issues": [ "gh-4882" ] + }, + "tree_pk.test.lua": { + "issues": [ "gh-4882" ] + }, + "alter_limits.test.lua": { + "issues": [ "gh-4926" ] + }, + "misc.test.lua": { + "issues": [ "gh-4982" ] + }, + "tuple.test.lua": { + "issues": [ "gh-4988" ] + }, + "transaction.test.lua": { + "issues": [ "gh-4990" ] + }, + "rtree_rect.test.lua": { + "issues": [ "gh-4994" ] + }, + "sequence.test.lua": { + "issues": [ "gh-4996" ] + }, + "on_replace.test.lua": { + "issues": [ "gh-4997" ] + }, + "role.test.lua": { + "issues": [ "gh-4998" ] + }, + "net.box_reconnect_after_gh-3164.test.lua": { + "issues": [ "gh-5081" ] + }, + "net.box_disconnect_gh-3859.test.lua": { + "issues": [ "gh-5156" ], + "checksums": [ "345f55562f152e52d79f78e2920b28a2" ] + } + } + } diff --git a/test/engine/suite.ini b/test/engine/suite.ini index 4ee5e94da..01899f088 100644 --- a/test/engine/suite.ini +++ b/test/engine/suite.ini @@ -10,5 +10,16 @@ config = engine.cfg lua_libs = conflict.lua ../box/lua/utils.lua ../box/lua/push.lua is_parallel = True pretest_clean = True -fragile = ddl.test.lua ; gh-4353 - gh-4973-concurrent-alter-fails.test.lua ; gh-5157 +fragile = { + "retries": 10, + "tests": { + "ddl.test.lua": { + "issues": [ "gh-4353" ], + "checksums": [ "dd8851d80183cc75052119ba646e295d" ] + }, + "gh-4973-concurrent-alter-fails.test.lua": { + "issues": [ "gh-5157" ], + "checksums": [ "4e797e63335cebe24dab15eae4aa8044" ] + } + } + } diff --git a/test/replication-py/suite.ini b/test/replication-py/suite.ini index 45add028c..78d69f4d9 100644 --- a/test/replication-py/suite.ini +++ b/test/replication-py/suite.ini @@ -3,6 +3,17 @@ core = tarantool script = master.lua description = tarantool/box, replication is_parallel = True -fragile = init_storage.test.py ; gh-4949 - conflict.test.py ; gh-4980 - cluster.test.py ; gh-5109 +fragile = { + "retries": 10, + "tests": { + "init_storage.test.py": { + "issues": [ "gh-4949" ] + }, + "conflict.test.py": { + "issues": [ "gh-4980" ] + }, + "cluster.test.py": { + "issues": [ "gh-5109" ] + } + } + } diff --git a/test/replication/suite.ini b/test/replication/suite.ini index 109850f6b..fe6b2c398 100644 --- a/test/replication/suite.ini +++ b/test/replication/suite.ini @@ -11,15 +11,49 @@ use_unix_sockets_iproto = True long_run = prune.test.lua is_parallel = True pretest_clean = True -fragile = errinj.test.lua ; gh-3870 - long_row_timeout.test.lua ; gh-4351 - gh-3160-misc-heartbeats-on-master-changes.test.lua ; gh-4940 - skip_conflict_row.test.lua ; gh-4958 - sync.test.lua ; gh-3835 - transaction.test.lua ; gh-4312 - autobootstrap.test.lua ; gh-4533 - replica_rejoin.test.lua ; gh-4985 - recover_missing_xlog.test.lua ; gh-4989 - box_set_replication_stress.test.lua ; gh-4992 gh-4986 - gh-4605-empty-password.test.lua ; gh-5030 - anon.test.lua ; gh-5058 +fragile = { + "retries": 10, + "tests": { + "errinj.test.lua": { + "issues": [ "gh-3870" ] + }, + "long_row_timeout.test.lua": { + "issues": [ "gh-4351" ] + }, + "gh-3160-misc-heartbeats-on-master-changes.test.lua": { + "issues": [ "gh-4940" ] + }, + "skip_conflict_row.test.lua": { + "issues": [ "gh-4958" ] + }, + "sync.test.lua": { + "issues": [ "gh-3835" ] + }, + "transaction.test.lua": { + "issues": [ "gh-4312" ] + }, + "autobootstrap.test.lua": { + "issues": [ "gh-4533" ] + }, + "replica_rejoin.test.lua": { + "issues": [ "gh-4985" ] + }, + "recover_missing_xlog.test.lua": { + "issues": [ "gh-4989" ] + }, + "box_set_replication_stress.test.lua": { + "issues": [ "gh-4992", "gh-4986" ], + "checksums": [ "58cd2e36248c230e96c02397af5d7dbd" ] + }, + "gh-4605-empty-password.test.lua": { + "issues": [ "gh-5030" ] + }, + "anon.test.lua": { + "issues": [ "gh-5058" ] + }, + "on_schema_init.test.lua": { + "issues": [ "gh-5291" ], + "checksums": [ "1cb01c103258e26c8a80458f6c40fd44" ] + } + } + } diff --git a/test/sql/suite.ini b/test/sql/suite.ini index 57733deec..de5ab1537 100644 --- a/test/sql/suite.ini +++ b/test/sql/suite.ini @@ -10,4 +10,11 @@ lua_libs = lua/sql_tokenizer.lua release_disabled = errinj.test.lua view_delayed_wal.test.lua disabled = sql-statN-index-drop.test.lua pretest_clean = True -fragile = func-recreate.test.lua ; gh-4384 +fragile = { + "retries": 10, + "tests": { + "func-recreate.test.lua": { + "issues": [ "gh-4384" ] + } + } + } diff --git a/test/vinyl/suite.ini b/test/vinyl/suite.ini index 3e11b087b..475e8b475 100644 --- a/test/vinyl/suite.ini +++ b/test/vinyl/suite.ini @@ -12,14 +12,41 @@ is_parallel = True # throttle.test.lua temporary disabled for gh-4168 disabled = upgrade.test.lua throttle.test.lua pretest_clean = True -fragile = errinj.test.lua ; gh-4346 - select_consistency.test.lua ; gh-4385 - throttle.test.lua ; gh-4168 - stat.test.lua ; gh-4951 - misc.test.lua ; gh-4979 - snapshot.test.lua ; gh-4984 - write_iterator.test.lua ; gh-4572 - errinj_ddl.test.lua ; gh-4993 - gh-4810-dump-during-index-build.test.lua ; gh-5031 - deferred_delete.test.lua ; gh-5089 - gh.test.lua ; gh-5141 +fragile = { + "retries": 10, + "tests": { + "errinj.test.lua": { + "issues": [ "gh-4346" ] + }, + "select_consistency.test.lua": { + "issues": [ "gh-4385" ] + }, + "throttle.test.lua": { + "issues": [ "gh-4168" ] + }, + "stat.test.lua": { + "issues": [ "gh-4951" ] + }, + "misc.test.lua": { + "issues": [ "gh-4979" ] + }, + "snapshot.test.lua": { + "issues": [ "gh-4984" ] + }, + "write_iterator.test.lua": { + "issues": [ "gh-4572" ] + }, + "errinj_ddl.test.lua": { + "issues": [ "gh-4993" ] + }, + "gh-4810-dump-during-index-build.test.lua": { + "issues": [ "gh-5031" ] + }, + "deferred_delete.test.lua": { + "issues": [ "gh-5089" ] + }, + "gh.test.lua": { + "issues": [ "gh-5141" ] + } + } + } diff --git a/test/wal_off/suite.ini b/test/wal_off/suite.ini index ed77be157..303584ed7 100644 --- a/test/wal_off/suite.ini +++ b/test/wal_off/suite.ini @@ -7,4 +7,11 @@ use_unix_sockets_iproto = True is_parallel = True pretest_clean = True disabled = iterator_lt_gt.test.lua -fragile = iterator_lt_gt.test.lua ; gh-3925 +fragile = { + "retries": 10, + "tests": { + "iterator_lt_gt.test.lua": { + "issues": [ "gh-3925" ] + } + } + } diff --git a/test/xlog/suite.ini b/test/xlog/suite.ini index 5143650e4..e99220242 100644 --- a/test/xlog/suite.ini +++ b/test/xlog/suite.ini @@ -10,5 +10,14 @@ use_unix_sockets_iproto = True long_run = snap_io_rate.test.lua is_parallel = True pretest_clean = True -fragile = checkpoint_daemon.test.lua ; gh-4952 - panic_on_broken_lsn.test.lua ; gh-4991 +fragile = { + "retries": 10, + "tests": { + "checkpoint_daemon.test.lua": { + "issues": [ "gh-4952" ] + }, + "panic_on_broken_lsn.test.lua": { + "issues": [ "gh-4991" ] + } + } + } -- 2.17.1