From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp41.i.mail.ru (smtp41.i.mail.ru [94.100.177.101]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 579314696C1 for ; Mon, 8 Jun 2020 18:27:41 +0300 (MSK) From: sergeyb@tarantool.org Date: Mon, 8 Jun 2020 18:25:32 +0300 Message-Id: <37fb3b360327ef0f93d5074153adda9e61eda697.1591629414.git.sergeyb@tarantool.org> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [PATCH v3 2/2] Add option to update file with reference output List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: tarantool-patches@dev.tarantool.org, alexander.turenko@tarantool.org From: Sergey Bronnikov New option covers two cases: - in case of test failure test-run.py create a file .reject with actual test output and one need to move .reject file to .result manually when test has a valid behaviour. With option --update-result test-run.py will do it automatically. - in case of abcense reference result file test-run.py forced to create such file in a source directory and set test status "new". This patch changes behaviour. With option --update-result test status is "new" and result file is created, without option test status is "fail" and result file is not created. Fixes https://github.com/tarantool/tarantool/issues/4654 Fixes https://github.com/tarantool/tarantool/issues/4258 Closes #194 --- lib/options.py | 8 ++++++++ lib/test.py | 22 +++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/options.py b/lib/options.py index 8bacb4a..47bbc0f 100644 --- a/lib/options.py +++ b/lib/options.py @@ -201,6 +201,14 @@ class Options: help="""Run the server under 'luacov'. Default: false.""") + parser.add_argument( + "--update-result", + dest="update_result", + action="store_true", + default=False, + help="""Update or create file with reference output (.result). + Default: false.""") + # XXX: We can use parser.parse_intermixed_args() on # Python 3.7 to understand commands like # ./test-run.py foo --exclude bar baz diff --git a/lib/test.py b/lib/test.py index 8bc1feb..be57224 100644 --- a/lib/test.py +++ b/lib/test.py @@ -152,7 +152,7 @@ class Test(object): it to stdout. Returns short status of the test as a string: 'skip', 'pass', - 'new', or 'fail'. There is also one possible value for + 'new', 'updated' or 'fail'. There is also one possible value for short_status, 'disabled', but it returned in the caller, TestSuite.run_test(). """ @@ -232,12 +232,18 @@ class Test(object): color_stdout("[ pass ]\n", schema='test_pass') if os.path.exists(self.tmp_result): os.remove(self.tmp_result) - elif (self.is_executed_ok and not - self.is_equal_result and not - os.path.isfile(self.result)) and not is_tap: + elif (self.is_executed_ok and + not self.is_equal_result and + not os.path.isfile(self.result) and not is_tap): shutil.copy(self.tmp_result, self.result) short_status = 'new' color_stdout("[ new ]\n", schema='test_new') + elif (self.is_executed_ok and + not self.is_equal_result and + lib.Options().args.update_result): + shutil.copy(self.tmp_result, self.result) + short_status = 'updated' + color_stdout("[ updated ]\n", schema='test_new') else: has_result = os.path.exists(self.tmp_result) if has_result: @@ -321,10 +327,16 @@ class Test(object): try: tap.parse(content) except ValueError as e: - color_stdout('\nTAP13 parse failed: %s\n' % str(e), + color_stdout('\nTAP13 parse failed (%s).\n' % str(e), schema='error') + color_stdout('\nNo result file (%s) found.\n' % self.result, + schema='error') + if not lib.Options().args.update_result: + msg = 'Run the test with --update-result option to write the new result file.\n' + color_stdout(msg, schema='error') self.is_crash_reported = True return False, False + is_ok = True for test_case in tap.tests: if test_case.result == 'ok': -- 2.23.0